pmret/papermario
1
0
Fork 0
mirror of https://github.com/pmret/papermario.git synced 2024-11-08 12:02:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

Snug bugs unrelated to and never been inside of a rug (#1082)

Browse Source 
* Fix enum name, offset

* Fix bugs

Some assets were slipping by the asset stack
Tex archve building wasn't respecting the asset stack (Fixes #1074)

* Fixes #1081

* fix paths kinda

* git subrepo pull --force tools/splat

subrepo:
  subdir:   "tools/splat"
  merged:   "818924683b"
upstream:
  origin:   "https://github.com/ethteck/splat.git"
  branch:   "master"
  commit:   "818924683b"
git-subrepo:
  version:  "0.4.5"
  origin:   "https://github.com/ingydotnet/git-subrepo"
  commit:   "aa416e4"

* Fix stuff after splupdate
This commit is contained in:
Ethan Roseman 2023-07-13 17:56:16 +09:00 committed by GitHub
parent 4f77ffbc3e
commit f91fe539a4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
29 changed files with 1096 additions and 462 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
include/enums.h
View File

@ -5941,7 +5941,7 @@ enum FileMenuMessages {
/* 33 */ FILE_MESSAGE_QUESTION, // ?[End] /* 33 */ FILE_MESSAGE_QUESTION, // ?[End]
/* 34 */ FILE_MESSAGE_PERIOD_34, // .[End] /* 34 */ FILE_MESSAGE_PERIOD_34, // .[End]
#if VERSION_PAL #if VERSION_PAL
UNK3, FILE_MESSAGE_BASE_UNK,
#endif #endif
}; };

6
src/filemenu/filemenu_main.c
View File

@ -567,7 +567,7 @@ void filemenu_draw_contents_file_info(s32 fileIdx,
#if VERSION_PAL #if VERSION_PAL
xOffset = D_filemenu_802508D8[gCurrentLanguage]; xOffset = D_filemenu_802508D8[gCurrentLanguage];
#else #else
xOffset = 0x22; xOffset = 34;
#endif #endif
filemenu_draw_message(filemenu_get_menu_message(FILE_MESSAGE_LEVEL), baseX + xOffset, baseY + 10, 255, 0xA, 1); filemenu_draw_message(filemenu_get_menu_message(FILE_MESSAGE_LEVEL), baseX + xOffset, baseY + 10, 255, 0xA, 1);
temp_s3_2 = gSaveSlotMetadata[fileIdx].level; temp_s3_2 = gSaveSlotMetadata[fileIdx].level;
@ -625,12 +625,12 @@ void filemenu_draw_contents_file_title(
filemenu_draw_message(filemenu_get_menu_message(FILE_MESSAGE_OK), baseX + FILE_X, baseY + 1, 255, 0, 1); filemenu_draw_message(filemenu_get_menu_message(FILE_MESSAGE_OK), baseX + FILE_X, baseY + 1, 255, 0, 1);
if (!gSaveSlotHasData[fileIdx]) { if (!gSaveSlotHasData[fileIdx]) {
filemenu_draw_message(filemenu_get_menu_message(fileIdx + UNK3), filemenu_draw_message(filemenu_get_menu_message(fileIdx + FILE_MESSAGE_BASE_UNK),
baseX + D_filemenu_802508D0[gCurrentLanguage], baseY + 1, 255, 0, 1); baseX + D_filemenu_802508D0[gCurrentLanguage], baseY + 1, 255, 0, 1);
} else { } else {
s32 tmp = D_filemenu_802508D0[gCurrentLanguage]; s32 tmp = D_filemenu_802508D0[gCurrentLanguage];
filemenu_draw_message(filemenu_get_menu_message(fileIdx + UNK3), filemenu_draw_message(filemenu_get_menu_message(fileIdx + FILE_MESSAGE_BASE_UNK),
baseX + tmp, baseY + 1, 255, 0, 1); baseX + tmp, baseY + 1, 255, 0, 1);
tmp += D_filemenu_802508D4[gCurrentLanguage]; tmp += D_filemenu_802508D4[gCurrentLanguage];

22
tools/build/configure.py
View File

@ -16,6 +16,9 @@ DO_SHA1_CHECK = True
# Paths: # Paths:
ROOT = Path(__file__).parent.parent.parent ROOT = Path(__file__).parent.parent.parent
if ROOT.is_absolute():
ROOT = ROOT.relative_to(Path.cwd())
BUILD_TOOLS = Path("tools/build") BUILD_TOOLS = Path("tools/build")
YAY0_COMPRESS_TOOL = f"{BUILD_TOOLS}/yay0/Yay0compress" YAY0_COMPRESS_TOOL = f"{BUILD_TOOLS}/yay0/Yay0compress"
CRC_TOOL = f"{BUILD_TOOLS}/rom/n64crc" CRC_TOOL = f"{BUILD_TOOLS}/rom/n64crc"
@ -241,7 +244,7 @@ def write_ninja_rules(
ninja.rule( ninja.rule(
"tex", "tex",
description="tex $out", description="tex $out",
command=f"$python {BUILD_TOOLS}/mapfs/tex.py $out $tex_dir", command=f"$python {BUILD_TOOLS}/mapfs/tex.py $out $tex_name $asset_stack",
) )
ninja.rule( ninja.rule(
@ -408,6 +411,9 @@ class Configure:
@lru_cache(maxsize=None) @lru_cache(maxsize=None)
def resolve_asset_path(self, path: Path) -> Path: def resolve_asset_path(self, path: Path) -> Path:
# Remove nonsense
path = Path(os.path.normpath(path))
parts = list(path.parts) parts = list(path.parts)
if parts[0] != "assets": if parts[0] != "assets":
@ -493,7 +499,7 @@ class Configure:
ninja.build( ninja.build(
outputs=object_strs, # $out outputs=object_strs, # $out
rule=task, rule=task,
inputs=self.resolve_src_paths(src_paths), # $in inputs=inputs, # $in
implicit=implicit, implicit=implicit,
order_only=order_only, order_only=order_only,
variables={"version": self.version, **variables}, variables={"version": self.version, **variables},
@ -743,6 +749,7 @@ class Configure:
"sprite_name": sprite_name, "sprite_name": sprite_name,
"asset_stack": ",".join(self.asset_stack), "asset_stack": ",".join(self.asset_stack),
}, },
asset_deps=[str(sprite_dir)],
) )
build(yay0_path, [bin_path], "yay0") build(yay0_path, [bin_path], "yay0")
@ -801,9 +808,8 @@ class Configure:
) )
build(entry.object_path, [entry.object_path.with_suffix(".bin")], "bin") build(entry.object_path, [entry.object_path.with_suffix(".bin")], "bin")
elif seg.type == "pm_map_data": elif seg.type == "pm_map_data":
bin_yay0s: List[ # flat list of (uncompressed path, compressed? path) pairs
Path bin_yay0s: List[Path] = []
] = [] # flat list of (uncompressed path, compressed? path) pairs
src_dir = Path("assets/x") / seg.name src_dir = Path("assets/x") / seg.name
for path in entry.src_paths: for path in entry.src_paths:
@ -906,7 +912,11 @@ class Configure:
bin_path, bin_path,
[tex_dir, path.parent / (name + ".json")], [tex_dir, path.parent / (name + ".json")],
"tex", "tex",
variables={"tex_dir": str(tex_dir)}, variables={
"tex_name": name,
"asset_stack": ",".join(self.asset_stack),
},
asset_deps=[f"mapfs/tex/{name}"],
) )
elif name.endswith("_shape"): elif name.endswith("_shape"):
map_name = "_".join(name.split("_")[:-1]) map_name = "_".join(name.split("_")[:-1])

190
tools/build/mapfs/tex.py
View File

@ -1,19 +1,199 @@
#!/usr/bin/env python3 #!/usr/bin/env python3
import argparse import argparse
import json
import os
from pathlib import Path from pathlib import Path
from sys import argv, path from sys import path
from typing import Tuple
path.append(str(Path(__file__).parent.parent.parent / "splat")) path.append(str(Path(__file__).parent.parent.parent / "splat"))
path.append(str(Path(__file__).parent.parent.parent / "splat_ext")) path.append(str(Path(__file__).parent.parent.parent / "build"))
from tex_archives import TexArchive
from common import get_asset_path
path.append(str(Path(__file__).parent.parent.parent))
from splat_ext.tex_archives import (
AUX_COMBINE_MODES_INV,
TILES_BASIC,
TILES_INDEPENDENT_AUX,
TILES_MIPMAPS,
TILES_SHARED_AUX,
TexImage,
get_format_code,
)
# read texture properties from dictionary and load images
def img_from_json(json_data, tex_name: str, asset_stack: Tuple[Path, ...]) -> TexImage:
ret = TexImage()
ret.img_name = json_data["name"]
if "ext" in json_data:
ret.raw_ext = json_data["ext"]
else:
ret.raw_ext = "tif"
# read data for main tile
main_data = json_data.get("main")
if main_data == None:
raise Exception(f"Texture {ret.img_name} has no definition for 'main'")
(main_fmt_name, ret.main_hwrap, ret.main_vwrap) = ret.read_json_img(
main_data, "main", ret.img_name
)
(ret.main_fmt, ret.main_depth) = get_format_code(main_fmt_name)
# read main image
img_path = get_asset_path(
Path(f"mapfs/tex/{tex_name}/{ret.img_name}.png"), asset_stack
)
if not os.path.isfile(img_path):
raise Exception(f"Could not find main image for texture: {ret.img_name}")
(
ret.main_img,
ret.main_pal,
ret.main_width,
ret.main_height,
) = ret.get_img_file(main_fmt_name, str(img_path))
# read data for aux tile
ret.has_aux = "aux" in json_data
if ret.has_aux:
aux_data = json_data.get("aux")
(aux_fmt_name, ret.aux_hwrap, ret.aux_vwrap) = ret.read_json_img(
aux_data, "aux", ret.img_name
)
if aux_fmt_name == "Shared":
# aux tiles have blank attributes in SHARED mode
aux_fmt_name = main_fmt_name
ret.aux_fmt = 0
ret.aux_depth = 0
ret.aux_hwrap = 0
ret.aux_vwrap = 0
ret.extra_tiles = TILES_SHARED_AUX
else:
(ret.aux_fmt, ret.aux_depth) = get_format_code(aux_fmt_name)
ret.extra_tiles = TILES_INDEPENDENT_AUX
# read aux image
img_path = get_asset_path(
Path(f"mapfs/tex/{tex_name}/{ret.img_name}_AUX.png"), asset_stack
)
if not os.path.isfile(img_path):
raise Exception(f"Could not find AUX image for texture: {ret.img_name}")
(
ret.aux_img,
ret.aux_pal,
ret.aux_width,
ret.aux_height,
) = ret.get_img_file(aux_fmt_name, str(img_path))
if ret.extra_tiles == TILES_SHARED_AUX:
# aux tiles have blank sizes in SHARED mode
ret.main_height *= 2
ret.aux_width = 0
ret.aux_height = 0
else:
ret.aux_fmt = 0
ret.aux_depth = 0
ret.aux_hwrap = 0
ret.aux_vwrap = 0
ret.aux_width = 0
ret.aux_height = 0
ret.extra_tiles = TILES_BASIC
# read mipmaps
ret.has_mipmaps = json_data.get("hasMipmaps", False)
if ret.has_mipmaps:
ret.mipmaps = []
mipmap_idx = 1
divisor = 2
if ret.main_width >= (32 >> ret.main_depth):
while True:
if (ret.main_width // divisor) <= 0:
break
mmw = ret.main_width // divisor
mmh = ret.main_height // divisor
img_path = get_asset_path(
Path(f"mapfs/tex/{tex_name}/{ret.img_name}_MM{mipmap_idx}.png"),
asset_stack,
)
if not os.path.isfile(img_path):
raise Exception(
f"Texture {ret.img_name} is missing mipmap level {mipmap_idx} (size = {mmw} x {mmh})"
)
(raster, pal, width, height) = ret.get_img_file(
main_fmt_name, str(img_path)
)
ret.mipmaps.append(raster)
if width != mmw or height != mmh:
raise Exception(
f"Texture {ret.img_name} has wrong size for mipmap level {mipmap_idx} \n"
+ f"MM{mipmap_idx} size = {width} x {height}, but should be = {mmw} x {mmh}"
)
divisor = divisor * 2
mipmap_idx += 1
if (ret.main_width // divisor) < (16 >> ret.main_depth):
break
ret.extra_tiles = TILES_MIPMAPS
# read filter mode
if json_data.get("filter", False):
ret.filter_mode = 2
else:
ret.filter_mode = 0
# read tile combine mode
combine_str = json_data.get("combine", "Missing")
ret.combine = AUX_COMBINE_MODES_INV.get(combine_str)
if ret.combine == None:
raise Exception(f"Texture {ret.img_name} has invalid 'combine'")
ret.is_variant = json_data.get("variant", False)
return ret
def build(
out_path: Path, tex_name: str, asset_stack: Tuple[Path, ...], endian: str = "big"
):
out_bytes = bytearray()
json_path = get_asset_path(Path(f"mapfs/tex/{tex_name}.json"), asset_stack)
with open(json_path) as json_file:
json_str = json_file.read()
json_data = json.loads(json_str)
if len(json_data) > 128:
raise Exception(
f"Maximum number of textures (128) exceeded by {tex_name} ({len(json_data)})`"
)
for img_data in json_data:
img = img_from_json(img_data, tex_name, asset_stack)
img.add_bytes(tex_name, out_bytes)
with open(out_path, "wb") as out_bin:
out_bin.write(out_bytes)
if __name__ == "__main__": if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Texture archives") parser = argparse.ArgumentParser(description="Texture archives")
parser.add_argument("bin_out", type=Path, help="Output binary file path") parser.add_argument("bin_out", type=Path, help="Output binary file path")
parser.add_argument("tex_dir", type=Path, help="File path to input tex subdirectory") parser.add_argument("name", help="Name of tex subdirectory")
parser.add_argument("asset_stack", help="comma-separated asset stack")
parser.add_argument( parser.add_argument(
"--endian", choices=["big", "little"], default="big", help="Output endianness" "--endian", choices=["big", "little"], default="big", help="Output endianness"
) )
args = parser.parse_args() args = parser.parse_args()
TexArchive.build(args.bin_out, args.tex_dir, args.endian) asset_stack = tuple(Path(d) for d in args.asset_stack.split(","))
build(args.bin_out, args.name, asset_stack, args.endian)

4
tools/splat/.gitrepo
View File

@ -6,7 +6,7 @@
[subrepo] [subrepo]
remote = https://github.com/ethteck/splat.git remote = https://github.com/ethteck/splat.git
branch = master branch = master
commit = e72a868f9f7e9da25f13194b51b93e64c2dcc83a commit = 818924683bfb1145129b5e43ff02abe7b4be37a3
parent = 4dfc35713736d65d8d607a0e0f4121bc6938d6e3 parent = c122fe97362f81b15d8ede79f24b28a62e859a68
method = merge method = merge
cmdver = 0.4.5 cmdver = 0.4.5

32
tools/splat/CHANGELOG.md
View File

@ -1,5 +1,29 @@
# splat Release Notes # splat Release Notes
### 0.15.1
* Made some modifications such that linker object paths should be simpler in some circumstances
### 0.15.0
* New options:
* `data_string_encoding` can be set at the global level (or `str_encoding` at the segment level) to specify the encoding using when guessing and disassembling strings the the data section. In spimdisasm this value defaults to ASCII.
* `rodata_string_guesser_level` changes the behaviour of the rodata string guesser. A higher value means more agressive guessing, while 0 and negative means no guessing at all. Even if the guesser feature is disabled, symbols manually marked as strings in the symbol_addrs.txt file will still be disassembled as strings. In spimdisasm this value defaults to 1.
* level 0: Completely disable the guessing feature.
* level 1: The most conservative guessing level. Imposes the following restrictions:
* Do not try to guess if the user provided a type for the symbol.
* Do no try to guess if type information for the symbol can be inferred by other means.
* A string symbol must be referenced only once.
* Strings must not be empty.
* level 2: A string no longer needs to be referenced only once to be considered a possible string. This can happen because of a deduplication optimization.
* level 3: Empty strings are allowed.
* level 4: Symbols with autodetected type information but no user type information can still be guessed as strings.
* `data_string_guesser_level` is similar to `rodata_string_guesser_level`, but for the data section instead. In spimdisasm this value defaults to 2.
* `asm_emit_size_directive` toggles the size directived emitted by the disassembler. In spimdisasm this defaults to True.
### 0.14.1
* Fix bug, cod cleanup
### 0.14.0 ### 0.14.0
* Add support for PSX's GTE instruction set * Add support for PSX's GTE instruction set
@ -40,7 +64,7 @@
### 0.13.3 ### 0.13.3
* Added a new symbol_addrs attribute `appears_after_overlays_addr:0x1234` which will modify the linker script such that the symbol's address is equal to the value of the end of the longest overlay starting with address 0x1234. It achieve this by writing a series of sym = MAX(sym, seg_vram_END) statements into the linker script. For some games, it's feasible to manually create such statements, but for games with hundreds of overlays at the same address, this is very tedious and prone to error. The new attribute allows you to have peace of mind that the symbol will end up after all of these overlays. * Added a new symbol_addrs attribute `appears_after_overlays_addr:0x1234` which will modify the linker script such that the symbol's address is equal to the value of the end of the longest overlay starting with address 0x1234. It achieves this by writing a series of sym = MAX(sym, seg_vram_END) statements into the linker script. For some games, it's feasible to manually create such statements, but for games with hundreds of overlays at the same address, this is very tedious and prone to error. The new attribute allows you to have peace of mind that the symbol will end up after all of these overlays.
### 0.13.2 ### 0.13.2
@ -61,7 +85,7 @@
### 0.12.14 ### 0.12.14
* New option: `pair_rodata_to_text`. * New option: `pair_rodata_to_text`.
* If enabled, splat will try to find to which text segment an unpaired rodata segment belongs and it will hint it to the user. * If enabled, splat will try to find to which text segment an unpaired rodata segment belongs, and it will hint it to the user.
### 0.12.13 ### 0.12.13
@ -91,7 +115,7 @@
### 0.12.8 ### 0.12.8
* The gfx and vtx segments now have a `data_only` option, which, if enabled, will emit only the plain data for the type and omit the enclosing symbol definition. This mode is useful when you want to manually declare the symbol and then #include the extracted data within the declaration. * The gfx and vtx segments now have a `data_only` option, which, if enabled, will emit only the plain data for the type and omit the enclosing symbol definition. This mode is useful when you want to manually declare the symbol and then #include the extracted data within the declaration.
* The gfx segment has a method, `format_sym_name()`, which will allow custom overriding of the output of symbol names by extending the `gfx` segment. For example, this can be used to transform context-specific symbol names like mac_01_vtx into N(vtx), where N() is a macro that applies the current "namespace" to the symbol. Paper Mario plans to use this so we can extract an asset once and then #include it in multiple places, while giving each inclusion unique symbol names for each component. * The gfx segment has a method, `format_sym_name()`, which will allow custom overriding of the output of symbol names by extending the `gfx` segment. For example, this can be used to transform context-specific symbol names like mac_01_vtx into N(vtx), where N() is a macro that applies the current "namespace" to the symbol. Paper Mario plans to use this, so we can extract an asset once and then #include it in multiple places, while giving each inclusion unique symbol names for each component.
### 0.12.7 ### 0.12.7
@ -115,7 +139,7 @@
* Update minimal spimdisasm version to 1.7.1. * Update minimal spimdisasm version to 1.7.1.
* Fix spimdisasm>=1.7.0 non being able to see symbols which only are referenced by other data symbols. * Fix spimdisasm>=1.7.0 non being able to see symbols which only are referenced by other data symbols.
* An check was added to prevent segments marked with `exclusive_ram_id` have a vram address range which overlaps with segments not marked with said tag. If this happens it will be warned to the user. * A check was added to prevent segments marked with `exclusive_ram_id` have a vram address range which overlaps with segments not marked with said tag. If this happens it will be warned to the user.
### 0.12.4 ### 0.12.4

17
tools/splat/disassembler/spimdisasm_disassembler.py
View File

@ -8,7 +8,7 @@ from typing import Set
class SpimdisasmDisassembler(disassembler.Disassembler): class SpimdisasmDisassembler(disassembler.Disassembler):
# This value should be kept in sync with the version listed on requirements.txt # This value should be kept in sync with the version listed on requirements.txt
SPIMDISASM_MIN = (1, 13, 0) SPIMDISASM_MIN = (1, 15, 0)
def configure(self, opts: SplatOpts): def configure(self, opts: SplatOpts):
# Configure spimdisasm # Configure spimdisasm
@ -28,6 +28,16 @@ class SpimdisasmDisassembler(disassembler.Disassembler):
spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO = False spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO = False
if opts.rodata_string_guesser_level is not None:
spimdisasm.common.GlobalConfig.RODATA_STRING_GUESSER_LEVEL = (
opts.rodata_string_guesser_level
)
if opts.data_string_guesser_level is not None:
spimdisasm.common.GlobalConfig.DATA_STRING_GUESSER_LEVEL = (
opts.data_string_guesser_level
)
rabbitizer.config.regNames_userFpcCsr = False rabbitizer.config.regNames_userFpcCsr = False
rabbitizer.config.regNames_vr4300Cop0NamedRegisters = False rabbitizer.config.regNames_vr4300Cop0NamedRegisters = False
@ -70,6 +80,11 @@ class SpimdisasmDisassembler(disassembler.Disassembler):
spimdisasm.common.GlobalConfig.ASM_DATA_LABEL = opts.asm_data_macro spimdisasm.common.GlobalConfig.ASM_DATA_LABEL = opts.asm_data_macro
spimdisasm.common.GlobalConfig.ASM_TEXT_END_LABEL = opts.asm_end_label spimdisasm.common.GlobalConfig.ASM_TEXT_END_LABEL = opts.asm_end_label
if opts.asm_emit_size_directive is not None:
spimdisasm.common.GlobalConfig.ASM_EMIT_SIZE_DIRECTIVE = (
opts.asm_emit_size_directive
)
if spimdisasm.common.GlobalConfig.ASM_TEXT_LABEL == ".globl": if spimdisasm.common.GlobalConfig.ASM_TEXT_LABEL == ".globl":
spimdisasm.common.GlobalConfig.ASM_TEXT_ENT_LABEL = ".ent" spimdisasm.common.GlobalConfig.ASM_TEXT_ENT_LABEL = ".ent"
spimdisasm.common.GlobalConfig.ASM_TEXT_FUNC_AS_LABEL = True spimdisasm.common.GlobalConfig.ASM_TEXT_FUNC_AS_LABEL = True

283
tools/splat/disassembler_section.py Normal file
View File

@ -0,0 +1,283 @@
import spimdisasm
from util import symbols
from typing import Optional, Set, Tuple
from segtypes.segment import Segment
from util import log, options, symbols
from abc import ABC, abstractmethod
from typing import Callable
class DisassemblerSection(ABC):
@abstractmethod
def disassemble(self):
raise NotImplementedError("disassemble")
@abstractmethod
def analyze(self):
raise NotImplementedError("analyze")
@abstractmethod
def set_comment_offset(self, rom_start: int):
raise NotImplementedError("set_comment_offset")
@abstractmethod
def make_bss_section(
self,
rom_start,
rom_end,
vram_start,
bss_end,
name,
segment_rom_start,
exclusive_ram_id,
):
raise NotImplementedError("make_bss_section")
@abstractmethod
def make_data_section(
self,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
):
raise NotImplementedError("make_data_section")
@abstractmethod
def get_section(self):
raise NotImplementedError("get_section")
@abstractmethod
def make_rodata_section(
self,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
):
raise NotImplementedError("make_rodata_section")
@abstractmethod
def make_text_section(
self,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
):
raise NotImplementedError("make_text_section")
class SpimdisasmDisassemberSection(DisassemblerSection):
def __init__(self):
self.spim_section: Optional[spimdisasm.mips.sections.SectionBase] = None
def disassemble(self) -> str:
assert self.spim_section is not None
return self.spim_section.disassemble()
def analyze(self):
assert self.spim_section is not None
self.spim_section.analyze()
def set_comment_offset(self, rom_start: int):
assert self.spim_section is not None
self.spim_section.setCommentOffset(rom_start)
def make_bss_section(
self,
rom_start: int,
rom_end: int,
vram_start: int,
bss_end: int,
name: str,
segment_rom_start: int,
exclusive_ram_id,
):
self.spim_section = spimdisasm.mips.sections.SectionBss(
symbols.spim_context,
rom_start,
rom_end,
vram_start,
bss_end,
name,
segment_rom_start,
exclusive_ram_id,
)
def make_data_section(
self,
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
):
self.spim_section = spimdisasm.mips.sections.SectionData(
symbols.spim_context,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
def get_section(self) -> Optional[spimdisasm.mips.sections.SectionBase]:
return self.spim_section
def make_rodata_section(
self,
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
):
self.spim_section = spimdisasm.mips.sections.SectionRodata(
symbols.spim_context,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
def make_text_section(
self,
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
):
self.spim_section = spimdisasm.mips.sections.SectionText(
symbols.spim_context,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
def make_disassembler_section() -> Optional[SpimdisasmDisassemberSection]:
if options.opts.platform in ["n64", "psx", "ps2"]:
return SpimdisasmDisassemberSection()
raise NotImplementedError("No disassembler section for requested platform")
return None
def make_text_section(
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
) -> DisassemblerSection:
section = make_disassembler_section()
assert section is not None
section.make_text_section(
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
return section
def make_data_section(
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
) -> DisassemblerSection:
section = make_disassembler_section()
assert section is not None
section.make_data_section(
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
return section
def make_rodata_section(
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
) -> DisassemblerSection:
section = make_disassembler_section()
assert section is not None
section.make_rodata_section(
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
return section
def make_bss_section(
rom_start: int,
rom_end: int,
vram_start: int,
bss_end: int,
name: str,
segment_rom_start: int,
exclusive_ram_id,
) -> DisassemblerSection:
section = make_disassembler_section()
assert section is not None
section.make_bss_section(
rom_start,
rom_end,
vram_start,
bss_end,
name,
segment_rom_start,
exclusive_ram_id,
)
return section

2
tools/splat/requirements.txt
View File

@ -4,7 +4,7 @@ tqdm
intervaltree intervaltree
colorama colorama
# This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py
spimdisasm>=1.13.0 spimdisasm>=1.15.0
rabbitizer>=1.7.0 rabbitizer>=1.7.0
pygfxd pygfxd
n64img>=0.1.4 n64img>=0.1.4

14
tools/splat/segtypes/common/bss.py
View File

@ -1,8 +1,9 @@
import spimdisasm
from util import options, symbols, log from util import options, symbols, log
from segtypes.common.data import CommonSegData from segtypes.common.data import CommonSegData
from disassembler_section import make_bss_section
class CommonSegBss(CommonSegData): class CommonSegBss(CommonSegData):
def get_linker_section(self) -> str: def get_linker_section(self) -> str:
@ -37,8 +38,7 @@ class CommonSegBss(CommonSegData):
bss_end = next_subsegment.vram_start bss_end = next_subsegment.vram_start
assert isinstance(bss_end, int), f"{self.name} {bss_end}" assert isinstance(bss_end, int), f"{self.name} {bss_end}"
self.spim_section = spimdisasm.mips.sections.SectionBss( self.spim_section = make_bss_section(
symbols.spim_context,
self.rom_start, self.rom_start,
self.rom_end, self.rom_end,
self.vram_start, self.vram_start,
@ -48,10 +48,12 @@ class CommonSegBss(CommonSegData):
self.get_exclusive_ram_id(), self.get_exclusive_ram_id(),
) )
self.spim_section.analyze() assert self.spim_section is not None
self.spim_section.setCommentOffset(self.rom_start)
for spim_sym in self.spim_section.symbolList: self.spim_section.analyze()
self.spim_section.set_comment_offset(self.rom_start)
for spim_sym in self.spim_section.get_section().symbolList:
symbols.create_symbol_from_spim_symbol( symbols.create_symbol_from_spim_symbol(
self.get_most_parent(), spim_sym.contextSym self.get_most_parent(), spim_sym.contextSym
) )

12
tools/splat/segtypes/common/c.py
View File

@ -153,10 +153,10 @@ class CommonSegC(CommonSegCodeSubsegment):
self.print_file_boundaries() self.print_file_boundaries()
assert self.spim_section is not None and isinstance( assert self.spim_section is not None and isinstance(
self.spim_section, spimdisasm.mips.sections.SectionText self.spim_section.get_section(), spimdisasm.mips.sections.SectionText
), f"{self.name}, rom_start:{self.rom_start}, rom_end:{self.rom_end}" ), f"{self.name}, rom_start:{self.rom_start}, rom_end:{self.rom_end}"
rodata_spim_segment = None rodata_spim_segment: Optional[spimdisasm.mips.sections.SectionRodata] = None
if ( if (
options.opts.migrate_rodata_to_functions options.opts.migrate_rodata_to_functions
and self.rodata_sibling is not None and self.rodata_sibling is not None
@ -166,15 +166,15 @@ class CommonSegC(CommonSegCodeSubsegment):
), self.rodata_sibling.type ), self.rodata_sibling.type
if self.rodata_sibling.spim_section is not None: if self.rodata_sibling.spim_section is not None:
assert isinstance( assert isinstance(
self.rodata_sibling.spim_section, self.rodata_sibling.spim_section.get_section(),
spimdisasm.mips.sections.SectionRodata, spimdisasm.mips.sections.SectionRodata,
) )
rodata_spim_segment = self.rodata_sibling.spim_section rodata_spim_segment = self.rodata_sibling.spim_section.get_section()
# Precompute function-rodata pairings # Precompute function-rodata pairings
symbols_entries = ( symbols_entries = (
spimdisasm.mips.FunctionRodataEntry.getAllEntriesFromSections( spimdisasm.mips.FunctionRodataEntry.getAllEntriesFromSections(
self.spim_section, rodata_spim_segment self.spim_section.get_section(), rodata_spim_segment
) )
) )
@ -319,7 +319,7 @@ class CommonSegC(CommonSegCodeSubsegment):
macro_name: str, macro_name: str,
) -> str: ) -> str:
if options.opts.compiler == IDO: if options.opts.compiler == IDO:
# IDO uses the asm processor to embeed assembly and it doesn't require a special directive to include symbols # IDO uses the asm processor to embeed assembly, and it doesn't require a special directive to include symbols
asm_outpath = Path( asm_outpath = Path(
os.path.join(asm_out_dir, self.name, spim_sym.getName() + ".s") os.path.join(asm_out_dir, self.name, spim_sym.getName() + ".s")
) )

17
tools/splat/segtypes/common/code.py
View File

@ -278,8 +278,17 @@ class CommonSegCode(CommonSegGroup):
) )
segment.sibling = base_segments.get(segment.name, None) segment.sibling = base_segments.get(segment.name, None)
if segment.is_rodata() and segment.sibling is not None:
segment.sibling.rodata_sibling = segment if segment.sibling is not None:
if self.section_order.index(".text") < self.section_order.index(
".rodata"
):
if segment.is_rodata():
segment.sibling.rodata_sibling = segment
else:
if segment.is_text() and segment.sibling.is_rodata():
segment.rodata_sibling = segment.sibling
segment.sibling.sibling = segment
segment.parent = self segment.parent = self
if segment.special_vram_segment: if segment.special_vram_segment:
@ -300,6 +309,10 @@ class CommonSegCode(CommonSegGroup):
if segment.is_text(): if segment.is_text():
base_segments[segment.name] = segment base_segments[segment.name] = segment
if self.section_order.index(".rodata") < self.section_order.index(".text"):
if segment.is_rodata() and segment.sibling is None:
base_segments[segment.name] = segment
prev_start = start prev_start = start
if end is not None: if end is not None:
last_rom_end = end last_rom_end = end

31
tools/splat/segtypes/common/codesubsegment.py
View File

@ -10,6 +10,8 @@ from segtypes.common.code import CommonSegCode
from segtypes.segment import Segment from segtypes.segment import Segment
from disassembler_section import DisassemblerSection, make_text_section
# abstract class for c, asm, data, etc # abstract class for c, asm, data, etc
class CommonSegCodeSubsegment(Segment): class CommonSegCodeSubsegment(Segment):
@ -24,7 +26,7 @@ class CommonSegCodeSubsegment(Segment):
self.yaml.get("str_encoding", None) if isinstance(self.yaml, dict) else None self.yaml.get("str_encoding", None) if isinstance(self.yaml, dict) else None
) )
self.spim_section: Optional[spimdisasm.mips.sections.SectionBase] = None self.spim_section: Optional[DisassemblerSection] = None
self.instr_category = rabbitizer.InstrCategory.CPU self.instr_category = rabbitizer.InstrCategory.CPU
if options.opts.platform == "ps2": if options.opts.platform == "ps2":
self.instr_category = rabbitizer.InstrCategory.R5900 self.instr_category = rabbitizer.InstrCategory.R5900
@ -56,8 +58,7 @@ class CommonSegCodeSubsegment(Segment):
f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'"
) )
self.spim_section = spimdisasm.mips.sections.SectionText( self.spim_section = make_text_section(
symbols.spim_context,
self.rom_start, self.rom_start,
self.rom_end, self.rom_end,
self.vram_start, self.vram_start,
@ -67,13 +68,15 @@ class CommonSegCodeSubsegment(Segment):
self.get_exclusive_ram_id(), self.get_exclusive_ram_id(),
) )
self.spim_section.isHandwritten = is_hasm assert self.spim_section is not None
self.spim_section.instrCat = self.instr_category
self.spim_section.get_section().isHandwritten = is_hasm
self.spim_section.get_section().instrCat = self.instr_category
self.spim_section.analyze() self.spim_section.analyze()
self.spim_section.setCommentOffset(self.rom_start) self.spim_section.set_comment_offset(self.rom_start)
for func in self.spim_section.symbolList: for func in self.spim_section.get_section().symbolList:
assert isinstance(func, spimdisasm.mips.symbols.SymbolFunction) assert isinstance(func, spimdisasm.mips.symbols.SymbolFunction)
self.process_insns(func) self.process_insns(func)
@ -85,7 +88,7 @@ class CommonSegCodeSubsegment(Segment):
jtbl_label_vram, True, type="jtbl_label", define=True jtbl_label_vram, True, type="jtbl_label", define=True
) )
sym.type = "jtbl_label" sym.type = "jtbl_label"
symbols.add_symbol_to_spim_section(self.spim_section, sym) symbols.add_symbol_to_spim_section(self.spim_section.get_section(), sym)
def process_insns( def process_insns(
self, self,
@ -103,7 +106,7 @@ class CommonSegCodeSubsegment(Segment):
# Gather symbols found by spimdisasm and create those symbols in splat's side # Gather symbols found by spimdisasm and create those symbols in splat's side
for referenced_vram in func_spim.instrAnalyzer.referencedVrams: for referenced_vram in func_spim.instrAnalyzer.referencedVrams:
context_sym = self.spim_section.getSymbol( context_sym = self.spim_section.get_section().getSymbol(
referenced_vram, tryPlusOffset=False referenced_vram, tryPlusOffset=False
) )
if context_sym is not None: if context_sym is not None:
@ -124,7 +127,7 @@ class CommonSegCodeSubsegment(Segment):
if instr_offset in func_spim.instrAnalyzer.symbolInstrOffset: if instr_offset in func_spim.instrAnalyzer.symbolInstrOffset:
sym_address = func_spim.instrAnalyzer.symbolInstrOffset[instr_offset] sym_address = func_spim.instrAnalyzer.symbolInstrOffset[instr_offset]
context_sym = self.spim_section.getSymbol( context_sym = self.spim_section.get_section().getSymbol(
sym_address, tryPlusOffset=False sym_address, tryPlusOffset=False
) )
if context_sym is not None: if context_sym is not None:
@ -141,7 +144,7 @@ class CommonSegCodeSubsegment(Segment):
assert isinstance(self.rom_start, int) assert isinstance(self.rom_start, int)
for in_file_offset in self.spim_section.fileBoundaries: for in_file_offset in self.spim_section.get_section().fileBoundaries:
if (in_file_offset % 16) != 0: if (in_file_offset % 16) != 0:
continue continue
@ -150,8 +153,10 @@ class CommonSegCodeSubsegment(Segment):
# Look up for the last symbol in this boundary # Look up for the last symbol in this boundary
sym_addr = 0 sym_addr = 0
for sym in self.spim_section.symbolList: for sym in self.spim_section.get_section().symbolList:
symOffset = sym.inFileOffset - self.spim_section.inFileOffset symOffset = (
sym.inFileOffset - self.spim_section.get_section().inFileOffset
)
if in_file_offset == symOffset: if in_file_offset == symOffset:
break break
sym_addr = sym.vram sym_addr = sym.vram

22
tools/splat/segtypes/common/data.py
View File

@ -7,6 +7,8 @@ from util import options, symbols, log
from segtypes.common.codesubsegment import CommonSegCodeSubsegment from segtypes.common.codesubsegment import CommonSegCodeSubsegment
from segtypes.common.group import CommonSegGroup from segtypes.common.group import CommonSegGroup
from disassembler_section import DisassemblerSection, make_data_section
class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup): class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup):
def out_path(self) -> Optional[Path]: def out_path(self) -> Optional[Path]:
@ -89,8 +91,7 @@ class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup):
f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'"
) )
self.spim_section = spimdisasm.mips.sections.SectionData( self.spim_section = make_data_section(
symbols.spim_context,
self.rom_start, self.rom_start,
self.rom_end, self.rom_end,
self.vram_start, self.vram_start,
@ -100,12 +101,25 @@ class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup):
self.get_exclusive_ram_id(), self.get_exclusive_ram_id(),
) )
assert self.spim_section is not None
# Set rodata string encoding
# First check the global configuration
if options.opts.data_string_encoding is not None:
self.spim_section.get_section().stringEncoding = (
options.opts.data_string_encoding
)
# Then check the per-segment configuration in case we want to override the global one
if self.str_encoding is not None:
self.spim_section.get_section().stringEncoding = self.str_encoding
self.spim_section.analyze() self.spim_section.analyze()
self.spim_section.setCommentOffset(self.rom_start) self.spim_section.set_comment_offset(self.rom_start)
rodata_encountered = False rodata_encountered = False
for symbol in self.spim_section.symbolList: for symbol in self.spim_section.get_section().symbolList:
symbols.create_symbol_from_spim_symbol( symbols.create_symbol_from_spim_symbol(
self.get_most_parent(), symbol.contextSym self.get_most_parent(), symbol.contextSym
) )

17
tools/splat/segtypes/common/rodata.py
View File

@ -5,6 +5,8 @@ from util import log, options, symbols
from segtypes.common.data import CommonSegData from segtypes.common.data import CommonSegData
from disassembler_section import make_rodata_section
class CommonSegRodata(CommonSegData): class CommonSegRodata(CommonSegData):
def get_linker_section(self) -> str: def get_linker_section(self) -> str:
@ -53,8 +55,7 @@ class CommonSegRodata(CommonSegData):
f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'"
) )
self.spim_section = spimdisasm.mips.sections.SectionRodata( self.spim_section = make_rodata_section(
symbols.spim_context,
self.rom_start, self.rom_start,
self.rom_end, self.rom_end,
self.vram_start, self.vram_start,
@ -64,21 +65,25 @@ class CommonSegRodata(CommonSegData):
self.get_exclusive_ram_id(), self.get_exclusive_ram_id(),
) )
assert self.spim_section is not None
# Set rodata string encoding # Set rodata string encoding
# First check the global configuration # First check the global configuration
if options.opts.string_encoding is not None: if options.opts.string_encoding is not None:
self.spim_section.stringEncoding = options.opts.string_encoding self.spim_section.get_section().stringEncoding = (
options.opts.string_encoding
)
# Then check the per-segment configuration in case we want to override the global one # Then check the per-segment configuration in case we want to override the global one
if self.str_encoding is not None: if self.str_encoding is not None:
self.spim_section.stringEncoding = self.str_encoding self.spim_section.get_section().stringEncoding = self.str_encoding
self.spim_section.analyze() self.spim_section.analyze()
self.spim_section.setCommentOffset(self.rom_start) self.spim_section.set_comment_offset(self.rom_start)
possible_text_segments: Set[Segment] = set() possible_text_segments: Set[Segment] = set()
for symbol in self.spim_section.symbolList: for symbol in self.spim_section.get_section().symbolList:
generated_symbol = symbols.create_symbol_from_spim_symbol( generated_symbol = symbols.create_symbol_from_spim_symbol(
self.get_most_parent(), symbol.contextSym self.get_most_parent(), symbol.contextSym
) )

5
tools/splat/segtypes/linker_entry.py
View File

@ -39,7 +39,10 @@ def path_to_object_path(path: Path) -> Path:
full_suffix = ".o" full_suffix = ".o"
else: else:
full_suffix = path.suffix + ".o" full_suffix = path.suffix + ".o"
return clean_up_path(options.opts.build_path / path.with_suffix(full_suffix))
if not str(path).startswith(str(options.opts.build_path)):
path = options.opts.build_path / path
return clean_up_path(path.with_suffix(full_suffix))
def write_file_if_different(path: Path, new_content: str): def write_file_if_different(path: Path, new_content: str):

4
tools/splat/split.py
View File

@ -19,7 +19,7 @@ from segtypes.linker_entry import (
from segtypes.segment import Segment from segtypes.segment import Segment
from util import log, options, palettes, symbols, relocs from util import log, options, palettes, symbols, relocs
VERSION = "0.14.0" VERSION = "0.15.1"
parser = argparse.ArgumentParser( parser = argparse.ArgumentParser(
description="Split a rom given a rom, a config, and output directory" description="Split a rom given a rom, a config, and output directory"
@ -349,7 +349,7 @@ def main(config_path, modes, verbose, use_cache=True, skip_version_check=False):
if ( if (
options.opts.is_mode_active("ld") and options.opts.platform != "gc" options.opts.is_mode_active("ld") and options.opts.platform != "gc"
): # TODO move this to platform initialization when it gets implemented ): # TODO move this to platform initialization when it gets implemented
# Calculate list of segments for which we need to find the largest so we can safely place the symbol after it # Calculate list of segments for which we need to find the largest, so we can safely place the symbol after it
max_vram_end_syms: Dict[str, List[Segment]] = {} max_vram_end_syms: Dict[str, List[Segment]] = {}
for sym in symbols.appears_after_overlays_syms: for sym in symbols.appears_after_overlays_syms:
max_vram_end_syms[sym.name] = [ max_vram_end_syms[sym.name] = [

272
tools/splat/test.py
View File

@ -1,8 +1,9 @@
from spimdisasm.common import FileSectionType
from split import * from split import *
import unittest import unittest
import io import io
import filecmp import filecmp
import pprint
from util import symbols, options from util import symbols, options
import spimdisasm import spimdisasm
from segtypes.common.rodata import CommonSegRodata from segtypes.common.rodata import CommonSegRodata
@ -10,7 +11,6 @@ from segtypes.common.code import CommonSegCode
from segtypes.common.c import CommonSegC from segtypes.common.c import CommonSegC
from segtypes.common.bss import CommonSegBss from segtypes.common.bss import CommonSegBss
import difflib import difflib
from segtypes.common.group import CommonSegGroup
class Testing(unittest.TestCase): class Testing(unittest.TestCase):
@ -73,8 +73,6 @@ class Testing(unittest.TestCase):
# can't diff binary # can't diff binary
if file[0] == ".splache": if file[0] == ".splache":
continue continue
file1_lines = []
file2_lines = []
with open(f"{file[1]}/{file[0]}") as file1: with open(f"{file[1]}/{file[0]}") as file1:
file1_lines = file1.readlines() file1_lines = file1.readlines()
with open(f"{file[2]}/{file[0]}") as file2: with open(f"{file[2]}/{file[0]}") as file2:
@ -126,12 +124,12 @@ class Symbols(unittest.TestCase):
disassembler_instance.create_disassembler_instance("n64") disassembler_instance.create_disassembler_instance("n64")
# first char is uppercase # first char is uppercase
assert symbols.check_valid_type("Symbol") == True assert symbols.check_valid_type("Symbol")
splat_sym_types = {"func", "jtbl", "jtbl_label", "label"} splat_sym_types = {"func", "jtbl", "jtbl_label", "label"}
for type in splat_sym_types: for type in splat_sym_types:
assert symbols.check_valid_type(type) == True assert symbols.check_valid_type(type)
spim_types = [ spim_types = [
"char*", "char*",
@ -151,20 +149,15 @@ class Symbols(unittest.TestCase):
] ]
for type in spim_types: for type in spim_types:
assert symbols.check_valid_type(type) == True assert symbols.check_valid_type(type)
def test_add_symbol_to_spim_segment(self): def test_add_symbol_to_spim_segment(self):
context = None
vromStart = 0x0
vromEnd = 0x10
vramStart = 0x40000000 + 0x0
vramEnd = 0x40000000 + 0x10
segment = spimdisasm.common.SymbolsSegment( segment = spimdisasm.common.SymbolsSegment(
context=context, context=spimdisasm.common.Context(),
vromStart=vromStart, vromStart=0x0,
vromEnd=vromEnd, vromEnd=0x10,
vramStart=vramStart, vramStart=0x40000000 + 0x0,
vramEnd=vramEnd, vramEnd=0x40000000 + 0x10,
) )
sym = symbols.Symbol(0x40000000) sym = symbols.Symbol(0x40000000)
sym.user_declared = False sym.user_declared = False
@ -177,16 +170,15 @@ class Symbols(unittest.TestCase):
assert sym.defined == result.isDefined assert sym.defined == result.isDefined
def test_add_symbol_to_spim_section(self): def test_add_symbol_to_spim_section(self):
context = spimdisasm.common.Context()
section = spimdisasm.mips.sections.SectionBase( section = spimdisasm.mips.sections.SectionBase(
context=context, context=spimdisasm.common.Context(),
vromStart=0x100, vromStart=0x0,
vromEnd=None, vromEnd=0x10,
vram=None, vram=0x40000000,
filename=None, filename="test",
words=None, words=[],
sectionType=None, sectionType=FileSectionType.Text,
segmentVromStart=None, segmentVromStart=0x0,
overlayCategory=None, overlayCategory=None,
) )
sym = symbols.Symbol(0x100) sym = symbols.Symbol(0x100)
@ -199,36 +191,28 @@ class Symbols(unittest.TestCase):
assert sym.defined == result.isDefined assert sym.defined == result.isDefined
def test_create_symbol_from_spim_symbol(self): def test_create_symbol_from_spim_symbol(self):
rom_start = 0x0
rom_end = 0x100
type = "func"
name = "MyFunc"
vram_start = 0x40000000
args = None
yaml = None
# need to init otherwise options.opts isn't defined. # need to init otherwise options.opts isn't defined.
# used in initializing a Segment # used in initializing a Segment
test_init() test_init()
segment = Segment( segment = Segment(
rom_start=rom_start, rom_start=0x0,
rom_end=rom_end, rom_end=0x100,
type=type, type="func",
name=name, name="MyFunc",
vram_start=vram_start, vram_start=0x40000000,
args=[], args=[],
yaml=yaml, yaml=None,
) )
context_sym = spimdisasm.common.ContextSymbol(address=0) context_sym = spimdisasm.common.ContextSymbol(address=0)
result = symbols.create_symbol_from_spim_symbol(segment, context_sym) result = symbols.create_symbol_from_spim_symbol(segment, context_sym)
assert result.referenced == True assert result.referenced
assert result.extract == True assert result.extract
assert result.name == "D_0" assert result.name == "D_0"
def get_yaml(): def get_yaml():
yaml = { return {
"name": "basic_app", "name": "basic_app",
"type": "code", "type": "code",
"start": 0, "start": 0,
@ -236,7 +220,6 @@ def get_yaml():
"subalign": 4, "subalign": 4,
"subsegments": [[0, "data"], [0x1DC, "c", "main"], [0x1FC, "data"]], "subsegments": [[0, "data"], [0x1DC, "c", "main"], [0x1FC, "data"]],
} }
return yaml
class Rodata(unittest.TestCase): class Rodata(unittest.TestCase):
@ -252,11 +235,11 @@ class Rodata(unittest.TestCase):
yaml=None, yaml=None,
) )
rom_data = [] rom_data = []
for i in range(0, 0x100): for i in range(0x100):
rom_data.append(i) rom_data.append(i)
common_seg_rodata.disassemble_data(bytes(rom_data)) common_seg_rodata.disassemble_data(bytes(rom_data))
assert common_seg_rodata.spim_section is not None assert common_seg_rodata.spim_section is not None
assert common_seg_rodata.spim_section.words[0] == 0x0010203 assert common_seg_rodata.spim_section.get_section().words[0] == 0x0010203
assert symbols.get_all_symbols()[0].vram_start == 0x400 assert symbols.get_all_symbols()[0].vram_start == 0x400
assert symbols.get_all_symbols()[0].segment == common_seg_rodata assert symbols.get_all_symbols()[0].segment == common_seg_rodata
assert symbols.get_all_symbols()[0].linker_section == ".rodata" assert symbols.get_all_symbols()[0].linker_section == ".rodata"
@ -270,11 +253,11 @@ class Rodata(unittest.TestCase):
rodata_sym = spimdisasm.mips.symbols.SymbolRodata( rodata_sym = spimdisasm.mips.symbols.SymbolRodata(
context=context, context=context,
vromStart=0x100, vromStart=0x100,
vromEnd=None, vromEnd=0x200,
inFileOffset=None, inFileOffset=0,
vram=0x100, vram=0x100,
words=[0, 1, 2, 3, 4, 5, 6, 7], words=[0, 1, 2, 3, 4, 5, 6, 7],
segmentVromStart=None, segmentVromStart=0,
overlayCategory=None, overlayCategory=None,
) )
rodata_sym.contextSym.forceMigration = True rodata_sym.contextSym.forceMigration = True
@ -282,7 +265,7 @@ class Rodata(unittest.TestCase):
context_sym = spimdisasm.common.ContextSymbol(address=0) context_sym = spimdisasm.common.ContextSymbol(address=0)
context_sym.address = result_symbol_addr context_sym.address = result_symbol_addr
rodata_sym.contextSym.referenceFunctions = [context_sym] rodata_sym.contextSym.referenceFunctions = {context_sym}
# Segment __init__ requires opts to be initialized # Segment __init__ requires opts to be initialized
test_init() test_init()
@ -340,9 +323,192 @@ class Bss(unittest.TestCase):
rom_bytes = bytes([0, 1, 2, 3, 4, 5, 6, 7]) rom_bytes = bytes([0, 1, 2, 3, 4, 5, 6, 7])
bss.disassemble_data(rom_bytes) bss.disassemble_data(rom_bytes)
assert isinstance(bss.spim_section, spimdisasm.mips.sections.SectionBss) assert bss.spim_section is not None
assert bss.spim_section.bssVramStart == 0x40000000
assert bss.spim_section.bssVramEnd == 0x300 assert isinstance(
bss.spim_section.get_section(), spimdisasm.mips.sections.SectionBss
)
assert bss.spim_section.get_section().bssVramStart == 0x40000000
assert bss.spim_section.get_section().bssVramEnd == 0x300
class SymbolsInitialize(unittest.TestCase):
def test_attrs(self):
import pathlib
symbols.reset_symbols()
test_init()
sym_addrs_lines = [
"func_1 = 0x100 // type:func size:10 rom:100 segment:test_segment name_end:the_name_end "
"appears_after_overlays_addr:1234"
]
all_segments = [
Segment(
rom_start=0x100,
rom_end=0x200,
type="func",
name="test_segment",
vram_start=0x300,
args=[],
yaml={},
)
]
symbols.handle_sym_addrs(
pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments
)
assert symbols.all_symbols[0].given_name == "func_1"
assert symbols.all_symbols[0].type == "func"
assert symbols.all_symbols[0].given_size == 10
assert symbols.all_symbols[0].rom == 100
assert symbols.all_symbols[0].segment == all_segments[0]
assert symbols.all_symbols[0].given_name_end == "the_name_end"
assert symbols.appears_after_overlays_syms[0] == symbols.all_symbols[0]
def test_boolean_attrs(self):
import pathlib
symbols.reset_symbols()
test_init()
sym_addrs_lines = [
"func_1 = 0x100 // dead:True defined:True extract:True force_migration:True force_not_migration:True "
"allow_addend:True dont_allow_addend:True"
]
all_segments = [
Segment(
rom_start=0x100,
rom_end=0x200,
type="func",
name="test_segment",
vram_start=0x300,
args=[],
yaml={},
)
]
symbols.handle_sym_addrs(
pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments
)
assert symbols.all_symbols[0].dead == True
assert symbols.all_symbols[0].defined == True
assert symbols.all_symbols[0].force_migration == True
assert symbols.all_symbols[0].force_not_migration == True
assert symbols.all_symbols[0].allow_addend == True
assert symbols.all_symbols[0].dont_allow_addend == True
# test spim ban range
def test_ignore(self):
import pathlib
symbols.reset_symbols()
test_init()
sym_addrs_lines = ["func_1 = 0x100 // ignore:True size:4"]
all_segments = [
Segment(
rom_start=0x100,
rom_end=0x200,
type="func",
name="test_segment",
vram_start=0x300,
args=[],
yaml={},
)
]
symbols.handle_sym_addrs(
pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments
)
assert symbols.spim_context.bannedRangedSymbols[0].start == 16
assert symbols.spim_context.bannedRangedSymbols[0].end == 20
class InitializeSpimContext(unittest.TestCase):
def test_overlay(self):
symbols.reset_symbols()
test_init()
yaml = {
"name": "boot",
"type": "code",
"start": 4096,
"vram": 2147484672,
"bss_size": 128,
"exclusive_ram_id": "overlay",
"subsegments": [
[4096, "c", "main"],
[4336, "hasm", "handwritten"],
[4352, "data", "main"],
[4368, "rodata", "main"],
{"type": "bss", "vram": 2147484992, "name": "main"},
],
}
all_segments: List["Segment"] = [
CommonSegCode(
rom_start=0x0,
rom_end=0x200,
type="code",
name="main",
vram_start=0x100,
args=[],
yaml=yaml,
)
]
# force this since it's hard to set up
all_segments[0].exclusive_ram_id = "overlay"
symbols.initialize_spim_context(all_segments)
# spim should have added something to overlaySegments
assert (
type(symbols.spim_context.overlaySegments["overlay"][0])
== spimdisasm.common.SymbolsSegment
)
# test globalSegment settings
def test_global(self):
symbols.reset_symbols()
test_init()
yaml = {
"name": "boot",
"type": "code",
"start": 4096,
"vram": 2147484672,
"bss_size": 128,
"exclusive_ram_id": "overlay",
"subsegments": [
[4096, "c", "main"],
[4336, "hasm", "handwritten"],
[4352, "data", "main"],
[4368, "rodata", "main"],
{"type": "bss", "vram": 2147484992, "name": "main"},
],
}
all_segments: List["Segment"] = [
CommonSegCode(
rom_start=0x0,
rom_end=0x200,
type="code",
name="main",
vram_start=0x100,
args=[],
yaml=yaml,
)
]
assert symbols.spim_context.globalSegment.vramStart == 2147483648
assert symbols.spim_context.globalSegment.vramEnd == 2147487744
symbols.initialize_spim_context(all_segments)
assert symbols.spim_context.globalSegment.vramStart == 256
assert symbols.spim_context.globalSegment.vramEnd == 896
if __name__ == "__main__": if __name__ == "__main__":

0
tools/splat/test/basic_app/build.sh Normal file → Executable file
View File

2
tools/splat/test/basic_app/expected/asm/data/main.data.s
View File

@ -4,8 +4,10 @@
glabel D_80000500 glabel D_80000500
/* 1100 80000500 */ .word 0x00000001 /* 1100 80000500 */ .word 0x00000001
.size D_80000500, . - D_80000500
glabel D_80000504 glabel D_80000504
/* 1104 80000504 */ .word 0x00000000 /* 1104 80000504 */ .word 0x00000000
/* 1108 80000508 */ .word 0x00000000 /* 1108 80000508 */ .word 0x00000000
/* 110C 8000050C */ .word 0x00000000 /* 110C 8000050C */ .word 0x00000000
.size D_80000504, . - D_80000504

1
tools/splat/test/basic_app/expected/asm/handwritten.s
View File

@ -13,3 +13,4 @@ glabel func_800004F0
/* 10F4 800004F4 03E00008 */ jr $ra /* 10F4 800004F4 03E00008 */ jr $ra
/* 10F8 800004F8 00000000 */ nop /* 10F8 800004F8 00000000 */ nop
/* 10FC 800004FC 00000000 */ nop /* 10FC 800004FC 00000000 */ nop
.size func_800004F0, . - func_800004F0

1
tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s
View File

@ -52,3 +52,4 @@ glabel .L80000480
/* 1094 80000494 8FBE0000 */ lw $fp, 0x0($sp) /* 1094 80000494 8FBE0000 */ lw $fp, 0x0($sp)
/* 1098 80000498 03E00008 */ jr $ra /* 1098 80000498 03E00008 */ jr $ra
/* 109C 8000049C 27BD0008 */ addiu $sp, $sp, 0x8 /* 109C 8000049C 27BD0008 */ addiu $sp, $sp, 0x8
.size func_80000400, . - func_80000400

1
tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s
View File

@ -23,3 +23,4 @@ glabel func_800004A0
/* 10E4 800004E4 03E00008 */ jr $ra /* 10E4 800004E4 03E00008 */ jr $ra
/* 10E8 800004E8 27BD0018 */ addiu $sp, $sp, 0x18 /* 10E8 800004E8 27BD0018 */ addiu $sp, $sp, 0x18
/* 10EC 800004EC 00000000 */ nop /* 10EC 800004EC 00000000 */ nop
.size func_800004A0, . - func_800004A0

8
tools/splat/util/color.py
View File

@ -17,11 +17,3 @@ def unpack_color(data):
b = ceil(0xFF * (b / 31)) b = ceil(0xFF * (b / 31))
return r, g, b, a return r, g, b, a
def pack_color(r, g, b, a):
r = r >> 3
g = g >> 3
b = b >> 3
a = a >> 7
return (r << 11) | (g << 6) | (b << 1) | a

2
tools/splat/util/n64/rominfo.py
View File

@ -106,7 +106,7 @@ class N64EntrypointInfo:
register_values[insn.rt.value] = insn.getProcessedImmediate() << 16 register_values[insn.rt.value] = insn.getProcessedImmediate() << 16
elif insn.canBeLo(): elif insn.canBeLo():
if insn.isLikelyHandwritten(): if insn.isLikelyHandwritten():
# Try to skip this instructions: # Try to skip these instructions:
# addi $t0, $t0, 0x8 # addi $t0, $t0, 0x8
# addi $t1, $t1, -0x8 # addi $t1, $t1, -0x8
pass pass

27
tools/splat/util/options.py
View File

@ -1,4 +1,5 @@
from dataclasses import dataclass from dataclasses import dataclass
import os
from pathlib import Path from pathlib import Path
from typing import cast, Dict, List, Literal, Mapping, Optional, Set, Type, TypeVar from typing import cast, Dict, List, Literal, Mapping, Optional, Set, Type, TypeVar
@ -98,7 +99,7 @@ class SplatOpts:
ld_section_labels: List[str] ld_section_labels: List[str]
# Determines whether to add wildcards for section linking in the linker script (.rodata* for example) # Determines whether to add wildcards for section linking in the linker script (.rodata* for example)
ld_wildcard_sections: bool ld_wildcard_sections: bool
# Determines whether to use use "follows" settings to determine locations of overlays in the linker script. # Determines whether to use "follows" settings to determine locations of overlays in the linker script.
# If disabled, this effectively ignores "follows" directives in the yaml. # If disabled, this effectively ignores "follows" directives in the yaml.
ld_use_follows: bool ld_use_follows: bool
# If enabled, the end symbol for each segment will be placed before the alignment directive for the segment # If enabled, the end symbol for each segment will be placed before the alignment directive for the segment
@ -141,6 +142,8 @@ class SplatOpts:
asm_data_macro: str asm_data_macro: str
# Determines the macro used at the end of a function, such as endlabel or .end # Determines the macro used at the end of a function, such as endlabel or .end
asm_end_label: str asm_end_label: str
# Toggles the .size directive emitted by the disassembler
asm_emit_size_directive: Optional[bool]
# Determines including the macro.inc file on non-migrated rodata variables # Determines including the macro.inc file on non-migrated rodata variables
include_macro_inc: bool include_macro_inc: bool
# Determines the number of characters to left align before the TODO finish documenting # Determines the number of characters to left align before the TODO finish documenting
@ -154,12 +157,18 @@ class SplatOpts:
# o32 is highly recommended, as it provides logically named registers for floating point instructions # o32 is highly recommended, as it provides logically named registers for floating point instructions
# For more info, see https://gist.github.com/EllipticEllipsis/27eef11205c7a59d8ea85632bc49224d # For more info, see https://gist.github.com/EllipticEllipsis/27eef11205c7a59d8ea85632bc49224d
mips_abi_float_regs: str mips_abi_float_regs: str
# Determines whether to ad ".set gp=64 to asm/hasm files" # Determines whether to add ".set gp=64" to asm/hasm files
add_set_gp_64: bool add_set_gp_64: bool
# Generate .asmproc.d dependency files for each C file which still reference functions in assembly files # Generate .asmproc.d dependency files for each C file which still reference functions in assembly files
create_asm_dependencies: bool create_asm_dependencies: bool
# Global option for rodata string encoding. This can be overriden per segment # Global option for rodata string encoding. This can be overriden per segment
string_encoding: Optional[str] string_encoding: Optional[str]
# Global option for data string encoding. This can be overriden per segment
data_string_encoding: Optional[str]
# Global option for the rodata string guesser. 0 disables the guesser completely.
rodata_string_guesser_level: Optional[int]
# Global option for the data string guesser. 0 disables the guesser completely.
data_string_guesser_level: Optional[int]
# Global option for allowing data symbols using addends on symbol references. It can be overriden per symbol # Global option for allowing data symbols using addends on symbol references. It can be overriden per symbol
allow_data_addends: bool allow_data_addends: bool
# Determines whether to include the "Generated by spimdisasm" text in the asm # Determines whether to include the "Generated by spimdisasm" text in the asm
@ -240,7 +249,7 @@ class OptParser:
def parse_path( def parse_path(
self, base_path: Path, opt: str, default: Optional[str] = None self, base_path: Path, opt: str, default: Optional[str] = None
) -> Path: ) -> Path:
return base_path / Path(self.parse_opt(opt, str, default)) return Path(os.path.normpath(base_path / self.parse_opt(opt, str, default)))
def parse_optional_path(self, base_path: Path, opt: str) -> Optional[Path]: def parse_optional_path(self, base_path: Path, opt: str) -> Optional[Path]:
if opt not in self._yaml: if opt not in self._yaml:
@ -275,7 +284,9 @@ def _parse_yaml(
platform = p.parse_opt_within("platform", str, ["n64", "psx", "gc", "ps2"]) platform = p.parse_opt_within("platform", str, ["n64", "psx", "gc", "ps2"])
comp = compiler.for_name(p.parse_opt("compiler", str, "IDO")) comp = compiler.for_name(p.parse_opt("compiler", str, "IDO"))
base_path = Path(config_paths[0]).parent / p.parse_opt("base_path", str) base_path = Path(
os.path.normpath(Path(config_paths[0]).parent / p.parse_opt("base_path", str))
)
asm_path: Path = p.parse_path(base_path, "asm_path", "asm") asm_path: Path = p.parse_path(base_path, "asm_path", "asm")
def parse_endianness() -> Literal["big", "little"]: def parse_endianness() -> Literal["big", "little"]:
@ -378,6 +389,7 @@ def _parse_yaml(
), ),
asm_data_macro=p.parse_opt("asm_data_macro", str, comp.asm_data_macro), asm_data_macro=p.parse_opt("asm_data_macro", str, comp.asm_data_macro),
asm_end_label=p.parse_opt("asm_end_label", str, comp.asm_end_label), asm_end_label=p.parse_opt("asm_end_label", str, comp.asm_end_label),
asm_emit_size_directive=p.parse_optional_opt("asm_emit_size_directive", bool),
include_macro_inc=p.parse_opt( include_macro_inc=p.parse_opt(
"include_macro_inc", bool, comp.include_macro_inc "include_macro_inc", bool, comp.include_macro_inc
), ),
@ -398,6 +410,13 @@ def _parse_yaml(
add_set_gp_64=p.parse_opt("add_set_gp_64", bool, True), add_set_gp_64=p.parse_opt("add_set_gp_64", bool, True),
create_asm_dependencies=p.parse_opt("create_asm_dependencies", bool, False), create_asm_dependencies=p.parse_opt("create_asm_dependencies", bool, False),
string_encoding=p.parse_optional_opt("string_encoding", str), string_encoding=p.parse_optional_opt("string_encoding", str),
data_string_encoding=p.parse_optional_opt("data_string_encoding", str),
rodata_string_guesser_level=p.parse_optional_opt(
"rodata_string_guesser_level", int
),
data_string_guesser_level=p.parse_optional_opt(
"data_string_guesser_level", int
),
allow_data_addends=p.parse_opt("allow_data_addends", bool, True), allow_data_addends=p.parse_opt("allow_data_addends", bool, True),
header_encoding=p.parse_opt("header_encoding", str, "ASCII"), header_encoding=p.parse_opt("header_encoding", str, "ASCII"),
gfx_ucode=p.parse_opt_within( gfx_ucode=p.parse_opt_within(

363
tools/splat/util/symbols.py
View File

@ -6,6 +6,7 @@ import spimdisasm
import tqdm import tqdm
from intervaltree import IntervalTree from intervaltree import IntervalTree
from disassembler import disassembler_instance from disassembler import disassembler_instance
from pathlib import Path
# circular import # circular import
if TYPE_CHECKING: if TYPE_CHECKING:
@ -71,6 +72,175 @@ def to_cname(symbol_name: str) -> str:
return symbol_name return symbol_name
def handle_sym_addrs(path: Path, sym_addrs_lines: List[str], all_segments):
def get_seg_for_name(name: str) -> Optional["Segment"]:
for segment in all_segments:
if segment.name == name:
return segment
return None
for line_num, line in enumerate(
tqdm.tqdm(sym_addrs_lines, desc=f"Loading symbols ({path.stem})")
):
line = line.strip()
if not line == "" and not line.startswith("//"):
comment_loc = line.find("//")
line_main = line
line_ext = ""
if comment_loc != -1:
line_ext = line[comment_loc + 2 :].strip()
line_main = line[:comment_loc].strip()
try:
line_split = line_main.split("=")
name = line_split[0].strip()
addr = int(line_split[1].strip()[:-1], 0)
except:
log.parsing_error_preamble(path, line_num, line)
log.write("Line should be of the form")
log.write("<function_name> = <address> // attr0:val0 attr1:val1 [...]")
log.write("with <address> in hex preceded by 0x, or dec")
log.write("")
raise
sym = Symbol(addr, given_name=name)
ignore_sym = False
if line_ext:
for info in line_ext.split(" "):
if ":" in info:
if info.count(":") > 1:
log.parsing_error_preamble(path, line_num, line)
log.write(f"Too many ':'s in '{info}'")
log.error("")
attr_name, attr_val = info.split(":")
if attr_name == "":
log.parsing_error_preamble(path, line_num, line)
log.write(
f"Missing attribute name in '{info}', is there extra whitespace?"
)
log.error("")
if attr_val == "":
log.parsing_error_preamble(path, line_num, line)
log.write(
f"Missing attribute value in '{info}', is there extra whitespace?"
)
log.error("")
# Non-Boolean attributes
try:
if attr_name == "type":
if not check_valid_type(attr_val):
log.parsing_error_preamble(path, line_num, line)
log.write(
f"Unrecognized symbol type in '{info}', it should be one of"
)
log.write(
[
*splat_sym_types,
*spimdisasm.common.gKnownTypes,
]
)
log.write(
"You may use a custom type that starts with a capital letter"
)
log.error("")
type = attr_val
sym.type = type
continue
if attr_name == "size":
size = int(attr_val, 0)
sym.given_size = size
continue
if attr_name == "rom":
rom_addr = int(attr_val, 0)
sym.rom = rom_addr
continue
if attr_name == "segment":
seg = get_seg_for_name(attr_val)
if seg is None:
log.parsing_error_preamble(path, line_num, line)
log.write(f"Cannot find segment '{attr_val}'")
log.error("")
else:
# Add segment to symbol
sym.segment = seg
continue
if attr_name == "name_end":
sym.given_name_end = attr_val
continue
if attr_name == "appears_after_overlays_addr":
sym.appears_after_overlays_addr = int(attr_val, 0)
appears_after_overlays_syms.append(sym)
continue
except:
log.parsing_error_preamble(path, line_num, line)
log.write(
f"value of attribute '{attr_name}' could not be read:"
)
log.write("")
raise
# Boolean attributes
tf_val = (
True
if is_truey(attr_val)
else False
if is_falsey(attr_val)
else None
)
if tf_val is None:
log.parsing_error_preamble(path, line_num, line)
log.write(
f"Invalid Boolean value '{attr_val}' for attribute '{attr_name}', should be one of"
)
log.write([*TRUEY_VALS, *FALSEY_VALS])
log.error("")
else:
if attr_name == "dead":
sym.dead = tf_val
continue
if attr_name == "defined":
sym.defined = tf_val
continue
if attr_name == "extract":
sym.extract = tf_val
continue
if attr_name == "ignore":
ignore_sym = tf_val
continue
if attr_name == "force_migration":
sym.force_migration = tf_val
continue
if attr_name == "force_not_migration":
sym.force_not_migration = tf_val
continue
if attr_name == "allow_addend":
sym.allow_addend = tf_val
continue
if attr_name == "dont_allow_addend":
sym.dont_allow_addend = tf_val
continue
if ignore_sym:
if sym.given_size is None or sym.given_size == 0:
ignored_addresses.add(sym.vram_start)
else:
spim_context.addBannedSymbolRangeBySize(
sym.vram_start, sym.given_size
)
continue
if sym.segment:
sym.segment.add_symbol(sym)
sym.user_declared = True
add_symbol(sym)
def initialize(all_segments: "List[Segment]"): def initialize(all_segments: "List[Segment]"):
global all_symbols global all_symbols
global all_symbols_dict global all_symbols_dict
@ -80,187 +250,12 @@ def initialize(all_segments: "List[Segment]"):
all_symbols_dict = {} all_symbols_dict = {}
all_symbols_ranges = IntervalTree() all_symbols_ranges = IntervalTree()
def get_seg_for_name(name: str) -> Optional["Segment"]:
for segment in all_segments:
if segment.name == name:
return segment
return None
# Manual list of func name / addrs # Manual list of func name / addrs
for path in options.opts.symbol_addrs_paths: for path in options.opts.symbol_addrs_paths:
if path.exists(): if path.exists():
with open(path) as f: with open(path) as f:
sym_addrs_lines = f.readlines() sym_addrs_lines = f.readlines()
for line_num, line in enumerate( handle_sym_addrs(path, sym_addrs_lines, all_segments)
tqdm.tqdm(sym_addrs_lines, desc=f"Loading symbols ({path.stem})")
):
line = line.strip()
if not line == "" and not line.startswith("//"):
comment_loc = line.find("//")
line_main = line
line_ext = ""
if comment_loc != -1:
line_ext = line[comment_loc + 2 :].strip()
line_main = line[:comment_loc].strip()
try:
line_split = line_main.split("=")
name = line_split[0].strip()
addr = int(line_split[1].strip()[:-1], 0)
except:
log.parsing_error_preamble(path, line_num, line)
log.write("Line should be of the form")
log.write(
"<function_name> = <address> // attr0:val0 attr1:val1 [...]"
)
log.write("with <address> in hex preceded by 0x, or dec")
log.write("")
raise
sym = Symbol(addr, given_name=name)
ignore_sym = False
if line_ext:
for info in line_ext.split(" "):
if ":" in info:
if info.count(":") > 1:
log.parsing_error_preamble(path, line_num, line)
log.write(f"Too many ':'s in '{info}'")
log.error("")
attr_name, attr_val = info.split(":")
if attr_name == "":
log.parsing_error_preamble(path, line_num, line)
log.write(
f"Missing attribute name in '{info}', is there extra whitespace?"
)
log.error("")
if attr_val == "":
log.parsing_error_preamble(path, line_num, line)
log.write(
f"Missing attribute value in '{info}', is there extra whitespace?"
)
log.error("")
# Non-Boolean attributes
try:
if attr_name == "type":
if not check_valid_type(attr_val):
log.parsing_error_preamble(
path, line_num, line
)
log.write(
f"Unrecognized symbol type in '{info}', it should be one of"
)
log.write(
[
*splat_sym_types,
*spimdisasm.common.gKnownTypes,
]
)
log.write(
"You may use a custom type that starts with a capital letter"
)
log.error("")
type = attr_val
sym.type = type
continue
if attr_name == "size":
size = int(attr_val, 0)
sym.given_size = size
continue
if attr_name == "rom":
rom_addr = int(attr_val, 0)
sym.rom = rom_addr
continue
if attr_name == "segment":
seg = get_seg_for_name(attr_val)
if seg is None:
log.parsing_error_preamble(
path, line_num, line
)
log.write(
f"Cannot find segment '{attr_val}'"
)
log.error("")
else:
# Add segment to symbol
sym.segment = seg
continue
if attr_name == "name_end":
sym.given_name_end = attr_val
continue
if attr_name == "appears_after_overlays_addr":
sym.appears_after_overlays_addr = int(
attr_val, 0
)
appears_after_overlays_syms.append(sym)
continue
except:
log.parsing_error_preamble(path, line_num, line)
log.write(
f"value of attribute '{attr_name}' could not be read:"
)
log.write("")
raise
# Boolean attributes
tf_val = (
True
if is_truey(attr_val)
else False
if is_falsey(attr_val)
else None
)
if tf_val is None:
log.parsing_error_preamble(path, line_num, line)
log.write(
f"Invalid Boolean value '{attr_val}' for attribute '{attr_name}', should be one of"
)
log.write([*TRUEY_VALS, *FALSEY_VALS])
log.error("")
else:
if attr_name == "dead":
sym.dead = tf_val
continue
if attr_name == "defined":
sym.defined = tf_val
continue
if attr_name == "extract":
sym.extract = tf_val
continue
if attr_name == "ignore":
ignore_sym = tf_val
continue
if attr_name == "force_migration":
sym.force_migration = tf_val
continue
if attr_name == "force_not_migration":
sym.force_not_migration = tf_val
continue
if attr_name == "allow_addend":
sym.allow_addend = tf_val
continue
if attr_name == "dont_allow_addend":
sym.dont_allow_addend = tf_val
continue
if ignore_sym:
if sym.given_size == None or sym.given_size == 0:
ignored_addresses.add(sym.vram_start)
else:
spim_context.addBannedSymbolRangeBySize(
sym.vram_start, sym.given_size
)
ignore_sym = False
continue
if sym.segment:
sym.segment.add_symbol(sym)
sym.user_declared = True
add_symbol(sym)
def initialize_spim_context(all_segments: "List[Segment]") -> None: def initialize_spim_context(all_segments: "List[Segment]") -> None:
@ -292,6 +287,7 @@ def initialize_spim_context(all_segments: "List[Segment]") -> None:
continue continue
ram_id = segment.get_exclusive_ram_id() ram_id = segment.get_exclusive_ram_id()
if ram_id is None: if ram_id is None:
if global_vram_start is None: if global_vram_start is None:
global_vram_start = segment.vram_start global_vram_start = segment.vram_start
@ -644,3 +640,18 @@ class Symbol:
def get_all_symbols(): def get_all_symbols():
global all_symbols global all_symbols
return all_symbols return all_symbols
def reset_symbols():
global all_symbols
global all_symbols_dict
global all_symbols_ranges
global ignored_addresses
global to_mark_as_defined
global appears_after_overlays_syms
all_symbols = []
all_symbols_dict = {}
all_symbols_ranges = IntervalTree()
ignored_addresses = set()
to_mark_as_defined = set()
appears_after_overlays_syms = []

18
tools/splat_ext/pm_map_data.py
View File

@ -1,9 +1,8 @@
from math import ceil
import os, sys import os, sys
from pathlib import Path from pathlib import Path
from typing import List
from segtypes.n64.segment import N64Segment from segtypes.n64.segment import N64Segment
from util.n64.Yay0decompress import Yay0Decompressor from util.n64.Yay0decompress import Yay0Decompressor
from util.color import unpack_color
from segtypes.n64.palette import iter_in_groups from segtypes.n64.palette import iter_in_groups
from util import options from util import options
import png # type: ignore import png # type: ignore
@ -30,6 +29,21 @@ def decode_null_terminated_ascii(data):
def parse_palette(data): def parse_palette(data):
palette = [] palette = []
# RRRRRGGG GGBBBBBA
def unpack_color(data):
s = int.from_bytes(data[0:2], byteorder="big")
r = (s >> 11) & 0x1F
g = (s >> 6) & 0x1F
b = (s >> 1) & 0x1F
a = (s & 1) * 0xFF
r = ceil(0xFF * (r / 31))
g = ceil(0xFF * (g / 31))
b = ceil(0xFF * (b / 31))
return r, g, b, a
for a, b in iter_in_groups(data, 2): for a, b in iter_in_groups(data, 2):
palette.append(unpack_color([a, b])) palette.append(unpack_color([a, b]))

183
tools/splat_ext/tex_archives.py
View File

@ -1,12 +1,11 @@
from dataclasses import dataclass from dataclasses import dataclass
import os from math import ceil
import struct import struct
import json import json
from pathlib import Path from pathlib import Path
import png import png
import n64img.image import n64img.image
from util.color import unpack_color, pack_color
from segtypes.n64.palette import iter_in_groups from segtypes.n64.palette import iter_in_groups
from sys import path from sys import path
@ -28,6 +27,21 @@ def decode_null_terminated_ascii(data):
def parse_palette(data): def parse_palette(data):
palette = [] palette = []
# RRRRRGGG GGBBBBBA
def unpack_color(data):
s = int.from_bytes(data[0:2], byteorder="big")
r = (s >> 11) & 0x1F
g = (s >> 6) & 0x1F
b = (s >> 1) & 0x1F
a = (s & 1) * 0xFF
r = ceil(0xFF * (r / 31))
g = ceil(0xFF * (g / 31))
b = ceil(0xFF * (b / 31))
return r, g, b, a
for a, b in iter_in_groups(data, 2): for a, b in iter_in_groups(data, 2):
palette.append(unpack_color([a, b])) palette.append(unpack_color([a, b]))
@ -50,13 +64,13 @@ TILES_MIPMAPS = 1
TILES_SHARED_AUX = 2 TILES_SHARED_AUX = 2
TILES_INDEPENDENT_AUX = 3 TILES_INDEPENDENT_AUX = 3
aux_combine_modes = { AUX_COMBINE_MODES = {
0x00: "None", # multiply main * prim, ignore aux 0x00: "None", # multiply main * prim, ignore aux
0x08: "Multiply", # multiply main * aux * prim 0x08: "Multiply", # multiply main * aux * prim
0x0D: "ModulateAlpha", # use prim color, but multiply alpha by the difference between main and aux red channels 0x0D: "ModulateAlpha", # use prim color, but multiply alpha by the difference between main and aux red channels
0x10: "LerpMainAux", # use prim alpha to lerp between main and aux color, use main alpha 0x10: "LerpMainAux", # use prim alpha to lerp between main and aux color, use main alpha
} }
aux_combine_modes_inv = {v: k for k, v in aux_combine_modes.items()} AUX_COMBINE_MODES_INV = {v: k for k, v in AUX_COMBINE_MODES.items()}
wrap_modes = { wrap_modes = {
0: "Repeat", 0: "Repeat",
@ -363,7 +377,7 @@ class TexImage:
if self.filter_mode == 2: if self.filter_mode == 2:
out["filter"] = True out["filter"] = True
out["combine"] = aux_combine_modes.get(self.combine_mode) out["combine"] = AUX_COMBINE_MODES.get(self.combine_mode)
if self.is_variant: if self.is_variant:
out["variant"] = True out["variant"] = True
@ -395,7 +409,15 @@ class TexImage:
return fmt_str, hwrap, vwrap return fmt_str, hwrap, vwrap
def get_img_file(self, fmt_str, img_file): def get_img_file(self, fmt_str, img_file: str):
def pack_color(r, g, b, a):
r = r >> 3
g = g >> 3
b = b >> 3
a = a >> 7
return (r << 11) | (g << 6) | (b << 1) | a
(out_img, out_w, out_h) = Converter( (out_img, out_w, out_h) = Converter(
mode=fmt_str.lower(), infile=img_file, flip_y=True mode=fmt_str.lower(), infile=img_file, flip_y=True
).convert() ).convert()
@ -414,132 +436,6 @@ class TexImage:
return (out_img, out_pal, out_w, out_h) return (out_img, out_pal, out_w, out_h)
# read texture properties from dictionary and load images
def from_json(self, tex_path: Path, json_data):
self.img_name = json_data["name"]
if "ext" in json_data:
self.raw_ext = json_data["ext"]
else:
self.raw_ext = "tif"
# read data for main tile
main_data = json_data.get("main")
if main_data == None:
raise Exception(f"Texture {self.img_name} has no definition for 'main'")
(main_fmt_name, self.main_hwrap, self.main_vwrap) = self.read_json_img(
main_data, "main", self.img_name
)
(self.main_fmt, self.main_depth) = get_format_code(main_fmt_name)
# read main image
img_path = str(tex_path / f"{self.img_name}.png")
if not os.path.isfile(img_path):
raise Exception(f"Could not find main image for texture: {self.img_name}")
(
self.main_img,
self.main_pal,
self.main_width,
self.main_height,
) = self.get_img_file(main_fmt_name, img_path)
# read data for aux tile
self.has_aux = "aux" in json_data
if self.has_aux:
aux_data = json_data.get("aux")
(aux_fmt_name, self.aux_hwrap, self.aux_vwrap) = self.read_json_img(
aux_data, "aux", self.img_name
)
if aux_fmt_name == "Shared":
# aux tiles have blank attributes in SHARED mode
aux_fmt_name = main_fmt_name
self.aux_fmt = 0
self.aux_depth = 0
self.aux_hwrap = 0
self.aux_vwrap = 0
self.extra_tiles = TILES_SHARED_AUX
else:
(self.aux_fmt, self.aux_depth) = get_format_code(aux_fmt_name)
self.extra_tiles = TILES_INDEPENDENT_AUX
# read aux image
img_path = str(tex_path / f"{self.img_name}_AUX.png")
if not os.path.isfile(img_path):
raise Exception(
f"Could not find AUX image for texture: {self.img_name}"
)
(
self.aux_img,
self.aux_pal,
self.aux_width,
self.aux_height,
) = self.get_img_file(aux_fmt_name, img_path)
if self.extra_tiles == TILES_SHARED_AUX:
# aux tiles have blank sizes in SHARED mode
self.main_height *= 2
self.aux_width = 0
self.aux_height = 0
else:
self.aux_fmt = 0
self.aux_depth = 0
self.aux_hwrap = 0
self.aux_vwrap = 0
self.aux_width = 0
self.aux_height = 0
self.extra_tiles = TILES_BASIC
# read mipmaps
self.has_mipmaps = json_data.get("hasMipmaps", False)
if self.has_mipmaps:
self.mipmaps = []
mipmap_idx = 1
divisor = 2
if self.main_width >= (32 >> self.main_depth):
while True:
if (self.main_width // divisor) <= 0:
break
mmw = self.main_width // divisor
mmh = self.main_height // divisor
img_path = str(tex_path / f"{self.img_name}_MM{mipmap_idx}.png")
if not os.path.isfile(img_path):
raise Exception(
f"Texture {self.img_name} is missing mipmap level {mipmap_idx} (size = {mmw} x {mmh})"
)
(raster, pal, width, height) = self.get_img_file(
main_fmt_name, img_path
)
self.mipmaps.append(raster)
if width != mmw or height != mmh:
raise Exception(
f"Texture {self.img_name} has wrong size for mipmap level {mipmap_idx} \n"
+ f"MM{mipmap_idx} size = {width} x {height}, but should be = {mmw} x {mmh}"
)
divisor = divisor * 2
mipmap_idx += 1
if (self.main_width // divisor) < (16 >> self.main_depth):
break
self.extra_tiles = TILES_MIPMAPS
# read filter mode
if json_data.get("filter", False):
self.filter_mode = 2
else:
self.filter_mode = 0
# read tile combine mode
combine_str = json_data.get("combine", "Missing")
self.combine = aux_combine_modes_inv.get(combine_str)
if self.combine == None:
raise Exception(f"Texture {self.img_name} has invalid 'combine'")
self.is_variant = json_data.get("variant", False)
# write texture header and image raster/palettes to byte array # write texture header and image raster/palettes to byte array
def add_bytes(self, tex_name: str, bytes: bytearray): def add_bytes(self, tex_name: str, bytes: bytearray):
# form raw name and write to header # form raw name and write to header
@ -617,26 +513,3 @@ class TexArchive:
json_fn = str(tex_path) + ".json" json_fn = str(tex_path) + ".json"
with open(json_fn, "w") as f: with open(json_fn, "w") as f:
f.write(json_out) f.write(json_out)
@staticmethod
def build(out_path: Path, tex_path: Path, endian: str = "big"):
out_bytes = bytearray()
tex_name = os.path.basename(tex_path)
json_fn = str(tex_path) + ".json"
with open(json_fn, "r") as json_file:
json_str = json_file.read()
json_data = json.loads(json_str)
if len(json_data) > 128:
raise Exception(
f"Maximum number of textures (128) exceeded by {tex_name} ({len(json_data)})`"
)
for img_data in json_data:
img = TexImage()
img.from_json(tex_path, img_data)
img.add_bytes(tex_name, out_bytes)
with open(out_path, "wb") as out_bin:
out_bin.write(out_bytes)