From f91fe539a4b92775b3c983e896f0fbc448cf0033 Mon Sep 17 00:00:00 2001 From: Ethan Roseman Date: Thu, 13 Jul 2023 17:56:16 +0900 Subject: [PATCH] Snug bugs unrelated to and never been inside of a rug (#1082) * Fix enum name, offset * Fix bugs Some assets were slipping by the asset stack Tex archve building wasn't respecting the asset stack (Fixes #1074) * Fixes #1081 * fix paths kinda * git subrepo pull --force tools/splat subrepo: subdir: "tools/splat" merged: "818924683b" upstream: origin: "https://github.com/ethteck/splat.git" branch: "master" commit: "818924683b" git-subrepo: version: "0.4.5" origin: "https://github.com/ingydotnet/git-subrepo" commit: "aa416e4" * Fix stuff after splupdate --- include/enums.h | 2 +- src/filemenu/filemenu_main.c | 6 +- tools/build/configure.py | 22 +- tools/build/mapfs/tex.py | 190 ++++++++- tools/splat/.gitrepo | 4 +- tools/splat/CHANGELOG.md | 32 +- .../disassembler/spimdisasm_disassembler.py | 17 +- tools/splat/disassembler_section.py | 283 ++++++++++++++ tools/splat/requirements.txt | 2 +- tools/splat/segtypes/common/bss.py | 14 +- tools/splat/segtypes/common/c.py | 12 +- tools/splat/segtypes/common/code.py | 17 +- tools/splat/segtypes/common/codesubsegment.py | 31 +- tools/splat/segtypes/common/data.py | 22 +- tools/splat/segtypes/common/rodata.py | 17 +- tools/splat/segtypes/linker_entry.py | 5 +- tools/splat/split.py | 4 +- tools/splat/test.py | 272 ++++++++++--- tools/splat/test/basic_app/build.sh | 0 .../basic_app/expected/asm/data/main.data.s | 2 + .../test/basic_app/expected/asm/handwritten.s | 1 + .../asm/nonmatchings/main/func_80000400.s | 1 + .../asm/nonmatchings/main/func_800004A0.s | 1 + tools/splat/util/color.py | 8 - tools/splat/util/n64/rominfo.py | 2 +- tools/splat/util/options.py | 27 +- tools/splat/util/symbols.py | 363 +++++++++--------- tools/splat_ext/pm_map_data.py | 18 +- tools/splat_ext/tex_archives.py | 183 ++------- 29 files changed, 1096 insertions(+), 462 deletions(-) create mode 100644 tools/splat/disassembler_section.py mode change 100644 => 100755 tools/splat/test/basic_app/build.sh diff --git a/include/enums.h b/include/enums.h index 7bf566fde1..2c52c4466b 100644 --- a/include/enums.h +++ b/include/enums.h @@ -5941,7 +5941,7 @@ enum FileMenuMessages { /* 33 */ FILE_MESSAGE_QUESTION, // ?[End] /* 34 */ FILE_MESSAGE_PERIOD_34, // .[End] #if VERSION_PAL - UNK3, + FILE_MESSAGE_BASE_UNK, #endif }; diff --git a/src/filemenu/filemenu_main.c b/src/filemenu/filemenu_main.c index f12aed4648..d5ab7ccd84 100644 --- a/src/filemenu/filemenu_main.c +++ b/src/filemenu/filemenu_main.c @@ -567,7 +567,7 @@ void filemenu_draw_contents_file_info(s32 fileIdx, #if VERSION_PAL xOffset = D_filemenu_802508D8[gCurrentLanguage]; #else - xOffset = 0x22; + xOffset = 34; #endif filemenu_draw_message(filemenu_get_menu_message(FILE_MESSAGE_LEVEL), baseX + xOffset, baseY + 10, 255, 0xA, 1); temp_s3_2 = gSaveSlotMetadata[fileIdx].level; @@ -625,12 +625,12 @@ void filemenu_draw_contents_file_title( filemenu_draw_message(filemenu_get_menu_message(FILE_MESSAGE_OK), baseX + FILE_X, baseY + 1, 255, 0, 1); if (!gSaveSlotHasData[fileIdx]) { - filemenu_draw_message(filemenu_get_menu_message(fileIdx + UNK3), + filemenu_draw_message(filemenu_get_menu_message(fileIdx + FILE_MESSAGE_BASE_UNK), baseX + D_filemenu_802508D0[gCurrentLanguage], baseY + 1, 255, 0, 1); } else { s32 tmp = D_filemenu_802508D0[gCurrentLanguage]; - filemenu_draw_message(filemenu_get_menu_message(fileIdx + UNK3), + filemenu_draw_message(filemenu_get_menu_message(fileIdx + FILE_MESSAGE_BASE_UNK), baseX + tmp, baseY + 1, 255, 0, 1); tmp += D_filemenu_802508D4[gCurrentLanguage]; diff --git a/tools/build/configure.py b/tools/build/configure.py index 0eb1805c66..c736e98856 100755 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -16,6 +16,9 @@ DO_SHA1_CHECK = True # Paths: ROOT = Path(__file__).parent.parent.parent +if ROOT.is_absolute(): + ROOT = ROOT.relative_to(Path.cwd()) + BUILD_TOOLS = Path("tools/build") YAY0_COMPRESS_TOOL = f"{BUILD_TOOLS}/yay0/Yay0compress" CRC_TOOL = f"{BUILD_TOOLS}/rom/n64crc" @@ -241,7 +244,7 @@ def write_ninja_rules( ninja.rule( "tex", description="tex $out", - command=f"$python {BUILD_TOOLS}/mapfs/tex.py $out $tex_dir", + command=f"$python {BUILD_TOOLS}/mapfs/tex.py $out $tex_name $asset_stack", ) ninja.rule( @@ -408,6 +411,9 @@ class Configure: @lru_cache(maxsize=None) def resolve_asset_path(self, path: Path) -> Path: + # Remove nonsense + path = Path(os.path.normpath(path)) + parts = list(path.parts) if parts[0] != "assets": @@ -493,7 +499,7 @@ class Configure: ninja.build( outputs=object_strs, # $out rule=task, - inputs=self.resolve_src_paths(src_paths), # $in + inputs=inputs, # $in implicit=implicit, order_only=order_only, variables={"version": self.version, **variables}, @@ -743,6 +749,7 @@ class Configure: "sprite_name": sprite_name, "asset_stack": ",".join(self.asset_stack), }, + asset_deps=[str(sprite_dir)], ) build(yay0_path, [bin_path], "yay0") @@ -801,9 +808,8 @@ class Configure: ) build(entry.object_path, [entry.object_path.with_suffix(".bin")], "bin") elif seg.type == "pm_map_data": - bin_yay0s: List[ - Path - ] = [] # flat list of (uncompressed path, compressed? path) pairs + # flat list of (uncompressed path, compressed? path) pairs + bin_yay0s: List[Path] = [] src_dir = Path("assets/x") / seg.name for path in entry.src_paths: @@ -906,7 +912,11 @@ class Configure: bin_path, [tex_dir, path.parent / (name + ".json")], "tex", - variables={"tex_dir": str(tex_dir)}, + variables={ + "tex_name": name, + "asset_stack": ",".join(self.asset_stack), + }, + asset_deps=[f"mapfs/tex/{name}"], ) elif name.endswith("_shape"): map_name = "_".join(name.split("_")[:-1]) diff --git a/tools/build/mapfs/tex.py b/tools/build/mapfs/tex.py index 113cbbf64a..e0b39a5981 100644 --- a/tools/build/mapfs/tex.py +++ b/tools/build/mapfs/tex.py @@ -1,19 +1,199 @@ #!/usr/bin/env python3 import argparse +import json +import os from pathlib import Path -from sys import argv, path +from sys import path +from typing import Tuple + path.append(str(Path(__file__).parent.parent.parent / "splat")) -path.append(str(Path(__file__).parent.parent.parent / "splat_ext")) -from tex_archives import TexArchive +path.append(str(Path(__file__).parent.parent.parent / "build")) + +from common import get_asset_path + +path.append(str(Path(__file__).parent.parent.parent)) +from splat_ext.tex_archives import ( + AUX_COMBINE_MODES_INV, + TILES_BASIC, + TILES_INDEPENDENT_AUX, + TILES_MIPMAPS, + TILES_SHARED_AUX, + TexImage, + get_format_code, +) + + +# read texture properties from dictionary and load images +def img_from_json(json_data, tex_name: str, asset_stack: Tuple[Path, ...]) -> TexImage: + ret = TexImage() + + ret.img_name = json_data["name"] + + if "ext" in json_data: + ret.raw_ext = json_data["ext"] + else: + ret.raw_ext = "tif" + + # read data for main tile + main_data = json_data.get("main") + if main_data == None: + raise Exception(f"Texture {ret.img_name} has no definition for 'main'") + + (main_fmt_name, ret.main_hwrap, ret.main_vwrap) = ret.read_json_img( + main_data, "main", ret.img_name + ) + (ret.main_fmt, ret.main_depth) = get_format_code(main_fmt_name) + + # read main image + img_path = get_asset_path( + Path(f"mapfs/tex/{tex_name}/{ret.img_name}.png"), asset_stack + ) + if not os.path.isfile(img_path): + raise Exception(f"Could not find main image for texture: {ret.img_name}") + ( + ret.main_img, + ret.main_pal, + ret.main_width, + ret.main_height, + ) = ret.get_img_file(main_fmt_name, str(img_path)) + + # read data for aux tile + ret.has_aux = "aux" in json_data + if ret.has_aux: + aux_data = json_data.get("aux") + (aux_fmt_name, ret.aux_hwrap, ret.aux_vwrap) = ret.read_json_img( + aux_data, "aux", ret.img_name + ) + + if aux_fmt_name == "Shared": + # aux tiles have blank attributes in SHARED mode + aux_fmt_name = main_fmt_name + ret.aux_fmt = 0 + ret.aux_depth = 0 + ret.aux_hwrap = 0 + ret.aux_vwrap = 0 + ret.extra_tiles = TILES_SHARED_AUX + else: + (ret.aux_fmt, ret.aux_depth) = get_format_code(aux_fmt_name) + ret.extra_tiles = TILES_INDEPENDENT_AUX + + # read aux image + img_path = get_asset_path( + Path(f"mapfs/tex/{tex_name}/{ret.img_name}_AUX.png"), asset_stack + ) + if not os.path.isfile(img_path): + raise Exception(f"Could not find AUX image for texture: {ret.img_name}") + ( + ret.aux_img, + ret.aux_pal, + ret.aux_width, + ret.aux_height, + ) = ret.get_img_file(aux_fmt_name, str(img_path)) + if ret.extra_tiles == TILES_SHARED_AUX: + # aux tiles have blank sizes in SHARED mode + ret.main_height *= 2 + ret.aux_width = 0 + ret.aux_height = 0 + + else: + ret.aux_fmt = 0 + ret.aux_depth = 0 + ret.aux_hwrap = 0 + ret.aux_vwrap = 0 + ret.aux_width = 0 + ret.aux_height = 0 + ret.extra_tiles = TILES_BASIC + + # read mipmaps + ret.has_mipmaps = json_data.get("hasMipmaps", False) + if ret.has_mipmaps: + ret.mipmaps = [] + mipmap_idx = 1 + divisor = 2 + if ret.main_width >= (32 >> ret.main_depth): + while True: + if (ret.main_width // divisor) <= 0: + break + mmw = ret.main_width // divisor + mmh = ret.main_height // divisor + + img_path = get_asset_path( + Path(f"mapfs/tex/{tex_name}/{ret.img_name}_MM{mipmap_idx}.png"), + asset_stack, + ) + if not os.path.isfile(img_path): + raise Exception( + f"Texture {ret.img_name} is missing mipmap level {mipmap_idx} (size = {mmw} x {mmh})" + ) + + (raster, pal, width, height) = ret.get_img_file( + main_fmt_name, str(img_path) + ) + ret.mipmaps.append(raster) + if width != mmw or height != mmh: + raise Exception( + f"Texture {ret.img_name} has wrong size for mipmap level {mipmap_idx} \n" + + f"MM{mipmap_idx} size = {width} x {height}, but should be = {mmw} x {mmh}" + ) + + divisor = divisor * 2 + mipmap_idx += 1 + if (ret.main_width // divisor) < (16 >> ret.main_depth): + break + ret.extra_tiles = TILES_MIPMAPS + + # read filter mode + if json_data.get("filter", False): + ret.filter_mode = 2 + else: + ret.filter_mode = 0 + + # read tile combine mode + combine_str = json_data.get("combine", "Missing") + ret.combine = AUX_COMBINE_MODES_INV.get(combine_str) + if ret.combine == None: + raise Exception(f"Texture {ret.img_name} has invalid 'combine'") + + ret.is_variant = json_data.get("variant", False) + + return ret + + +def build( + out_path: Path, tex_name: str, asset_stack: Tuple[Path, ...], endian: str = "big" +): + out_bytes = bytearray() + + json_path = get_asset_path(Path(f"mapfs/tex/{tex_name}.json"), asset_stack) + + with open(json_path) as json_file: + json_str = json_file.read() + json_data = json.loads(json_str) + + if len(json_data) > 128: + raise Exception( + f"Maximum number of textures (128) exceeded by {tex_name} ({len(json_data)})`" + ) + + for img_data in json_data: + img = img_from_json(img_data, tex_name, asset_stack) + img.add_bytes(tex_name, out_bytes) + + with open(out_path, "wb") as out_bin: + out_bin.write(out_bytes) + if __name__ == "__main__": parser = argparse.ArgumentParser(description="Texture archives") parser.add_argument("bin_out", type=Path, help="Output binary file path") - parser.add_argument("tex_dir", type=Path, help="File path to input tex subdirectory") + parser.add_argument("name", help="Name of tex subdirectory") + parser.add_argument("asset_stack", help="comma-separated asset stack") parser.add_argument( "--endian", choices=["big", "little"], default="big", help="Output endianness" ) args = parser.parse_args() - TexArchive.build(args.bin_out, args.tex_dir, args.endian) + asset_stack = tuple(Path(d) for d in args.asset_stack.split(",")) + + build(args.bin_out, args.name, asset_stack, args.endian) diff --git a/tools/splat/.gitrepo b/tools/splat/.gitrepo index 0b8a57334a..6ed00eb1d1 100644 --- a/tools/splat/.gitrepo +++ b/tools/splat/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/ethteck/splat.git branch = master - commit = e72a868f9f7e9da25f13194b51b93e64c2dcc83a - parent = 4dfc35713736d65d8d607a0e0f4121bc6938d6e3 + commit = 818924683bfb1145129b5e43ff02abe7b4be37a3 + parent = c122fe97362f81b15d8ede79f24b28a62e859a68 method = merge cmdver = 0.4.5 diff --git a/tools/splat/CHANGELOG.md b/tools/splat/CHANGELOG.md index 4ea2ade118..95a106e878 100644 --- a/tools/splat/CHANGELOG.md +++ b/tools/splat/CHANGELOG.md @@ -1,5 +1,29 @@ # splat Release Notes +### 0.15.1 +* Made some modifications such that linker object paths should be simpler in some circumstances + +### 0.15.0 + +* New options: + * `data_string_encoding` can be set at the global level (or `str_encoding` at the segment level) to specify the encoding using when guessing and disassembling strings the the data section. In spimdisasm this value defaults to ASCII. + * `rodata_string_guesser_level` changes the behaviour of the rodata string guesser. A higher value means more agressive guessing, while 0 and negative means no guessing at all. Even if the guesser feature is disabled, symbols manually marked as strings in the symbol_addrs.txt file will still be disassembled as strings. In spimdisasm this value defaults to 1. + * level 0: Completely disable the guessing feature. + * level 1: The most conservative guessing level. Imposes the following restrictions: + * Do not try to guess if the user provided a type for the symbol. + * Do no try to guess if type information for the symbol can be inferred by other means. + * A string symbol must be referenced only once. + * Strings must not be empty. + * level 2: A string no longer needs to be referenced only once to be considered a possible string. This can happen because of a deduplication optimization. + * level 3: Empty strings are allowed. + * level 4: Symbols with autodetected type information but no user type information can still be guessed as strings. + * `data_string_guesser_level` is similar to `rodata_string_guesser_level`, but for the data section instead. In spimdisasm this value defaults to 2. + * `asm_emit_size_directive` toggles the size directived emitted by the disassembler. In spimdisasm this defaults to True. + +### 0.14.1 + +* Fix bug, cod cleanup + ### 0.14.0 * Add support for PSX's GTE instruction set @@ -40,7 +64,7 @@ ### 0.13.3 -* Added a new symbol_addrs attribute `appears_after_overlays_addr:0x1234` which will modify the linker script such that the symbol's address is equal to the value of the end of the longest overlay starting with address 0x1234. It achieve this by writing a series of sym = MAX(sym, seg_vram_END) statements into the linker script. For some games, it's feasible to manually create such statements, but for games with hundreds of overlays at the same address, this is very tedious and prone to error. The new attribute allows you to have peace of mind that the symbol will end up after all of these overlays. +* Added a new symbol_addrs attribute `appears_after_overlays_addr:0x1234` which will modify the linker script such that the symbol's address is equal to the value of the end of the longest overlay starting with address 0x1234. It achieves this by writing a series of sym = MAX(sym, seg_vram_END) statements into the linker script. For some games, it's feasible to manually create such statements, but for games with hundreds of overlays at the same address, this is very tedious and prone to error. The new attribute allows you to have peace of mind that the symbol will end up after all of these overlays. ### 0.13.2 @@ -61,7 +85,7 @@ ### 0.12.14 * New option: `pair_rodata_to_text`. - * If enabled, splat will try to find to which text segment an unpaired rodata segment belongs and it will hint it to the user. + * If enabled, splat will try to find to which text segment an unpaired rodata segment belongs, and it will hint it to the user. ### 0.12.13 @@ -91,7 +115,7 @@ ### 0.12.8 * The gfx and vtx segments now have a `data_only` option, which, if enabled, will emit only the plain data for the type and omit the enclosing symbol definition. This mode is useful when you want to manually declare the symbol and then #include the extracted data within the declaration. -* The gfx segment has a method, `format_sym_name()`, which will allow custom overriding of the output of symbol names by extending the `gfx` segment. For example, this can be used to transform context-specific symbol names like mac_01_vtx into N(vtx), where N() is a macro that applies the current "namespace" to the symbol. Paper Mario plans to use this so we can extract an asset once and then #include it in multiple places, while giving each inclusion unique symbol names for each component. +* The gfx segment has a method, `format_sym_name()`, which will allow custom overriding of the output of symbol names by extending the `gfx` segment. For example, this can be used to transform context-specific symbol names like mac_01_vtx into N(vtx), where N() is a macro that applies the current "namespace" to the symbol. Paper Mario plans to use this, so we can extract an asset once and then #include it in multiple places, while giving each inclusion unique symbol names for each component. ### 0.12.7 @@ -115,7 +139,7 @@ * Update minimal spimdisasm version to 1.7.1. * Fix spimdisasm>=1.7.0 non being able to see symbols which only are referenced by other data symbols. -* An check was added to prevent segments marked with `exclusive_ram_id` have a vram address range which overlaps with segments not marked with said tag. If this happens it will be warned to the user. +* A check was added to prevent segments marked with `exclusive_ram_id` have a vram address range which overlaps with segments not marked with said tag. If this happens it will be warned to the user. ### 0.12.4 diff --git a/tools/splat/disassembler/spimdisasm_disassembler.py b/tools/splat/disassembler/spimdisasm_disassembler.py index 4e3ffdcb06..399b4e3f08 100644 --- a/tools/splat/disassembler/spimdisasm_disassembler.py +++ b/tools/splat/disassembler/spimdisasm_disassembler.py @@ -8,7 +8,7 @@ from typing import Set class SpimdisasmDisassembler(disassembler.Disassembler): # This value should be kept in sync with the version listed on requirements.txt - SPIMDISASM_MIN = (1, 13, 0) + SPIMDISASM_MIN = (1, 15, 0) def configure(self, opts: SplatOpts): # Configure spimdisasm @@ -28,6 +28,16 @@ class SpimdisasmDisassembler(disassembler.Disassembler): spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO = False + if opts.rodata_string_guesser_level is not None: + spimdisasm.common.GlobalConfig.RODATA_STRING_GUESSER_LEVEL = ( + opts.rodata_string_guesser_level + ) + + if opts.data_string_guesser_level is not None: + spimdisasm.common.GlobalConfig.DATA_STRING_GUESSER_LEVEL = ( + opts.data_string_guesser_level + ) + rabbitizer.config.regNames_userFpcCsr = False rabbitizer.config.regNames_vr4300Cop0NamedRegisters = False @@ -70,6 +80,11 @@ class SpimdisasmDisassembler(disassembler.Disassembler): spimdisasm.common.GlobalConfig.ASM_DATA_LABEL = opts.asm_data_macro spimdisasm.common.GlobalConfig.ASM_TEXT_END_LABEL = opts.asm_end_label + if opts.asm_emit_size_directive is not None: + spimdisasm.common.GlobalConfig.ASM_EMIT_SIZE_DIRECTIVE = ( + opts.asm_emit_size_directive + ) + if spimdisasm.common.GlobalConfig.ASM_TEXT_LABEL == ".globl": spimdisasm.common.GlobalConfig.ASM_TEXT_ENT_LABEL = ".ent" spimdisasm.common.GlobalConfig.ASM_TEXT_FUNC_AS_LABEL = True diff --git a/tools/splat/disassembler_section.py b/tools/splat/disassembler_section.py new file mode 100644 index 0000000000..2eb745a32a --- /dev/null +++ b/tools/splat/disassembler_section.py @@ -0,0 +1,283 @@ +import spimdisasm +from util import symbols +from typing import Optional, Set, Tuple +from segtypes.segment import Segment +from util import log, options, symbols + + +from abc import ABC, abstractmethod +from typing import Callable + + +class DisassemblerSection(ABC): + @abstractmethod + def disassemble(self): + raise NotImplementedError("disassemble") + + @abstractmethod + def analyze(self): + raise NotImplementedError("analyze") + + @abstractmethod + def set_comment_offset(self, rom_start: int): + raise NotImplementedError("set_comment_offset") + + @abstractmethod + def make_bss_section( + self, + rom_start, + rom_end, + vram_start, + bss_end, + name, + segment_rom_start, + exclusive_ram_id, + ): + raise NotImplementedError("make_bss_section") + + @abstractmethod + def make_data_section( + self, + rom_start, + rom_end, + vram_start, + name, + rom_bytes, + segment_rom_start, + exclusive_ram_id, + ): + raise NotImplementedError("make_data_section") + + @abstractmethod + def get_section(self): + raise NotImplementedError("get_section") + + @abstractmethod + def make_rodata_section( + self, + rom_start, + rom_end, + vram_start, + name, + rom_bytes, + segment_rom_start, + exclusive_ram_id, + ): + raise NotImplementedError("make_rodata_section") + + @abstractmethod + def make_text_section( + self, + rom_start, + rom_end, + vram_start, + name, + rom_bytes, + segment_rom_start, + exclusive_ram_id, + ): + raise NotImplementedError("make_text_section") + + +class SpimdisasmDisassemberSection(DisassemblerSection): + def __init__(self): + self.spim_section: Optional[spimdisasm.mips.sections.SectionBase] = None + + def disassemble(self) -> str: + assert self.spim_section is not None + return self.spim_section.disassemble() + + def analyze(self): + assert self.spim_section is not None + self.spim_section.analyze() + + def set_comment_offset(self, rom_start: int): + assert self.spim_section is not None + self.spim_section.setCommentOffset(rom_start) + + def make_bss_section( + self, + rom_start: int, + rom_end: int, + vram_start: int, + bss_end: int, + name: str, + segment_rom_start: int, + exclusive_ram_id, + ): + self.spim_section = spimdisasm.mips.sections.SectionBss( + symbols.spim_context, + rom_start, + rom_end, + vram_start, + bss_end, + name, + segment_rom_start, + exclusive_ram_id, + ) + + def make_data_section( + self, + rom_start: int, + rom_end: int, + vram_start: int, + name: str, + rom_bytes: bytes, + segment_rom_start: int, + exclusive_ram_id, + ): + self.spim_section = spimdisasm.mips.sections.SectionData( + symbols.spim_context, + rom_start, + rom_end, + vram_start, + name, + rom_bytes, + segment_rom_start, + exclusive_ram_id, + ) + + def get_section(self) -> Optional[spimdisasm.mips.sections.SectionBase]: + return self.spim_section + + def make_rodata_section( + self, + rom_start: int, + rom_end: int, + vram_start: int, + name: str, + rom_bytes: bytes, + segment_rom_start: int, + exclusive_ram_id, + ): + self.spim_section = spimdisasm.mips.sections.SectionRodata( + symbols.spim_context, + rom_start, + rom_end, + vram_start, + name, + rom_bytes, + segment_rom_start, + exclusive_ram_id, + ) + + def make_text_section( + self, + rom_start: int, + rom_end: int, + vram_start: int, + name: str, + rom_bytes: bytes, + segment_rom_start: int, + exclusive_ram_id, + ): + self.spim_section = spimdisasm.mips.sections.SectionText( + symbols.spim_context, + rom_start, + rom_end, + vram_start, + name, + rom_bytes, + segment_rom_start, + exclusive_ram_id, + ) + + +def make_disassembler_section() -> Optional[SpimdisasmDisassemberSection]: + if options.opts.platform in ["n64", "psx", "ps2"]: + return SpimdisasmDisassemberSection() + + raise NotImplementedError("No disassembler section for requested platform") + return None + + +def make_text_section( + rom_start: int, + rom_end: int, + vram_start: int, + name: str, + rom_bytes: bytes, + segment_rom_start: int, + exclusive_ram_id, +) -> DisassemblerSection: + section = make_disassembler_section() + assert section is not None + section.make_text_section( + rom_start, + rom_end, + vram_start, + name, + rom_bytes, + segment_rom_start, + exclusive_ram_id, + ) + return section + + +def make_data_section( + rom_start: int, + rom_end: int, + vram_start: int, + name: str, + rom_bytes: bytes, + segment_rom_start: int, + exclusive_ram_id, +) -> DisassemblerSection: + section = make_disassembler_section() + assert section is not None + section.make_data_section( + rom_start, + rom_end, + vram_start, + name, + rom_bytes, + segment_rom_start, + exclusive_ram_id, + ) + return section + + +def make_rodata_section( + rom_start: int, + rom_end: int, + vram_start: int, + name: str, + rom_bytes: bytes, + segment_rom_start: int, + exclusive_ram_id, +) -> DisassemblerSection: + section = make_disassembler_section() + assert section is not None + section.make_rodata_section( + rom_start, + rom_end, + vram_start, + name, + rom_bytes, + segment_rom_start, + exclusive_ram_id, + ) + return section + + +def make_bss_section( + rom_start: int, + rom_end: int, + vram_start: int, + bss_end: int, + name: str, + segment_rom_start: int, + exclusive_ram_id, +) -> DisassemblerSection: + section = make_disassembler_section() + assert section is not None + section.make_bss_section( + rom_start, + rom_end, + vram_start, + bss_end, + name, + segment_rom_start, + exclusive_ram_id, + ) + return section diff --git a/tools/splat/requirements.txt b/tools/splat/requirements.txt index 77cb923673..56576c295b 100644 --- a/tools/splat/requirements.txt +++ b/tools/splat/requirements.txt @@ -4,7 +4,7 @@ tqdm intervaltree colorama # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py -spimdisasm>=1.13.0 +spimdisasm>=1.15.0 rabbitizer>=1.7.0 pygfxd n64img>=0.1.4 diff --git a/tools/splat/segtypes/common/bss.py b/tools/splat/segtypes/common/bss.py index ef79311fec..4bc74d1fc8 100644 --- a/tools/splat/segtypes/common/bss.py +++ b/tools/splat/segtypes/common/bss.py @@ -1,8 +1,9 @@ -import spimdisasm from util import options, symbols, log from segtypes.common.data import CommonSegData +from disassembler_section import make_bss_section + class CommonSegBss(CommonSegData): def get_linker_section(self) -> str: @@ -37,8 +38,7 @@ class CommonSegBss(CommonSegData): bss_end = next_subsegment.vram_start assert isinstance(bss_end, int), f"{self.name} {bss_end}" - self.spim_section = spimdisasm.mips.sections.SectionBss( - symbols.spim_context, + self.spim_section = make_bss_section( self.rom_start, self.rom_end, self.vram_start, @@ -48,10 +48,12 @@ class CommonSegBss(CommonSegData): self.get_exclusive_ram_id(), ) - self.spim_section.analyze() - self.spim_section.setCommentOffset(self.rom_start) + assert self.spim_section is not None - for spim_sym in self.spim_section.symbolList: + self.spim_section.analyze() + self.spim_section.set_comment_offset(self.rom_start) + + for spim_sym in self.spim_section.get_section().symbolList: symbols.create_symbol_from_spim_symbol( self.get_most_parent(), spim_sym.contextSym ) diff --git a/tools/splat/segtypes/common/c.py b/tools/splat/segtypes/common/c.py index f824382ad9..5990b6c78f 100644 --- a/tools/splat/segtypes/common/c.py +++ b/tools/splat/segtypes/common/c.py @@ -153,10 +153,10 @@ class CommonSegC(CommonSegCodeSubsegment): self.print_file_boundaries() assert self.spim_section is not None and isinstance( - self.spim_section, spimdisasm.mips.sections.SectionText + self.spim_section.get_section(), spimdisasm.mips.sections.SectionText ), f"{self.name}, rom_start:{self.rom_start}, rom_end:{self.rom_end}" - rodata_spim_segment = None + rodata_spim_segment: Optional[spimdisasm.mips.sections.SectionRodata] = None if ( options.opts.migrate_rodata_to_functions and self.rodata_sibling is not None @@ -166,15 +166,15 @@ class CommonSegC(CommonSegCodeSubsegment): ), self.rodata_sibling.type if self.rodata_sibling.spim_section is not None: assert isinstance( - self.rodata_sibling.spim_section, + self.rodata_sibling.spim_section.get_section(), spimdisasm.mips.sections.SectionRodata, ) - rodata_spim_segment = self.rodata_sibling.spim_section + rodata_spim_segment = self.rodata_sibling.spim_section.get_section() # Precompute function-rodata pairings symbols_entries = ( spimdisasm.mips.FunctionRodataEntry.getAllEntriesFromSections( - self.spim_section, rodata_spim_segment + self.spim_section.get_section(), rodata_spim_segment ) ) @@ -319,7 +319,7 @@ class CommonSegC(CommonSegCodeSubsegment): macro_name: str, ) -> str: if options.opts.compiler == IDO: - # IDO uses the asm processor to embeed assembly and it doesn't require a special directive to include symbols + # IDO uses the asm processor to embeed assembly, and it doesn't require a special directive to include symbols asm_outpath = Path( os.path.join(asm_out_dir, self.name, spim_sym.getName() + ".s") ) diff --git a/tools/splat/segtypes/common/code.py b/tools/splat/segtypes/common/code.py index dc9a5bce57..a3f614743f 100644 --- a/tools/splat/segtypes/common/code.py +++ b/tools/splat/segtypes/common/code.py @@ -278,8 +278,17 @@ class CommonSegCode(CommonSegGroup): ) segment.sibling = base_segments.get(segment.name, None) - if segment.is_rodata() and segment.sibling is not None: - segment.sibling.rodata_sibling = segment + + if segment.sibling is not None: + if self.section_order.index(".text") < self.section_order.index( + ".rodata" + ): + if segment.is_rodata(): + segment.sibling.rodata_sibling = segment + else: + if segment.is_text() and segment.sibling.is_rodata(): + segment.rodata_sibling = segment.sibling + segment.sibling.sibling = segment segment.parent = self if segment.special_vram_segment: @@ -300,6 +309,10 @@ class CommonSegCode(CommonSegGroup): if segment.is_text(): base_segments[segment.name] = segment + if self.section_order.index(".rodata") < self.section_order.index(".text"): + if segment.is_rodata() and segment.sibling is None: + base_segments[segment.name] = segment + prev_start = start if end is not None: last_rom_end = end diff --git a/tools/splat/segtypes/common/codesubsegment.py b/tools/splat/segtypes/common/codesubsegment.py index 7507673a0b..a26f2cbe9d 100644 --- a/tools/splat/segtypes/common/codesubsegment.py +++ b/tools/splat/segtypes/common/codesubsegment.py @@ -10,6 +10,8 @@ from segtypes.common.code import CommonSegCode from segtypes.segment import Segment +from disassembler_section import DisassemblerSection, make_text_section + # abstract class for c, asm, data, etc class CommonSegCodeSubsegment(Segment): @@ -24,7 +26,7 @@ class CommonSegCodeSubsegment(Segment): self.yaml.get("str_encoding", None) if isinstance(self.yaml, dict) else None ) - self.spim_section: Optional[spimdisasm.mips.sections.SectionBase] = None + self.spim_section: Optional[DisassemblerSection] = None self.instr_category = rabbitizer.InstrCategory.CPU if options.opts.platform == "ps2": self.instr_category = rabbitizer.InstrCategory.R5900 @@ -56,8 +58,7 @@ class CommonSegCodeSubsegment(Segment): f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" ) - self.spim_section = spimdisasm.mips.sections.SectionText( - symbols.spim_context, + self.spim_section = make_text_section( self.rom_start, self.rom_end, self.vram_start, @@ -67,13 +68,15 @@ class CommonSegCodeSubsegment(Segment): self.get_exclusive_ram_id(), ) - self.spim_section.isHandwritten = is_hasm - self.spim_section.instrCat = self.instr_category + assert self.spim_section is not None + + self.spim_section.get_section().isHandwritten = is_hasm + self.spim_section.get_section().instrCat = self.instr_category self.spim_section.analyze() - self.spim_section.setCommentOffset(self.rom_start) + self.spim_section.set_comment_offset(self.rom_start) - for func in self.spim_section.symbolList: + for func in self.spim_section.get_section().symbolList: assert isinstance(func, spimdisasm.mips.symbols.SymbolFunction) self.process_insns(func) @@ -85,7 +88,7 @@ class CommonSegCodeSubsegment(Segment): jtbl_label_vram, True, type="jtbl_label", define=True ) sym.type = "jtbl_label" - symbols.add_symbol_to_spim_section(self.spim_section, sym) + symbols.add_symbol_to_spim_section(self.spim_section.get_section(), sym) def process_insns( self, @@ -103,7 +106,7 @@ class CommonSegCodeSubsegment(Segment): # Gather symbols found by spimdisasm and create those symbols in splat's side for referenced_vram in func_spim.instrAnalyzer.referencedVrams: - context_sym = self.spim_section.getSymbol( + context_sym = self.spim_section.get_section().getSymbol( referenced_vram, tryPlusOffset=False ) if context_sym is not None: @@ -124,7 +127,7 @@ class CommonSegCodeSubsegment(Segment): if instr_offset in func_spim.instrAnalyzer.symbolInstrOffset: sym_address = func_spim.instrAnalyzer.symbolInstrOffset[instr_offset] - context_sym = self.spim_section.getSymbol( + context_sym = self.spim_section.get_section().getSymbol( sym_address, tryPlusOffset=False ) if context_sym is not None: @@ -141,7 +144,7 @@ class CommonSegCodeSubsegment(Segment): assert isinstance(self.rom_start, int) - for in_file_offset in self.spim_section.fileBoundaries: + for in_file_offset in self.spim_section.get_section().fileBoundaries: if (in_file_offset % 16) != 0: continue @@ -150,8 +153,10 @@ class CommonSegCodeSubsegment(Segment): # Look up for the last symbol in this boundary sym_addr = 0 - for sym in self.spim_section.symbolList: - symOffset = sym.inFileOffset - self.spim_section.inFileOffset + for sym in self.spim_section.get_section().symbolList: + symOffset = ( + sym.inFileOffset - self.spim_section.get_section().inFileOffset + ) if in_file_offset == symOffset: break sym_addr = sym.vram diff --git a/tools/splat/segtypes/common/data.py b/tools/splat/segtypes/common/data.py index e97ce6dece..b4c366fcbc 100644 --- a/tools/splat/segtypes/common/data.py +++ b/tools/splat/segtypes/common/data.py @@ -7,6 +7,8 @@ from util import options, symbols, log from segtypes.common.codesubsegment import CommonSegCodeSubsegment from segtypes.common.group import CommonSegGroup +from disassembler_section import DisassemblerSection, make_data_section + class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup): def out_path(self) -> Optional[Path]: @@ -89,8 +91,7 @@ class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup): f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" ) - self.spim_section = spimdisasm.mips.sections.SectionData( - symbols.spim_context, + self.spim_section = make_data_section( self.rom_start, self.rom_end, self.vram_start, @@ -100,12 +101,25 @@ class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup): self.get_exclusive_ram_id(), ) + assert self.spim_section is not None + + # Set rodata string encoding + # First check the global configuration + if options.opts.data_string_encoding is not None: + self.spim_section.get_section().stringEncoding = ( + options.opts.data_string_encoding + ) + + # Then check the per-segment configuration in case we want to override the global one + if self.str_encoding is not None: + self.spim_section.get_section().stringEncoding = self.str_encoding + self.spim_section.analyze() - self.spim_section.setCommentOffset(self.rom_start) + self.spim_section.set_comment_offset(self.rom_start) rodata_encountered = False - for symbol in self.spim_section.symbolList: + for symbol in self.spim_section.get_section().symbolList: symbols.create_symbol_from_spim_symbol( self.get_most_parent(), symbol.contextSym ) diff --git a/tools/splat/segtypes/common/rodata.py b/tools/splat/segtypes/common/rodata.py index a189671721..bac0826b16 100644 --- a/tools/splat/segtypes/common/rodata.py +++ b/tools/splat/segtypes/common/rodata.py @@ -5,6 +5,8 @@ from util import log, options, symbols from segtypes.common.data import CommonSegData +from disassembler_section import make_rodata_section + class CommonSegRodata(CommonSegData): def get_linker_section(self) -> str: @@ -53,8 +55,7 @@ class CommonSegRodata(CommonSegData): f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" ) - self.spim_section = spimdisasm.mips.sections.SectionRodata( - symbols.spim_context, + self.spim_section = make_rodata_section( self.rom_start, self.rom_end, self.vram_start, @@ -64,21 +65,25 @@ class CommonSegRodata(CommonSegData): self.get_exclusive_ram_id(), ) + assert self.spim_section is not None + # Set rodata string encoding # First check the global configuration if options.opts.string_encoding is not None: - self.spim_section.stringEncoding = options.opts.string_encoding + self.spim_section.get_section().stringEncoding = ( + options.opts.string_encoding + ) # Then check the per-segment configuration in case we want to override the global one if self.str_encoding is not None: - self.spim_section.stringEncoding = self.str_encoding + self.spim_section.get_section().stringEncoding = self.str_encoding self.spim_section.analyze() - self.spim_section.setCommentOffset(self.rom_start) + self.spim_section.set_comment_offset(self.rom_start) possible_text_segments: Set[Segment] = set() - for symbol in self.spim_section.symbolList: + for symbol in self.spim_section.get_section().symbolList: generated_symbol = symbols.create_symbol_from_spim_symbol( self.get_most_parent(), symbol.contextSym ) diff --git a/tools/splat/segtypes/linker_entry.py b/tools/splat/segtypes/linker_entry.py index b907c24fd9..981d3d312e 100644 --- a/tools/splat/segtypes/linker_entry.py +++ b/tools/splat/segtypes/linker_entry.py @@ -39,7 +39,10 @@ def path_to_object_path(path: Path) -> Path: full_suffix = ".o" else: full_suffix = path.suffix + ".o" - return clean_up_path(options.opts.build_path / path.with_suffix(full_suffix)) + + if not str(path).startswith(str(options.opts.build_path)): + path = options.opts.build_path / path + return clean_up_path(path.with_suffix(full_suffix)) def write_file_if_different(path: Path, new_content: str): diff --git a/tools/splat/split.py b/tools/splat/split.py index 4bc36346d2..cf002997d1 100755 --- a/tools/splat/split.py +++ b/tools/splat/split.py @@ -19,7 +19,7 @@ from segtypes.linker_entry import ( from segtypes.segment import Segment from util import log, options, palettes, symbols, relocs -VERSION = "0.14.0" +VERSION = "0.15.1" parser = argparse.ArgumentParser( description="Split a rom given a rom, a config, and output directory" @@ -349,7 +349,7 @@ def main(config_path, modes, verbose, use_cache=True, skip_version_check=False): if ( options.opts.is_mode_active("ld") and options.opts.platform != "gc" ): # TODO move this to platform initialization when it gets implemented - # Calculate list of segments for which we need to find the largest so we can safely place the symbol after it + # Calculate list of segments for which we need to find the largest, so we can safely place the symbol after it max_vram_end_syms: Dict[str, List[Segment]] = {} for sym in symbols.appears_after_overlays_syms: max_vram_end_syms[sym.name] = [ diff --git a/tools/splat/test.py b/tools/splat/test.py index 963bf841f2..3685541145 100644 --- a/tools/splat/test.py +++ b/tools/splat/test.py @@ -1,8 +1,9 @@ +from spimdisasm.common import FileSectionType + from split import * import unittest import io import filecmp -import pprint from util import symbols, options import spimdisasm from segtypes.common.rodata import CommonSegRodata @@ -10,7 +11,6 @@ from segtypes.common.code import CommonSegCode from segtypes.common.c import CommonSegC from segtypes.common.bss import CommonSegBss import difflib -from segtypes.common.group import CommonSegGroup class Testing(unittest.TestCase): @@ -73,8 +73,6 @@ class Testing(unittest.TestCase): # can't diff binary if file[0] == ".splache": continue - file1_lines = [] - file2_lines = [] with open(f"{file[1]}/{file[0]}") as file1: file1_lines = file1.readlines() with open(f"{file[2]}/{file[0]}") as file2: @@ -126,12 +124,12 @@ class Symbols(unittest.TestCase): disassembler_instance.create_disassembler_instance("n64") # first char is uppercase - assert symbols.check_valid_type("Symbol") == True + assert symbols.check_valid_type("Symbol") splat_sym_types = {"func", "jtbl", "jtbl_label", "label"} for type in splat_sym_types: - assert symbols.check_valid_type(type) == True + assert symbols.check_valid_type(type) spim_types = [ "char*", @@ -151,20 +149,15 @@ class Symbols(unittest.TestCase): ] for type in spim_types: - assert symbols.check_valid_type(type) == True + assert symbols.check_valid_type(type) def test_add_symbol_to_spim_segment(self): - context = None - vromStart = 0x0 - vromEnd = 0x10 - vramStart = 0x40000000 + 0x0 - vramEnd = 0x40000000 + 0x10 segment = spimdisasm.common.SymbolsSegment( - context=context, - vromStart=vromStart, - vromEnd=vromEnd, - vramStart=vramStart, - vramEnd=vramEnd, + context=spimdisasm.common.Context(), + vromStart=0x0, + vromEnd=0x10, + vramStart=0x40000000 + 0x0, + vramEnd=0x40000000 + 0x10, ) sym = symbols.Symbol(0x40000000) sym.user_declared = False @@ -177,16 +170,15 @@ class Symbols(unittest.TestCase): assert sym.defined == result.isDefined def test_add_symbol_to_spim_section(self): - context = spimdisasm.common.Context() section = spimdisasm.mips.sections.SectionBase( - context=context, - vromStart=0x100, - vromEnd=None, - vram=None, - filename=None, - words=None, - sectionType=None, - segmentVromStart=None, + context=spimdisasm.common.Context(), + vromStart=0x0, + vromEnd=0x10, + vram=0x40000000, + filename="test", + words=[], + sectionType=FileSectionType.Text, + segmentVromStart=0x0, overlayCategory=None, ) sym = symbols.Symbol(0x100) @@ -199,36 +191,28 @@ class Symbols(unittest.TestCase): assert sym.defined == result.isDefined def test_create_symbol_from_spim_symbol(self): - rom_start = 0x0 - rom_end = 0x100 - type = "func" - name = "MyFunc" - vram_start = 0x40000000 - args = None - yaml = None - # need to init otherwise options.opts isn't defined. # used in initializing a Segment test_init() segment = Segment( - rom_start=rom_start, - rom_end=rom_end, - type=type, - name=name, - vram_start=vram_start, + rom_start=0x0, + rom_end=0x100, + type="func", + name="MyFunc", + vram_start=0x40000000, args=[], - yaml=yaml, + yaml=None, ) context_sym = spimdisasm.common.ContextSymbol(address=0) result = symbols.create_symbol_from_spim_symbol(segment, context_sym) - assert result.referenced == True - assert result.extract == True + assert result.referenced + assert result.extract assert result.name == "D_0" def get_yaml(): - yaml = { + return { "name": "basic_app", "type": "code", "start": 0, @@ -236,7 +220,6 @@ def get_yaml(): "subalign": 4, "subsegments": [[0, "data"], [0x1DC, "c", "main"], [0x1FC, "data"]], } - return yaml class Rodata(unittest.TestCase): @@ -252,11 +235,11 @@ class Rodata(unittest.TestCase): yaml=None, ) rom_data = [] - for i in range(0, 0x100): + for i in range(0x100): rom_data.append(i) common_seg_rodata.disassemble_data(bytes(rom_data)) assert common_seg_rodata.spim_section is not None - assert common_seg_rodata.spim_section.words[0] == 0x0010203 + assert common_seg_rodata.spim_section.get_section().words[0] == 0x0010203 assert symbols.get_all_symbols()[0].vram_start == 0x400 assert symbols.get_all_symbols()[0].segment == common_seg_rodata assert symbols.get_all_symbols()[0].linker_section == ".rodata" @@ -270,11 +253,11 @@ class Rodata(unittest.TestCase): rodata_sym = spimdisasm.mips.symbols.SymbolRodata( context=context, vromStart=0x100, - vromEnd=None, - inFileOffset=None, + vromEnd=0x200, + inFileOffset=0, vram=0x100, words=[0, 1, 2, 3, 4, 5, 6, 7], - segmentVromStart=None, + segmentVromStart=0, overlayCategory=None, ) rodata_sym.contextSym.forceMigration = True @@ -282,7 +265,7 @@ class Rodata(unittest.TestCase): context_sym = spimdisasm.common.ContextSymbol(address=0) context_sym.address = result_symbol_addr - rodata_sym.contextSym.referenceFunctions = [context_sym] + rodata_sym.contextSym.referenceFunctions = {context_sym} # Segment __init__ requires opts to be initialized test_init() @@ -340,9 +323,192 @@ class Bss(unittest.TestCase): rom_bytes = bytes([0, 1, 2, 3, 4, 5, 6, 7]) bss.disassemble_data(rom_bytes) - assert isinstance(bss.spim_section, spimdisasm.mips.sections.SectionBss) - assert bss.spim_section.bssVramStart == 0x40000000 - assert bss.spim_section.bssVramEnd == 0x300 + assert bss.spim_section is not None + + assert isinstance( + bss.spim_section.get_section(), spimdisasm.mips.sections.SectionBss + ) + assert bss.spim_section.get_section().bssVramStart == 0x40000000 + assert bss.spim_section.get_section().bssVramEnd == 0x300 + + +class SymbolsInitialize(unittest.TestCase): + def test_attrs(self): + import pathlib + + symbols.reset_symbols() + test_init() + + sym_addrs_lines = [ + "func_1 = 0x100 // type:func size:10 rom:100 segment:test_segment name_end:the_name_end " + "appears_after_overlays_addr:1234" + ] + + all_segments = [ + Segment( + rom_start=0x100, + rom_end=0x200, + type="func", + name="test_segment", + vram_start=0x300, + args=[], + yaml={}, + ) + ] + + symbols.handle_sym_addrs( + pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments + ) + assert symbols.all_symbols[0].given_name == "func_1" + assert symbols.all_symbols[0].type == "func" + assert symbols.all_symbols[0].given_size == 10 + assert symbols.all_symbols[0].rom == 100 + assert symbols.all_symbols[0].segment == all_segments[0] + assert symbols.all_symbols[0].given_name_end == "the_name_end" + assert symbols.appears_after_overlays_syms[0] == symbols.all_symbols[0] + + def test_boolean_attrs(self): + import pathlib + + symbols.reset_symbols() + test_init() + + sym_addrs_lines = [ + "func_1 = 0x100 // dead:True defined:True extract:True force_migration:True force_not_migration:True " + "allow_addend:True dont_allow_addend:True" + ] + + all_segments = [ + Segment( + rom_start=0x100, + rom_end=0x200, + type="func", + name="test_segment", + vram_start=0x300, + args=[], + yaml={}, + ) + ] + + symbols.handle_sym_addrs( + pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments + ) + assert symbols.all_symbols[0].dead == True + assert symbols.all_symbols[0].defined == True + assert symbols.all_symbols[0].force_migration == True + assert symbols.all_symbols[0].force_not_migration == True + assert symbols.all_symbols[0].allow_addend == True + assert symbols.all_symbols[0].dont_allow_addend == True + + # test spim ban range + def test_ignore(self): + import pathlib + + symbols.reset_symbols() + test_init() + + sym_addrs_lines = ["func_1 = 0x100 // ignore:True size:4"] + + all_segments = [ + Segment( + rom_start=0x100, + rom_end=0x200, + type="func", + name="test_segment", + vram_start=0x300, + args=[], + yaml={}, + ) + ] + + symbols.handle_sym_addrs( + pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments + ) + assert symbols.spim_context.bannedRangedSymbols[0].start == 16 + assert symbols.spim_context.bannedRangedSymbols[0].end == 20 + + +class InitializeSpimContext(unittest.TestCase): + def test_overlay(self): + symbols.reset_symbols() + test_init() + + yaml = { + "name": "boot", + "type": "code", + "start": 4096, + "vram": 2147484672, + "bss_size": 128, + "exclusive_ram_id": "overlay", + "subsegments": [ + [4096, "c", "main"], + [4336, "hasm", "handwritten"], + [4352, "data", "main"], + [4368, "rodata", "main"], + {"type": "bss", "vram": 2147484992, "name": "main"}, + ], + } + + all_segments: List["Segment"] = [ + CommonSegCode( + rom_start=0x0, + rom_end=0x200, + type="code", + name="main", + vram_start=0x100, + args=[], + yaml=yaml, + ) + ] + + # force this since it's hard to set up + all_segments[0].exclusive_ram_id = "overlay" + + symbols.initialize_spim_context(all_segments) + # spim should have added something to overlaySegments + assert ( + type(symbols.spim_context.overlaySegments["overlay"][0]) + == spimdisasm.common.SymbolsSegment + ) + + # test globalSegment settings + def test_global(self): + symbols.reset_symbols() + test_init() + + yaml = { + "name": "boot", + "type": "code", + "start": 4096, + "vram": 2147484672, + "bss_size": 128, + "exclusive_ram_id": "overlay", + "subsegments": [ + [4096, "c", "main"], + [4336, "hasm", "handwritten"], + [4352, "data", "main"], + [4368, "rodata", "main"], + {"type": "bss", "vram": 2147484992, "name": "main"}, + ], + } + + all_segments: List["Segment"] = [ + CommonSegCode( + rom_start=0x0, + rom_end=0x200, + type="code", + name="main", + vram_start=0x100, + args=[], + yaml=yaml, + ) + ] + + assert symbols.spim_context.globalSegment.vramStart == 2147483648 + assert symbols.spim_context.globalSegment.vramEnd == 2147487744 + symbols.initialize_spim_context(all_segments) + assert symbols.spim_context.globalSegment.vramStart == 256 + assert symbols.spim_context.globalSegment.vramEnd == 896 if __name__ == "__main__": diff --git a/tools/splat/test/basic_app/build.sh b/tools/splat/test/basic_app/build.sh old mode 100644 new mode 100755 diff --git a/tools/splat/test/basic_app/expected/asm/data/main.data.s b/tools/splat/test/basic_app/expected/asm/data/main.data.s index 90ebeaf570..1ed582a025 100644 --- a/tools/splat/test/basic_app/expected/asm/data/main.data.s +++ b/tools/splat/test/basic_app/expected/asm/data/main.data.s @@ -4,8 +4,10 @@ glabel D_80000500 /* 1100 80000500 */ .word 0x00000001 +.size D_80000500, . - D_80000500 glabel D_80000504 /* 1104 80000504 */ .word 0x00000000 /* 1108 80000508 */ .word 0x00000000 /* 110C 8000050C */ .word 0x00000000 +.size D_80000504, . - D_80000504 diff --git a/tools/splat/test/basic_app/expected/asm/handwritten.s b/tools/splat/test/basic_app/expected/asm/handwritten.s index 02f374624b..539d92ac11 100644 --- a/tools/splat/test/basic_app/expected/asm/handwritten.s +++ b/tools/splat/test/basic_app/expected/asm/handwritten.s @@ -13,3 +13,4 @@ glabel func_800004F0 /* 10F4 800004F4 03E00008 */ jr $ra /* 10F8 800004F8 00000000 */ nop /* 10FC 800004FC 00000000 */ nop +.size func_800004F0, . - func_800004F0 diff --git a/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s b/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s index a31fa98a05..21d0010cbd 100644 --- a/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s +++ b/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s @@ -52,3 +52,4 @@ glabel .L80000480 /* 1094 80000494 8FBE0000 */ lw $fp, 0x0($sp) /* 1098 80000498 03E00008 */ jr $ra /* 109C 8000049C 27BD0008 */ addiu $sp, $sp, 0x8 +.size func_80000400, . - func_80000400 diff --git a/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s b/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s index a11c09f88d..112eb68090 100644 --- a/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s +++ b/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s @@ -23,3 +23,4 @@ glabel func_800004A0 /* 10E4 800004E4 03E00008 */ jr $ra /* 10E8 800004E8 27BD0018 */ addiu $sp, $sp, 0x18 /* 10EC 800004EC 00000000 */ nop +.size func_800004A0, . - func_800004A0 diff --git a/tools/splat/util/color.py b/tools/splat/util/color.py index fc3b242316..6d75cbd7cd 100644 --- a/tools/splat/util/color.py +++ b/tools/splat/util/color.py @@ -17,11 +17,3 @@ def unpack_color(data): b = ceil(0xFF * (b / 31)) return r, g, b, a - -def pack_color(r, g, b, a): - r = r >> 3 - g = g >> 3 - b = b >> 3 - a = a >> 7 - - return (r << 11) | (g << 6) | (b << 1) | a diff --git a/tools/splat/util/n64/rominfo.py b/tools/splat/util/n64/rominfo.py index 396d16e34f..cede4debf0 100755 --- a/tools/splat/util/n64/rominfo.py +++ b/tools/splat/util/n64/rominfo.py @@ -106,7 +106,7 @@ class N64EntrypointInfo: register_values[insn.rt.value] = insn.getProcessedImmediate() << 16 elif insn.canBeLo(): if insn.isLikelyHandwritten(): - # Try to skip this instructions: + # Try to skip these instructions: # addi $t0, $t0, 0x8 # addi $t1, $t1, -0x8 pass diff --git a/tools/splat/util/options.py b/tools/splat/util/options.py index dd293358ef..2c0936389b 100644 --- a/tools/splat/util/options.py +++ b/tools/splat/util/options.py @@ -1,4 +1,5 @@ from dataclasses import dataclass +import os from pathlib import Path from typing import cast, Dict, List, Literal, Mapping, Optional, Set, Type, TypeVar @@ -98,7 +99,7 @@ class SplatOpts: ld_section_labels: List[str] # Determines whether to add wildcards for section linking in the linker script (.rodata* for example) ld_wildcard_sections: bool - # Determines whether to use use "follows" settings to determine locations of overlays in the linker script. + # Determines whether to use "follows" settings to determine locations of overlays in the linker script. # If disabled, this effectively ignores "follows" directives in the yaml. ld_use_follows: bool # If enabled, the end symbol for each segment will be placed before the alignment directive for the segment @@ -141,6 +142,8 @@ class SplatOpts: asm_data_macro: str # Determines the macro used at the end of a function, such as endlabel or .end asm_end_label: str + # Toggles the .size directive emitted by the disassembler + asm_emit_size_directive: Optional[bool] # Determines including the macro.inc file on non-migrated rodata variables include_macro_inc: bool # Determines the number of characters to left align before the TODO finish documenting @@ -154,12 +157,18 @@ class SplatOpts: # o32 is highly recommended, as it provides logically named registers for floating point instructions # For more info, see https://gist.github.com/EllipticEllipsis/27eef11205c7a59d8ea85632bc49224d mips_abi_float_regs: str - # Determines whether to ad ".set gp=64 to asm/hasm files" + # Determines whether to add ".set gp=64" to asm/hasm files add_set_gp_64: bool # Generate .asmproc.d dependency files for each C file which still reference functions in assembly files create_asm_dependencies: bool # Global option for rodata string encoding. This can be overriden per segment string_encoding: Optional[str] + # Global option for data string encoding. This can be overriden per segment + data_string_encoding: Optional[str] + # Global option for the rodata string guesser. 0 disables the guesser completely. + rodata_string_guesser_level: Optional[int] + # Global option for the data string guesser. 0 disables the guesser completely. + data_string_guesser_level: Optional[int] # Global option for allowing data symbols using addends on symbol references. It can be overriden per symbol allow_data_addends: bool # Determines whether to include the "Generated by spimdisasm" text in the asm @@ -240,7 +249,7 @@ class OptParser: def parse_path( self, base_path: Path, opt: str, default: Optional[str] = None ) -> Path: - return base_path / Path(self.parse_opt(opt, str, default)) + return Path(os.path.normpath(base_path / self.parse_opt(opt, str, default))) def parse_optional_path(self, base_path: Path, opt: str) -> Optional[Path]: if opt not in self._yaml: @@ -275,7 +284,9 @@ def _parse_yaml( platform = p.parse_opt_within("platform", str, ["n64", "psx", "gc", "ps2"]) comp = compiler.for_name(p.parse_opt("compiler", str, "IDO")) - base_path = Path(config_paths[0]).parent / p.parse_opt("base_path", str) + base_path = Path( + os.path.normpath(Path(config_paths[0]).parent / p.parse_opt("base_path", str)) + ) asm_path: Path = p.parse_path(base_path, "asm_path", "asm") def parse_endianness() -> Literal["big", "little"]: @@ -378,6 +389,7 @@ def _parse_yaml( ), asm_data_macro=p.parse_opt("asm_data_macro", str, comp.asm_data_macro), asm_end_label=p.parse_opt("asm_end_label", str, comp.asm_end_label), + asm_emit_size_directive=p.parse_optional_opt("asm_emit_size_directive", bool), include_macro_inc=p.parse_opt( "include_macro_inc", bool, comp.include_macro_inc ), @@ -398,6 +410,13 @@ def _parse_yaml( add_set_gp_64=p.parse_opt("add_set_gp_64", bool, True), create_asm_dependencies=p.parse_opt("create_asm_dependencies", bool, False), string_encoding=p.parse_optional_opt("string_encoding", str), + data_string_encoding=p.parse_optional_opt("data_string_encoding", str), + rodata_string_guesser_level=p.parse_optional_opt( + "rodata_string_guesser_level", int + ), + data_string_guesser_level=p.parse_optional_opt( + "data_string_guesser_level", int + ), allow_data_addends=p.parse_opt("allow_data_addends", bool, True), header_encoding=p.parse_opt("header_encoding", str, "ASCII"), gfx_ucode=p.parse_opt_within( diff --git a/tools/splat/util/symbols.py b/tools/splat/util/symbols.py index 1be57b3432..9a6bc68488 100644 --- a/tools/splat/util/symbols.py +++ b/tools/splat/util/symbols.py @@ -6,6 +6,7 @@ import spimdisasm import tqdm from intervaltree import IntervalTree from disassembler import disassembler_instance +from pathlib import Path # circular import if TYPE_CHECKING: @@ -71,6 +72,175 @@ def to_cname(symbol_name: str) -> str: return symbol_name +def handle_sym_addrs(path: Path, sym_addrs_lines: List[str], all_segments): + def get_seg_for_name(name: str) -> Optional["Segment"]: + for segment in all_segments: + if segment.name == name: + return segment + return None + + for line_num, line in enumerate( + tqdm.tqdm(sym_addrs_lines, desc=f"Loading symbols ({path.stem})") + ): + line = line.strip() + if not line == "" and not line.startswith("//"): + comment_loc = line.find("//") + line_main = line + line_ext = "" + + if comment_loc != -1: + line_ext = line[comment_loc + 2 :].strip() + line_main = line[:comment_loc].strip() + + try: + line_split = line_main.split("=") + name = line_split[0].strip() + addr = int(line_split[1].strip()[:-1], 0) + except: + log.parsing_error_preamble(path, line_num, line) + log.write("Line should be of the form") + log.write(" =
// attr0:val0 attr1:val1 [...]") + log.write("with
in hex preceded by 0x, or dec") + log.write("") + raise + + sym = Symbol(addr, given_name=name) + + ignore_sym = False + if line_ext: + for info in line_ext.split(" "): + if ":" in info: + if info.count(":") > 1: + log.parsing_error_preamble(path, line_num, line) + log.write(f"Too many ':'s in '{info}'") + log.error("") + + attr_name, attr_val = info.split(":") + if attr_name == "": + log.parsing_error_preamble(path, line_num, line) + log.write( + f"Missing attribute name in '{info}', is there extra whitespace?" + ) + log.error("") + if attr_val == "": + log.parsing_error_preamble(path, line_num, line) + log.write( + f"Missing attribute value in '{info}', is there extra whitespace?" + ) + log.error("") + + # Non-Boolean attributes + try: + if attr_name == "type": + if not check_valid_type(attr_val): + log.parsing_error_preamble(path, line_num, line) + log.write( + f"Unrecognized symbol type in '{info}', it should be one of" + ) + log.write( + [ + *splat_sym_types, + *spimdisasm.common.gKnownTypes, + ] + ) + log.write( + "You may use a custom type that starts with a capital letter" + ) + log.error("") + type = attr_val + sym.type = type + continue + if attr_name == "size": + size = int(attr_val, 0) + sym.given_size = size + continue + if attr_name == "rom": + rom_addr = int(attr_val, 0) + sym.rom = rom_addr + continue + if attr_name == "segment": + seg = get_seg_for_name(attr_val) + if seg is None: + log.parsing_error_preamble(path, line_num, line) + log.write(f"Cannot find segment '{attr_val}'") + log.error("") + else: + # Add segment to symbol + sym.segment = seg + continue + if attr_name == "name_end": + sym.given_name_end = attr_val + continue + if attr_name == "appears_after_overlays_addr": + sym.appears_after_overlays_addr = int(attr_val, 0) + appears_after_overlays_syms.append(sym) + continue + except: + log.parsing_error_preamble(path, line_num, line) + log.write( + f"value of attribute '{attr_name}' could not be read:" + ) + log.write("") + raise + + # Boolean attributes + tf_val = ( + True + if is_truey(attr_val) + else False + if is_falsey(attr_val) + else None + ) + if tf_val is None: + log.parsing_error_preamble(path, line_num, line) + log.write( + f"Invalid Boolean value '{attr_val}' for attribute '{attr_name}', should be one of" + ) + log.write([*TRUEY_VALS, *FALSEY_VALS]) + log.error("") + else: + if attr_name == "dead": + sym.dead = tf_val + continue + if attr_name == "defined": + sym.defined = tf_val + continue + if attr_name == "extract": + sym.extract = tf_val + continue + if attr_name == "ignore": + ignore_sym = tf_val + continue + if attr_name == "force_migration": + sym.force_migration = tf_val + continue + if attr_name == "force_not_migration": + sym.force_not_migration = tf_val + continue + if attr_name == "allow_addend": + sym.allow_addend = tf_val + continue + if attr_name == "dont_allow_addend": + sym.dont_allow_addend = tf_val + continue + + if ignore_sym: + if sym.given_size is None or sym.given_size == 0: + ignored_addresses.add(sym.vram_start) + else: + spim_context.addBannedSymbolRangeBySize( + sym.vram_start, sym.given_size + ) + + continue + + if sym.segment: + sym.segment.add_symbol(sym) + + sym.user_declared = True + add_symbol(sym) + + def initialize(all_segments: "List[Segment]"): global all_symbols global all_symbols_dict @@ -80,187 +250,12 @@ def initialize(all_segments: "List[Segment]"): all_symbols_dict = {} all_symbols_ranges = IntervalTree() - def get_seg_for_name(name: str) -> Optional["Segment"]: - for segment in all_segments: - if segment.name == name: - return segment - return None - # Manual list of func name / addrs for path in options.opts.symbol_addrs_paths: if path.exists(): with open(path) as f: sym_addrs_lines = f.readlines() - for line_num, line in enumerate( - tqdm.tqdm(sym_addrs_lines, desc=f"Loading symbols ({path.stem})") - ): - line = line.strip() - if not line == "" and not line.startswith("//"): - comment_loc = line.find("//") - line_main = line - line_ext = "" - - if comment_loc != -1: - line_ext = line[comment_loc + 2 :].strip() - line_main = line[:comment_loc].strip() - - try: - line_split = line_main.split("=") - name = line_split[0].strip() - addr = int(line_split[1].strip()[:-1], 0) - except: - log.parsing_error_preamble(path, line_num, line) - log.write("Line should be of the form") - log.write( - " =
// attr0:val0 attr1:val1 [...]" - ) - log.write("with
in hex preceded by 0x, or dec") - log.write("") - raise - - sym = Symbol(addr, given_name=name) - - ignore_sym = False - if line_ext: - for info in line_ext.split(" "): - if ":" in info: - if info.count(":") > 1: - log.parsing_error_preamble(path, line_num, line) - log.write(f"Too many ':'s in '{info}'") - log.error("") - - attr_name, attr_val = info.split(":") - if attr_name == "": - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Missing attribute name in '{info}', is there extra whitespace?" - ) - log.error("") - if attr_val == "": - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Missing attribute value in '{info}', is there extra whitespace?" - ) - log.error("") - - # Non-Boolean attributes - try: - if attr_name == "type": - if not check_valid_type(attr_val): - log.parsing_error_preamble( - path, line_num, line - ) - log.write( - f"Unrecognized symbol type in '{info}', it should be one of" - ) - log.write( - [ - *splat_sym_types, - *spimdisasm.common.gKnownTypes, - ] - ) - log.write( - "You may use a custom type that starts with a capital letter" - ) - log.error("") - type = attr_val - sym.type = type - continue - if attr_name == "size": - size = int(attr_val, 0) - sym.given_size = size - continue - if attr_name == "rom": - rom_addr = int(attr_val, 0) - sym.rom = rom_addr - continue - if attr_name == "segment": - seg = get_seg_for_name(attr_val) - if seg is None: - log.parsing_error_preamble( - path, line_num, line - ) - log.write( - f"Cannot find segment '{attr_val}'" - ) - log.error("") - else: - # Add segment to symbol - sym.segment = seg - continue - if attr_name == "name_end": - sym.given_name_end = attr_val - continue - if attr_name == "appears_after_overlays_addr": - sym.appears_after_overlays_addr = int( - attr_val, 0 - ) - appears_after_overlays_syms.append(sym) - continue - except: - log.parsing_error_preamble(path, line_num, line) - log.write( - f"value of attribute '{attr_name}' could not be read:" - ) - log.write("") - raise - - # Boolean attributes - tf_val = ( - True - if is_truey(attr_val) - else False - if is_falsey(attr_val) - else None - ) - if tf_val is None: - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Invalid Boolean value '{attr_val}' for attribute '{attr_name}', should be one of" - ) - log.write([*TRUEY_VALS, *FALSEY_VALS]) - log.error("") - else: - if attr_name == "dead": - sym.dead = tf_val - continue - if attr_name == "defined": - sym.defined = tf_val - continue - if attr_name == "extract": - sym.extract = tf_val - continue - if attr_name == "ignore": - ignore_sym = tf_val - continue - if attr_name == "force_migration": - sym.force_migration = tf_val - continue - if attr_name == "force_not_migration": - sym.force_not_migration = tf_val - continue - if attr_name == "allow_addend": - sym.allow_addend = tf_val - continue - if attr_name == "dont_allow_addend": - sym.dont_allow_addend = tf_val - continue - if ignore_sym: - if sym.given_size == None or sym.given_size == 0: - ignored_addresses.add(sym.vram_start) - else: - spim_context.addBannedSymbolRangeBySize( - sym.vram_start, sym.given_size - ) - - ignore_sym = False - continue - - if sym.segment: - sym.segment.add_symbol(sym) - - sym.user_declared = True - add_symbol(sym) + handle_sym_addrs(path, sym_addrs_lines, all_segments) def initialize_spim_context(all_segments: "List[Segment]") -> None: @@ -292,6 +287,7 @@ def initialize_spim_context(all_segments: "List[Segment]") -> None: continue ram_id = segment.get_exclusive_ram_id() + if ram_id is None: if global_vram_start is None: global_vram_start = segment.vram_start @@ -644,3 +640,18 @@ class Symbol: def get_all_symbols(): global all_symbols return all_symbols + + +def reset_symbols(): + global all_symbols + global all_symbols_dict + global all_symbols_ranges + global ignored_addresses + global to_mark_as_defined + global appears_after_overlays_syms + all_symbols = [] + all_symbols_dict = {} + all_symbols_ranges = IntervalTree() + ignored_addresses = set() + to_mark_as_defined = set() + appears_after_overlays_syms = [] diff --git a/tools/splat_ext/pm_map_data.py b/tools/splat_ext/pm_map_data.py index 6ba7d8f724..9653002770 100644 --- a/tools/splat_ext/pm_map_data.py +++ b/tools/splat_ext/pm_map_data.py @@ -1,9 +1,8 @@ +from math import ceil import os, sys from pathlib import Path -from typing import List from segtypes.n64.segment import N64Segment from util.n64.Yay0decompress import Yay0Decompressor -from util.color import unpack_color from segtypes.n64.palette import iter_in_groups from util import options import png # type: ignore @@ -30,6 +29,21 @@ def decode_null_terminated_ascii(data): def parse_palette(data): palette = [] + # RRRRRGGG GGBBBBBA + def unpack_color(data): + s = int.from_bytes(data[0:2], byteorder="big") + + r = (s >> 11) & 0x1F + g = (s >> 6) & 0x1F + b = (s >> 1) & 0x1F + a = (s & 1) * 0xFF + + r = ceil(0xFF * (r / 31)) + g = ceil(0xFF * (g / 31)) + b = ceil(0xFF * (b / 31)) + + return r, g, b, a + for a, b in iter_in_groups(data, 2): palette.append(unpack_color([a, b])) diff --git a/tools/splat_ext/tex_archives.py b/tools/splat_ext/tex_archives.py index 7d5f422faf..67cbd5e064 100644 --- a/tools/splat_ext/tex_archives.py +++ b/tools/splat_ext/tex_archives.py @@ -1,12 +1,11 @@ from dataclasses import dataclass -import os +from math import ceil import struct import json from pathlib import Path import png import n64img.image -from util.color import unpack_color, pack_color from segtypes.n64.palette import iter_in_groups from sys import path @@ -28,6 +27,21 @@ def decode_null_terminated_ascii(data): def parse_palette(data): palette = [] + # RRRRRGGG GGBBBBBA + def unpack_color(data): + s = int.from_bytes(data[0:2], byteorder="big") + + r = (s >> 11) & 0x1F + g = (s >> 6) & 0x1F + b = (s >> 1) & 0x1F + a = (s & 1) * 0xFF + + r = ceil(0xFF * (r / 31)) + g = ceil(0xFF * (g / 31)) + b = ceil(0xFF * (b / 31)) + + return r, g, b, a + for a, b in iter_in_groups(data, 2): palette.append(unpack_color([a, b])) @@ -50,13 +64,13 @@ TILES_MIPMAPS = 1 TILES_SHARED_AUX = 2 TILES_INDEPENDENT_AUX = 3 -aux_combine_modes = { +AUX_COMBINE_MODES = { 0x00: "None", # multiply main * prim, ignore aux 0x08: "Multiply", # multiply main * aux * prim 0x0D: "ModulateAlpha", # use prim color, but multiply alpha by the difference between main and aux red channels 0x10: "LerpMainAux", # use prim alpha to lerp between main and aux color, use main alpha } -aux_combine_modes_inv = {v: k for k, v in aux_combine_modes.items()} +AUX_COMBINE_MODES_INV = {v: k for k, v in AUX_COMBINE_MODES.items()} wrap_modes = { 0: "Repeat", @@ -363,7 +377,7 @@ class TexImage: if self.filter_mode == 2: out["filter"] = True - out["combine"] = aux_combine_modes.get(self.combine_mode) + out["combine"] = AUX_COMBINE_MODES.get(self.combine_mode) if self.is_variant: out["variant"] = True @@ -395,7 +409,15 @@ class TexImage: return fmt_str, hwrap, vwrap - def get_img_file(self, fmt_str, img_file): + def get_img_file(self, fmt_str, img_file: str): + def pack_color(r, g, b, a): + r = r >> 3 + g = g >> 3 + b = b >> 3 + a = a >> 7 + + return (r << 11) | (g << 6) | (b << 1) | a + (out_img, out_w, out_h) = Converter( mode=fmt_str.lower(), infile=img_file, flip_y=True ).convert() @@ -414,132 +436,6 @@ class TexImage: return (out_img, out_pal, out_w, out_h) - # read texture properties from dictionary and load images - def from_json(self, tex_path: Path, json_data): - self.img_name = json_data["name"] - - if "ext" in json_data: - self.raw_ext = json_data["ext"] - else: - self.raw_ext = "tif" - - # read data for main tile - main_data = json_data.get("main") - if main_data == None: - raise Exception(f"Texture {self.img_name} has no definition for 'main'") - - (main_fmt_name, self.main_hwrap, self.main_vwrap) = self.read_json_img( - main_data, "main", self.img_name - ) - (self.main_fmt, self.main_depth) = get_format_code(main_fmt_name) - - # read main image - img_path = str(tex_path / f"{self.img_name}.png") - if not os.path.isfile(img_path): - raise Exception(f"Could not find main image for texture: {self.img_name}") - ( - self.main_img, - self.main_pal, - self.main_width, - self.main_height, - ) = self.get_img_file(main_fmt_name, img_path) - - # read data for aux tile - self.has_aux = "aux" in json_data - if self.has_aux: - aux_data = json_data.get("aux") - (aux_fmt_name, self.aux_hwrap, self.aux_vwrap) = self.read_json_img( - aux_data, "aux", self.img_name - ) - - if aux_fmt_name == "Shared": - # aux tiles have blank attributes in SHARED mode - aux_fmt_name = main_fmt_name - self.aux_fmt = 0 - self.aux_depth = 0 - self.aux_hwrap = 0 - self.aux_vwrap = 0 - self.extra_tiles = TILES_SHARED_AUX - else: - (self.aux_fmt, self.aux_depth) = get_format_code(aux_fmt_name) - self.extra_tiles = TILES_INDEPENDENT_AUX - - # read aux image - img_path = str(tex_path / f"{self.img_name}_AUX.png") - if not os.path.isfile(img_path): - raise Exception( - f"Could not find AUX image for texture: {self.img_name}" - ) - ( - self.aux_img, - self.aux_pal, - self.aux_width, - self.aux_height, - ) = self.get_img_file(aux_fmt_name, img_path) - if self.extra_tiles == TILES_SHARED_AUX: - # aux tiles have blank sizes in SHARED mode - self.main_height *= 2 - self.aux_width = 0 - self.aux_height = 0 - - else: - self.aux_fmt = 0 - self.aux_depth = 0 - self.aux_hwrap = 0 - self.aux_vwrap = 0 - self.aux_width = 0 - self.aux_height = 0 - self.extra_tiles = TILES_BASIC - - # read mipmaps - self.has_mipmaps = json_data.get("hasMipmaps", False) - if self.has_mipmaps: - self.mipmaps = [] - mipmap_idx = 1 - divisor = 2 - if self.main_width >= (32 >> self.main_depth): - while True: - if (self.main_width // divisor) <= 0: - break - mmw = self.main_width // divisor - mmh = self.main_height // divisor - - img_path = str(tex_path / f"{self.img_name}_MM{mipmap_idx}.png") - if not os.path.isfile(img_path): - raise Exception( - f"Texture {self.img_name} is missing mipmap level {mipmap_idx} (size = {mmw} x {mmh})" - ) - - (raster, pal, width, height) = self.get_img_file( - main_fmt_name, img_path - ) - self.mipmaps.append(raster) - if width != mmw or height != mmh: - raise Exception( - f"Texture {self.img_name} has wrong size for mipmap level {mipmap_idx} \n" - + f"MM{mipmap_idx} size = {width} x {height}, but should be = {mmw} x {mmh}" - ) - - divisor = divisor * 2 - mipmap_idx += 1 - if (self.main_width // divisor) < (16 >> self.main_depth): - break - self.extra_tiles = TILES_MIPMAPS - - # read filter mode - if json_data.get("filter", False): - self.filter_mode = 2 - else: - self.filter_mode = 0 - - # read tile combine mode - combine_str = json_data.get("combine", "Missing") - self.combine = aux_combine_modes_inv.get(combine_str) - if self.combine == None: - raise Exception(f"Texture {self.img_name} has invalid 'combine'") - - self.is_variant = json_data.get("variant", False) - # write texture header and image raster/palettes to byte array def add_bytes(self, tex_name: str, bytes: bytearray): # form raw name and write to header @@ -617,26 +513,3 @@ class TexArchive: json_fn = str(tex_path) + ".json" with open(json_fn, "w") as f: f.write(json_out) - - @staticmethod - def build(out_path: Path, tex_path: Path, endian: str = "big"): - out_bytes = bytearray() - tex_name = os.path.basename(tex_path) - - json_fn = str(tex_path) + ".json" - with open(json_fn, "r") as json_file: - json_str = json_file.read() - json_data = json.loads(json_str) - - if len(json_data) > 128: - raise Exception( - f"Maximum number of textures (128) exceeded by {tex_name} ({len(json_data)})`" - ) - - for img_data in json_data: - img = TexImage() - img.from_json(tex_path, img_data) - img.add_bytes(tex_name, out_bytes) - - with open(out_path, "wb") as out_bin: - out_bin.write(out_bytes)