diff --git a/.vscode/settings.json b/.vscode/settings.json index 290eec4db1..9ca2f99876 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -36,8 +36,8 @@ "./tools", "./tools/build", "./tools/build/sprite", - "./tools/splat", - "./tools/build/imgfx" + "./tools/build/imgfx", + "./tools/splat/src" ], "[c]": { "editor.wordSeparators": "`~!@#%^&*()-=+[{]}\\|;:'\",.<>/?", // no $, for scripts diff --git a/requirements.txt b/requirements.txt index 9ca34d04ef..29866d7833 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,3 +12,5 @@ intervaltree rabbitizer n64img python-githooks +crunch64>=0.2.0 +splat64>=0.21.0 diff --git a/tools/build/configure.py b/tools/build/configure.py index dbec80e84f..f3888ae0d5 100755 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -376,7 +376,7 @@ class Configure: self.linker_entries = None def split(self, assets: bool, code: bool, shift: bool, debug: bool): - import split + import splat.scripts.split as split modes = ["ld"] if assets: @@ -500,19 +500,6 @@ class Configure: modern_gcc: bool, c_maps: bool = False, ): - import segtypes - import segtypes.common.asm - import segtypes.common.bin - import segtypes.common.c - import segtypes.common.data - import segtypes.common.group - import segtypes.common.rodatabin - import segtypes.common.textbin - import segtypes.n64.header - import segtypes.n64.img - import segtypes.n64.palette - import segtypes.n64.yay0 - assert self.linker_entries is not None built_objects = set() @@ -632,6 +619,8 @@ class Configure: "actor_types", ) + import splat + # Build objects for entry in self.linker_entries: seg = entry.segment @@ -641,16 +630,16 @@ class Configure: assert entry.object_path is not None - if isinstance(seg, segtypes.n64.header.N64SegHeader): + if isinstance(seg, splat.segtypes.n64.header.N64SegHeader): build(entry.object_path, entry.src_paths, "as") - elif isinstance(seg, segtypes.common.asm.CommonSegAsm) or ( - isinstance(seg, segtypes.common.data.CommonSegData) and not seg.type[0] == "." + elif isinstance(seg, splat.segtypes.common.asm.CommonSegAsm) or ( + isinstance(seg, splat.segtypes.common.data.CommonSegData) and not seg.type[0] == "." ): build(entry.object_path, entry.src_paths, "as") elif seg.type in ["pm_effect_loads", "pm_effect_shims"]: build(entry.object_path, entry.src_paths, "as") - elif isinstance(seg, segtypes.common.c.CommonSegC) or ( - isinstance(seg, segtypes.common.data.CommonSegData) and seg.type[0] == "." + elif isinstance(seg, splat.segtypes.common.c.CommonSegC) or ( + isinstance(seg, splat.segtypes.common.data.CommonSegData) and seg.type[0] == "." ): cflags = None if isinstance(seg.yaml, dict): @@ -733,9 +722,9 @@ class Configure: ) # images embedded inside data aren't linked, but they do need to be built into .inc.c files - if isinstance(seg, segtypes.common.group.CommonSegGroup): + if isinstance(seg, splat.segtypes.common.group.CommonSegGroup): for seg in seg.subsegments: - if isinstance(seg, segtypes.n64.img.N64SegImg): + if isinstance(seg, splat.segtypes.n64.img.N64SegImg): flags = "" if seg.n64img.flip_h: flags += "--flip-x " @@ -780,7 +769,7 @@ class Configure: "bin_inc_c", vars, ) - elif isinstance(seg, segtypes.n64.palette.N64SegPalette): + elif isinstance(seg, splat.segtypes.n64.palette.N64SegPalette): src_paths = [seg.out_path().relative_to(ROOT)] inc_dir = self.build_path() / "include" / seg.dir bin_path = self.build_path() / seg.dir / (seg.name + ".pal.bin") @@ -810,16 +799,16 @@ class Configure: vars, ) elif ( - isinstance(seg, segtypes.common.bin.CommonSegBin) - or isinstance(seg, segtypes.common.textbin.CommonSegTextbin) - or isinstance(seg, segtypes.common.rodatabin.CommonSegRodatabin) + isinstance(seg, splat.segtypes.common.bin.CommonSegBin) + or isinstance(seg, splat.segtypes.common.textbin.CommonSegTextbin) + or isinstance(seg, splat.segtypes.common.rodatabin.CommonSegRodatabin) ): build(entry.object_path, entry.src_paths, "bin") - elif isinstance(seg, segtypes.n64.yay0.N64SegYay0): + elif isinstance(seg, splat.segtypes.n64.yay0.N64SegYay0): compressed_path = entry.object_path.with_suffix("") # remove .o build(compressed_path, entry.src_paths, "yay0") build(entry.object_path, [compressed_path], "bin") - elif isinstance(seg, segtypes.n64.img.N64SegImg): + elif isinstance(seg, splat.segtypes.n64.img.N64SegImg): flags = "" if seg.n64img.flip_h: flags += "--flip-x " @@ -845,7 +834,7 @@ class Configure: # ) # vars = {"c_name": c_sym.name} build(inc_dir / (seg.name + ".png.h"), entry.src_paths, "img_header") - elif isinstance(seg, segtypes.n64.palette.N64SegPalette): + elif isinstance(seg, splat.segtypes.n64.palette.N64SegPalette): bin_path = entry.object_path.with_suffix(".bin") build( @@ -1382,7 +1371,7 @@ if __name__ == "__main__": extra_cflags += " -Wmissing-braces -Wimplicit -Wredundant-decls -Wstrict-prototypes -Wno-redundant-decls" # add splat to python import path - sys.path.insert(0, str((ROOT / args.splat).resolve())) + sys.path.insert(0, str((ROOT / args.splat / "src").resolve())) ninja = ninja_syntax.Writer(open(str(ROOT / "build.ninja"), "w"), width=9999) diff --git a/tools/splat/.github/workflows/black.yml b/tools/splat/.github/workflows/black.yml deleted file mode 100644 index 138bd3ad6f..0000000000 --- a/tools/splat/.github/workflows/black.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: black - -on: - push: - pull_request: - -jobs: - black_checks: - runs-on: ubuntu-latest - name: black - steps: - - uses: actions/checkout@v1 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install Dependencies - run: | - pip install black - pip install -r requirements.txt - pip install types-PyYAML - - name: black - run: black --check . diff --git a/tools/splat/.github/workflows/mypy.yml b/tools/splat/.github/workflows/mypy.yml deleted file mode 100644 index ba18d96f27..0000000000 --- a/tools/splat/.github/workflows/mypy.yml +++ /dev/null @@ -1,23 +0,0 @@ -name: mypy - -on: - push: - pull_request: - -jobs: - mypy_checks: - runs-on: ubuntu-latest - name: mypy - steps: - - uses: actions/checkout@v1 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install Dependencies - run: | - pip install mypy - pip install -r requirements.txt - pip install types-PyYAML - - name: mypy - run: mypy --show-column-numbers --hide-error-context . diff --git a/tools/splat/.github/workflows/publish_docs_to_wiki.yml b/tools/splat/.github/workflows/publish_docs_to_wiki.yml deleted file mode 100644 index 41a15b23a5..0000000000 --- a/tools/splat/.github/workflows/publish_docs_to_wiki.yml +++ /dev/null @@ -1,54 +0,0 @@ -# Based on script from https://github.com/orgs/community/discussions/25929 - -name: Publish docs to Wiki - -# Trigger this action only if there are changes pushed to the docs/** directory under the main branch -on: - push: - paths: - - docs/** # This includes all sub folders - branches: - - main # This can be changed to any branch of your preference - -jobs: - publish_docs_to_wiki: - name: Publish docs to Wiki - runs-on: ubuntu-latest - steps: - # Clone the wiki repository - - name: Checkout Wiki repository - uses: actions/checkout@v4 - with: - repository: ${{ github.event.repository.owner.name }}/${{ github.event.repository.name }}.wiki - path: wiki_repo - - # Clone the main repository - - name: Checkout main repository - uses: actions/checkout@v4 - with: - repository: ${{ github.event.repository.owner.name }}/${{ github.event.repository.name }} - path: splat_repo - - - name: Get the new Wiki files - run: | - cd wiki_repo - rm *.md - cp ../splat_repo/docs/* . - - # `git log -1 --pretty=%aN` prints the current commit's author name - # `git log -1 --pretty=%aE` prints the current commit's author mail - - name: Stage new files - run: | - cd wiki_repo - git config user.name $(git log -1 --pretty=%aN) - git config user.email $(git log -1 --pretty=%aE) - git add . - - # `git diff-index --quiet HEAD` returns non-zero if there are any changes. - # This allows to avoid making a commit/push if there are no changes to the Wiki files - - # `git log -1 --pretty=%B` prints the current commit's message - - name: Push new files to the Wiki - run: | - cd wiki_repo - git diff-index --quiet HEAD || (git commit -m "$(git log -1 --pretty=%B)" && git push) diff --git a/tools/splat/.github/workflows/unit_tests.yml b/tools/splat/.github/workflows/unit_tests.yml deleted file mode 100644 index 9cdb1f5da7..0000000000 --- a/tools/splat/.github/workflows/unit_tests.yml +++ /dev/null @@ -1,20 +0,0 @@ -name: unit_tests - -on: - push: - pull_request: - -jobs: - unit_tests: - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v1 - - name: Set up Python 3.8 - uses: actions/setup-python@v1 - with: - python-version: 3.8 - - name: Install Dependencies - run: pip install -r requirements.txt - - name: Run unit tests - run: sh run_tests.sh diff --git a/tools/splat/.gitignore b/tools/splat/.gitignore deleted file mode 100644 index 38da5f7169..0000000000 --- a/tools/splat/.gitignore +++ /dev/null @@ -1,10 +0,0 @@ -.idea/ -venv/ -.vscode/ -__pycache__/ -.mypy_cache/ -util/n64/Yay0decompress -*.ld -*.n64 -*.yaml -*.z64 diff --git a/tools/splat/.gitrepo b/tools/splat/.gitrepo deleted file mode 100644 index 22f18caca7..0000000000 --- a/tools/splat/.gitrepo +++ /dev/null @@ -1,12 +0,0 @@ -; DO NOT EDIT (unless you know what you are doing) -; -; This subdirectory is a git "subrepo", and this file is maintained by the -; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme -; -[subrepo] - remote = https://github.com/ethteck/splat.git - branch = main - commit = d6490bb877f692f5e20a7846cabd5f793b314054 - parent = c829fde77b3ccc15761582437966317d3d5b13d6 - method = merge - cmdver = 0.4.5 diff --git a/tools/splat/CHANGELOG.md b/tools/splat/CHANGELOG.md deleted file mode 100644 index 422a8dbe88..0000000000 --- a/tools/splat/CHANGELOG.md +++ /dev/null @@ -1,565 +0,0 @@ -# splat Release Notes - -### 0.19.0: vram_classes - -* New top-level yaml feature: `vram_classes`. This allows you to make common definitions for vram locations that can be applied to multiple segments. Please see the [documentation](docs/VramClasses.md) for more details! - * Renamed `ld_use_follows` to `ld_use_symbolic_vram_addresses` to more accurately describe what it's doing - * Renamed `vram_of_symbol` segment option to `vram_symbol` to provide consistency between the segment-level option and the vram class field. - * Removed `appears_after_overlays_addr` symbol_addrs option in favor of specifying this behavior with `vram_classes` -* Removed `dead` symbol_addrs option -* A warning is now emitted when the `sha1` top-level yaml option is not provided. Adding this is highly recommended, as it prevents errors using splat in which the wrong binary is provided. - -### 0.18.3 - -* splat now will emit a `FILL(0)` statement on each segment of a linker script by default, to customize this behavior use the `ld_fill_value` yaml option or the per-segment `ld_fill_value` option. -* New yaml option: `ld_fill_value` - * Allows to specify the value of the `FILL` statement generated on every segment of the linker script. - * It must be either an integer, which will be used as the parameter for the `FILL` statement, or `null`, which tells splat to not emit `FILL` statements. - * This behavior can be customized per segment too. -* New per segment option: `ld_fill_value` - * Allows to specify the value of the `FILL` statement generated for this specific top-level segment of the linker script, ignoring the global configuration. - * If not set, then the global configuration is used. - -### 0.18.2 - -* Fix rodata migration for `.rdata` sections (and other rodata sections that don't use the name `.rodata`) -* `spimdisasm` 1.18.0 or above is now required. - -### 0.18.1 - -* New yaml options: `check_consecutive_segment_types` - * Allows to turn off checking for segment types not being in a consecutive order -* New option for segments: `linker_section_order` and `linker_section` - * `linker_section_order`: Allows overriding the section order used for linker script generation. Useful when a section of a file is not between the other sections of the same type in the ROM, for example a file having its data section between other files's rodata. - * `linker_section`: Allows to override the `.section` directive that will be used when generating the disassembly of the corresponding section, without needing to write an extension segment. This also affects the section name that will be used during link time. Useful for sections with special names, like an executable section named `.start` - -### 0.18.0 - -* `symbol_addrs` parsing checks: - * Enforce lines contain a single `;` - * Enforce no duplicates (same vram, same rom) - -### 0.17.3 - -* Move wiki to the `docs` folder -* Added the ability to specify `find_file_boundaries` on a per segment basis -* Fix `cpp` segment not symbolizing rodata symbols properly - -### 0.17.2 - -* Added more support for PS2 elf files - -### 0.17.1 - -* New yaml options: `ld_sections_allowlist` and `ld_sections_denylist` - * `ld_sections_allowlist`: A list of sections to preserve during link time. It can be useful to preserve debugging sections. - * `ld_sections_denylist`: A list of sections to discard during link time. It can be useful to avoid using the wildcard discard. Note that this option does not turn off `ld_discard_section`. - -### 0.17.0 - -* BREAKING: Linker script generation now imposes the specified `section_order`, which may not completely reflect the yaml order. - * In case this new linker script generation can't be properly adapted to a repo, the old generation can be reenabled by using the `ld_legacy_generation` flag as a temporary solution. Keep in mind this option may be removed in the future. -* New yaml options related to linker script generation: `ld_partial_linking`, `ld_partial_scripts_path`, `ld_partial_build_segments_path`, `elf_path`, `ld_dependencies` - * `ld_partial_linking`: Changes how the linker script is generated, allowing partially linking each segment. This allows for faster linking times when making changes to files at the cost of a slower build time from a clean build and loosing filepaths in the mapfile. This is also known as "incremental linking". This option requires both `ld_partial_scripts_path` and `ld_partial_build_segments_path`. - * `ld_partial_scripts_path`: Folder were each intermediary linker script will be written to. - * `ld_partial_build_segments_path`: Folder where the built partially linked segments will be placed by the build system. - * `elf_path`: Path to the final elf target. - * `ld_dependencies`: Generate a dependency file for every linker script generated, including the main linker script and the ones for partial linking. Dependency files will have the same path and name as the corresponding linker script, but changing the extension to `.d`. Requires `elf_path` to be set. -* New misc yaml options: `asm_function_alt_macro` and `ique_symbols` - * `asm_function_alt_macro`: Allows to use a different label on symbols that are in the middle of functions (that are not branch targets of any kind) than the one used for the label for functions, allowing for alternative function entrypoints. - * `ique_symbols` Automatically fills libultra symbols that are exclusive for iQue. This option is ignored if platform is not N64. -* New "incbin" segments: `textbin`, `databin` and `rodatabin` - * Allows to specify binary blobs to be linked in a specific section instead of the data default. - * If a `textbin` section has a corresponding `databin` and/or `rodatabin` section with the same name then those will be included in the same generated assembly file. - * If a known symbol matches the vram of a incbin section then it will be emitted properly, allowing for better integration with the rest of splat's symbol system. -* `spimdisasm` 1.17.0 or above is now required. - -### 0.16.10 - -* Produce an error if subsegments do not have an ascending vram order. - * This can happen because bss subsegments need their vram to be specified explicitly. - -### 0.16.9 - -* Add command line argument `--disassemble-all`, which has the same effect as the `disassemble_all` yaml option so will disamble already matched functions as well as migrated data. - * Note: the command line argument takes precedence over the yaml, so will take effect even if the yaml option is set to false. - -### 0.16.8 - -* Avoid ignoring the `align` defined in a segment for `code` segments - -### 0.16.7 - -* Use `pylibyaml` to speed-up yaml parsing - -### 0.16.6 - -* Add option `ld_rom_start`. - * Allows offsetting rom address linker symbols by some arbitrary value. - * Useful for SN64 games which often have rom addresses offset by 0xB0000000. - * Defaults to 0. - -### 0.16.5 - -* Add option `segment_symbols_style`. - * Allows changing the style of the generated segment symbols in the linker script. - * Possible values: - * `splat`: The current style for segment symbols. - * `makerom`: Style that aims to be compatible with makerom generated symbols. - * Defaults to `splat`. - -### 0.16.4 - -* Add `get_section_flags` method to the `Segment` class. - * Useful for providing linker section flags when creating a custom section when making splat extensions. - * This may be necessary for some custom section types, because sections unrecognized by the linker will not link its data properly. - * More info about section flags: - -### 0.16.3 - -* Add `--stdout-only` flag. Redirects the progress bar output to `stdout` instead of `stderr`. -* Add a check to prevent relocs with duplicated rom addresses. -* Check empty functions only have 2 instructions before autodecompiling them. - -### 0.16.2 - -* Add option `disassemble_all`. If enabled then already matched functions and migrated data will be disassembled to files anyways. - -### 0.16.1 - -* Various changes so that series of image and palette subsegments can have `auto` rom addresses (as long as the first can find its rom address from the parent segment or its own definition) - -### 0.16.0 - -* Add option `detect_redundant_function_end`. It tries to detect redundant and unreferenced functions ends and merge them together. - * This option is ignored if the compiler is not set to IDO. - * This type of codegen is only affected by flags `-g`, `-g1` and `-g2`. - * This option can also be overriden per file. -* Disable `include_macro_inc` by default for IDO projects. -* Disable `asm_emit_size_directive` by default for SN64 projects. -* `spimdisasm` 1.16.0 or above is now required. - -### 0.15.4 - -* Try to assign a segment to an user-declared symbol if the user declared the rom address. - * Helps to disambiguate symbols for same-address overlays. - -### 0.15.3 - -* Disabled `asm_emit_size_directive` by default for IDO projects. - -### 0.15.2 - -* Various cleanup and fixes to support more liberal use of `auto` for rom addresses - -### 0.15.1 - -* Made some modifications such that linker object paths should be simpler in some circumstances - -### 0.15.0 - -* New options: - * `data_string_encoding` can be set at the global level (or `str_encoding` at the segment level) to specify the encoding using when guessing and disassembling strings the the data section. In spimdisasm this value defaults to ASCII. - * `rodata_string_guesser_level` changes the behaviour of the rodata string guesser. A higher value means more agressive guessing, while 0 and negative means no guessing at all. Even if the guesser feature is disabled, symbols manually marked as strings in the symbol_addrs.txt file will still be disassembled as strings. In spimdisasm this value defaults to 1. - * level 0: Completely disable the guessing feature. - * level 1: The most conservative guessing level. Imposes the following restrictions: - * Do not try to guess if the user provided a type for the symbol. - * Do no try to guess if type information for the symbol can be inferred by other means. - * A string symbol must be referenced only once. - * Strings must not be empty. - * level 2: A string no longer needs to be referenced only once to be considered a possible string. This can happen because of a deduplication optimization. - * level 3: Empty strings are allowed. - * level 4: Symbols with autodetected type information but no user type information can still be guessed as strings. - * `data_string_guesser_level` is similar to `rodata_string_guesser_level`, but for the data section instead. In spimdisasm this value defaults to 2. - * `asm_emit_size_directive` toggles the size directived emitted by the disassembler. In spimdisasm this defaults to True. - -### 0.14.1 - -* Fix bug, cod cleanup - -### 0.14.0 - -* Add support for PSX's GTE instruction set - -### 0.13.10 - -* New option `disasm_unknown` (False by default) - * If enabled it tells the disassembler to try disassembling functions with unknown instructions instead of falling back to disassembling as raw data - -### 0.13.9 - -* New segment option `linker_entry` (true by default). - * If disabled, this segment will not produce entries in the linker script. - -### 0.13.8 - -* New option `segment_end_before_align`. - * If enabled, the end symbol for each segment will be placed before the alignment directive for the segment - -### 0.13.7 - -* Severely sped-up linker entry writing by using a dict instead of a list. Symbol headers will no longer be in any specific order (which shouldn't matter, because they're headers). - -### 0.13.6 - -* Changed CI image processing so that their data is fetched during the scan phase, supporting palettes that come before CI images. - -### 0.13.5 - -* An error will be produced if a symbol is declared with an unknown type in the symbol_addrs file. - * The current list of known symbols is `'func', 'label', 'jtbl', 'jtbl_label', 's8', 'u8', 's16', 'u16', 's32', 'u32', 's64', 'u64', 'f32', 'f64', 'Vec3f', 'asciz', 'char*', 'char'`. - * Custom types are allowed if they start with a capital letter. - -### 0.13.4 - -* Renamed `follows_vram_symbol` segment option to `vram_of_symbol` to more accurately reflect what it's used for - to set the segment's vram based on a symbol. -* Refactored the `appears_after_overlays_addr` feature so that expressions are written at the latest possible moment in the linker script. This fixes errors and warnings regarding forward references to later symbols. - -### 0.13.3 - -* Added a new symbol_addrs attribute `appears_after_overlays_addr:0x1234` which will modify the linker script such that the symbol's address is equal to the value of the end of the longest overlay starting with address 0x1234. It achieves this by writing a series of sym = MAX(sym, seg_vram_END) statements into the linker script. For some games, it's feasible to manually create such statements, but for games with hundreds of overlays at the same address, this is very tedious and prone to error. The new attribute allows you to have peace of mind that the symbol will end up after all of these overlays. - -### 0.13.2 - -* Actually implemented `ld_use_follows`. Oopz - -### 0.13.1 - -* Added `ld_wildcard_sections` option (disabled by default), which adds a wildcard to the linker script for section linking. This can be helpful for modern GCC, which creates additional rodata sections such as ".rodata.xyz". -* Added `ld_use_follows` option (enabled by default), which, if disabled, makes splat ignore follows_vram and follows_symbols. This helps for fixing matching builds while being able to add infrastructure to the yaml for non-matching builds by just re-enabling the option. - -### 0.13.0 - -* Automatically generate `INCLUDE_RODATA`/`#pragma GLOBAL_ASM` directives for non migrated rodata symbols when creating new C files. -* Non migrated rodata symbols will now only be produced if the C file has a corresponding rodata file with the same name and the C file has a `INCLUDE_RODATA`/`#pragma GLOBAL_ASM` directive referencing the symbol, similar to how functions are disassembled. - * Because of this, the `partial_migration` attribute has lost its purpose and has been removed. -* Rodata symbol files are now included in the autogenerated dependency files too. - -### 0.12.14 - -* New option: `pair_rodata_to_text`. - * If enabled, splat will try to find to which text segment an unpaired rodata segment belongs, and it will hint it to the user. - -### 0.12.13 - -* bss segments can now omit the rom offset. - -### 0.12.12 - -* Try to detect and warn to the user if a gap between two migrated rodata symbols is detected and suggest possible solutions to the user. - -### 0.12.11 - -* New disassembly option in the yaml: `allow_data_addends`. - * Allows enabling/disabling using addends on all `.data` symbols. -* Three new options for symbols: `name_end`, `allow_addend` and `dont_allow_addend`. - * `name_end`: allows to provide a closing name for any symbol. Useful for handwritten asm which usually have an "end" name. - * `allow_addend` and `dont_allow_addend`: Allow overriding the global `allow_data_addends` option for allowing addends on data symbols. - -### 0.12.10 - -* Allows passing user-created relocs to the disassembler via the `reloc_addrs.txt` file, allowing to improve the automatic disassembly. -* Multiple reloc_addrs files can be specified in the yaml with the `reloc_addrs_path` option. - -### 0.12.9 - -* Added `format_sym_name()` to the vtx segment so it, too, can be extended - -### 0.12.8 - -* The gfx and vtx segments now have a `data_only` option, which, if enabled, will emit only the plain data for the type and omit the enclosing symbol definition. This mode is useful when you want to manually declare the symbol and then #include the extracted data within the declaration. -* The gfx segment has a method, `format_sym_name()`, which will allow custom overriding of the output of symbol names by extending the `gfx` segment. For example, this can be used to transform context-specific symbol names like mac_01_vtx into N(vtx), where N() is a macro that applies the current "namespace" to the symbol. Paper Mario plans to use this, so we can extract an asset once and then #include it in multiple places, while giving each inclusion unique symbol names for each component. - -### 0.12.7 - -* Allow setting a different macro for jumptable labels with `asm_jtbl_label_macro` - * The currently recommended one is `jlabel` instead of `glabel` -* Two new options for symbols: `force_migration` and `force_not_migration` - * Useful for weird cases where the disassembler decided a rodata symbol must (or must not) be migrated when it really shouldn't (or should) -* Fix `str_encoding` defaulting to `False` instead of `None` -* Output empty rules in generated dependency files to avoid issues when the function file does not exist anymore (i.e. when it gets matched) -* Allow changing the `include_macro_inc` option in the yaml - -### 0.12.6 - -* Adds two new N64-specific segments: - * IPL3: Allows setting its correct VRAM address without messing the global segment detection - * RSP: Allows disassembling using the RSP instruction set instead of the default one -* PS2 was added as a new platform option. - * When this is selected the R5900 instruction set will be used when disassembling instead of the default one. - -### 0.12.5 - -* Update minimal spimdisasm version to 1.7.1. -* Fix spimdisasm>=1.7.0 non being able to see symbols which only are referenced by other data symbols. -* A check was added to prevent segments marked with `exclusive_ram_id` have a vram address range which overlaps with segments not marked with said tag. If this happens it will be warned to the user. - -### 0.12.4 - -* Fixed a bug involving the order of attributes in symbol_addrs preventing proper range searching during calls to `get_symbol` - -### 0.12.3: Initial Gamecube Support -Initial support for Gamecube disk images has been set up! Disassembly is not currently supported, and a more comprehensive explanation of Gamecube support will come once that is finished. - -* The Symbol class is now hashable -* Added the ability for segments to specify a file path (`path`) to receive that file's contents as their split input -* The `generated_s_preamble` option now will be applied to data files created by spimdisasm -* Rewrote symbol range check code to be more efficient -* Fixed bug that allowed empty top-level segments of type `code`. -* Fixed progress bars to properly update their descriptions -* Fixed bug pertaining to symbols getting assigned to segments they shouldn't if their segment is given in symbol_addrs (`segment:`) - -### 0.12.2 -* Fixed bug where `given_dir` was possibly not a `Path` - -### 0.12.1 -* The constructor for `Segment` takes far fewer arguments now, which will affect (and hopefully simplify) any custom segments that are implemented. - -* The new option `string_encoding` can be set at the global or segment level and will influence the encoding for strings in rodata during disassembly. The default encoding used is EUC-JP, as it was previously. - -## 0.12.0: Performance Boost - -In this release, we bring many performance improvements, making splat dramatically faster. We have observed speedups of 10-20x, though your results may vary. - -* Linker script `_romPos` alignment statements now take a form that is friendlier to different assemblers. - -* Fixed the default value of `use_legacy_include_asm` to be what it was before 0.11.2 - -### 0.11.2 -* The way options are parsed and accessed has been completely refactored. The following option names have changed: - -`linker_symbol_header_path` -> `ld_symbol_header_path` - -`asm_endlabels` -> `asm_end_label` - -Additionally, any custom segments or code that needs to read options will have to accommodate the new API for doing so. Options are now fields of an object named `opts` within the existing `options` namespace. Because the options are fields, `get_` is no longer necessary. To give an example: - -Before: `options.get_asm_path()` - -After: `options.opts.asm_path` - -The clean_up_path function in linker_entry.py now uses a cache, offering a small performance improvement during the linker script writing phase. - -### 0.11.1 -* The linker script now includes a `_SIZE` symbol for each segment. -* The new `create_asm_dependencies`, if enabled, will cause splat to create `.asmproc.d` files that can inform a build system which asm files a c file depends upon. If your build system is configured correctly, this can allow triggering a rebuild of a C file when its included asm files are modified. -* Splat no longer depends directly on pypng and now instead uses [n64img](https://github.com/decompals/n64img). Currently, all image behavior uses the exact same code. Eventually, n64img will be implemented in C and support rebuilding images as well. - -## 0.11.0: Spimdisasm Returns - -Spimdisasm now handles data (data, rodata, bss) disassembly in splat! This includes a few changes in behavior: - -* Rodata will be migrated to c files' asm function files when a .rodata subsegment is used that corresponds with an identically-named c file. Some symbols may not be automatically migrated to functions when it is not clear if they belong to the function itself (an example of which being const arrays). In this case, the `partial_migration` option can be enabled for the given .rodata subsegment and splat will create .s files for these unmigrated rodata symbols. These files can then be included in your c files, or you can go ahead and migrate these symbols to c and disable the `partial_migration` feature. - -* BSS can now be disassembled as well, and the size of a code segment's bss section can be specified with the `bss_size` option. This option will tell splat how large the bss section is in bytes so BSS can properly be handled during disassembly. For bss subsegments, the rom address will of course not change, but the vram address should still be specified. This currently can only be done in the dict form of segment representation, rather than the list form. - -Thanks again to [AngheloAlf](https://github.com/AngheloAlf) for adding this functionality and continuing to improve splat's disassembler. - -## 0.10.0: The Linker Script Update - -Linker scripts splat produces are now capable of being shift-friendly. Rom addresses will automatically shift, and ram addresses will still be hard-coded unless the new segment option `follows_vram` is specified. The value of this option should be the name of a segment (a) that this segment (b) should follow in memory. If a grows or shrinks, b's start address will also do so to accommodate it. - -The `enable_ld_alignment_hack` option and corresponding behavior has been removed. This proved to add too much complexity to the linker script generation code and was becoming quite a burden to keep dealing with. Apologies for any inconvenience this may cause. But trust me: in the long run, it's good you won't be depending on that madness. - -### 0.9.5 -* Changes have been made to the linker script such that it is more shiftable. Rather than setting the rom position to hard-coded addresses, it increments the position by the size of the previous segment. Some projects may experience some alignment-related issues after this change. If specified, the new segment option `align: n` will add an `ALIGN(n)` directive for that section's linker segment. - -### 0.9.4 -* A new linker script section is now automatically created when the .bss section begins, using NOLOAD as opposed to the previous hacky rom rewinding we were previously doing. Additionally, `ld_section_labels` now includes `.rodata` by default. - -### 0.9.3 -* Added `add_set_gp_64` option (true by default), which allows controlling whether to add ".set gp=64" to asm/hasm files - -### 0.9.2 -* Added "palette" argument to ci4/ci8 segments so that segments' palettes can be manually specified - -### 0.9.1 -* Fixed a bug in which local labels and jump table labels could replace raw words in data blobs during data disassembly - -## 0.9.0: The Big Update -### Introducing [spimdisasm](https://github.com/Decompollaborate/spimdisasm)! -* Thanks to [AngheloAlf](https://github.com/AngheloAlf), we now have a much better MIPS disassembler in splat! spimdisasm has much better hi/lo matching, much lower ram usage, and plenty of other goodies. - -We plan to roll this out in phases. Currently, it only handles actual code disassembly. Later on, we will probably migrate our current data assembly code to use spimdisasm as well. - -**NOTICE**: This integration has been tested on a variety of games and configurations. However, with any giant change to the platform like this, there are bound to be things we didn't catch. Please be patient with us as we handle these remaining issues. Though from what we've seen already, the slight bugs one may come across are totally worth the much improved disassembly. - -### gfx segment type -* A new `gfx` segment type is available, which creates a c file containing a disassembled display list according to the segment's start and end offsets. Thanks to [Glank](https://github.com/glankk) and [Tharo](https://github.com/thar0/) for their work on [libgfxd](https://github.com/glankk/libgfxd) and [pygfxd](https://github.com/thar0/pygfxd/), respectively, for helping make this a possibility in splat. - -### API breaking changes -* Some `Segment()` arguments have changed, which may cause extensions to break. Please see the `__init__` function for `Segment` for more details. - -### symbol_addrs.txt changes -* symbol_addrs now supports the `segment:` attribute, which allows specifying the symbol's top-level segment. This can be helpful for symbol resolution when overlays use overlapping vram ranges. See `exclusive_ram_id` below for more information. - -### Global options changes - -The new `symbol_name_format` option allows specification of how symbols will be named. This can be set as a global option and also changed per-segment. `symbol_name_format_no_rom` is used when the symbol does not have a rom address (BSS). - - The following substitutions are allowed: - -`$ROM` - the rom address of the symbol, hex-formatted and padded to 6 characters (ABCF10, 000030, 123456) (note: only for `symbol_name_format`, usage in `symbol_name_format_no_rom` will cause an error) - -`$VRAM` - the vram address of the symbol, hex-formatted and padded to 8 characters (00030010, 00020015, ABCDEF10) - -`$SEG` - the name of the top-level segment in which the symbol resides - -The default values for these options are as follows - -`symbol_name_format` : `$VRAM` - -`symbol_name_format_no_rom` : `$VRAM_$SEG` - -The appropriate prefix string will still automatically be applied depending on the type of the symbol: `D_` for data, `jtbl_` for jump tables, and `func_` for functions. This functionality may be customizable in the future. - ----- -The `auto_all_sections` option now should be a list of section names (`[".data", ".rodata", ".bss"]` by default) indicating the sections that should be linked from .o files built from source files (.c or asm/hasm .s files), when no subsegment explicitly indicates linking this type of section. - -For example, if any subsegment of a code segment is of segment type `data` or `.data`, the `.data` section from all `c`/`asm`/`hasm` subsegments will not be linked unless explicitly indicated with a relevant `.data` subsegment. - -Previously, this option was a bool, and it enabled this feature for all sections specified in `section_order`. Now, the desired sections must be specified manually. The default value for this option retains previous behavior. - ----- -The new `mips_abi_float_regs` option allows for changing the format of float registers for MIPS disassembly. The default value does not change any prior behavior, but `o32` is heavily encouraged and may become the default option in the future. For more information, see this [great writeup](https://gist.github.com/EllipticEllipsis/27eef11205c7a59d8ea85632bc49224d). - ----- -The new `gfx_ucode` option allows for specifying the target for the graphics macro format, which is used in the gfx segment type. The default is `f3dex2`. - - -### Segment options changes - -The new `exclusive_ram_id` segment option allows specifying an identifer that will prevent the segment from seeing any symbols from other segments with the same identifer. This is useful when multiple segments are mapped to the same vram address at runtime and should never be able to refer to each other's symbols. Setting all of these segments to have the same value for this option will prevent their symbols from clashing / meshing unexpectedly. - ----- - -The `overlay` setting on segments has been removed. Please see `symbol_name_format` above for info on how to influence the names of symbols, which can be applied at the segment level as well as the global level. - ----- -## 0.8.0: Arbitrary Section Order -* You can now use the option `section_order` to define the binary section order for your target binary. By default, this is `[".text", ".data", ".rodata", ".bss"]`. See options.py for more details -* Documented all options in options.py -* Support for SN64 games (thanks Wiseguy!) -* More consistent handling of paths (thanks Mkst!) -* Various other cleanup and fixes across the board - -### 0.7.10: WIP PSX support -* WIP PSX support has been added, thanks to @mkst! (https://github.com/ethteck/splat/pull/99) - * Many segments have moved to a "common" package - * Endianness of the input binary is now a configurable option -* Linker hack restored but is now optional and off by default - -### 0.7.9 -* Finally removed the dumb linker section alignment hack -* Added version number to output on execution - -### 0.7.8 - -* Fixed a bug relating to a linker section alignment hack (thanks Wiseguy!) -* Fixed a bug in linker_entry.py's clean_up_path that should make this function more versatile (thanks Wiseguy!) - -### 0.7.7 - -* Disassembly now reads the `size` property of a function in symbol_addrs.txt to disassemble `size / 4` number of instructions. Feel free to specify the size of your functions in your symbol_addrs file if splat's disassembly is chopping a function too short or making a function too long. - -### 0.7.6 - -* Fixed a bug involving detection of defined functions in c files for GLOBAL_ASM-using projects -* Added options to disable the creation of undefined_funcs/syms_auto.txt files -* Added a Vtx segment type for creating c files containg model vertex data in the n64 libultra Vtx format -* Added a `cpp` segment type which is identical to `c` but looks for a file with the extension ".cpp" instead of ".c". - -### 0.7.5: all_ types and auto_all_sections - -If you have a group segment with multiple c files and want splat to automatically create linker entries at a given position for each code object (c, asm, hasm) in the segment, you can use an `all_` type for that section. For example, you can add `[auto, all_bss]` as the last subsegment in a segment. This will direct splat to create a linker entry for each code object in the segment. This saves a lot of time when it comes to manually adding .bss subsegments for bss support, for example. The same thing can be done for data and rodata sections, but note this should probably be done later into a project when all data / rodata is migrated to c files, as the `all_` types lose the rom positioning information that's necessary for splat to do proper disassembly. - -The `auto_all_sections` option, when set to true, will automatically add `all_` types into every group. This is only done for a section in a group if no other manual declarations for that section exist. For example, if you have 30 c files in a group and a .data later on for one of them, `auto_all_sections` will not interfere with your `.data` subsegment. If you remove this, however, splat will use `auto_all_sections` to implicitly `.data` subsegments for all of your code objects behind the scenes. This feature is again particualrly helpful for bss support, as it will create bss linker entries for every file in your project (assuming you don't have any manual .bss subsegments), which eliminates the need to create dummy .bss subsegments just for the sake of configuring the linker script. - -### 0.7.2 - -* Data disassembly changes: - * String detection has been improved. Please send me false positives / negatives as you see them and I can try to improve it further! - * Symbols in a data segment pointed to by other symbols will now properly be split out as their own symbols - -### 0.7.1 - -* Image segment changes: - * Added `flip_x` and `flip_y` boolean parameters to replace `flip`. - * `flip` is deprecated and will produce a warning when used. - * Fixed flipping of `ci4` and `ci8` images. - * Fixed `extract: false` (and `start: auto`) behaviour. - -## 0.7.0: The Path Update - -* Significantly better performance, especially when using the cache feature (`--use-cache` CLI arg). -* BREAKING: Some cli args for splat have been renamed. Please consult the usage output (-h or no args) for more information. - * `--new` has been renamed to `--use-cache` - * `--modes` arg changes: - * Image modes have been combined into the `img` mode - * Code and ASM modes have been combined into the `code` mode -* BREAKING: The `name` attribute of a segment now should no longer be a subdirectory but rather a meaningful name for the segment which will be used as the name of the linker section. If your `name` was previously a directory, please change it into a `dir`. -* BREAKING: `subsections` has been renamed to `subsegments` -* New `dir` segment attribute specifies a subdirectory into which files will be saved. You can combine `dir` ("foo") with a subsegment name containing a subdirectory ("bar/out"), and the paths will be joined (foo/bar/out.c) - * If the `dir` attribute is specified but the `name` isn't, the `name` becomes `dir` with directory separation slashes replaced with underscores (foo/bar/baz -> foo_bar_baz) -* BREAKING: Many configuration options have been renamed. `_dir` options have been changed to the suffix `_path`. -* BREAKING: Assets (non-code, like `bin` and images) are now placed in the directory `asset_path` (defaults to `assets`). -* Linker symbol header generation. Set the `linker_symbol_header_path` option to use. - * `typedef u8[] Addr;` is recommended in your `common.h` header. -* You can now provide `auto` as the `start` attribute for a segment, e.g. `[auto, c, my_file]`. This causes the segment to not be extracted, but linked. This feature is intended for modding. -* Providing just a ROM address but no type or name for a segment is now valid anywhere in `segments` or `subsegments` rather than just at the end of the ROM. It specifies the end of the previous segment for types that need it (`palette`, `bin`, `Yay0`) and causes the linker to simply write padding until that address. -* The linker script file is left untouched if the contents have not changed since the previous split. -* You can now group together segments with `type: group` (similar to `code`). Note that any ASM or C segments must live under a `type: code` segment, not a basic `group`. - -### 0.6.5: Bugfixes, rodata migration, and made options static - -If you wrote a custom extension, options should be imported and statically referenced -`from util import options` - -see options.py for more info on how to now get and set options - -BREAKING: vram can only be specified on a segment if the segment is defined as a dict in the config - -### 0.6.3: More refactoring -**Breaking Change**: The command line args to split.py have changed. Currently, only the config path is now a required argument to splat. The old `rom` and `outdir` parameters are now optional (`--rom`, `--outdir`). Now, you can add rom and out directory paths in the yaml. - -The `out_dir` option specifies a directory relative to the config file. If your config file is in a subdirectory of the main repo, you can set `out_dir: ../`, for example. - -The `target_path` option spcifies a path to the binary file to split, relative to the `out_dir`. If your `baserom.z64` is in the top-level of the repo, you can set `target_path: baserom.z64`, for example. - -### 0.6.2: subsegments -I've begun a refactor of the code "files" code, which makes everything cleaner and easier to extend. -There's also a new option, `create_new_c_files`, which disables the creation of nonexistent c files. This behavior is on by default, but if you want to disable it for any reason, you now have the option to do so. - -I am also working on adding bss support as well. It should almost be all set, aside from the changes needed in the linker script. - -**Breaking change**: The `files` field in `code` segments should now be renamed to `subsegments`. - -### 0.6.1: `assets_dir` option - -This release adds a new `assets_dir` option in `splat.yaml`s that allows you to override the default `img`, `bin`, and other directories that segments output to. - -Want to interdisperse split assets with your sourcecode? `assets_dir: src`! -Want to have all assets live in a single directory? `assets_dir: assets`! - -## 0.6: The Symbol Update -Internally, there's a new Symbol class which stores information about a symbol and is stored in a couple places during disassembly. Many things should be improved, such as reconciling symbols within overlays, things being named functions vs data symbols, and more. - -**Breaking change**: The format to symbol_addrs.txt has been updated. After specifying the name and address of a symbol (`symbol = addr;`), optional properties of symbols can be set via inline comment, space delimited, in any order. The properties are of the format `name:value` - * `type:` supports `func` mostly right now but will support `label` and `data` later on. Internally, `jtbl` is used as well, for jump tables. Splat uses type information during disassembly to disambiguate symbols with the same addresses. - * `rom:` is for the hex rom address of the symbol, beginning with `0x`. If available, this information is extremely valuable for use in disambiguating symbols. - * `size:` specifies the size of the symbol, which splat will use to generate offsets during disassembly. Uses the same format as `rom:` - -**function example**: `FuncNameHere = 0x80023423; // type:func rom:0x10023` - -**data example**: `gSomeDataVar = 0x80024233; // type:data size:0x100` - -## 0.5 The Rename Update -* n64splat name changed to splat - * Some refactoring was done to support other platforms besides n64 in the future - * New `platform` option, which defaults to `n64` - * This will cause breaking changes in custom segments, so please refer to one of the changes in one of the n64 base segments for details -* Support for custom artifact paths - * New `undefined_syms_auto_path` option - * New `undefined_funcs_auto_path` option - * New `cache_path` option - * (All path-like options' names now end with `_path`) diff --git a/tools/splat/Dockerfile b/tools/splat/Dockerfile deleted file mode 100644 index 0c84c24135..0000000000 --- a/tools/splat/Dockerfile +++ /dev/null @@ -1,8 +0,0 @@ -FROM ubuntu:22.04 -RUN apt-get update -RUN apt install -y binutils-mips-linux-gnu -ADD https://github.com/decompals/mips-binutils-2.6/releases/download/main/binutils-2.6-linux.tar.gz . -ADD https://github.com/decompals/mips-gcc-2.7.2/releases/download/main/gcc-2.7.2-linux.tar.gz . -RUN mkdir -p ./gcc-2.7.2 && \ - tar -xvf gcc-2.7.2-linux.tar.gz -C ./gcc-2.7.2 && \ - tar -xvf binutils-2.6-linux.tar.gz -C ./gcc-2.7.2 diff --git a/tools/splat/LICENSE b/tools/splat/LICENSE deleted file mode 100644 index 42621fb707..0000000000 --- a/tools/splat/LICENSE +++ /dev/null @@ -1,21 +0,0 @@ -MIT License - -Copyright (c) 2021 Ethan Roseman - -Permission is hereby granted, free of charge, to any person obtaining a copy -of this software and associated documentation files (the "Software"), to deal -in the Software without restriction, including without limitation the rights -to use, copy, modify, merge, publish, distribute, sublicense, and/or sell -copies of the Software, and to permit persons to whom the Software is -furnished to do so, subject to the following conditions: - -The above copyright notice and this permission notice shall be included in all -copies or substantial portions of the Software. - -THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR -IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE -AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER -LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, -OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -SOFTWARE. diff --git a/tools/splat/Makefile b/tools/splat/Makefile deleted file mode 100644 index b72aafecba..0000000000 --- a/tools/splat/Makefile +++ /dev/null @@ -1,11 +0,0 @@ -UTIL_DIR := util - -default: all - -all: Yay0decompress - -Yay0decompress: - gcc $(UTIL_DIR)/n64/Yay0decompress.c -fPIC -shared -O3 -Wall -Wextra -o $(UTIL_DIR)/n64/Yay0decompress - -clean: - rm -f $(UTIL_DIR)/n64/Yay0decompress diff --git a/tools/splat/README.md b/tools/splat/README.md deleted file mode 100644 index 219be683b3..0000000000 --- a/tools/splat/README.md +++ /dev/null @@ -1,9 +0,0 @@ -# splat -A binary splitting tool to assist with decompilation and modding projects - -Currently, only N64, PSX, and PS2 binaries are supported. - -Please check out the [wiki](https://github.com/ethteck/splat/wiki) for more information including [examples](https://github.com/ethteck/splat/wiki/Examples) of projects that use splat. - -### Requirements -splat requires Python 3.8+. Package requirements can be installed via `pip3 install -U -r requirements.txt` diff --git a/tools/splat/create_config.py b/tools/splat/create_config.py deleted file mode 100755 index 92898589cf..0000000000 --- a/tools/splat/create_config.py +++ /dev/null @@ -1,320 +0,0 @@ -#! /usr/bin/env python3 - -import argparse -import sys -from pathlib import Path - -from util.gc import gcinfo -from util.n64 import find_code_length, rominfo -from util.psx import psxexeinfo - -parser = argparse.ArgumentParser( - description="Create a splat config from an N64 ROM, PSX executable, or a GameCube disc image." -) -parser.add_argument( - "file", help="Path to a .z64/.n64 ROM, PSX executable, or .iso/.gcm GameCube image" -) - - -def main(file_path: Path): - if not file_path.exists(): - sys.exit(f"File {file_path} does not exist ({file_path.absolute()})") - if file_path.is_dir(): - sys.exit(f"Path {file_path} is a directory ({file_path.absolute()})") - - # Check for N64 ROM - if file_path.suffix.lower() == ".n64" or file_path.suffix.lower() == ".z64": - create_n64_config(file_path) - return - - file_bytes = file_path.read_bytes() - - # Check for GC disc image - if int.from_bytes(file_bytes[0x1C:0x20], byteorder="big") == 0xC2339F3D: - create_gc_config(file_path, file_bytes) - return - - # Check for PSX executable - if file_bytes[0:8] == b"PS-X EXE": - create_psx_config(file_path, file_bytes) - return - - -def create_n64_config(rom_path: Path): - rom_bytes = rominfo.read_rom(rom_path) - - rom = rominfo.get_info(rom_path, rom_bytes) - basename = rom.name.replace(" ", "").lower() - - header = f"""\ -name: {rom.name.title()} ({rom.get_country_name()}) -sha1: {rom.sha1} -options: - basename: {basename} - target_path: {rom_path.with_suffix(".z64")} - elf_path: build/{basename}.elf - base_path: . - platform: n64 - compiler: {rom.compiler} - - # asm_path: asm - # src_path: src - # build_path: build - # create_asm_dependencies: True - - ld_script_path: {basename}.ld - ld_dependencies: True - - find_file_boundaries: True - header_encoding: {rom.header_encoding} - - o_as_suffix: True - use_legacy_include_asm: False - mips_abi_float_regs: o32 - - asm_function_macro: glabel - asm_jtbl_label_macro: jlabel - asm_data_macro: dlabel - - # section_order: [".text", ".data", ".rodata", ".bss"] - # auto_all_sections: [".data", ".rodata", ".bss"] - - symbol_addrs_path: - - symbol_addrs.txt - reloc_addrs_path: - - reloc_addrs.txt - - # undefined_funcs_auto_path: undefined_funcs_auto.txt - # undefined_syms_auto_path: undefined_syms_auto.txt - - extensions_path: tools/splat_ext - - # string_encoding: ASCII - # data_string_encoding: ASCII - rodata_string_guesser_level: 2 - data_string_guesser_level: 2 - # libultra_symbols: True - # hardware_regs: True - # gfx_ucode: # one of [f3d, f3db, f3dex, f3dexb, f3dex2] -""" - - first_section_end = find_code_length.run(rom_bytes, 0x1000, rom.entry_point) - - segments = f"""\ -segments: - - name: header - type: header - start: 0x0 - - - name: boot - type: bin - start: 0x40 - - - name: entry - type: code - start: 0x1000 - vram: 0x{rom.entry_point:X} - subsegments: - - [0x1000, hasm] - - - name: main - type: code - start: 0x{0x1000 + rom.entrypoint_info.entry_size:X} - vram: 0x{rom.entry_point + rom.entrypoint_info.entry_size:X} - follows_vram: entry -""" - - if rom.entrypoint_info.bss_size is not None: - segments += f"""\ - bss_size: 0x{rom.entrypoint_info.bss_size:X} -""" - - segments += f"""\ - subsegments: - - [0x{0x1000 + rom.entrypoint_info.entry_size:X}, asm] -""" - - if ( - rom.entrypoint_info.bss_size is not None - and rom.entrypoint_info.bss_start_address is not None - ): - bss_start = rom.entrypoint_info.bss_start_address - rom.entry_point + 0x1000 - # first_section_end points to the start of data - segments += f"""\ - - [0x{first_section_end:X}, data] - - {{ start: 0x{bss_start:X}, type: bss, vram: 0x{rom.entrypoint_info.bss_start_address:08X} }} -""" - # Point next segment to the detected end of the main one - first_section_end = bss_start - - segments += f"""\ - - - type: bin - start: 0x{first_section_end:X} - follows_vram: main - - [0x{rom.size:X}] -""" - - out_file = f"{basename}.yaml" - with open(out_file, "w", newline="\n") as f: - print(f"Writing config to {out_file}") - f.write(header) - f.write(segments) - - -def create_gc_config(iso_path: Path, iso_bytes: bytes): - gc = gcinfo.get_info(iso_path, iso_bytes) - basename = gc.system_code + gc.game_code + gc.region_code + gc.publisher_code - - header = f"""\ -name: \"{gc.name.title()} ({gc.get_region_name()})\" -system_code: {gc.system_code} -game_code: {gc.game_code} -region_code: {gc.region_code} -publisher_code: {gc.publisher_code} -sha1: {gc.sha1} -options: - filesystem_path: filesystem - basename: {basename} - target_path: {iso_path.with_suffix(".iso")} - base_path: . - compiler: {gc.compiler} - platform: gc - # undefined_funcs_auto: True - # undefined_funcs_auto_path: undefined_funcs_auto.txt - # undefined_syms_auto: True - # undefined_syms_auto_path: undefined_syms_auto.txt - # symbol_addrs_path: symbol_addrs.txt - # asm_path: asm - # src_path: src - # build_path: build - # extensions_path: tools/splat_ext - # section_order: [".text", ".data", ".rodata", ".bss"] - # auto_all_sections: [".data", ".rodata", ".bss"] -""" - - segments = f"""\ -segments: - - name: filesystem - type: fst - path: filesystem/sys/fst.bin - - name: bootinfo - type: bootinfo - path: filesystem/sys/boot.bin - - name: bi2 - type: bi2 - path: filesystem/sys/bi2.bin - - name: apploader - type: apploader - path: filesystem/sys/apploader.img - - name: main - type: dol - path: filesystem/sys/main.dol -""" - - out_file = f"{basename}.yaml" - with open(out_file, "w", newline="\n") as f: - print(f"Writing config to {out_file}") - f.write(header) - f.write(segments) - - -def create_psx_config(exe_path: Path, exe_bytes: bytes): - exe = psxexeinfo.PsxExe.get_info(exe_path, exe_bytes) - basename = exe_path.name.replace(" ", "").lower() - - header = f"""\ -name: {exe_path.name} -sha1: {exe.sha1} -options: - basename: {basename} - target_path: {exe_path} - base_path: . - platform: psx - compiler: GCC - - # asm_path: asm - # src_path: src - # build_path: build - # create_asm_dependencies: True - - ld_script_path: {basename}.ld - - find_file_boundaries: False - gp_value: 0x{exe.initial_gp:08X} - - o_as_suffix: True - use_legacy_include_asm: False - - asm_function_macro: glabel - asm_jtbl_label_macro: jlabel - asm_data_macro: dlabel - - section_order: [".rodata", ".text", ".data", ".bss"] - # auto_all_sections: [".data", ".rodata", ".bss"] - - symbol_addrs_path: - - symbol_addrs.txt - reloc_addrs_path: - - reloc_addrs.txt - - # undefined_funcs_auto_path: undefined_funcs_auto.txt - # undefined_syms_auto_path: undefined_syms_auto.txt - - extensions_path: tools/splat_ext - - subalign: 2 - - string_encoding: ASCII - data_string_encoding: ASCII - rodata_string_guesser_level: 2 - data_string_guesser_level: 2 -""" - - segments = f"""\ -segments: - - name: header - type: header - start: 0x0 - - - name: main - type: code - start: 0x800 - vram: 0x{exe.destination_vram:X} - bss_size: 0x{exe.bss_size:X} - subsegments: -""" - text_offset = exe.text_offset - if text_offset != 0x800: - segments += f"""\ - - [0x800, rodata, 800] -""" - segments += f"""\ - - [0x{text_offset:X}, asm, {text_offset:X}] -""" - - if exe.data_vram != 0 and exe.data_size != 0: - data_offset = exe.data_offset - segments += f"""\ - - [0x{data_offset:X}, data, {data_offset:X}] -""" - - if exe.bss_size != 0: - segments += f"""\ - - {{ start: 0x{exe.size:X}, type: bss, name: {exe.bss_vram:X}, vram: 0x{exe.bss_vram:X} }} -""" - - segments += f"""\ - - [0x{exe.size:X}] -""" - - out_file = f"{basename}.yaml" - with open(out_file, "w", newline="\n") as f: - print(f"Writing config to {out_file}") - f.write(header) - f.write(segments) - - -if __name__ == "__main__": - args = parser.parse_args() - main(Path(args.file)) diff --git a/tools/splat/disassembler/__init__.py b/tools/splat/disassembler/__init__.py deleted file mode 100644 index 09c2ca5ffe..0000000000 --- a/tools/splat/disassembler/__init__.py +++ /dev/null @@ -1,2 +0,0 @@ -from .disassembler import Disassembler -from .spimdisasm_disassembler import SpimdisasmDisassembler diff --git a/tools/splat/disassembler/disassembler.py b/tools/splat/disassembler/disassembler.py deleted file mode 100644 index e64aa26163..0000000000 --- a/tools/splat/disassembler/disassembler.py +++ /dev/null @@ -1,17 +0,0 @@ -from abc import ABC, abstractmethod -from util.options import SplatOpts -from typing import Set - - -class Disassembler(ABC): - @abstractmethod - def configure(self, options: SplatOpts): - raise NotImplementedError("configure") - - @abstractmethod - def check_version(self, skip_version_check: bool, splat_version: str): - raise NotImplementedError("check_version") - - @abstractmethod - def known_types(self) -> Set[str]: - raise NotImplementedError("known_types") diff --git a/tools/splat/disassembler/disassembler_instance.py b/tools/splat/disassembler/disassembler_instance.py deleted file mode 100644 index 988fe3ceee..0000000000 --- a/tools/splat/disassembler/disassembler_instance.py +++ /dev/null @@ -1,26 +0,0 @@ -from .disassembler import Disassembler -from .spimdisasm_disassembler import SpimdisasmDisassembler -from .null_disassembler import NullDisassembler - -__instance: Disassembler = NullDisassembler() -__initialized = False - - -def create_disassembler_instance(platform: str): - global __instance - global __initialized - if platform in ["n64", "psx", "ps2"]: - __instance = SpimdisasmDisassembler() - __initialized = True - return - - raise NotImplementedError("No disassembler for requested platform") - - -def get_instance() -> Disassembler: - global __instance - global __initialized - if not __initialized: - raise Exception("Disassembler instance not initialized") - return None - return __instance diff --git a/tools/splat/disassembler/null_disassembler.py b/tools/splat/disassembler/null_disassembler.py deleted file mode 100644 index 046725e9e1..0000000000 --- a/tools/splat/disassembler/null_disassembler.py +++ /dev/null @@ -1,14 +0,0 @@ -from disassembler import disassembler -from util.options import SplatOpts -from typing import Set - - -class NullDisassembler(disassembler.Disassembler): - def configure(self, opts: SplatOpts): - pass - - def check_version(self, skip_version_check: bool, splat_version: str): - pass - - def known_types(self) -> Set[str]: - return set() diff --git a/tools/splat/disassembler/spimdisasm_disassembler.py b/tools/splat/disassembler/spimdisasm_disassembler.py deleted file mode 100644 index 335c9d2c87..0000000000 --- a/tools/splat/disassembler/spimdisasm_disassembler.py +++ /dev/null @@ -1,124 +0,0 @@ -from disassembler import disassembler -import spimdisasm -import rabbitizer -from util import log, compiler -from util.options import SplatOpts -from typing import Set - - -class SpimdisasmDisassembler(disassembler.Disassembler): - # This value should be kept in sync with the version listed on requirements.txt - SPIMDISASM_MIN = (1, 18, 0) - - def configure(self, opts: SplatOpts): - # Configure spimdisasm - spimdisasm.common.GlobalConfig.PRODUCE_SYMBOLS_PLUS_OFFSET = True - spimdisasm.common.GlobalConfig.TRUST_USER_FUNCTIONS = True - spimdisasm.common.GlobalConfig.TRUST_JAL_FUNCTIONS = True - spimdisasm.common.GlobalConfig.GLABEL_ASM_COUNT = False - - if opts.rom_address_padding: - spimdisasm.common.GlobalConfig.ASM_COMMENT_OFFSET_WIDTH = 6 - else: - spimdisasm.common.GlobalConfig.ASM_COMMENT_OFFSET_WIDTH = 0 - - # spimdisasm is not performing any analyzis on non-text sections so enabling this options is pointless - spimdisasm.common.GlobalConfig.AUTOGENERATED_NAMES_BASED_ON_SECTION_TYPE = False - spimdisasm.common.GlobalConfig.AUTOGENERATED_NAMES_BASED_ON_DATA_TYPE = False - - spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO = False - - if opts.rodata_string_guesser_level is not None: - spimdisasm.common.GlobalConfig.RODATA_STRING_GUESSER_LEVEL = ( - opts.rodata_string_guesser_level - ) - - if opts.data_string_guesser_level is not None: - spimdisasm.common.GlobalConfig.DATA_STRING_GUESSER_LEVEL = ( - opts.data_string_guesser_level - ) - - rabbitizer.config.regNames_userFpcCsr = False - rabbitizer.config.regNames_vr4300Cop0NamedRegisters = False - - rabbitizer.config.misc_opcodeLJust = opts.mnemonic_ljust - 1 - - rabbitizer.config.regNames_gprAbiNames = rabbitizer.Abi.fromStr( - opts.mips_abi_gpr - ) - rabbitizer.config.regNames_fprAbiNames = rabbitizer.Abi.fromStr( - opts.mips_abi_float_regs - ) - - if opts.endianness == "big": - spimdisasm.common.GlobalConfig.ENDIAN = spimdisasm.common.InputEndian.BIG - else: - spimdisasm.common.GlobalConfig.ENDIAN = spimdisasm.common.InputEndian.LITTLE - - rabbitizer.config.pseudos_pseudoMove = False - - selected_compiler = opts.compiler - if selected_compiler == compiler.SN64: - rabbitizer.config.regNames_namedRegisters = False - rabbitizer.config.toolchainTweaks_sn64DivFix = True - rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch = True - spimdisasm.common.GlobalConfig.ASM_COMMENT = False - spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO = ( - False - ) - spimdisasm.common.GlobalConfig.COMPILER = spimdisasm.common.Compiler.SN64 - elif selected_compiler == compiler.GCC: - rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch = True - spimdisasm.common.GlobalConfig.COMPILER = spimdisasm.common.Compiler.GCC - elif selected_compiler == compiler.IDO: - spimdisasm.common.GlobalConfig.COMPILER = spimdisasm.common.Compiler.IDO - - spimdisasm.common.GlobalConfig.DETECT_REDUNDANT_FUNCTION_END = ( - opts.detect_redundant_function_end - ) - - spimdisasm.common.GlobalConfig.GP_VALUE = opts.gp - - spimdisasm.common.GlobalConfig.ASM_TEXT_LABEL = opts.asm_function_macro - spimdisasm.common.GlobalConfig.ASM_TEXT_ALT_LABEL = opts.asm_function_alt_macro - spimdisasm.common.GlobalConfig.ASM_JTBL_LABEL = opts.asm_jtbl_label_macro - spimdisasm.common.GlobalConfig.ASM_DATA_LABEL = opts.asm_data_macro - spimdisasm.common.GlobalConfig.ASM_TEXT_END_LABEL = opts.asm_end_label - - if opts.asm_emit_size_directive is not None: - spimdisasm.common.GlobalConfig.ASM_EMIT_SIZE_DIRECTIVE = ( - opts.asm_emit_size_directive - ) - - if spimdisasm.common.GlobalConfig.ASM_TEXT_LABEL == ".globl": - spimdisasm.common.GlobalConfig.ASM_TEXT_ENT_LABEL = ".ent" - spimdisasm.common.GlobalConfig.ASM_TEXT_FUNC_AS_LABEL = True - - if spimdisasm.common.GlobalConfig.ASM_DATA_LABEL == ".globl": - spimdisasm.common.GlobalConfig.ASM_DATA_SYM_AS_LABEL = True - - spimdisasm.common.GlobalConfig.LINE_ENDS = opts.c_newline - - spimdisasm.common.GlobalConfig.ALLOW_ALL_ADDENDS_ON_DATA = ( - opts.allow_data_addends - ) - - spimdisasm.common.GlobalConfig.DISASSEMBLE_UNKNOWN_INSTRUCTIONS = ( - opts.disasm_unknown - ) - - if opts.compiler == compiler.GCC and opts.platform == "ps2": - rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch = False - - def check_version(self, skip_version_check: bool, splat_version: str): - if not skip_version_check and spimdisasm.__version_info__ < self.SPIMDISASM_MIN: - log.error( - f"splat {splat_version} requires as minimum spimdisasm {self.SPIMDISASM_MIN}, but the installed version is {spimdisasm.__version_info__}" - ) - - log.write( - f"splat {splat_version} (powered by spimdisasm {spimdisasm.__version__})" - ) - - def known_types(self) -> Set[str]: - return spimdisasm.common.gKnownTypes diff --git a/tools/splat/disassembler_section.py b/tools/splat/disassembler_section.py deleted file mode 100644 index b9e1cada0d..0000000000 --- a/tools/splat/disassembler_section.py +++ /dev/null @@ -1,280 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Optional - -import spimdisasm - -from util import options, symbols - - -class DisassemblerSection(ABC): - @abstractmethod - def disassemble(self): - raise NotImplementedError("disassemble") - - @abstractmethod - def analyze(self): - raise NotImplementedError("analyze") - - @abstractmethod - def set_comment_offset(self, rom_start: int): - raise NotImplementedError("set_comment_offset") - - @abstractmethod - def make_bss_section( - self, - rom_start, - rom_end, - vram_start, - bss_end, - name, - segment_rom_start, - exclusive_ram_id, - ): - raise NotImplementedError("make_bss_section") - - @abstractmethod - def make_data_section( - self, - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ): - raise NotImplementedError("make_data_section") - - @abstractmethod - def get_section(self): - raise NotImplementedError("get_section") - - @abstractmethod - def make_rodata_section( - self, - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ): - raise NotImplementedError("make_rodata_section") - - @abstractmethod - def make_text_section( - self, - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ): - raise NotImplementedError("make_text_section") - - -class SpimdisasmDisassemberSection(DisassemblerSection): - def __init__(self): - self.spim_section: Optional[spimdisasm.mips.sections.SectionBase] = None - - def disassemble(self) -> str: - assert self.spim_section is not None - return self.spim_section.disassemble() - - def analyze(self): - assert self.spim_section is not None - self.spim_section.analyze() - - def set_comment_offset(self, rom_start: int): - assert self.spim_section is not None - self.spim_section.setCommentOffset(rom_start) - - def make_bss_section( - self, - rom_start: int, - rom_end: int, - vram_start: int, - bss_end: int, - name: str, - segment_rom_start: int, - exclusive_ram_id, - ): - self.spim_section = spimdisasm.mips.sections.SectionBss( - symbols.spim_context, - rom_start, - rom_end, - vram_start, - bss_end, - name, - segment_rom_start, - exclusive_ram_id, - ) - - def make_data_section( - self, - rom_start: int, - rom_end: int, - vram_start: int, - name: str, - rom_bytes: bytes, - segment_rom_start: int, - exclusive_ram_id, - ): - self.spim_section = spimdisasm.mips.sections.SectionData( - symbols.spim_context, - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ) - - def get_section(self) -> Optional[spimdisasm.mips.sections.SectionBase]: - return self.spim_section - - def make_rodata_section( - self, - rom_start: int, - rom_end: int, - vram_start: int, - name: str, - rom_bytes: bytes, - segment_rom_start: int, - exclusive_ram_id, - ): - self.spim_section = spimdisasm.mips.sections.SectionRodata( - symbols.spim_context, - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ) - - def make_text_section( - self, - rom_start: int, - rom_end: int, - vram_start: int, - name: str, - rom_bytes: bytes, - segment_rom_start: int, - exclusive_ram_id, - ): - self.spim_section = spimdisasm.mips.sections.SectionText( - symbols.spim_context, - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ) - - -def make_disassembler_section() -> Optional[SpimdisasmDisassemberSection]: - if options.opts.platform in ["n64", "psx", "ps2"]: - return SpimdisasmDisassemberSection() - - raise NotImplementedError("No disassembler section for requested platform") - return None - - -def make_text_section( - rom_start: int, - rom_end: int, - vram_start: int, - name: str, - rom_bytes: bytes, - segment_rom_start: int, - exclusive_ram_id, -) -> DisassemblerSection: - section = make_disassembler_section() - assert section is not None - section.make_text_section( - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ) - return section - - -def make_data_section( - rom_start: int, - rom_end: int, - vram_start: int, - name: str, - rom_bytes: bytes, - segment_rom_start: int, - exclusive_ram_id, -) -> DisassemblerSection: - section = make_disassembler_section() - assert section is not None - section.make_data_section( - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ) - return section - - -def make_rodata_section( - rom_start: int, - rom_end: int, - vram_start: int, - name: str, - rom_bytes: bytes, - segment_rom_start: int, - exclusive_ram_id, -) -> DisassemblerSection: - section = make_disassembler_section() - assert section is not None - section.make_rodata_section( - rom_start, - rom_end, - vram_start, - name, - rom_bytes, - segment_rom_start, - exclusive_ram_id, - ) - return section - - -def make_bss_section( - rom_start: int, - rom_end: int, - vram_start: int, - bss_end: int, - name: str, - segment_rom_start: int, - exclusive_ram_id, -) -> DisassemblerSection: - section = make_disassembler_section() - assert section is not None - section.make_bss_section( - rom_start, - rom_end, - vram_start, - bss_end, - name, - segment_rom_start, - exclusive_ram_id, - ) - return section diff --git a/tools/splat/docs/Adding-Symbols.md b/tools/splat/docs/Adding-Symbols.md deleted file mode 100644 index 8bb0debe1d..0000000000 --- a/tools/splat/docs/Adding-Symbols.md +++ /dev/null @@ -1,136 +0,0 @@ -Symbols (i.e. labelling a function or variable) are controlled by the `symbols_addrs.txt` file. - -The format for defining symbols is: - -```ini -symbol = address; // option1:value1 option2:value2 -``` -e.g. -```ini -osInitialize = 0x801378C0; // type:func -``` - -:information_source: The file used can be overridden via the `symbol_addrs_path` setting in the global `options` section of the splat yaml. This option can also accept a list of paths, allowing for symbols to be organized in multiple files. - -## symbol - -This is the name of the symbol and can be any valid C variable name, e.g. `myCoolFunction` or `gReticulatedSplineCounter` - -## address - -This is the VRAM address expressed in hexadecimal, e.g. `0x80001050` - -## options - -An optional `key:pair` list of settings, note that each option should be separated by whitespace, but there should be no whitespace between the key:value pairs themselves. - -### `type` - -Override splat's automatic type detection, possible values are: -- `func`: Functions -- `jtbl`: Jumptables -- `jtbl_label`: Jumptables labels (inside functions) -- `label`: Branch labels (inside functions) -- `s8`, `u8`: To specify data/rodata to be disassembled as `.byte`s -- `s16`, `u16`: To specify data/rodata to be disassembled as `.short`s -- `s32`, `u32`: To specify data/rodata to be disassembled as `.word`s (the default) -- `s64`, `u64`: :man_shrugging: -- `f32`, `Vec3f`: To specify data/rodata to be disassembled as `.float`s -- `f64`: To specify data/rodata to be disassembled as `.double`s -- `asciz`, `char*`, `char`: C strings (disassembled as `.asciz`) -- Any custom type starting with a capital letter (will default to `.word`s) - -Any other type will produce an error. - -**Example:** -```ini -minFrameSize = 0x80241D08; // type:s32 -``` - -### `size` - -The size of the function or the size of the data depending on the type of the symbol. It specifies a size in bytes. e.g. `size:0x10`. - -**Example:** -```ini -RawHuffmanTable = 0x8022E0E0; // type:symbol size:0x100 -``` - -### `rom` - -The ROM offset for the symbol, useful (potentially mandatory) for symbols in overlays where multiple symbols could share the same VRAM address. - -**Example:** -```ini -create_particle_effect = 0x802D5F4C; // type:func rom:0x6E75FC -``` - -### `segment` - -Allows specifying to which specific segment this symbol belongs to, useful to disambiguate symbols from segments that share the same VRAM address. This name must be the same as the name of a segment listed in the yaml. - -**Example:** -```ini -sMenuTexture = 0x06004040; // segment:menu_assets -``` - -### `name_end` - -Emits a symbol after the end of the data of the current symbol. Useful to reference the end of an assembly symbol, like RSP data. - -**Example:** -```ini -rspbootTextStart = 0x80084690; // name_end:rspbootTextEnd -``` - -### `defined` - -Forces the symbol to be defined - i.e. prevent it from appearing in `undefined_syms_auto.txt` should splat not encounter the symbol during the symbol detection phase. - -**Example:** -```ini -__osDpDeviceBusy = 0x8014B3D0; // defined:true -``` - -### `extract` - -TBD - -### `ignore` - -Prevents an address from being symbolized and referenced. Useful to get a finer control over the disassembled output. - -**Example:** -```ini -D_A0000000 = 0xA0000000; // ignore:true -``` - -It can also be combined with the `size` attribute to avoid a range of addresses of being symbolized. - -**Example:** -```ini -D_80000000 = 0x80000000; // ignore:true size:0x10 -``` - -### `force_migration` and `force_not_migration` - -Grants a finer control over the automatic rodata migration to functions. This may be required because of the migration heuristic failing to migrate (or to not migrate) a symbol, producing a disordered rodata section. Forcing the migration of a rodata symbol to a function will only work if that function references said rodata symbol. Forcing the not-migration of a rodata symbol always works. - -This attribute is ignored if the `migrate_rodata_to_functions` option is disabled. - -**Example:** -```ini -jtbl_800B13D0 = 0x800B13D0; // type:jtbl force_migration:True -STR_800B32A8 = 0x800C9520; // type:asciz force_not_migration:True -``` - -### `allow_addend` and `dont_allow_addend` - -Allows this symbol to reference (or not reference) other symbols with an addend. - -This attribute overrides the global `allow_data_addends` option. - -**Example:** -```ini -aspMainTextStart = 0x80084760; // dont_allow_addend:True -``` diff --git a/tools/splat/docs/Advanced.md b/tools/splat/docs/Advanced.md deleted file mode 100644 index fe9b4974fb..0000000000 --- a/tools/splat/docs/Advanced.md +++ /dev/null @@ -1,7 +0,0 @@ -## Writing custom segment handler - -The following list contains examples of custom segments: - -- [RNC](https://github.com/mkst/sssv/blob/master/tools/splat_ext/rnc.py) -- [Vtx](https://github.com/mkst/sssv/blob/master/tools/splat_ext/sssv_vtx.py) -- [Multiple](https://github.com/pmret/papermario/tree/main/tools/splat_ext) diff --git a/tools/splat/docs/Configuration.md b/tools/splat/docs/Configuration.md deleted file mode 100644 index 6ee550062d..0000000000 --- a/tools/splat/docs/Configuration.md +++ /dev/null @@ -1,612 +0,0 @@ -Splat has various options for configuration, all of which are listed under the `options` section of the yaml file. - -## Project configuration - -### base_path - -Path that all other configured paths are relative to. - -#### Usage - -```yaml -base_path: path/to/base/folder -``` - -#### Default - -`.` *(Current directory)* - - -### target_path - -Path to target binary. - -#### Usage - -```yaml -target_path: path/to/target/binary -``` - -### elf_path - -Path to the final elf target - -#### Default -Path to the binary that was used as the input to `create_config.py` - - -### platform - -The target platform for the binary. Options are -- `n64` (Nintendo 64) -- `psx` (PlayStation 1) -- `ps2` (PlayStation 2) -- `gc` (GameCube) - -#### Usage -```yaml -platform: psx -``` - - -### compiler - -Compiler used to build the binary. - -splat recognizes the following compilers, and it will adapt it behavior accordingly for them, but unknown compilers can be passed as well: -- GCC -- SN64 -- IDO - -#### Usage -```yaml -compiler: IDO -``` - -#### Default -`ido` - - -### endianness - -Determines the endianness of the target binary. If not set, the endiannesss will be guessed from the selected platform. - -Valid values: -- big -- little - - -### section_order - -Determines the default section order of the target binary. This can be overridden per-segment. - -Expects a list of strings. - - -### generated_c_preamble - -String that is placed before the contents of newly-generated `.c` files. - -#### Usage - -```yaml -generated_c_preamble: #include "header.h" -``` - -#### Default - -`#include "common.h"` - - -### generated_s_preamble - -String that is placed before the contents of newly-generated assembly (`.s`) files. - -#### Usage - -```yaml -generated_s_preamble: .set fp=64 -``` - - -### o_as_suffix - -Determines whether to replace the suffix of the file to `.o` or to append `.o` to the suffix of the file. - - -### gp_value - -The value of the `$gp` register to correctly calculate offset to `%gp_rel` relocs. - - -### check_consecutive_segment_types - -By default splat will check and error if there are any non consecutive segment types. -This option disables said feature. - -#### Usage - -```yaml -# Disable checking for non-consecutive segments -check_consecutive_segment_types: False -``` - - -## Paths - - -### asset_path - -Path to output split asset files. - -#### Usage - -```yaml -asset_path: path/to/assets/folder -``` - -#### Default - -`assets` - - -### symbol_addrs_path - -Determines the path to the symbol addresses file(s). A `symbol_addrs` file is to be updated/curated manually and contains addresses of symbols as well as optional metadata such as rom address, type, and more - -It's possible to use more than one file by supplying a list instead of a string - -#### Usage -```yaml -symbol_addrs_path: path/to/symbol_addrs -``` - -#### Default -`symbol_addrs.txt` - - - -### reloc_addrs_paths - - - -### build_path -Path that built files will be found. Used for generation of the linker script. - -#### Usage -```yaml -build_path: path/to/build/folder -``` - -#### Default -`build` - - -### src_path -Path to split `.c` files. - -#### Usage -```yaml -src_path: path/to/src/folder -``` - -#### Default -`src` - - -### asm_path -Path to output split assembly files. - -#### Usage -```yaml -asm_path: path/to/asm/folder -``` - -#### Default -`asm` - - -### data_path - -Determines the path to the asm data directory - - -### nonmatchings_path - -Determines the path to the asm nonmatchings directory - - -### cache_path -Path to splat cache - -#### Usage -```yaml -cache_path: path/to/splat/cache -``` - -#### Default -`.splat_cache` - - -### create_undefined_funcs_auto -If `True`, splat will generate an `undefined_funcs_auto.txt` file. - -#### Usage -```yaml -create_undefined_funcs_auto: False -``` - -#### Default -`True` - - -### undefined_funcs_auto_path -Path to file containing automatically defined functions. - -#### Usage -```yaml -undefined_funcs_auto_path: path/to/undefined_funcs_auto.txt -``` - -#### Default -`undefined_funcs_auto.txt` - - - -### create_undefined_syms_auto -If `True`, splat will generate an `undefined_syms_auto.txt` file. - -#### Usage -```yaml -create_undefined_syms_auto: False -``` - -#### Default -`True` - - -### undefined_syms_auto_path -Path to file containing automatically defined symbols. - -#### Usage -```yaml -undefined_syms_auto_path: path/to/undefined_syms_auto.txt -``` - -#### Default -`undefined_syms_auto.txt` - - -### extensions_path -If you are using splat extension(s), this is the path they will be loaded from. - -#### Usage -```yaml -extensions_path: path/to/extensions/folder -``` - -#### Default -`tools/splat_ext` - - -### lib_path - -Determines the path to library files that are to be linked into the target binary - - -### elf_section_list_path -Path to file containing elf section list. - -#### Usage -```yaml -elf_section_list_path: path/to/elf_sections -``` - -#### Default -`elf_sections.txt` - - -## Linker script - - -### subalign - -Sub-alignment (in bytes) of sections. - -#### Usage -```yaml -subalign: 4 -``` - -#### Default -`16` - - -### auto_all_sections - -TODO - -### ld_script_path - -Path to output ld script. - -#### Usage - -```yaml -ld_script_path: path/to/ld/script.ld -``` - -#### Default - -`{basename}.ld` - - -### ld_symbol_header_path - -Path to output a header containing linker symbols. - -#### Usage -```yaml -ld_symbol_header_path: path/to/linker_symbol_header -``` - -### ld_discard_section - -Determines whether to add a discard section to the linker script - -### ld_section_labels - -Determines the list of section labels that are to be added to the linker script - -### ld_wildcard_sections - -Determines whether to add wildcards for section linking in the linker script (.rodata* for example) - -### ld_use_symbolic_vram_addreses - -Determines whether to use `follows_vram` (segment option) and `vram_symbol` / `follows_classes` (vram_class options) to calculate vram addresses in the linker script. -Enabled by default. If disabled, this uses the plain integer values for vram addresses defined in the yaml. - -### ld_partial_linking - -Change linker script generation to allow partially linking segments. Requires both `ld_partial_scripts_path` and `ld_partial_build_segments_path` to be set. - -### ld_partial_scripts_path - -Folder were each intermediary linker script will be written to. - -### ld_partial_build_segments_path - -Folder where the built partially linked segments will be placed by the build system. - -### ld_dependencies - -Generate a dependency file for every linker script generated. Dependency files will have the same path and name as the corresponding linker script, but changing the extension to `.d`. Requires `elf_path` to be set. - -### ld_legacy_generation - -Legacy linker script generation does not impose the section_order specified in the yaml options or per-segment options. - -### segment_end_before_align - -If enabled, the end symbol for each segment will be placed before the alignment directive for the segment - -### segment_symbols_style - -Controls the style of the auto-generated segment symbols in the linker script. - -Possible values: -- splat -- makerom - - -### ld_rom_start - -Specifies the starting offset for rom address symbols in the linker script. - - -### ld_fill_value - -Allows to specify the value of the `FILL` statement generated on every segment of the linker script. - -It must be either an integer, which will be used as the parameter for the `FILL` statement, or `null`, which tells splat to not emit `FILL` statements. - -This behavior can be customized per segment too. See [ld_fill_value](Segments.md#ld_fill_value) on the Segments section. - -Defaults to 0. - - -## C file options - -### create_c_files - -Determines whether to create new c files if they don't exist - -### auto_decompile_empty_functions - -Determines whether to "auto-decompile" empty functions - -### do_c_func_detection - -Determines whether to detect matched/unmatched functions in existing c files so we can avoid creating `.s` files for already-decompiled functions. - -### c_newline - -Determines the newline char(s) to be used in c files - - -## (Dis)assembly-related options - -### symbol_name_format - -Determine the format that symbols should be named by default - -### symbol_name_format_no_rom - -Same as `symbol_name_format` but for symbols with no rom address - -### find_file_boundaries - -Determines whether to detect and hint to the user about likely file splits when disassembling. - -This setting can also be set on a per segment basis, if you'd like to enable or disable detection for specific segments. This could be useful when you are confident you identified all subsegments in a segment, yet `splat` still hints that subsegments could be split. - -### pair_rodata_to_text - -Determines whether to detect and hint to the user about possible rodata sections corresponding to a text section - -### migrate_rodata_to_functions - -Determines whether to attempt to automatically migrate rodata into functions - -### asm_inc_header - -Determines the header to be used in every asm file that's included from c files - -### asm_function_macro - -Determines the macro used to declare functions in asm files - -### asm_function_alt_macro - -Determines the macro used to declare symbols in the middle of functions in asm files (which may be alternative entries) - -### asm_jtbl_label_macro - -Determines the macro used to declare jumptable labels in asm files - -### asm_data_macro - -Determines the macro used to declare data symbols in asm files - -### asm_end_label - -Determines the macro used at the end of a function, such as endlabel or .end - -### asm_emit_size_directive - -Toggles the .size directive emitted by the disassembler - -### include_macro_inc - -Determines including the macro.inc file on non-migrated rodata variables - -### mnemonic_ljust - -Determines the number of characters to left align before the instruction - -### rom_address_padding - -Determines whether to pad the rom address - -### mips_abi_gpr - -Determines which ABI names to use for general purpose registers - -### mips_abi_float_regs - -Determines which ABI names to use for floating point registers. - -Valid values: -- numeric -- o32 -- n32 -- n64 - -`o32`` is highly recommended, as it provides logically named registers for floating point instructions. -For more info, see https://gist.github.com/EllipticEllipsis/27eef11205c7a59d8ea85632bc49224d - -### add_set_gp_64 - -Determines whether to add ".set gp=64" to asm/hasm files - -### create_asm_dependencies - -Generate `.asmproc.d` dependency files for each C file which still reference functions in assembly files - -### string_encoding - -Global option for rodata string encoding. This can be overriden per segment - -### data_string_encoding - -Global option for data string encoding. This can be overriden per segment - -### rodata_string_guesser_level - -Global option for the rodata string guesser. 0 disables the guesser completely. - -### data_string_guesser_level - -Global option for the data string guesser. 0 disables the guesser completely. - -### allow_data_addends - -Global option for allowing data symbols using addends on symbol references. It can be overriden per symbol - -### disasm_unknown - -Tells the disassembler to try disassembling functions with unknown instructions instead of falling back to disassembling as raw data - -### detect_redundant_function_end - -Tries to detect redundant and unreferenced functions ends and merge them together. This option is ignored if the compiler is not set to IDO. - -### disassemble_all - -Don't skip disassembling already matched functions and migrated sections - - -## N64-specific options - -### header_encoding - -Used to specify what encoding should be used used when parsing the N64 ROM header. - -#### Default - -`ASCII` - - -### gfx_ucode - -Determines the type gfx ucode (used by gfx segments) - -Valid options are: -- f3d -- f3db -- f3dex -- f3dexb -- f3dex2 - -### libultra_symbols - -Use named libultra symbols by default. Those will need to be added to a linker script manually by the user - -### ique_symbols - -Use named libultra symbols by default. Those will need to be added to a linker script manually by the user - -### hardware_regs - -Use named hardware register symbols by default. Those will need to be added to a linker script manually by the user - - -## Gamecube-specific options - -### filesystem_path - -Path where the iso's filesystem will be extracted to - -## Compiler-specific options - -### use_legacy_include_asm -If `True`, generate c files using the longer `INCLUDE_ASM` macro. This is defaulted to `True` to by-default support projects using the longer macro. - -#### Usage -```yaml -use_legacy_include_asm: False -``` - -#### Default -`True` diff --git a/tools/splat/docs/Examples.md b/tools/splat/docs/Examples.md deleted file mode 100644 index 82bf8cb47c..0000000000 --- a/tools/splat/docs/Examples.md +++ /dev/null @@ -1,32 +0,0 @@ -The following is a list of projects known to be using **splat** along with the compilers used: - -## N64 Projects - -- [Aidyn Chronicles](https://github.com/blackgamma7/Aidyn) `unknown` (does not build source) -- [Animal Forest](https://github.com/zeldaret/af) `ido7.1` -- [Banjo Kazooie](https://gitlab.com/banjo.decomp/banjo-kazooie) `ido5.3` -- [Conker's Bad Fur Day](https://github.com/mkst/conker) `ido5.3` -- [Dinosaur Planet](https://github.com/zestydevy/dinosaur-planet) `ido5.3` -- [Dr. Mario 64](https://github.com/AngheloAlf/drmario64) `kmc gcc2.7.2` & `egcs gcc2.91.66` -- [Gauntlet Legends](https://github.com/Drahsid/gauntlet-legends) `kmc gcc2.7.2` (uses old KMC GCC wrapper - do not use for reference) -- [Mario Party 3](https://github.com/PartyPlanner64/mp3) `gcc2.7.2 TBD` -- [Mischief Makers](https://github.com/Drahsid/mischief-makers) `ido5.3` -- [Neon Genesis Evangelion 64](https://github.com/farisawan-2000/evangelion) `kmc gcc2.7.2` -- [Paper Mario](https://github.com/pmret/papermario) `gcc2.8.1` -- [Pokemon Snap](https://github.com/ethteck/pokemonsnap) `ido7.1` -- [Pokemon Stadium](https://github.com/ethteck/pokemonstadium) `ido7.1` -- [Pokémon Puzzle League](https://github.com/AngheloAlf/puzzleleague64) `kmc gcc2.7.2` -- [Rocket Robot on Wheels](https://github.com/RocketRet/Rocket-Robot-On-Wheels) `SN64 (build 970404)` -- [Space Station Silicon Valley](https://github.com/mkst/sssv) `ido5.3` -- [Turok 3](https://github.com/drahsid/turok3) `psyq gcc2.8.0` -- [Yoshi's Story](https://github.com/decompals/yoshis-story) `ido7.1` - -## PS1 Projects - -- [Evo's Space Adventures](https://github.com/mkst/esa) `psyq 4.6 (gcc2.95)` -- [Final Fantasy 7](https://github.com/Drahsid/ffvii) `psyq <= 4.1 (gcc2.7.2)` -- [Silent Hill](https://github.com/Vatuu/silent-hill-decomp) `psyq <= 4.1 (gcc2.7.2) TBD` - -## PS2 Projects - -- [Kingdom Hearts](https://github.com/ethteck/kh1) TBD diff --git a/tools/splat/docs/General-Workflow.md b/tools/splat/docs/General-Workflow.md deleted file mode 100644 index 947f2b305a..0000000000 --- a/tools/splat/docs/General-Workflow.md +++ /dev/null @@ -1,179 +0,0 @@ -This describes an example of how to iteratively edit the splat segments config in order to maximise code and data migration from the binary. - -# 1 Initial configuration - -After successfully following the [Quickstart](https://github.com/ethteck/splat/wiki/Quickstart), you should have an initial configuration like the one below: - -```yaml -- name: main - type: code - start: 0x1060 - vram: 0x80070C60 - follows_vram: entry - bss_size: 0x3AE70 - subsegments: - - [0x1060, asm] - # ... a lot of additional `asm` sections - # This section is found out to contain __osViSwapContext - - [0x25C20, asm, energy_orb_wave] - # ... a lot of additional `asm` sections - - [0x2E450, data] - - - [0x3E330, rodata] - # ... a lot of additional `rodata` sections - - { start: 0x3F1B0, type: bss, vram: 0x800E9C20 } - -- [0x3F1B0, bin] -``` - -## 1.1 Match `rodata` to `asm` sections - -It's good practice to start pairing `rodata` sections with `asm` sections _before_ changing the `asm` sections into `c` files. This is because rodata may need to be explicitly included within the `c` file (via `INCLUDE_RODATA` or `GLOBAL_ASM` macros). - -`splat` provides hints about which `rodata` segments are referenced by which `asm` segments based on references to these symbols within the disassembled functions. - -These messages are output when splitting and look like: - -``` -Rodata segment '3EE10' may belong to the text segment 'energy_orb_wave' - Based on the usage from the function func_0xXXXXXXXX to the symbol D_800AEA10 -``` - -To pair these two sections, simply add the _name_ of the suggested text (i.e. `asm`) segment to the `rodata` segment: - -```yaml -- [0x3EE10, rodata, energy_orb_wave] # segment will be paired with a text (i.e. asm or c) segment named "energy_orb_wave" -``` - -**NOTE:** - -By default `migrate_rodata_to_functions` functionality is enabled. This causes splat to include paired rodata along with the disassembled assembly code, allowing it to be linked via `.rodata` segments from the get-go. This guide assumes that you will disable this functionality until you have successfully paired up the segments. - -### Troubleshooting - -#### Multiple `rodata` segments for a single text segment - -Using the following configuration: -```yaml -# ... -- [0x3E900, rodata] -- [0x3E930, rodata] -# ... -``` - -`splat` outputs a hint that doesn't immediately seem to make sense: - -``` -Rodata segment '3E900' may belong to the text segment '16100' - Based on the usage from the function func_80085DA0 to the symbol jtbl_800AE500 - -Rodata segment '3E930' may belong to the text segment '16100' - Based on the usage from the function func_800862C0 to the symbol jtbl_800AE530 -``` - -This hint tells you that `splat` believes one text segment references two `rodata` sections. This usually means that either the `rodata` should not be split at `0x3E930`, or that there is a missing split in the `asm` at `0x16100`, as a text segment can only have one `rodata` segment. - -If we assume that the rodata split is incorrect, we can remove the extraneous split: - -```yaml -# ... -- [0x3E900, rodata, "16100"] -# ... -``` - -**NOTE:** Splat uses heuristics to determine `rodata` and `asm` splits and is not perfect - false positives are possible and, if in doubt, double-check the assembly yourself before changing the splits. - - -### Multiple `asm` segments referring to the same `rodata` segment - -Sometimes the opposite is true, and `splat` believes two `asm` segments belong to a single `rodata` segment. In this case, you can split the `asm` segment to make sure two files are not paired with the same `rodata`. Note that this too can be a false positive. - - -# 2 Disassemble text, data, rodata - -Let's say you want to start decompiling the subsegment at `0x25C20` (`energy_orb_wave`). Start by replacing the `asm` type with `c`, and then re-run splat. - -```yaml -- [0x25C20, c, energy_orb_wave] -# ... -- [0x3EE10, rodata, energy_orb_wave] -``` - -This will disassemble the ROM at `0x25C20` as code, creating individual `.s` files for each function found. The output will be located in `{asm_path}/nonmatchings/energy_orb_wave/.s`. - -Assuming `data` and `rodata` segments have been paired with the `c` segment, splat will generate `{asm_path}/energy_orb_wave.data.s` and `{asm_path}/energy_orb_wave.rodata.s` respectively. - -Finally, splat will generate a C file, at `{src_path}/energy_orb_wave.c` containing macros that will be used to include all disassembled function assembly. - -**NOTE:** -- the path for where assembly is written can be configured via `asm_path`, the default is `{base_dir}/asm` -- the source code path can be configured via `src_path`, the default is `{base_path}/src` - -## Macros - -The macros to include text/rodata assembly are different for GCC vs IDO compiler: - -**GCC**: `INCLUDE_ASM` & `INCLUDE_RODATA` (text/rodata respectively) -**IDO**: `GLOBAL_ASM` - -These macros must be defined in an included header, which splat currently does not produce. - -For a GCC example, see the [include.h](https://github.com/AngheloAlf/drmario64/blob/master/include/include_asm.h) from the Dr. Mario project. - -For IDO, you will need to use [asm-processor](https://github.com/simonlindholm/asm-processor) in order to include assembly code within the c files. - - -# 3 Decompile text - -This involved back and forth between `.c` and `.s` files: - -- editing the `data.s`, `rodata.s` files to add/fixup symbols at the proper locations -- decompiling functions, declaring symbols (`extern`s) in the `.c` - -The linker script links -- `.text` (only) from the `.o` built from `energy_orb_wave.c` -- `.data` (only) from the `.o` built from `energy_orb_wave.data.s` -- `.rodata` (only) from the `.o` built from `energy_orb_wave.rodata.s` - -# 4 Decompile (ro)data - -Migrate data to the .c file, using raw values, lists or structs as appropriate code. - -Once you have paired the rodata and text segments together, you can enabled `migrate_rodata_to_functions`. This will add the paired rodata into each individual function's assembly file, and therefore, the rodata will end up in the compiled .o file. - -To link the .data/.rodata from the .o built from the .c file (instead of from the .s files), the subsegments must be changed from: - -```yaml -- [0x42100, c, energy_orb_wave] -- [0x42200, data, energy_orb_wave] # extract data at this ROM address as energy_orb_wave.data.s -- [0x42300, rodata, energy_orb_wave] # extract rodata at this ROM address as energy_orb_wave.rodata.s -``` - -to: - -```yaml -- [0x42100, c, energy_orb_wave] -- [0x42200, .data, energy_orb_wave] # take the .data section from the compiled c file named energy_orb_wave -- [0x42300, .rodata, energy_orb_wave] # take the .rodata section from the compiled c file named energy_orb_wave -``` - - -**NOTE:** -If using `auto_all_section` and there are no other `data`/`.data`/`rodata`/`.rodata` in the subsegments in the code segment, the subsegments can also be changed to - -```yaml -- [0x42100, c, energy_orb_wave] -- [0x42200] -``` - -# 5 Decompile bss - -`bss` works in a similar way to data/rodata. However, `bss` is usually discarded from the final binary, which makes it somewhat tricker to migrate. - -The `bss` segment will create assembly files that are full of `space`. The `.bss` segment will link the `.bss` section of the referenced `c` file. - -# 6 Done! - -`.text`, `.data`, `.rodata` and `.bss` are linked from the .o built from `energy_orb_wave.c` which now has everything to match when building - -The assembly files (functions .s, data.s and rodata.s files) can be deleted diff --git a/tools/splat/docs/Home.md b/tools/splat/docs/Home.md deleted file mode 100644 index 33307acb3a..0000000000 --- a/tools/splat/docs/Home.md +++ /dev/null @@ -1,25 +0,0 @@ -### What is splat? - -**splat** is a binary splitting tool, written in Python. Its goal is to support the successful disassembly and then rebuilding of binary data. - -It is the spiritual successor to [n64split](https://github.com/queueRAM/sm64tools/blob/master/n64split.c), originally written to handle N64 ROMs, it now has limited support for PSX and PS2 binaries. - -MIPS code disassembly is handled via [spimdisasm](https://github.com/Decompollaborate/spimdisasm/). - -There are a number of asset types built-in (e.g. various image formats, N64 Vtx data, etc), and it is designed to be simple to extend by writing your own custom types that can do anything you want as part of the **splat** pipeline. - - -### How does it work? - -**splat** takes a [yaml](https://en.wikipedia.org/wiki/YAML) configuration file which tells it *where* and *how* to split a given file. Symbols can be mapped to addresses (and their types provided) via an optional "symbol_addrs" file. - -**splat** runs two distinct phases: scan and split. - -The _scan_ phase makes a first pass over the data and performs the initial disassembly of code and data. During the _split_ phase, information gathered during the _scan_ phase is used and files & data are written out to disk. - -After scanning and splitting, **splat** will output a linker script that can be used as part of re-building the input file. - - -### Sounds great, how do I get started? - -Have a look at the [Quickstart](https://github.com/ethteck/splat/wiki/Quickstart), or check out the [Examples](https://github.com/ethteck/splat/wiki/Examples) page to see projects that are using **splat**. diff --git a/tools/splat/docs/Quickstart.md b/tools/splat/docs/Quickstart.md deleted file mode 100644 index f8711446e1..0000000000 --- a/tools/splat/docs/Quickstart.md +++ /dev/null @@ -1,153 +0,0 @@ -> **Note**: This quickstart is written with N64 ROMs in mind, and the assumption that you are using Ubuntu 20.04 either natively, via WSL2 or via Docker. - -For the purposes of this quickstart, we will assume that we are going to split a game called `mygame` and we have the ROM in `.z64` format named `baserom.z64`. - -Create a directory for `~/mygame` and `cd` into it: - -```sh -mkdir -p ${HOME}/mygame && cd ${HOME}/mygame -``` - -Copy the `baserom.z64` file into the `mygame` directory inside your home directory. - -### System packages - -#### Python 3.8 - -Ensure you are have **Python 3.8** or higher installed: - -```sh -$ python3 --version -Python 3.8.10 -``` - -If you get `bash: python3: command not found` install it with the following command: - -```sh -sudo apt-get update && sudo apt-get install -y python3 python3-pip -``` - -#### Git - -Ensure you have **git**: - -```sh -$ git --version -``` - -If you get `bash: git: command not found`, install it with the following command: - -```sh -sudo apt-get update && sudo apt-get install -y git -``` - -## Checkout the repository - -We will clone **splat** into a `tools` directory to keep things organised: - -```sh -git clone https://github.com/ethteck/splat.git tools/splat -``` - -## Python packages - -Run the following to install the prerequisite Python packages: - -```sh -python3 -m pip install -r ./tools/splat/requirements.txt -``` - -## Create a config file for your baserom - -**splat** has a script that will generate a `yaml` file for your ROM. - -```sh -python3 tools/splat/create_config.py baserom.z64 -``` - -The `yaml` file generated will be named based upon the name of the ROM (taken from its header). The example below is for Super Mario 64: - -```yaml -$ cat supermario64.yaml -name: Super Mario 64 (North America) -sha1: 9bef1128717f958171a4afac3ed78ee2bb4e86ce -options: - basename: supermario64 - target_path: baserom.z64 - base_path: . - compiler: IDO - find_file_boundaries: True - # platform: n64 - # undefined_funcs_auto_path: undefined_funcs_auto.txt - # undefined_syms_auto_path: undefined_syms_auto.txt - # symbol_addrs_path: symbol_addrs.txt - # undefined_syms_path: undefined_syms.txt - # asm_path: asm - # src_path: src - # build_path: build - # extensions_path: tools/splat_ext - # auto_all_sections: True -segments: - - name: header - type: header - start: 0x0 - - name: boot - type: bin - start: 0x40 - - name: main - type: code - start: 0x1000 - vram: 0x80246000 - subsegments: - - [0x1000, asm] - - type: bin - start: 0xE6430 - - [0x800000] -``` - -This is a bare-bones configuration and there is a lot of work required to map out the different sections of the ROM. - -## Run splat with your configuration - -```sh -python3 tools/splat/split.py supermario64.yaml -``` - -The output will look something like this: -``` -splat 0.7.10.1 -Loading and processing symbols -Starting scan -..Segment 1000, function at vram 80246DF8 ends with extra nops, indicating a likely file split. -File split suggestions for this segment will follow in config yaml format: - - [0x1E70, asm] - - [0x3C40, asm] - - [0x45E0, asm] - - [0x6FF0, asm] -# < -- snip --> - - [0xE6060, asm] - - [0xE61F0, asm] - - [0xE6200, asm] - - [0xE6260, asm] -.. -Starting split -.... -Split 943 KB (11.24%) in defined segments - header: 64 B (0.00%) 1 split, 0 cached - bin: 4 KB (0.05%) 1 split, 0 cached - code: 939 KB (11.19%) 1 split, 0 cached - unknown: 7 MB (88.76%) from unknown bin files -``` - -Notice that **splat** has found some potential file splits (function start/end with 16 byte alignment padded with nops). - -It's up to you to figure out the layout of the ROM. - - -## Next Steps - -The reassembly of the ROM is currently out of scope of this quickstart, as is switching out the `asm` segments for `c`. - -You can find a general workflow for using `splat` at [General Workflow](https://github.com/ethteck/splat/wiki/General-Workflow) - -Please feel free to improve this guide! diff --git a/tools/splat/docs/Segments.md b/tools/splat/docs/Segments.md deleted file mode 100644 index c5346e3e32..0000000000 --- a/tools/splat/docs/Segments.md +++ /dev/null @@ -1,312 +0,0 @@ -# Segments - -The configuration file for **splat** consists of a number of well-defined segments. - -Most segments can be defined as a either a dictionary or a list, however the list syntax is only suitable for simple cases as it does not allow for specifying many of the options a segment type has to offer. - -Splat segments' behavior generally falls under two categories: extraction and linking. Some segments will only do extraction, some will only do linking, some both, and some neither. Generally, segments will describe both extraction and linking behavior. Additionally, a segment type whose name starts with a dot (.) will only focus on linking. - -## `asm` - -**Description:** - -Segments designated Assembly, `asm`, will be disassembled via [spimdisasm](https://github.com/Decompollaborate/spimdisasm) and enriched with Symbols based on the contents of the `symbol_addrs` configuration. - -**Example:** - -```yaml -# as list -- [0xABC, asm, filepath1] -- [0xABC, asm, dir1/filepath2] # this will create filepath2.s inside a directory named dir1 - -# as dictionary -- name: filepath - type: asm - start: 0xABC -``` - -### `hasm` - -**Description:** - -Hand-written Assembly, `hasm`, similar to `asm` except it will not overwrite any existing files. Useful when assembly has been manually edited. - -**Example:** - -```yaml -# as list -- [0xABC, hasm, filepath] - -# as dictionary -- name: filepath - type: hasm - start: 0xABC -``` - -## `bin` - -**Description:** - -The `bin`(ary) segment type is for raw data, or data where the type is yet to be determined, data will be written out as raw `.bin` files. - -**Example:** - -```yaml -# as list -- [0xABC, bin, filepath] - -# as dictionary -- name: filepath - type: bin - start: 0xABC -``` - -## `code` - -**Description:** - -The 'code' segment type, `code` is a group that can have many `subsegments`. Useful to group sections of code together (e.g. all files part of the same overlay). - -**Example:** - -```yaml -# must be a dictionary -- name: main - type: code - start: 0x00001000 - vram: 0x80125900 - subsegments: - - [0x1000, asm, entrypoint] - - [0x1050, c, main] -``` - -## `c` - -**Description:** - -The C code segments have two behaviors: - -- If the target `.c` file does not exist, a new file will be generated with macros to include the original assembly (macros differ for IDO vs GCC compiler). -- Otherwise the target `.c` file is scanned to determine what assembly needs to be extracted from the ROM. - -Assembly that is extracted due to a `c` segment will be written to a `nonmatchings` folder, with one function per file. - -**Example:** - -```yaml -# as list -- [0xABC, c, filepath] - -# as dictionary -- name: filepath - type: c - start: 0xABC -``` - -## `header` - -**Description:** - -This is platform specific; parses the data and interprets as a header for e.g. N64 or PS1 elf. - -**Example:** - -```yaml -# as list -- [0xABC, header, filepath] - -# as dictionary -- name: filepath - type: header - start: 0xABC -``` - -## `data` - -**Description:** - -Data located in the ROM. Extracted as assembly; integer, float and string types will be attempted to be inferred by the disassembler. - -**Example:** - -```yaml -# as list -- [0xABC, data, filepath] - -# as dictionary -- name: filepath - type: data - start: 0xABC -``` - -This will created `filepath.data.s` within the `asm` folder. - -## `.data` - -**Description:** - -Data located in the ROM that is linked from a C file. Use the `.data` segment to tell the linker to pull the `.data` section from the compiled object of corresponding `c` segment. - -**Example:** - -```yaml -# as list -- [0xABC, .data, filepath] - -# as dictionary -- name: filepath - type: .data - start: 0xABC -``` - -**NOTE:** `splat` will not generate any `.data.s` files for these `.` (dot) sections. - -## `rodata` - -**Description:** - -Read-only data located in the ROM, e.g. floats, strings and jump tables. Extracted as assembly; integer, float and string types will be attempted to be inferred by the disassembler. - -**Example:** - -```yaml -# as list -- [0xABC, rodata, filepath] - -# as dictionary -- name: filepath - type: rodata - start: 0xABC -``` - -This will created `filepath.rodata.s` within the `asm` folder. - -## `.rodata` - -**Description:** - -Read-only data located in the ROM, linked to a C file. Use the `.rodata` segment to tell the linker to pull the `.rodata` section from the compiled object of corresponding `c` segment. - -**Example:** - -```yaml -# as list -- [0xABC, .rodata, filepath] - -# as dictionary -- name: filepath - type: .rodata - start: 0xABC -``` - -**NOTE:** `splat` will not generate any `.rodata.s` files for these `.` (dot) sections. - -## `bss` - -**Description:** - -`bss` is where variables are placed that have been declared but are not given an initial value. These sections are usually discarded from the final binary (although PSX binaries seem to include them!). - -Note that the `bss_size` option needs to be set at segment level for `bss` segments to work correctly. - -**Example:** - -```yaml -- { start: 0x7D1AD0, type: bss, name: filepath, vram: 0x803C0420 } -``` - -## `.bss` - -**Description:** - -Links the `.bss` section of the associated `c` file. - -**Example:** - -```yaml -- { start: 0x7D1AD0, type: .bss, name: filepath, vram: 0x803C0420 } -``` - -## Images - -**Description:** - -**splat** supports most of the [N64 image formats](https://n64squid.com/homebrew/n64-sdk/textures/image-formats/): - -- `i`, i.e. `i4` and `i8` -- `ia`, i.e. `ia4`, `ia8`, and `ia16` -- `ci`, i.e. `ci4` and `ci8` -- `rgb`, i.e. `rgba32` and `rgba16` - -These segments will parse the image data and dump out a `png` file. - -**Note:** Using the dictionary syntax allows for richer configuration. - -**Example:** - -```yaml -# as list -- [0xABC, i4, filename, width, height] -# as a dictionary -- name: filename - type: i4 - start: 0xABC - width: 64 - height: 64 - flip_x: yes - flip_y: no -``` - -## General segment options - -All splat's segments can be passed extra options for finer configuration. Note that those extra options require to rewrite the entry using the dictionary yaml notation instead of the list one. - -### `linker_section_order` - -**Description:** - -Allows overriding the section order used for linker script generation. - -Useful when a section of a file is not between the other sections of the same type in the ROM, for example a file having its data section between other files's rodata. - -Take in mind this option may need the [`check_consecutive_segment_types`](Configuration.md#check_consecutive_segment_types) yaml option to be turned off. - -**Example:** - -```yaml -- [0x400, data, file1] -# data ends - -# rodata starts -- [0x800, rodata, file2] -- { start: 0xA00, type: data, name: file3, linker_section_order: .rodata } -- [0xC00, rodata, file4] -``` - -This will created `file3.data.s` within the `asm` folder, but won't be reordered in the generated linker script to be placed on the data section. - -### `linker_section` - -**Description:** - -Allows to override the `.section` directive that will be used when generating the disassembly of the corresponding section, without needing to write an extension segment. This also affects the section name that will be used during link time. - -Useful for sections with special names, like an executable section named `.start` - -**Example:** - -```yaml -- { start: 0x1000, type: asm, name: snmain, linker_section: .start } -- [0x1070, rdata, libc] -- [0x10A0, rdata, main_030] -``` - -### `ld_fill_value` - -Allows to specify the value of the `FILL` statement generated for this specific top-level segment of the linker script, ignoring the global configuration. - -It must be either an integer, which will be used as the parameter for the `FILL` statement, or `null`, which tells splat to not emit a `FILL` statement for this segment. - -If not set, then the global configuration is used. See [ld_fill_value](Configuration.md#ld_fill_value) on the Configuration section. - -Defaults to the value of the global option. diff --git a/tools/splat/docs/VramClasses.md b/tools/splat/docs/VramClasses.md deleted file mode 100644 index c889919c85..0000000000 --- a/tools/splat/docs/VramClasses.md +++ /dev/null @@ -1,59 +0,0 @@ -# vram classes - -Version 0.19.0 introduced `vram_classes`, a new top-level yaml section that can be used to help reduce duplicated data in your yaml and more clearly organize its memory layout. - -## Introduction -Before vram classes, you might have had something like this in your yaml: - -```yaml -- type: code - start: 0x4269D0 - vram: 0x802A9000 - vram_symbol: battle_move_end - subsegments: ... -- type: code - start: 0x4273B0 - vram: 0x802A9000 # notice same `vram` and `vram_symbol` for both segments - vram_symbol: battle_move_end - subsegments: ... -``` - -Having to duplicate the vram address and vram_symbol properties for potentially dozens of hundreds of overlay segments is tedious and pollutes your yaml with repeated information that can become out of sync. Enter vram_classes! - -```yaml -- type: code - start: 0x4269D0 - vram_class: maps - subsegments: ... -- type: code - start: 0x4273B0 - vram_class: maps - subsegments: ... -``` - -Here, we are telling splat that both of these segments use the `maps` vram class. We are now effectively pointing both segments to the same source of information. Now let's look at how vram classes are defined: - -## Format - -```yaml -options: - ... - ld_use_symbolic_vram_addresses: True - ... -vram_classes: - - { name: maps, vram: 0x802A9000, vram_symbol: battle_move_end } -``` - -`vram_classes` is a top-level yaml section that contains a list of vram classes. You can either define them in dict form (as seen above) or list form. However, for list form, only `name` and `vram` are supported (`[maps, 0x802A9000]`). If you want to specify other options, please use the dict form. The fields supported are as follows: - -- `name` (required): The name of the class - -- `vram` (required): The vram address to be used during disasembly. If `ld_use_symbolic_vram_addresses` is disabled or no `vram_symbol` or `follows_classes` properties are provided, this address will be used in the linker script. - -The following properties are optional and only take effect if `ld_use_symbolic_vram_addresses` is enabled: - -- `vram_symbol`: The name of the symbol to use in the linker script for this class. - -- `follows_classes`: A list of vram class names that this class must come after in memory. If we added `follows_classes: [apples, bananas]` to our above vram_class, this would make all `maps` segments start at the end of all `apples` and `bananas` segments. - -The internal linker script symbol name that is chosen for `follows_classes` is the name of the class followed by `_CLASS_VRAM`. You can override this by also specifying `vram_symbol`. \ No newline at end of file diff --git a/tools/splat/mypy.ini b/tools/splat/mypy.ini deleted file mode 100644 index 6e7c2ef479..0000000000 --- a/tools/splat/mypy.ini +++ /dev/null @@ -1,4 +0,0 @@ -[mypy] -ignore_missing_imports = True -check_untyped_defs = True -mypy_path = stubs diff --git a/tools/splat/platforms/gc.py b/tools/splat/platforms/gc.py deleted file mode 100644 index af08a283c7..0000000000 --- a/tools/splat/platforms/gc.py +++ /dev/null @@ -1,5 +0,0 @@ -from util.gc import gcfst - - -def init(target_bytes: bytes): - gcfst.split_iso(target_bytes) diff --git a/tools/splat/platforms/n64.py b/tools/splat/platforms/n64.py deleted file mode 100644 index 5836b66617..0000000000 --- a/tools/splat/platforms/n64.py +++ /dev/null @@ -1,12 +0,0 @@ -from util import options, symbols - - -def init(target_bytes: bytes): - symbols.spim_context.fillDefaultBannedSymbols() - - if options.opts.libultra_symbols: - symbols.spim_context.globalSegment.fillLibultraSymbols() - if options.opts.ique_symbols: - symbols.spim_context.globalSegment.fillIQueSymbols() - if options.opts.hardware_regs: - symbols.spim_context.globalSegment.fillHardwareRegs(True) diff --git a/tools/splat/platforms/ps2.py b/tools/splat/platforms/ps2.py deleted file mode 100644 index 09da192fac..0000000000 --- a/tools/splat/platforms/ps2.py +++ /dev/null @@ -1,2 +0,0 @@ -def init(target_bytes: bytes): - pass diff --git a/tools/splat/platforms/psx.py b/tools/splat/platforms/psx.py deleted file mode 100644 index 09da192fac..0000000000 --- a/tools/splat/platforms/psx.py +++ /dev/null @@ -1,2 +0,0 @@ -def init(target_bytes: bytes): - pass diff --git a/tools/splat/requirements.txt b/tools/splat/requirements.txt deleted file mode 100644 index 66374ab243..0000000000 --- a/tools/splat/requirements.txt +++ /dev/null @@ -1,10 +0,0 @@ -PyYAML -pylibyaml -tqdm -intervaltree -colorama -# This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py -spimdisasm>=1.18.0 -rabbitizer>=1.7.0 -pygfxd -n64img>=0.1.4 diff --git a/tools/splat/run_tests.sh b/tools/splat/run_tests.sh deleted file mode 100644 index e57f8b3df9..0000000000 --- a/tools/splat/run_tests.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/bin/bash - -# docker build container -docker build --tag splat-build:latest . && \ -# get compilers and tools -# clean -cd test/basic_app && sh clean.sh && cd ../../ && \ -# build -docker run --rm -v $(pwd):/splat -w /splat/test/basic_app splat-build sh build.sh && \ -# test -python3 test.py diff --git a/tools/splat/segtypes/__init__.py b/tools/splat/segtypes/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/splat/segtypes/address_range.py b/tools/splat/segtypes/address_range.py deleted file mode 100644 index 0c57cc0d94..0000000000 --- a/tools/splat/segtypes/address_range.py +++ /dev/null @@ -1,10 +0,0 @@ -from dataclasses import dataclass - - -@dataclass -class AddressRange: - start: int - end: int - - def contains(self, addr: int) -> bool: - return self.start <= addr < self.end diff --git a/tools/splat/segtypes/common/__init__.py b/tools/splat/segtypes/common/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/splat/segtypes/common/asm.py b/tools/splat/segtypes/common/asm.py deleted file mode 100644 index db7a184042..0000000000 --- a/tools/splat/segtypes/common/asm.py +++ /dev/null @@ -1,39 +0,0 @@ -from pathlib import Path -from typing import Optional - -from util import options - -from segtypes.common.codesubsegment import CommonSegCodeSubsegment - - -class CommonSegAsm(CommonSegCodeSubsegment): - @staticmethod - def is_text() -> bool: - return True - - def out_path(self) -> Optional[Path]: - return options.opts.asm_path / self.dir / f"{self.name}.s" - - def scan(self, rom_bytes: bytes): - if ( - self.rom_start is not None - and self.rom_end is not None - and self.rom_start != self.rom_end - ): - self.scan_code(rom_bytes) - - def get_file_header(self): - return [] - - def split(self, rom_bytes: bytes): - if not self.rom_start == self.rom_end and self.spim_section is not None: - out_path = self.out_path() - if out_path: - out_path.parent.mkdir(parents=True, exist_ok=True) - - self.print_file_boundaries() - - with open(out_path, "w", newline="\n") as f: - for line in self.get_file_header(): - f.write(line + "\n") - f.write(self.spim_section.disassemble()) diff --git a/tools/splat/segtypes/common/bin.py b/tools/splat/segtypes/common/bin.py deleted file mode 100644 index b6b64a575f..0000000000 --- a/tools/splat/segtypes/common/bin.py +++ /dev/null @@ -1,32 +0,0 @@ -from pathlib import Path -from typing import Optional - -from util import log, options - -from segtypes.common.segment import CommonSegment - - -class CommonSegBin(CommonSegment): - @staticmethod - def is_data() -> bool: - return True - - def out_path(self) -> Optional[Path]: - return options.opts.asset_path / self.dir / f"{self.name}.bin" - - def split(self, rom_bytes): - path = self.out_path() - assert path is not None - path.parent.mkdir(parents=True, exist_ok=True) - - if self.rom_end is None: - log.error( - f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it" - ) - - with open(path, "wb") as f: - assert isinstance(self.rom_start, int) - assert isinstance(self.rom_end, int) - - f.write(rom_bytes[self.rom_start : self.rom_end]) - self.log(f"Wrote {self.name} to {path}") diff --git a/tools/splat/segtypes/common/bss.py b/tools/splat/segtypes/common/bss.py deleted file mode 100644 index 4bc74d1fc8..0000000000 --- a/tools/splat/segtypes/common/bss.py +++ /dev/null @@ -1,62 +0,0 @@ -from util import options, symbols, log - -from segtypes.common.data import CommonSegData - -from disassembler_section import make_bss_section - - -class CommonSegBss(CommonSegData): - def get_linker_section(self) -> str: - return ".bss" - - @staticmethod - def is_noload() -> bool: - return True - - def disassemble_data(self, rom_bytes: bytes): - if not isinstance(self.rom_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'" - ) - - # Supposedly logic error, not user error - assert isinstance(self.rom_end, int), f"{self.name} {self.rom_end}" - - # Supposedly logic error, not user error - segment_rom_start = self.get_most_parent().rom_start - assert isinstance(segment_rom_start, int), f"{self.name} {segment_rom_start}" - - if not isinstance(self.vram_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" - ) - - next_subsegment = self.parent.get_next_subsegment_for_ram(self.vram_start) - if next_subsegment is None: - bss_end = self.get_most_parent().vram_end - else: - bss_end = next_subsegment.vram_start - assert isinstance(bss_end, int), f"{self.name} {bss_end}" - - self.spim_section = make_bss_section( - self.rom_start, - self.rom_end, - self.vram_start, - bss_end, - self.name, - segment_rom_start, - self.get_exclusive_ram_id(), - ) - - assert self.spim_section is not None - - self.spim_section.analyze() - self.spim_section.set_comment_offset(self.rom_start) - - for spim_sym in self.spim_section.get_section().symbolList: - symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), spim_sym.contextSym - ) - - def should_scan(self) -> bool: - return options.opts.is_mode_active("code") and self.vram_start is not None diff --git a/tools/splat/segtypes/common/c.py b/tools/splat/segtypes/common/c.py deleted file mode 100644 index 6662fb7cda..0000000000 --- a/tools/splat/segtypes/common/c.py +++ /dev/null @@ -1,439 +0,0 @@ -import os -import re -from pathlib import Path -from typing import Optional, Set, List - -import spimdisasm - -from util import log, options, symbols -from util.compiler import GCC, SN64, IDO -from util.symbols import Symbol - -from segtypes.common.codesubsegment import CommonSegCodeSubsegment -from segtypes.common.rodata import CommonSegRodata - - -STRIP_C_COMMENTS_RE = re.compile( - r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', - re.DOTALL | re.MULTILINE, -) - -C_FUNC_RE = re.compile( - r"^(?:static\s+)?[^\s]+\s+([^\s(]+)\(([^;)]*)\)[^;]+?{", re.MULTILINE -) - -C_GLOBAL_ASM_IDO_RE = re.compile(r"GLOBAL_ASM\(\"(\w+\/)*(\w+)\.s\"\)", re.MULTILINE) - - -class CommonSegC(CommonSegCodeSubsegment): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.defined_funcs: Set[str] = set() - self.global_asm_funcs: Set[str] = set() - self.global_asm_rodata_syms: Set[str] = set() - - self.file_extension = "c" - - @staticmethod - def strip_c_comments(text): - def replacer(match): - s = match.group(0) - if s.startswith("/"): - return " " - else: - return s - - return re.sub(STRIP_C_COMMENTS_RE, replacer, text) - - @staticmethod - def get_funcs_defined_in_c(c_file: Path) -> Set[str]: - with open(c_file, "r") as f: - text = CommonSegC.strip_c_comments(f.read()) - - return set(m.group(1) for m in C_FUNC_RE.finditer(text)) - - @staticmethod - def find_all_instances(string: str, sub: str): - start = 0 - while True: - start = string.find(sub, start) - if start == -1: - return - yield start - start += len(sub) - - @staticmethod - def get_close_parenthesis(string: str, pos: int): - paren_count = 0 - while True: - cur_char = string[pos] - if cur_char == "(": - paren_count += 1 - elif cur_char == ")": - if paren_count == 0: - return pos + 1 - else: - paren_count -= 1 - pos += 1 - - @staticmethod - def find_include_macro(text: str, macro_name: str): - for pos in CommonSegC.find_all_instances(text, f"{macro_name}("): - close_paren_pos = CommonSegC.get_close_parenthesis( - text, pos + len(f"{macro_name}(") - ) - macro_contents = text[pos:close_paren_pos] - macro_args = macro_contents.split(",") - if options.opts.use_legacy_include_asm: - if len(macro_args) >= 3: - yield macro_args[2].strip(" )") - else: - if len(macro_args) >= 2: - yield macro_args[1].strip(" )") - - @staticmethod - def find_include_asm(text: str): - return CommonSegC.find_include_macro(text, "INCLUDE_ASM") - - @staticmethod - def find_include_rodata(text: str): - return CommonSegC.find_include_macro(text, "INCLUDE_RODATA") - - @staticmethod - def get_global_asm_funcs(c_file: Path) -> Set[str]: - with c_file.open() as f: - text = CommonSegC.strip_c_comments(f.read()) - if options.opts.compiler in [GCC, SN64]: - return set(CommonSegC.find_include_asm(text)) - else: - return set(m.group(2) for m in C_GLOBAL_ASM_IDO_RE.finditer(text)) - - @staticmethod - def get_global_asm_rodata_syms(c_file: Path) -> Set[str]: - with c_file.open() as f: - text = CommonSegC.strip_c_comments(f.read()) - if options.opts.compiler in [GCC, SN64]: - return set(CommonSegC.find_include_rodata(text)) - else: - return set(m.group(2) for m in C_GLOBAL_ASM_IDO_RE.finditer(text)) - - @staticmethod - def is_text() -> bool: - return True - - def out_path(self) -> Optional[Path]: - return options.opts.src_path / self.dir / f"{self.name}.{self.file_extension}" - - def scan(self, rom_bytes: bytes): - if ( - self.rom_start is not None - and self.rom_end is not None - and self.rom_start != self.rom_end - ): - path = self.out_path() - if path: - if options.opts.do_c_func_detection and os.path.exists(path): - # TODO run cpp? - self.defined_funcs = self.get_funcs_defined_in_c(path) - self.global_asm_funcs = self.get_global_asm_funcs(path) - self.global_asm_rodata_syms = self.get_global_asm_rodata_syms(path) - symbols.to_mark_as_defined.update(self.defined_funcs) - symbols.to_mark_as_defined.update(self.global_asm_funcs) - symbols.to_mark_as_defined.update(self.global_asm_rodata_syms) - - self.scan_code(rom_bytes) - - def split(self, rom_bytes: bytes): - if self.rom_start != self.rom_end: - asm_out_dir = options.opts.nonmatchings_path / self.dir - asm_out_dir.mkdir(parents=True, exist_ok=True) - - self.print_file_boundaries() - - assert self.spim_section is not None and isinstance( - self.spim_section.get_section(), spimdisasm.mips.sections.SectionText - ), f"{self.name}, rom_start:{self.rom_start}, rom_end:{self.rom_end}" - - rodata_section_type = "" - rodata_spim_segment: Optional[spimdisasm.mips.sections.SectionRodata] = None - if ( - options.opts.migrate_rodata_to_functions - and self.rodata_sibling is not None - ): - assert isinstance( - self.rodata_sibling, CommonSegRodata - ), self.rodata_sibling.type - rodata_section_type = ( - self.rodata_sibling.get_linker_section_linksection() - ) - if self.rodata_sibling.spim_section is not None: - assert isinstance( - self.rodata_sibling.spim_section.get_section(), - spimdisasm.mips.sections.SectionRodata, - ) - rodata_spim_segment = self.rodata_sibling.spim_section.get_section() - - # Precompute function-rodata pairings - symbols_entries = ( - spimdisasm.mips.FunctionRodataEntry.getAllEntriesFromSections( - self.spim_section.get_section(), rodata_spim_segment - ) - ) - - is_new_c_file = False - - # Check and create the C file - c_path = self.out_path() - if c_path: - if not c_path.exists() and options.opts.create_c_files: - self.create_c_file(asm_out_dir, c_path, symbols_entries) - is_new_c_file = True - - self.create_asm_dependencies_file( - c_path, asm_out_dir, is_new_c_file, symbols_entries - ) - - # Produce the asm files for functions - for entry in symbols_entries: - entry.sectionText = self.get_linker_section_linksection() - entry.sectionRodata = rodata_section_type - if entry.function is not None: - if ( - entry.function.getName() in self.global_asm_funcs - or is_new_c_file - or options.opts.disassemble_all - ): - func_sym = self.get_symbol( - entry.function.vram, - in_segment=True, - type="func", - local_only=True, - ) - assert func_sym is not None - - self.create_c_asm_file(entry, asm_out_dir, func_sym) - else: - for spim_rodata_sym in entry.rodataSyms: - if ( - spim_rodata_sym.getName() in self.global_asm_rodata_syms - or is_new_c_file - or options.opts.disassemble_all - ): - rodata_sym = self.get_symbol( - spim_rodata_sym.vram, in_segment=True, local_only=True - ) - assert rodata_sym is not None - - self.create_unmigrated_rodata_file( - spim_rodata_sym, asm_out_dir, rodata_sym - ) - - def get_c_preamble(self): - ret = [] - - preamble = options.opts.generated_c_preamble - ret.append(preamble) - ret.append("") - - return ret - - def check_gaps_in_migrated_rodata( - self, - func: spimdisasm.mips.symbols.SymbolFunction, - rodata_list: List[spimdisasm.mips.symbols.SymbolBase], - ): - for index in range(len(rodata_list) - 1): - rodata_sym = rodata_list[index] - next_rodata_sym = rodata_list[index + 1] - - if rodata_sym.vramEnd != next_rodata_sym.vram: - log.write( - f"\nA gap was detected in migrated rodata symbols!", status="warn" - ) - log.write( - f"\t In function '{func.getName()}' (0x{func.vram:08X}), gap detected between '{rodata_sym.getName()}' (0x{rodata_sym.vram:08X}) and '{next_rodata_sym.getName()}' (0x{next_rodata_sym.vram:08X})" - ) - log.write( - f"\t The address of the missing rodata symbol is 0x{rodata_sym.vramEnd:08X}" - ) - log.write( - f"\t Try to force the migration of that symbol with `force_migration:True` in the symbol_addrs.txt file; or avoid the migration of symbols around this address with `force_not_migration:True`" - ) - - def create_c_asm_file( - self, - func_rodata_entry: spimdisasm.mips.FunctionRodataEntry, - out_dir: Path, - func_sym: Symbol, - ): - outpath = out_dir / self.name / (func_sym.name + ".s") - - # Skip extraction if the file exists and the symbol is marked as extract=false - if outpath.exists() and not func_sym.extract: - return - - outpath.parent.mkdir(parents=True, exist_ok=True) - - with outpath.open("w", newline="\n") as f: - if options.opts.asm_inc_header: - f.write( - options.opts.c_newline.join(options.opts.asm_inc_header.split("\n")) - ) - - func_rodata_entry.writeToFile(f) - - if func_rodata_entry.function is not None: - self.check_gaps_in_migrated_rodata( - func_rodata_entry.function, func_rodata_entry.rodataSyms - ) - self.check_gaps_in_migrated_rodata( - func_rodata_entry.function, func_rodata_entry.lateRodataSyms - ) - - self.log(f"Disassembled {func_sym.name} to {outpath}") - - def create_unmigrated_rodata_file( - self, - spim_rodata_sym: spimdisasm.mips.symbols.SymbolBase, - out_dir: Path, - rodata_sym: Symbol, - ): - outpath = out_dir / self.name / (rodata_sym.name + ".s") - - # Skip extraction if the file exists and the symbol is marked as extract=false - if outpath.exists() and not rodata_sym.extract: - return - - outpath.parent.mkdir(parents=True, exist_ok=True) - - with outpath.open("w", newline="\n") as f: - if options.opts.include_macro_inc: - f.write('.include "macro.inc"\n\n') - preamble = options.opts.generated_s_preamble - if preamble: - f.write(preamble + "\n") - assert rodata_sym.linker_section is not None, rodata_sym.name - f.write(f".section {rodata_sym.linker_section}\n\n") - f.write(spim_rodata_sym.disassemble()) - - self.log(f"Disassembled {rodata_sym.name} to {outpath}") - - def get_c_line_include_macro( - self, - spim_sym: spimdisasm.mips.symbols.SymbolBase, - asm_out_dir: Path, - macro_name: str, - ) -> str: - if options.opts.compiler == IDO: - # IDO uses the asm processor to embeed assembly, and it doesn't require a special directive to include symbols - asm_outpath = Path( - os.path.join(asm_out_dir, self.name, spim_sym.getName() + ".s") - ) - rel_asm_outpath = os.path.relpath(asm_outpath, options.opts.base_path) - return f'#pragma GLOBAL_ASM("{rel_asm_outpath}")' - - if options.opts.use_legacy_include_asm: - rel_asm_out_dir = asm_out_dir.relative_to(options.opts.nonmatchings_path) - return f'{macro_name}(const s32, "{rel_asm_out_dir / self.name}", {spim_sym.getName()});' - - return f'{macro_name}("{asm_out_dir / self.name}", {spim_sym.getName()});' - - def get_c_lines_for_function( - self, func: spimdisasm.mips.symbols.SymbolFunction, asm_out_dir: Path - ) -> List[str]: - c_lines = [] - - # Terrible hack to "auto-decompile" empty functions - if ( - options.opts.auto_decompile_empty_functions - and len(func.instructions) == 2 - and func.instructions[0].isReturn() - and func.instructions[1].isNop() - ): - c_lines.append("void " + func.getName() + "(void) {") - c_lines.append("}") - else: - c_lines.append( - self.get_c_line_include_macro(func, asm_out_dir, "INCLUDE_ASM") - ) - c_lines.append("") - return c_lines - - def get_c_lines_for_rodata_sym( - self, rodata_sym: spimdisasm.mips.symbols.SymbolBase, asm_out_dir: Path - ): - c_lines = [ - self.get_c_line_include_macro(rodata_sym, asm_out_dir, "INCLUDE_RODATA") - ] - c_lines.append("") - return c_lines - - def create_c_file( - self, - asm_out_dir: Path, - c_path: Path, - symbols_entries: List[spimdisasm.mips.FunctionRodataEntry], - ): - c_lines = self.get_c_preamble() - - for entry in symbols_entries: - if entry.function is not None: - c_lines += self.get_c_lines_for_function(entry.function, asm_out_dir) - else: - for rodata_sym in entry.rodataSyms: - c_lines += self.get_c_lines_for_rodata_sym(rodata_sym, asm_out_dir) - - c_path.parent.mkdir(parents=True, exist_ok=True) - with c_path.open("w") as f: - f.write("\n".join(c_lines)) - log.write(f"Wrote {self.name} to {c_path}") - - def create_asm_dependencies_file( - self, - c_path: Path, - asm_out_dir: Path, - is_new_c_file: bool, - symbols_entries: List[spimdisasm.mips.FunctionRodataEntry], - ): - if not options.opts.create_asm_dependencies: - return - if ( - len(self.global_asm_funcs) + len(self.global_asm_rodata_syms) - ) == 0 and not is_new_c_file: - return - - assert self.spim_section is not None - - build_path = options.opts.build_path - - dep_path = build_path / c_path.with_suffix(".asmproc.d") - dep_path.parent.mkdir(parents=True, exist_ok=True) - with dep_path.open("w") as f: - if options.opts.use_o_as_suffix: - o_path = build_path / c_path.with_suffix(".o") - else: - o_path = build_path / c_path.with_suffix(c_path.suffix + ".o") - f.write(f"{o_path}:") - depend_list = [] - for entry in symbols_entries: - if entry.function is not None: - func_name = entry.function.getName() - - if func_name in self.global_asm_funcs or is_new_c_file: - outpath = asm_out_dir / self.name / (func_name + ".s") - depend_list.append(outpath) - f.write(f" \\\n {outpath}") - else: - for rodata_sym in entry.rodataSyms: - rodata_name = rodata_sym.getName() - - if rodata_name in self.global_asm_rodata_syms or is_new_c_file: - outpath = asm_out_dir / self.name / (rodata_name + ".s") - depend_list.append(outpath) - f.write(f" \\\n {outpath}") - - f.write("\n") - - for depend_file in depend_list: - f.write(f"{depend_file}:\n") diff --git a/tools/splat/segtypes/common/code.py b/tools/splat/segtypes/common/code.py deleted file mode 100644 index 3c2cf25641..0000000000 --- a/tools/splat/segtypes/common/code.py +++ /dev/null @@ -1,419 +0,0 @@ -import typing -from collections import OrderedDict -from typing import Dict, List, Optional, Tuple, Set - -from util import log, options -from util.range import Range -from util.symbols import Symbol - -from segtypes.common.group import CommonSegGroup -from segtypes.segment import Segment, parse_segment_align - - -def dotless_type(type: str) -> str: - return type[1:] if type[0] == "." else type - - -# code group -class CommonSegCode(CommonSegGroup): - def __init__( - self, - rom_start: Optional[int], - rom_end: Optional[int], - type: str, - name: str, - vram_start: Optional[int], - args: list, - yaml, - ): - self.bss_size: int = yaml.get("bss_size", 0) if isinstance(yaml, dict) else 0 - - super().__init__( - rom_start, - rom_end, - type, - name, - vram_start, - args=args, - yaml=yaml, - ) - - self.reported_file_split = False - self.jtbl_glabels_to_add: Set[int] = set() - self.jumptables: Dict[int, Tuple[int, int]] = {} - self.rodata_syms: Dict[int, List[Symbol]] = {} - - self.align = parse_segment_align(yaml) - if self.align is None: - self.align = 0x10 - - @property - def needs_symbols(self) -> bool: - return True - - @property - def vram_end(self) -> Optional[int]: - if self.vram_start is not None and self.size is not None: - return self.vram_start + self.size + self.bss_size - else: - return None - - def check_rodata_sym_impl(self, func_addr: int, sym: Symbol, rodata_section: Range): - if rodata_section.is_complete(): - assert rodata_section.start is not None - assert rodata_section.end is not None - - rodata_start: int = rodata_section.start - rodata_end: int = rodata_section.end - if rodata_start <= sym.vram_start < rodata_end: - if func_addr not in self.rodata_syms: - self.rodata_syms[func_addr] = [] - self.rodata_syms[func_addr].append(sym) - - # Prepare symbol for migration to the function - def check_rodata_sym(self, func_addr: int, sym: Symbol): - rodata_section = self.section_boundaries.get(".rodata") - if rodata_section is not None: - self.check_rodata_sym_impl(func_addr, sym, rodata_section) - rodata_section = self.section_boundaries.get(".rdata") - if rodata_section is not None: - self.check_rodata_sym_impl(func_addr, sym, rodata_section) - - def handle_alls(self, segs: List[Segment], base_segs) -> bool: - for i, elem in enumerate(segs): - if elem.type.startswith("all_"): - alls = [] - - rep_type = f"{elem.type[4:]}" - replace_class = Segment.get_class_for_type(rep_type) - - for base in base_segs.items(): - if isinstance(elem.rom_start, int) and isinstance( - self.rom_start, int - ): - # Shoddy rom to ram - assert self.vram_start is not None, self.vram_start - vram_start = elem.rom_start - self.rom_start + self.vram_start - else: - vram_start = None - rep: Segment = replace_class( - rom_start=elem.rom_start, - rom_end=elem.rom_end, - type=rep_type, - name=base[0], - vram_start=vram_start, - args=[], - yaml={}, - ) - rep.extract = False - rep.given_subalign = self.given_subalign - rep.exclusive_ram_id = self.get_exclusive_ram_id() - rep.given_dir = self.given_dir - rep.given_symbol_name_format = self.symbol_name_format - rep.given_symbol_name_format_no_rom = self.symbol_name_format_no_rom - rep.sibling = base[1] - rep.parent = self - if rep.special_vram_segment: - self.special_vram_segment = True - alls.append(rep) - - # Insert alls into segs at i - del segs[i] - segs[i:i] = alls - return True - return False - - # Find places we should automatically add "all_data" / "all_rodata" / "all_bss" - def find_inserts( - self, found_sections: typing.OrderedDict[str, Range] - ) -> "OrderedDict[str, int]": - inserts: OrderedDict[str, int] = OrderedDict() - - section_order = self.section_order.copy() - section_order.remove(".text") - - for i, section in enumerate(section_order): - if section not in options.opts.auto_all_sections: - continue - - if not found_sections[section].has_start(): - search_done = False - for j in range(i - 1, -1, -1): - end = found_sections[section_order[j]].end - if end is not None: - inserts[section] = end - search_done = True - break - if not search_done: - inserts[section] = -1 - pass - - return inserts - - def parse_subsegments(self, segment_yaml) -> List[Segment]: - if "subsegments" not in segment_yaml: - if not self.parent: - raise Exception( - f"No subsegments provided in top-level code segment {self.name}" - ) - return [] - - base_segments: OrderedDict[str, Segment] = OrderedDict() - ret = [] - prev_start: Optional[int] = -1 - prev_vram: Optional[int] = -1 - inserts: OrderedDict[ - str, int - ] = ( - OrderedDict() - ) # Used to manually add "all_" types for sections not otherwise defined in the yaml - - self.section_boundaries = OrderedDict( - (s_name, Range()) for s_name in options.opts.section_order - ) - - found_sections = OrderedDict( - (s_name, Range()) for s_name in self.section_boundaries - ) # Stores yaml index where a section was first found - found_sections.pop(".text") - - # Mark any manually added dot types - cur_section = None - - for i, subsegment_yaml in enumerate(segment_yaml["subsegments"]): - # endpos marker - if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1: - continue - - typ = Segment.parse_segment_type(subsegment_yaml) - if typ.startswith("all_"): - typ = typ[4:] - if not typ.startswith("."): - typ = f".{typ}" - - if typ in found_sections: - if cur_section is None: - # Starting point - found_sections[typ].start = i - cur_section = typ - else: - if cur_section != typ: - # We're changing sections - - if options.opts.check_consecutive_segment_types: - if found_sections[cur_section].has_end(): - log.error( - f"Section {cur_section} end encountered but was already ended earlier!" - ) - if found_sections[typ].has_start(): - log.error( - f"Section {typ} start encounted but has already started earlier!" - ) - - # End the current section - found_sections[cur_section].end = i - - # Start the next section - found_sections[typ].start = i - cur_section = typ - - if cur_section is not None: - found_sections[cur_section].end = -1 - - inserts = self.find_inserts(found_sections) - - last_rom_end = None - - for i, subsegment_yaml in enumerate(segment_yaml["subsegments"]): - # endpos marker - if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1: - continue - - typ = Segment.parse_segment_type(subsegment_yaml) - start = Segment.parse_segment_start(subsegment_yaml) - - # Add dummy segments to be expanded later - if typ.startswith("all_"): - dummy_seg = Segment( - rom_start=start, - rom_end=None, - type=typ, - name="", - vram_start=None, - args=[], - yaml={}, - ) - dummy_seg.given_subalign = self.given_subalign - dummy_seg.exclusive_ram_id = self.exclusive_ram_id - dummy_seg.given_dir = self.given_dir - dummy_seg.given_symbol_name_format = self.symbol_name_format - dummy_seg.given_symbol_name_format_no_rom = ( - self.symbol_name_format_no_rom - ) - ret.append(dummy_seg) - continue - - segment_class = Segment.get_class_for_type(typ) - - end = self.get_next_seg_start(i, segment_yaml["subsegments"]) - - if start is None: - # Attempt to infer the start address - if i == 0: - # The start address of this segment is the start address of the group - start = self.rom_start - else: - # The start address is the end address of the previous segment - start = last_rom_end - - if start is not None and end is None: - est_size = segment_class.estimate_size(subsegment_yaml) - if est_size is not None: - end = start + est_size - - if start is not None and prev_start is not None and start < prev_start: - log.error( - f"Error: Group segment '{self.name}' contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})" - ) - - vram = None - if start is not None: - assert isinstance(start, int) - vram = self.get_most_parent().rom_to_ram(start) - - if segment_class.is_noload(): - # Pretend bss's rom address is after the last actual rom segment - start = last_rom_end - # and it has a rom size of zero - end = last_rom_end - - segment: Segment = Segment.from_yaml( - segment_class, subsegment_yaml, start, end, vram - ) - - if ( - segment.vram_start is not None - and prev_vram is not None - and segment.vram_start < prev_vram - ): - log.error( - f"Error: Group segment '{self.name}' contains subsegments which are out of ascending vram order (0x{prev_vram:X} followed by 0x{segment.vram_start:X}).\n" - + f"Detected when processing file '{segment.name}' of type '{segment.type}'" - ) - - segment.sibling = base_segments.get(segment.name, None) - - if segment.sibling is not None: - if self.section_order.index(".text") < self.section_order.index( - ".rodata" - ): - if segment.is_rodata(): - segment.sibling.rodata_sibling = segment - else: - if segment.is_text() and segment.sibling.is_rodata(): - segment.rodata_sibling = segment.sibling - segment.sibling.sibling = segment - - if self.section_order.index(".text") < self.section_order.index( - ".data" - ): - if segment.is_data(): - segment.sibling.data_sibling = segment - else: - if segment.is_text() and segment.sibling.is_data(): - segment.data_sibling = segment.sibling - segment.sibling.sibling = segment - - segment.parent = self - if segment.special_vram_segment: - self.special_vram_segment = True - - for i, section in enumerate(self.section_order): - if not self.section_boundaries[section].has_start() and dotless_type( - section - ) == dotless_type(segment.type): - if i > 0: - prev_section = self.section_order[i - 1] - self.section_boundaries[prev_section].end = segment.vram_start - self.section_boundaries[section].start = segment.vram_start - - segment.bss_contains_common = self.bss_contains_common - ret.append(segment) - - if segment.is_text(): - base_segments[segment.name] = segment - - if self.section_order.index(".rodata") < self.section_order.index(".text"): - if segment.is_rodata() and segment.sibling is None: - base_segments[segment.name] = segment - - prev_start = start - prev_vram = segment.vram_start - if end is not None: - last_rom_end = end - - # Add the automatic all_ sections - orig_len = len(ret) - for section in reversed(inserts): - idx = inserts[section] - - if idx == -1: - idx = orig_len - - # bss hack TODO maybe rethink - if ( - section == "bss" - and self.vram_start is not None - and self.rom_end is not None - and self.rom_start is not None - ): - rom_start = self.rom_end - vram_start = self.vram_start + self.rom_end - self.rom_start - else: - rom_start = None - vram_start = None - - new_seg = Segment( - rom_start=rom_start, - rom_end=None, - type="all_" + section, - name="", - vram_start=vram_start, - args=[], - yaml={}, - ) - new_seg.given_subalign = self.given_subalign - new_seg.exclusive_ram_id = self.exclusive_ram_id - new_seg.given_dir = self.given_dir - new_seg.given_symbol_name_format = self.symbol_name_format - new_seg.given_symbol_name_format_no_rom = self.symbol_name_format_no_rom - ret.insert(idx, new_seg) - - check = True - while check: - check = self.handle_alls(ret, base_segments) - - # TODO why is this necessary? - rodata_section = self.section_boundaries.get( - ".rodata" - ) or self.section_boundaries.get(".rdata") - if ( - rodata_section is not None - and rodata_section.has_start() - and not rodata_section.has_end() - ): - assert self.vram_end is not None - rodata_section.end = self.vram_end - - return ret - - def scan(self, rom_bytes): - # Always scan code first - for sub in self.subsegments: - if sub.is_text() and sub.should_scan(): - sub.scan(rom_bytes) - - # Scan everyone else - for sub in self.subsegments: - if not sub.is_text() and sub.should_scan(): - sub.scan(rom_bytes) diff --git a/tools/splat/segtypes/common/codesubsegment.py b/tools/splat/segtypes/common/codesubsegment.py deleted file mode 100644 index 08f732762f..0000000000 --- a/tools/splat/segtypes/common/codesubsegment.py +++ /dev/null @@ -1,214 +0,0 @@ -from typing import Optional - -import spimdisasm -import rabbitizer - -from util import options, symbols, log - -from segtypes import segment -from segtypes.common.code import CommonSegCode - -from segtypes.segment import Segment - -from disassembler_section import DisassemblerSection, make_text_section - - -# abstract class for c, asm, data, etc -class CommonSegCodeSubsegment(Segment): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - vram = segment.parse_segment_vram(self.yaml) - if vram is not None: - self.vram_start = vram - - self.str_encoding: Optional[str] = ( - self.yaml.get("str_encoding", None) if isinstance(self.yaml, dict) else None - ) - - self.spim_section: Optional[DisassemblerSection] = None - self.instr_category = rabbitizer.InstrCategory.CPU - if options.opts.platform == "ps2": - self.instr_category = rabbitizer.InstrCategory.R5900 - elif options.opts.platform == "psx": - self.instr_category = rabbitizer.InstrCategory.R3000GTE - - self.detect_redundant_function_end: Optional[bool] = ( - self.yaml.get("detect_redundant_function_end", None) - if isinstance(self.yaml, dict) - else None - ) - - @property - def needs_symbols(self) -> bool: - return True - - def get_linker_section(self) -> str: - return ".text" - - def scan_code(self, rom_bytes, is_hasm=False): - if not isinstance(self.rom_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'" - ) - - # Supposedly logic error, not user error - assert isinstance(self.rom_end, int), self.rom_end - - # Supposedly logic error, not user error - segment_rom_start = self.get_most_parent().rom_start - assert isinstance(segment_rom_start, int), segment_rom_start - - if not isinstance(self.vram_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" - ) - - self.spim_section = make_text_section( - self.rom_start, - self.rom_end, - self.vram_start, - self.name, - rom_bytes, - segment_rom_start, - self.get_exclusive_ram_id(), - ) - - assert self.spim_section is not None - - self.spim_section.get_section().isHandwritten = is_hasm - self.spim_section.get_section().instrCat = self.instr_category - self.spim_section.get_section().detectRedundantFunctionEnd = ( - self.detect_redundant_function_end - ) - - self.spim_section.analyze() - self.spim_section.set_comment_offset(self.rom_start) - - for func in self.spim_section.get_section().symbolList: - assert isinstance(func, spimdisasm.mips.symbols.SymbolFunction) - - self.process_insns(func) - - # Process jumptable labels and pass them to spimdisasm - self.gather_jumptable_labels(rom_bytes) - for jtbl_label_vram in self.parent.jtbl_glabels_to_add: - sym = self.create_symbol( - jtbl_label_vram, True, type="jtbl_label", define=True - ) - sym.type = "jtbl_label" - symbols.add_symbol_to_spim_section(self.spim_section.get_section(), sym) - - def process_insns( - self, - func_spim: spimdisasm.mips.symbols.SymbolFunction, - ): - assert isinstance(self.parent, CommonSegCode) - assert func_spim.vram is not None - assert func_spim.vramEnd is not None - assert self.spim_section is not None - self.parent: CommonSegCode = self.parent - - symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), func_spim.contextSym - ) - - # Gather symbols found by spimdisasm and create those symbols in splat's side - for referenced_vram in func_spim.instrAnalyzer.referencedVrams: - context_sym = self.spim_section.get_section().getSymbol( - referenced_vram, tryPlusOffset=False - ) - if context_sym is not None: - if context_sym.type == spimdisasm.common.SymbolSpecialType.jumptable: - self.parent.jumptables[referenced_vram] = ( - func_spim.vram, - func_spim.vramEnd, - ) - symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), context_sym - ) - - # Main loop - for i, insn in enumerate(func_spim.instructions): - instr_offset = i * 4 - - # update pointer accesses from this function - if instr_offset in func_spim.instrAnalyzer.symbolInstrOffset: - sym_address = func_spim.instrAnalyzer.symbolInstrOffset[instr_offset] - - context_sym = self.spim_section.get_section().getSymbol( - sym_address, tryPlusOffset=False - ) - if context_sym is not None: - sym = symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), context_sym - ) - - if self.parent: - self.parent.check_rodata_sym(func_spim.vram, sym) - - def print_file_boundaries(self): - if not self.show_file_boundaries or not self.spim_section: - return - - assert isinstance(self.rom_start, int) - - for in_file_offset in self.spim_section.get_section().fileBoundaries: - if (in_file_offset % 16) != 0: - continue - - if not self.parent.reported_file_split: - self.parent.reported_file_split = True - - # Look up for the last symbol in this boundary - sym_addr = 0 - for sym in self.spim_section.get_section().symbolList: - symOffset = ( - sym.inFileOffset - self.spim_section.get_section().inFileOffset - ) - if in_file_offset == symOffset: - break - sym_addr = sym.vram - - print( - f"\nSegment {self.name}, symbol at vram {sym_addr:X} ends with extra nops, indicating a likely file split." - ) - print( - "File split suggestions for this segment will follow in config yaml format:" - ) - print(f" - [0x{self.rom_start+in_file_offset:X}, {self.type}]") - - def gather_jumptable_labels(self, rom_bytes): - assert isinstance(self.rom_start, int) - assert isinstance(self.vram_start, int) - - # TODO: use the seg_symbols for this - # jumptables = [j.type == "jtbl" for j in self.seg_symbols] - for jumptable in self.parent.jumptables: - start, end = self.parent.jumptables[jumptable] - rom_offset = self.rom_start + jumptable - self.vram_start - - if rom_offset <= 0: - return - - while rom_offset: - word = rom_bytes[rom_offset : rom_offset + 4] - word_int = int.from_bytes(word, options.opts.endianness) - if word_int >= start and word_int <= end: - self.parent.jtbl_glabels_to_add.add(word_int) - else: - break - - rom_offset += 4 - - def should_scan(self) -> bool: - return ( - options.opts.is_mode_active("code") - and self.rom_start is not None - and self.rom_end is not None - ) - - def should_split(self) -> bool: - return ( - self.extract and options.opts.is_mode_active("code") and self.should_scan() - ) # only split if the segment was scanned first diff --git a/tools/splat/segtypes/common/cpp.py b/tools/splat/segtypes/common/cpp.py deleted file mode 100644 index c163059704..0000000000 --- a/tools/splat/segtypes/common/cpp.py +++ /dev/null @@ -1,8 +0,0 @@ -from segtypes.common.c import CommonSegC - - -class CommonSegCpp(CommonSegC): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.file_extension = "cpp" diff --git a/tools/splat/segtypes/common/data.py b/tools/splat/segtypes/common/data.py deleted file mode 100644 index c96c787b62..0000000000 --- a/tools/splat/segtypes/common/data.py +++ /dev/null @@ -1,151 +0,0 @@ -from pathlib import Path -from typing import Optional - -from util import options, symbols, log - -from segtypes.common.codesubsegment import CommonSegCodeSubsegment -from segtypes.common.group import CommonSegGroup - -from disassembler_section import make_data_section - - -class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup): - @staticmethod - def is_data() -> bool: - return True - - def asm_out_path(self) -> Path: - typ = self.type - if typ.startswith("."): - typ = typ[1:] - - return options.opts.data_path / self.dir / f"{self.name}.{typ}.s" - - def out_path(self) -> Optional[Path]: - if self.type.startswith("."): - if self.sibling: - # C file - return self.sibling.out_path() - else: - # Implied C file - return options.opts.src_path / self.dir / f"{self.name}.c" - else: - # ASM - return self.asm_out_path() - - def scan(self, rom_bytes: bytes): - CommonSegGroup.scan(self, rom_bytes) - - if self.rom_start is not None and self.rom_end is not None: - self.disassemble_data(rom_bytes) - - def split(self, rom_bytes: bytes): - super().split(rom_bytes) - - if self.type.startswith(".") and not options.opts.disassemble_all: - return - - if self.spim_section is None or not self.should_self_split(): - return - - path = self.asm_out_path() - - path.parent.mkdir(parents=True, exist_ok=True) - - self.print_file_boundaries() - - with path.open("w", newline="\n") as f: - f.write('.include "macro.inc"\n\n') - preamble = options.opts.generated_s_preamble - if preamble: - f.write(preamble + "\n") - - f.write(f".section {self.get_linker_section()}") - section_flags = self.get_section_flags() - if section_flags: - f.write(f', "{section_flags}"') - f.write("\n\n") - - f.write(self.spim_section.disassemble()) - - def should_self_split(self) -> bool: - return options.opts.is_mode_active("data") - - def should_scan(self) -> bool: - return True - - def should_split(self) -> bool: - return True - - def cache(self): - return [CommonSegCodeSubsegment.cache(self), CommonSegGroup.cache(self)] - - def get_linker_section(self) -> str: - return ".data" - - def get_linker_entries(self): - return CommonSegCodeSubsegment.get_linker_entries(self) - - def disassemble_data(self, rom_bytes): - if not isinstance(self.rom_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'" - ) - - # Supposedly logic error, not user error - assert isinstance(self.rom_end, int), self.rom_end - - # Supposedly logic error, not user error - segment_rom_start = self.get_most_parent().rom_start - assert isinstance(segment_rom_start, int), segment_rom_start - - if not isinstance(self.vram_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" - ) - - self.spim_section = make_data_section( - self.rom_start, - self.rom_end, - self.vram_start, - self.name, - rom_bytes, - segment_rom_start, - self.get_exclusive_ram_id(), - ) - - assert self.spim_section is not None - - # Set rodata string encoding - # First check the global configuration - if options.opts.data_string_encoding is not None: - self.spim_section.get_section().stringEncoding = ( - options.opts.data_string_encoding - ) - - # Then check the per-segment configuration in case we want to override the global one - if self.str_encoding is not None: - self.spim_section.get_section().stringEncoding = self.str_encoding - - self.spim_section.analyze() - self.spim_section.set_comment_offset(self.rom_start) - - rodata_encountered = False - - for symbol in self.spim_section.get_section().symbolList: - symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), symbol.contextSym - ) - - # Hint to the user that we are now in the .rodata section and no longer in the .data section (assuming rodata follows data) - if not rodata_encountered and self.get_most_parent().rodata_follows_data: - if symbol.contextSym.isJumpTable(): - rodata_encountered = True - print( - f"Data segment {self.name}, symbol at vram {symbol.contextSym.vram:X} is a jumptable, indicating the start of the rodata section _may_ be near here." - ) - print( - "Please note the real start of the rodata section may be way before this point." - ) - if symbol.contextSym.vromAddress is not None: - print(f" - [0x{symbol.contextSym.vromAddress:X}, rodata]") diff --git a/tools/splat/segtypes/common/databin.py b/tools/splat/segtypes/common/databin.py deleted file mode 100644 index 616288652f..0000000000 --- a/tools/splat/segtypes/common/databin.py +++ /dev/null @@ -1,45 +0,0 @@ -from pathlib import Path -from typing import Optional - -from util import log, options - -from segtypes.common.textbin import CommonSegTextbin - - -class CommonSegDatabin(CommonSegTextbin): - @staticmethod - def is_text() -> bool: - return False - - @staticmethod - def is_data() -> bool: - return True - - def get_linker_section(self) -> str: - return ".data" - - def get_section_flags(self) -> Optional[str]: - return "wa" - - def split(self, rom_bytes): - if self.rom_end is None: - log.error( - f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it" - ) - - self.write_bin(rom_bytes) - - if self.sibling is None: - # textbin will write the incbin instead - - s_path = self.out_path() - assert s_path is not None - s_path.parent.mkdir(parents=True, exist_ok=True) - - with s_path.open("w") as f: - f.write('.include "macro.inc"\n\n') - preamble = options.opts.generated_s_preamble - if preamble: - f.write(preamble + "\n") - - self.write_asm_contents(rom_bytes, f) diff --git a/tools/splat/segtypes/common/decompressor.py b/tools/splat/segtypes/common/decompressor.py deleted file mode 100644 index 152521ef94..0000000000 --- a/tools/splat/segtypes/common/decompressor.py +++ /dev/null @@ -1,50 +0,0 @@ -from typing import Optional, Any - -from util import log, options -from util.n64.decompressor import Decompressor - -from segtypes.n64.segment import N64Segment - - -class CommonSegDecompressor(N64Segment): - decompressor: Decompressor - compression_type = "" # "Mio0" -> filename.Mio0.o - - def split(self, rom_bytes): - if self.decompressor is None: - log.error("Decompressor is not a standalone segment type") - - out_dir = options.opts.asset_path / self.dir - out_dir.mkdir(parents=True, exist_ok=True) - - if self.rom_end is None: - log.error( - f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it" - ) - - out_path = out_dir / f"{self.name}.bin" - with open(out_path, "wb") as f: - assert isinstance(self.rom_start, int) - assert isinstance(self.rom_end, int) - - self.log(f"Decompressing {self.name}") - compressed_bytes = rom_bytes[self.rom_start : self.rom_end] - decompressed_bytes = self.decompressor.decompress(compressed_bytes) - f.write(decompressed_bytes) - self.log(f"Wrote {self.name} to {out_path}") - - def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry - - return [ - LinkerEntry( - self, - [options.opts.asset_path / self.dir / f"{self.name}.bin"], - options.opts.asset_path - / self.dir - / f"{self.name}.{self.compression_type}", - self.get_linker_section_order(), - self.get_linker_section_linksection(), - self.is_noload(), - ) - ] diff --git a/tools/splat/segtypes/common/group.py b/tools/splat/segtypes/common/group.py deleted file mode 100644 index 1242b22d7d..0000000000 --- a/tools/splat/segtypes/common/group.py +++ /dev/null @@ -1,160 +0,0 @@ -from typing import List, Optional - -from util import log - -from segtypes.common.segment import CommonSegment -from segtypes.segment import Segment - - -class CommonSegGroup(CommonSegment): - def __init__( - self, - rom_start: Optional[int], - rom_end: Optional[int], - type: str, - name: str, - vram_start: Optional[int], - args: list, - yaml, - ): - super().__init__( - rom_start, - rom_end, - type, - name, - vram_start, - args=args, - yaml=yaml, - ) - - self.subsegments: List[Segment] = self.parse_subsegments(yaml) - - def get_next_seg_start(self, i, subsegment_yamls): - return ( - self.rom_end - if i == len(subsegment_yamls) - 1 - else Segment.parse_segment_start(subsegment_yamls[i + 1]) - ) - - def parse_subsegments(self, yaml) -> List[Segment]: - ret: List[Segment] = [] - - if not yaml or "subsegments" not in yaml: - return ret - - prev_start: Optional[int] = -1 - last_rom_end = 0 - - for i, subsegment_yaml in enumerate(yaml["subsegments"]): - # endpos marker - if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1: - continue - - typ = Segment.parse_segment_type(subsegment_yaml) - start = Segment.parse_segment_start(subsegment_yaml) - - segment_class = Segment.get_class_for_type(typ) - - end = self.get_next_seg_start(i, yaml["subsegments"]) - - if start is None: - # Attempt to infer the start address - if i == 0: - # The start address of this segment is the start address of the group - start = self.rom_start - else: - # The start address is the end address of the previous segment - start = last_rom_end - - if start is not None and end is None: - est_size = segment_class.estimate_size(subsegment_yaml) - if est_size is not None: - end = start + est_size - - if start is not None and prev_start is not None and start < prev_start: - log.error( - f"Error: Group segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})" - ) - - vram = None - if start is not None: - most_parent = self.get_most_parent() - if ( - most_parent.vram_start is not None - and most_parent.rom_start is not None - ): - vram = most_parent.vram_start + start - most_parent.rom_start - - if segment_class.is_noload(): - # Pretend bss's rom address is after the last actual rom segment - start = last_rom_end - # and it has a rom size of zero - end = last_rom_end - - segment: Segment = Segment.from_yaml( - segment_class, subsegment_yaml, start, end, vram - ) - segment.parent = self - if segment.special_vram_segment: - self.special_vram_segment = True - - ret.append(segment) - prev_start = start - if end is not None: - last_rom_end = end - - return ret - - @property - def needs_symbols(self) -> bool: - for seg in self.subsegments: - if seg.needs_symbols: - return True - return False - - def get_linker_entries(self): - return [entry for sub in self.subsegments for entry in sub.get_linker_entries()] - - def scan(self, rom_bytes): - for sub in self.subsegments: - if sub.should_scan(): - sub.scan(rom_bytes) - - def split(self, rom_bytes): - for sub in self.subsegments: - if sub.should_split(): - sub.split(rom_bytes) - - def should_split(self) -> bool: - return self.extract - - def should_scan(self) -> bool: - return self.extract - - def cache(self): - c = [] - - for sub in self.subsegments: - c.append(sub.cache()) - - return c - - def get_subsegment_for_ram(self, addr: int) -> Optional[Segment]: - for sub in self.subsegments: - if sub.contains_vram(addr): - return sub - return None - - def get_next_subsegment_for_ram(self, addr: int) -> Optional[Segment]: - """ - Returns the first subsegment which comes after the specified address, - or None in case this address belongs to the last subsegment of this group - """ - - for sub in self.subsegments: - if sub.vram_start is None: - continue - assert isinstance(sub.vram_start, int) - if sub.vram_start > addr: - return sub - return None diff --git a/tools/splat/segtypes/common/hasm.py b/tools/splat/segtypes/common/hasm.py deleted file mode 100644 index de7c54c1df..0000000000 --- a/tools/splat/segtypes/common/hasm.py +++ /dev/null @@ -1,24 +0,0 @@ -from segtypes.common.asm import CommonSegAsm - - -class CommonSegHasm(CommonSegAsm): - def scan(self, rom_bytes: bytes): - if ( - self.rom_start is not None - and self.rom_end is not None - and self.rom_start != self.rom_end - ): - self.scan_code(rom_bytes, is_hasm=True) - - def split(self, rom_bytes: bytes): - if not self.rom_start == self.rom_end and self.spim_section is not None: - out_path = self.out_path() - if out_path and not out_path.exists(): - out_path.parent.mkdir(parents=True, exist_ok=True) - - self.print_file_boundaries() - - with open(out_path, "w", newline="\n") as f: - for line in self.get_file_header(): - f.write(line + "\n") - f.write(self.spim_section.disassemble()) diff --git a/tools/splat/segtypes/common/header.py b/tools/splat/segtypes/common/header.py deleted file mode 100644 index 55bfc25a07..0000000000 --- a/tools/splat/segtypes/common/header.py +++ /dev/null @@ -1,46 +0,0 @@ -from pathlib import Path - -from util import options - -from segtypes.common.segment import CommonSegment - - -class CommonSegHeader(CommonSegment): - @staticmethod - def is_data() -> bool: - return True - - def should_split(self): - return self.extract and options.opts.is_mode_active("code") - - @staticmethod - def get_line(typ, data, comment): - if typ == "ascii": - text = data.decode("ASCII").strip() - text = text.replace("\x00", "\\0") # escape NUL chars - dstr = '"' + text + '"' - else: # .word, .byte - dstr = "0x" + data.hex().upper() - - dstr = dstr.ljust(20 - len(typ)) - - return f".{typ} {dstr} /* {comment} */" - - def out_path(self) -> Path: - return options.opts.asm_path / self.dir / f"{self.name}.s" - - def parse_header(self, rom_bytes): - return [] - - def split(self, rom_bytes): - header_lines = self.parse_header(rom_bytes) - - src_path = self.out_path() - src_path.parent.mkdir(parents=True, exist_ok=True) - with open(src_path, "w", newline="\n") as f: - f.write("\n".join(header_lines)) - self.log(f"Wrote {self.name} to {src_path}") - - @staticmethod - def get_default_name(addr): - return "header" diff --git a/tools/splat/segtypes/common/lib.py b/tools/splat/segtypes/common/lib.py deleted file mode 100644 index 17d4c12b4e..0000000000 --- a/tools/splat/segtypes/common/lib.py +++ /dev/null @@ -1,62 +0,0 @@ -from pathlib import Path -from typing import Optional - -from util import log, options - -from segtypes.linker_entry import LinkerEntry -from segtypes.n64.segment import N64Segment - - -class CommonSegLib(N64Segment): - def __init__( - self, - rom_start: Optional[int], - rom_end: Optional[int], - type: str, - name: str, - vram_start: Optional[int], - args: list, - yaml, - ): - super().__init__( - rom_start, - rom_end, - type, - name, - vram_start, - args=args, - yaml=yaml, - ) - - if isinstance(yaml, dict): - log.error("Error: 'dict' not currently supported for 'lib' segment") - return - if len(args) < 1: - log.error(f"Error: {self.name} is missing object file") - return - - self.extract = False - - if len(args) > 1: - self.object, self.section = args[0], args[1] - else: - self.object, self.section = args[0], ".text" - - def get_linker_section(self) -> str: - return self.section - - def get_linker_entries(self): - path = options.opts.lib_path / self.name - - object_path = Path(f"{path}.a:{self.object}.o") - - return [ - LinkerEntry( - self, - [path], - object_path, - self.get_linker_section_order(), - self.get_linker_section_linksection(), - self.is_noload(), - ) - ] diff --git a/tools/splat/segtypes/common/rdata.py b/tools/splat/segtypes/common/rdata.py deleted file mode 100644 index 37adbef22e..0000000000 --- a/tools/splat/segtypes/common/rdata.py +++ /dev/null @@ -1,6 +0,0 @@ -from segtypes.common.rodata import CommonSegRodata - - -class CommonSegRdata(CommonSegRodata): - def get_linker_section(self) -> str: - return ".rdata" diff --git a/tools/splat/segtypes/common/rodata.py b/tools/splat/segtypes/common/rodata.py deleted file mode 100644 index 599aacb2fe..0000000000 --- a/tools/splat/segtypes/common/rodata.py +++ /dev/null @@ -1,106 +0,0 @@ -from typing import Optional, Set, Tuple -import spimdisasm -from segtypes.segment import Segment -from util import log, options, symbols - -from segtypes.common.data import CommonSegData - -from disassembler_section import make_rodata_section - - -class CommonSegRodata(CommonSegData): - def get_linker_section(self) -> str: - return ".rodata" - - @staticmethod - def is_data() -> bool: - return False - - @staticmethod - def is_rodata() -> bool: - return True - - def get_possible_text_subsegment_for_symbol( - self, rodata_sym: spimdisasm.mips.symbols.SymbolBase - ) -> Optional[Tuple[Segment, spimdisasm.common.ContextSymbol]]: - # Check if this rodata segment does not have a corresponding code file, try to look for one - - if self.sibling is not None or not options.opts.pair_rodata_to_text: - return None - - if not rodata_sym.shouldMigrate(): - return None - - if len(rodata_sym.contextSym.referenceFunctions) != 1: - return None - - func = list(rodata_sym.contextSym.referenceFunctions)[0] - text_segment = self.parent.get_subsegment_for_ram(func.vram) - - if text_segment is None or not text_segment.is_text(): - return None - return text_segment, func - - def disassemble_data(self, rom_bytes): - if not isinstance(self.rom_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'" - ) - - # Supposedly logic error, not user error - assert isinstance(self.rom_end, int), self.rom_end - - # Supposedly logic error, not user error - segment_rom_start = self.get_most_parent().rom_start - assert isinstance(segment_rom_start, int), segment_rom_start - - if not isinstance(self.vram_start, int): - log.error( - f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'" - ) - - self.spim_section = make_rodata_section( - self.rom_start, - self.rom_end, - self.vram_start, - self.name, - rom_bytes, - segment_rom_start, - self.get_exclusive_ram_id(), - ) - - assert self.spim_section is not None - - # Set rodata string encoding - # First check the global configuration - if options.opts.string_encoding is not None: - self.spim_section.get_section().stringEncoding = ( - options.opts.string_encoding - ) - - # Then check the per-segment configuration in case we want to override the global one - if self.str_encoding is not None: - self.spim_section.get_section().stringEncoding = self.str_encoding - - self.spim_section.analyze() - self.spim_section.set_comment_offset(self.rom_start) - - possible_text_segments: Set[Segment] = set() - - for symbol in self.spim_section.get_section().symbolList: - generated_symbol = symbols.create_symbol_from_spim_symbol( - self.get_most_parent(), symbol.contextSym - ) - generated_symbol.linker_section = self.get_linker_section() - - possible_text = self.get_possible_text_subsegment_for_symbol(symbol) - if possible_text is not None: - text_segment, refenceeFunction = possible_text - if text_segment not in possible_text_segments: - print( - f"\nRodata segment '{self.name}' may belong to the text segment '{text_segment.name}'" - ) - print( - f" Based on the usage from the function {refenceeFunction.getName()} to the symbol {symbol.getName()}" - ) - possible_text_segments.add(text_segment) diff --git a/tools/splat/segtypes/common/rodatabin.py b/tools/splat/segtypes/common/rodatabin.py deleted file mode 100644 index 7cd5916792..0000000000 --- a/tools/splat/segtypes/common/rodatabin.py +++ /dev/null @@ -1,45 +0,0 @@ -from pathlib import Path -from typing import Optional - -from util import log, options - -from segtypes.common.textbin import CommonSegTextbin - - -class CommonSegRodatabin(CommonSegTextbin): - @staticmethod - def is_text() -> bool: - return False - - @staticmethod - def is_rodata() -> bool: - return True - - def get_linker_section(self) -> str: - return ".rodata" - - def get_section_flags(self) -> Optional[str]: - return "a" - - def split(self, rom_bytes): - if self.rom_end is None: - log.error( - f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it" - ) - - self.write_bin(rom_bytes) - - if self.sibling is None: - # textbin will write the incbin instead - - s_path = self.out_path() - assert s_path is not None - s_path.parent.mkdir(parents=True, exist_ok=True) - - with s_path.open("w") as f: - f.write('.include "macro.inc"\n\n') - preamble = options.opts.generated_s_preamble - if preamble: - f.write(preamble + "\n") - - self.write_asm_contents(rom_bytes, f) diff --git a/tools/splat/segtypes/common/sbss.py b/tools/splat/segtypes/common/sbss.py deleted file mode 100644 index cbd004670d..0000000000 --- a/tools/splat/segtypes/common/sbss.py +++ /dev/null @@ -1,6 +0,0 @@ -from segtypes.common.data import CommonSegData - - -class CommonSegSbss(CommonSegData): - def get_linker_section(self) -> str: - return ".sbss" diff --git a/tools/splat/segtypes/common/sdata.py b/tools/splat/segtypes/common/sdata.py deleted file mode 100644 index a3a0e528a3..0000000000 --- a/tools/splat/segtypes/common/sdata.py +++ /dev/null @@ -1,6 +0,0 @@ -from segtypes.common.data import CommonSegData - - -class CommonSegSdata(CommonSegData): - def get_linker_section(self) -> str: - return ".sdata" diff --git a/tools/splat/segtypes/common/segment.py b/tools/splat/segtypes/common/segment.py deleted file mode 100644 index 10e2b850bd..0000000000 --- a/tools/splat/segtypes/common/segment.py +++ /dev/null @@ -1,5 +0,0 @@ -from segtypes.segment import Segment - - -class CommonSegment(Segment): - pass diff --git a/tools/splat/segtypes/common/textbin.py b/tools/splat/segtypes/common/textbin.py deleted file mode 100644 index 5edf7aa380..0000000000 --- a/tools/splat/segtypes/common/textbin.py +++ /dev/null @@ -1,114 +0,0 @@ -from pathlib import Path -from typing import Optional, TextIO - -from util import log, options - -from segtypes.common.segment import CommonSegment - - -class CommonSegTextbin(CommonSegment): - @staticmethod - def is_text() -> bool: - return True - - def get_linker_section(self) -> str: - return ".text" - - def get_section_flags(self) -> Optional[str]: - return "ax" - - def out_path(self) -> Optional[Path]: - return options.opts.data_path / self.dir / f"{self.name}.s" - - def bin_path(self) -> Path: - typ = self.type - if typ.startswith("."): - typ = typ[1:] - - return options.opts.asset_path / self.dir / f"{self.name}.{typ}.bin" - - def write_bin(self, rom_bytes): - binpath = self.bin_path() - binpath.parent.mkdir(parents=True, exist_ok=True) - - assert isinstance(self.rom_start, int) - assert isinstance(self.rom_end, int) - - binpath.write_bytes(rom_bytes[self.rom_start : self.rom_end]) - - self.log(f"Wrote {self.name} to {binpath}") - - def write_asm_contents(self, rom_bytes, f: TextIO): - binpath = self.bin_path() - asm_label = options.opts.asm_function_macro - if not self.is_text(): - asm_label = options.opts.asm_data_macro - - assert isinstance(self.rom_start, int) - assert isinstance(self.rom_end, int) - - f.write(f".section {self.get_linker_section()}") - section_flags = self.get_section_flags() - if section_flags: - f.write(f', "{section_flags}"') - f.write("\n\n") - - # Check if there's a symbol at this address - sym = None - vram = self.rom_to_ram(self.rom_start) - if vram is not None: - sym = self.get_symbol(vram, in_segment=True) - - if sym is not None: - f.write(f"{asm_label} {sym.name}\n") - sym.defined = True - - f.write(f'.incbin "{binpath}"\n') - - if sym is not None: - if self.is_text() and options.opts.asm_end_label != "": - f.write(f"{options.opts.asm_end_label} {sym.name}\n") - - if sym.given_name_end is not None: - if ( - sym.given_size is None - or sym.given_size == self.rom_end - self.rom_start - ): - f.write(f"{asm_label} {sym.given_name_end}\n") - - def split(self, rom_bytes): - if self.rom_end is None: - log.error( - f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it" - ) - - self.write_bin(rom_bytes) - - s_path = self.out_path() - assert s_path is not None - s_path.parent.mkdir(parents=True, exist_ok=True) - - with s_path.open("w") as f: - f.write('.include "macro.inc"\n\n') - preamble = options.opts.generated_s_preamble - if preamble: - f.write(preamble + "\n") - - self.write_asm_contents(rom_bytes, f) - - # We check against CommonSegTextbin instead of the specific type because the other incbins inherit from this class - if isinstance(self.data_sibling, CommonSegTextbin): - f.write("\n") - self.data_sibling.write_asm_contents(rom_bytes, f) - - if isinstance(self.rodata_sibling, CommonSegTextbin): - f.write("\n") - self.rodata_sibling.write_asm_contents(rom_bytes, f) - - def should_scan(self) -> bool: - return self.rom_start is not None and self.rom_end is not None - - def should_split(self) -> bool: - return ( - self.extract and self.should_scan() - ) # only split if the segment was scanned first diff --git a/tools/splat/segtypes/gc/apploader.py b/tools/splat/segtypes/gc/apploader.py deleted file mode 100644 index 810e822577..0000000000 --- a/tools/splat/segtypes/gc/apploader.py +++ /dev/null @@ -1,8 +0,0 @@ -import struct -from pathlib import Path - -from segtypes.gc.segment import GCSegment - - -class GcSegApploader(GCSegment): - pass diff --git a/tools/splat/segtypes/gc/bi2.py b/tools/splat/segtypes/gc/bi2.py deleted file mode 100644 index 2bf6a97898..0000000000 --- a/tools/splat/segtypes/gc/bi2.py +++ /dev/null @@ -1,67 +0,0 @@ -import struct -from pathlib import Path - -from util import options - -from segtypes.gc.segment import GCSegment - - -class GcSegBi2(GCSegment): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def split(self, bi2_bytes): - lines = [] - - # Gathering variables - debug_monitor_size = struct.unpack_from(">I", bi2_bytes, 0x00)[0] - simulated_memory_size = struct.unpack_from(">I", bi2_bytes, 0x04)[0] - - argument_offset = struct.unpack_from(">I", bi2_bytes, 0x08)[0] - - debug_flag = struct.unpack_from(">I", bi2_bytes, 0x0C)[0] - - track_offset = struct.unpack_from(">I", bi2_bytes, 0x10)[0] - track_size = struct.unpack_from(">I", bi2_bytes, 0x14)[0] - - country_code_bi2 = struct.unpack_from(">I", bi2_bytes, 0x18)[0] - - unk_int = struct.unpack_from(">I", bi2_bytes, 0x1C)[0] - unk_int_2 = struct.unpack_from(">I", bi2_bytes, 0x20)[0] - - # Outputting .s file - lines.append(f"# GameCube disc image bi2 data, located at 0x440 in the disc.\n") - lines.append(f"# Generated by splat.\n\n") - - lines.append(f".section .data\n\n") - - lines.append(f"debug_monitor_size: .long 0x{debug_monitor_size:08X}\n") - lines.append(f"simulated_memory_size: .long 0x{simulated_memory_size:08X}\n\n") - - lines.append(f"argument_offset: .long 0x{argument_offset:08X}\n\n") - - lines.append(f"debug_flag: .long 0x{debug_flag:08X}\n\n") - - lines.append(f"track_offset: .long 0x{track_offset:08X}\n") - lines.append(f"track_size: .long 0x{track_size:08X}\n\n") - - lines.append(f"country_code_bi2: .long 0x{country_code_bi2:08X}\n\n") - - lines.append(f"ukn_int_bi2: .long 0x{unk_int:08X}\n") - lines.append(f"ukn_int_bi2_2: .long 0x{unk_int_2:08X}\n\n") - - lines.append(f".fill 0x00001FDC\n\n") - - out_path = self.out_path() - out_path.parent.mkdir(parents=True, exist_ok=True) - - with open(out_path, "w", encoding="utf-8") as f: - f.writelines(lines) - - return - - def should_split(self) -> bool: - return True - - def out_path(self) -> Path: - return options.opts.asm_path / "sys" / "bi2.s" diff --git a/tools/splat/segtypes/gc/bootinfo.py b/tools/splat/segtypes/gc/bootinfo.py deleted file mode 100644 index 28a93fd749..0000000000 --- a/tools/splat/segtypes/gc/bootinfo.py +++ /dev/null @@ -1,117 +0,0 @@ -import struct -from pathlib import Path - -from util import options - -from segtypes.gc.segment import GCSegment - - -class GcSegBootinfo(GCSegment): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def split(self, iso_bytes): - lines = [] - - gc_dvd_magic = struct.unpack_from(">I", iso_bytes, 0x1C)[0] - assert gc_dvd_magic == 0xC2339F3D - - # Gathering variables - system_code = chr(iso_bytes[0x00]) - game_code = iso_bytes[0x01:0x03].decode("utf-8") - region_code = chr(iso_bytes[0x03]) - publisher_code = iso_bytes[0x04:0x06].decode("utf-8") - - disc_id = iso_bytes[0x06] - game_version = iso_bytes[0x07] - audio_streaming = iso_bytes[0x08] - stream_buffer_size = iso_bytes[0x09] - - name = iso_bytes[0x20:0x400].decode("utf-8").strip("\x00") - name_padding_len = 0x3E0 - len(name) - - # The following is from YAGCD, don't know what they were for: - # https://web.archive.org/web/20220528011846/http://hitmen.c02.at/files/yagcd/yagcd/chap13.html#sec13.1 - apploader_size = struct.unpack_from(">I", iso_bytes, 0x400)[0] - debug_monitor_address = struct.unpack_from(">I", iso_bytes, 0x404)[0] - - # These on the other hand are easy to understand - dol_offset = struct.unpack_from(">I", iso_bytes, 0x420)[0] - fst_offset = struct.unpack_from(">I", iso_bytes, 0x424)[0] - fst_size = struct.unpack_from(">I", iso_bytes, 0x428)[0] - fst_max_size = struct.unpack_from(">I", iso_bytes, 0x42C)[0] - - user_position = struct.unpack_from(">I", iso_bytes, 0x430)[0] - user_length = struct.unpack_from(">I", iso_bytes, 0x434)[0] - unk_int = struct.unpack_from(">I", iso_bytes, 0x438)[0] - - # Outputting .s file - lines.append(f"# GameCube disc image boot data, located at 0x00 in the disc.\n") - lines.append(f"# Generated by splat.\n\n") - - lines.append(f".section .data\n\n") - - # Game ID stuff - lines.append(f'system_code: .ascii "{system_code}"\n') - lines.append(f'game_code: .ascii "{game_code}"\n') - lines.append(f'region_code: .ascii "{region_code}"\n') - lines.append(f'publisher_code: .ascii "{publisher_code}"\n\n') - - lines.append(f"disc_id: .byte {disc_id:X}\n") - lines.append(f"game_version: .byte {game_version:X}\n") - lines.append(f"audio_streaming: .byte {audio_streaming:X}\n") - lines.append(f"stream_buffer_size: .byte {stream_buffer_size:X}\n\n") - - # padding - lines.append(f".fill 0x12\n\n") - - # GC magic number - lines.append(f"gc_magic: .long 0xC2339F3D\n\n") - - # Long game name - lines.append(f'game_name: .ascii "{name}"\n') - lines.append(f".org 0x400\n\n") - - lines.append(f"apploader_size: .long 0x{apploader_size:08X}\n\n") - - # Unknown stuff gleaned from YAGCD - lines.append(f"debug_monitor_address: .long 0x{debug_monitor_address:08X}\n\n") - - # More padding - lines.append(f".fill 0x18\n\n") - - # DOL and FST data - lines.append(f"dol_offset: .long 0x{dol_offset:08X}\n") - lines.append(f"fst_offset: .long 0x{fst_offset:08X}\n\n") - - lines.append( - f"# The FST is only allocated once per game boot, even in games with multiple disks. fst_max_size is used to ensure that\n" - ) - lines.append( - f"# there is enough space allocated for the FST in the event that a game spans multiple disks, and one disk has a larger FST than another.\n" - ) - lines.append(f"fst_size: .long 0x{fst_size:08X}\n") - lines.append(f"fst_max_size: .long 0x{fst_max_size:08X}\n\n") - - # Honestly not sure what this data is for - lines.append(f"# Not even YAGCD knows what these are for.\n") - lines.append(f"user_position: .long 0x{user_position:08X}\n") - lines.append(f"user_length: .long 0x{user_length:08X}\n") - lines.append(f"unk_int: .long 0x{unk_int:08X}\n\n") - - # Final padding - lines.append(f".word 0\n") - out_path = self.out_path() - - out_path.parent.mkdir(parents=True, exist_ok=True) - - with open(out_path, "w", encoding="utf-8") as f: - f.writelines(lines) - - return - - def should_split(self) -> bool: - return True - - def out_path(self) -> Path: - return options.opts.asm_path / "sys" / "boot.s" diff --git a/tools/splat/segtypes/gc/dol.py b/tools/splat/segtypes/gc/dol.py deleted file mode 100644 index 3a92bda841..0000000000 --- a/tools/splat/segtypes/gc/dol.py +++ /dev/null @@ -1,8 +0,0 @@ -import struct -from pathlib import Path - -from segtypes.gc.segment import GCSegment - - -class GcSegDol(GCSegment): - pass diff --git a/tools/splat/segtypes/gc/dolheader.py b/tools/splat/segtypes/gc/dolheader.py deleted file mode 100644 index f8a060a660..0000000000 --- a/tools/splat/segtypes/gc/dolheader.py +++ /dev/null @@ -1,68 +0,0 @@ -from util import options - -from segtypes.common.header import CommonSegHeader - - -class DolSegHeader(CommonSegHeader): - def parse_header(self, dol_bytes): - header_lines = [] - header_lines.append(".section .data\n") - - # Text file offsets - for i in range(0x00, 0x1C, 4): - header_lines.append( - self.get_line("word", dol_bytes[i : i + 4], f"Text {i / 4} Offset") - ) - # Data file offsets - for i in range(0x1C, 0x48, 4): - header_lines.append( - self.get_line("word", dol_bytes[i : i + 4], f"Data {i / 4} Offset") - ) - - # Text RAM addresses - for i in range(0x48, 0x64, 4): - header_lines.append( - self.get_line( - "word", - dol_bytes[i : i + 4], - f"Text {(i - 0x48) / 4} Address", - ) - ) - # Data RAM addresses - for i in range(0x64, 0x90, 4): - header_lines.append( - self.get_line( - "word", - dol_bytes[i : i + 4], - f"Data {(i - 0x64) / 4} Address", - ) - ) - - # Text file sizes - for i in range(0x90, 0xAC, 4): - header_lines.append( - self.get_line( - "word", - dol_bytes[i : i + 4], - f"Text {(i - 0x90) / 4} Size", - ) - ) - # Data file sizes - for i in range(0xAC, 0xD8, 4): - header_lines.append( - self.get_line( - "word", - dol_bytes[i : i + 4], - f"Data {(i - 0xAC) / 4} Size", - ) - ) - - # BSS RAM address - header_lines.append(self.get_line("word", dol_bytes[0xD8:0xDC], "BSS Address")) - # BSS size - header_lines.append(self.get_line("word", dol_bytes[0xDC:0xE0], "BSS Size")) - - # Entry point - header_lines.append(self.get_line("word", dol_bytes[0xE0:0xE4], "Entry Point")) - - return header_lines diff --git a/tools/splat/segtypes/gc/fst.py b/tools/splat/segtypes/gc/fst.py deleted file mode 100644 index be091feea0..0000000000 --- a/tools/splat/segtypes/gc/fst.py +++ /dev/null @@ -1,8 +0,0 @@ -import struct -from pathlib import Path - -from segtypes.gc.segment import GCSegment - - -class GcSegFst(GCSegment): - pass diff --git a/tools/splat/segtypes/gc/rarc.py b/tools/splat/segtypes/gc/rarc.py deleted file mode 100644 index bea41ad97a..0000000000 --- a/tools/splat/segtypes/gc/rarc.py +++ /dev/null @@ -1,321 +0,0 @@ -import struct -from enum import IntEnum -from pathlib import Path - -from typing import List, Optional - -from util import options -from util.gc.gcutil import read_string_from_bytes -from util.n64.Yay0decompress import Yay0Decompressor - -from segtypes.gc.segment import GCSegment - - -# Represents the RARC archive format used by first-party Nintendo games. -class GcSegRarc(GCSegment): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - def split(self, file_bytes): - assert self.file_path is not None - - archive = GCRARCArchive(self.file_path, file_bytes) - archive.build_hierarchy(file_bytes) - - archive.emit(file_bytes) - - def should_split(self) -> bool: - return True - - -class GCRARCArchive: - def __init__(self, file_path: Path, file_bytes): - self.file_path = file_path - self.compression = "none" - file_bytes = self.try_decompress_archive(file_bytes) - - self.magic = struct.unpack_from(">I", file_bytes, 0x0000)[0] - self.file_size = struct.unpack_from(">I", file_bytes, 0x0004)[0] - self.data_header_offset = struct.unpack_from(">I", file_bytes, 0x0008)[0] - self.file_data_offset = struct.unpack_from(">I", file_bytes, 0x000C)[0] + 0x0020 - self.total_file_data_size = struct.unpack_from(">I", file_bytes, 0x0010)[0] - - self.mram_preload_size = struct.unpack_from(">I", file_bytes, 0x0014)[0] - self.aram_preload_size = struct.unpack_from(">I", file_bytes, 0x0018)[0] - - self.data_header = GCRARCDataHeader(self.data_header_offset, file_bytes) - self.nodes: List[GCRARCNode] = [] - - def try_decompress_archive(self, file_bytes): - compression_scheme = struct.unpack_from(">I", file_bytes, 0x0000)[0] - - # Yaz0 - if compression_scheme == 0x59617A30: - self.compression = "yaz0" - return file_bytes - # Yay0 - elif compression_scheme == 0x59617930: - self.compression = "yay0" - return Yay0Decompressor.decompress(file_bytes) - # Not compressed! - else: - return file_bytes - - def build_hierarchy(self, file_bytes): - string_table_offset = self.data_header.string_table_offset - string_table_size = self.data_header.string_table_size - - string_table_bytes = file_bytes[ - string_table_offset : string_table_offset + string_table_size - ] - - # Load the file entries into their corresponding nodes. - for i in range(self.data_header.node_count): - offset = self.data_header.node_offset + i * 0x10 - - new_node = GCRARCNode(offset, file_bytes, string_table_bytes) - new_node.get_entries( - self.data_header.file_entry_offset, file_bytes, string_table_bytes - ) - - self.nodes.append(new_node) - - # Now, organize the nodes into a hierarchy. - for n in self.nodes: - for e in n.entries: - # We're only looking for directory nodes, so ignore files. - if e.flags & int(GCRARCFlags.IS_FILE) != 0x00: - continue - - if e.name == "." or e.name == "..": - continue - - # This is the node that the current entry corresponds to. - dir_node = self.nodes[e.data_offset] - - # Set up hierarchy relationship. - dir_node.parent = n - n.children.append(dir_node) - - def emit(self, file_bytes): - assert options.opts.filesystem_path is not None - - rel_path = self.file_path.relative_to(options.opts.filesystem_path / "files") - arc_root_path = options.opts.asset_path / rel_path.with_suffix("") - - self.nodes[0].emit_to_filesystem_recursive( - arc_root_path, self.file_data_offset, file_bytes - ) - self.emit_config(arc_root_path) - - def emit_config(self, config_path: Path): - lines = [] - - lines.append(f'name: "{self.file_path.name}"\n') - - if self.compression != "none": - lines.append(f"compression: {self.compression}\n") - - lines.append(f"next_file_id: 0x{self.data_header.next_free_file_id:04X}\n") - lines.append( - f"sync_file_ids_to_indices: {self.data_header.sync_file_ids_to_indices}\n" - ) - - root_node = self.nodes[0] - - lines.append("root_dir:\n") - lines.append(f' res_type: "{root_node.resource_type}"\n') - lines.append(f' name: "{root_node.name}"\n') - - if len(root_node.entries) != 0: - lines.append(" entries:\n") - for e in root_node.entries: - entry_config = e.emit_config(2) - if entry_config != None: - lines.extend(entry_config) - - if len(root_node.children) != 0: - lines.append(" subdirs:\n") - for n in root_node.children: - node_config = n.emit_config(2) - if node_config != None: - lines.extend(node_config) - - with open(config_path / "arcinfo.yaml", "w", newline="\n") as f: - f.writelines(lines) - - -class GCRARCDataHeader: - def __init__(self, offset, file_bytes): - self.node_count = struct.unpack_from(">I", file_bytes, offset + 0x0000)[0] - self.node_offset = ( - struct.unpack_from(">I", file_bytes, offset + 0x0004)[0] + 0x0020 - ) - - self.file_entry_count = struct.unpack_from(">I", file_bytes, offset + 0x0008)[0] - self.file_entry_offset = ( - struct.unpack_from(">I", file_bytes, offset + 0x000C)[0] + 0x0020 - ) - - self.string_table_size = struct.unpack_from(">I", file_bytes, offset + 0x0010)[ - 0 - ] - self.string_table_offset = ( - struct.unpack_from(">I", file_bytes, offset + 0x0014)[0] + 0x0020 - ) - - self.next_free_file_id = struct.unpack_from(">H", file_bytes, offset + 0x0018)[ - 0 - ] - self.sync_file_ids_to_indices = bool(file_bytes[offset + 0x001A]) - - -class GCRARCNode: - def __init__(self, offset, file_bytes, string_table_bytes): - self.resource_type = file_bytes[offset + 0x0000 : offset + 0x0004].decode( - "utf-8" - ) - self.name_offset = struct.unpack_from(">I", file_bytes, offset + 0x0004)[0] - self.name_hash = struct.unpack_from(">H", file_bytes, offset + 0x0008)[0] - self.file_entry_count = struct.unpack_from(">H", file_bytes, offset + 0x000A)[0] - self.first_file_entry_index = struct.unpack_from( - ">I", file_bytes, offset + 0x000C - )[0] - - self.name = read_string_from_bytes(self.name_offset, string_table_bytes) - self.entries = [] - - self.parent: Optional[GCRARCNode] = None - self.children = [] - - def get_entries(self, file_entry_offset, file_bytes, string_table_bytes): - for i in range(self.file_entry_count): - entry_offset = file_entry_offset + (self.first_file_entry_index + i) * 0x14 - - new_entry = GCRARCFileEntry(entry_offset, file_bytes, string_table_bytes) - new_entry.parent_node = self - - self.entries.append(new_entry) - - def emit_to_filesystem_recursive( - self, root_path: Path, file_data_offset, file_bytes - ): - dir_path = root_path / self.get_full_directory_path() - dir_path.mkdir(parents=True, exist_ok=True) - - for n in self.children: - n.emit_to_filesystem_recursive(root_path, file_data_offset, file_bytes) - - for e in self.entries: - e.emit_to_filesystem(root_path, file_data_offset, file_bytes) - - def emit_config(self, level): - lines = [] - - lines.append(" " * level + f'- res_type: "{self.resource_type}"\n') - lines.append(" " * level + f' name: "{self.name}"\n') - - if len(self.entries) != 0: - lines.append(" " * level + " entries:\n") - for e in self.entries: - entry_config = e.emit_config(level + 1) - if entry_config != None: - lines.extend(entry_config) - - if len(self.children) != 0: - lines.append(" " * level + " subdirs:\n") - for n in self.children: - node_config = n.emit_config(level + 1) - if node_config != None: - lines.extend(node_config) - - return lines - - def print_recursive(self, level): - print((" " * level) + self.name) - - for n in self.children: - n.print_recursive(level + 1) - - def get_full_directory_path(self): - path_components: List[str] = [] - - node: Optional[GCRARCNode] = self - while node is not None: - path_components.insert(0, node.name) - node = node.parent - - return Path(*path_components) - - -class GCRARCFileEntry: - def __init__(self, offset, file_bytes, string_table_bytes): - self.file_id = struct.unpack_from(">H", file_bytes, offset + 0x0000)[0] - self.name_hash = struct.unpack_from(">H", file_bytes, offset + 0x0002)[0] - self.flags = file_bytes[offset + 0x0004] - self.name_offset = ( - struct.unpack_from(">I", file_bytes, offset + 0x0004)[0] & 0x00FFFFFF - ) - self.data_offset = struct.unpack_from(">I", file_bytes, offset + 0x0008)[0] - self.data_size = struct.unpack_from(">I", file_bytes, offset + 0x000C)[0] - - self.name = read_string_from_bytes(self.name_offset, string_table_bytes) - self.parent_node: Optional[GCRARCNode] = None - - def emit_to_filesystem(self, dir_path: Path, file_data_offset, file_bytes): - if self.flags & int(GCRARCFlags.IS_DIR) != 0x00: - return - - file_path = dir_path / self.get_full_file_path() - - file_data = file_bytes[ - file_data_offset - + self.data_offset : file_data_offset - + self.data_offset - + self.data_size - ] - with open(file_path, "wb") as f: - f.write(file_data) - - def emit_config(self, level): - if self.flags & int(GCRARCFlags.IS_DIR) != 0x00: - return - - lines = [] - - lines.append(" " * level + f' - name: "{self.name}"\n') - lines.append(" " * level + f" file_id: 0x{self.file_id:04X}\n") - - if self.flags & int(GCRARCFlags.IS_COMPRESSED) != 0x00: - if self.flags & int(GCRARCFlags.IS_YAZ0_COMPRESSED) != 0x00: - lines.append(" " * level + f" compression: yaz0\n") - else: - lines.append(" " * level + f" compression: yay0\n") - - if self.flags & int(GCRARCFlags.PRELOAD_TO_MRAM) == 0x00: - if self.flags & int(GCRARCFlags.PRELOAD_TO_ARAM) != 0x00: - lines.append(" " * level + f" preload_type: aram\n") - else: - lines.append(" " * level + f" preload_type: dvd\n") - - return lines - - def get_full_file_path(self): - path_components = [self.name] - - node = self.parent_node - while node is not None: - path_components.insert(0, node.name) - node = node.parent - - return Path("/".join(path_components)) - - -class GCRARCFlags(IntEnum): - IS_FILE = 0x01 - IS_DIR = 0x02 - IS_COMPRESSED = 0x04 - PRELOAD_TO_MRAM = 0x10 - PRELOAD_TO_ARAM = 0x20 - LOAD_FROM_DVD = 0x40 - IS_YAZ0_COMPRESSED = 0x80 diff --git a/tools/splat/segtypes/gc/relheader.py b/tools/splat/segtypes/gc/relheader.py deleted file mode 100644 index e92fc636e1..0000000000 --- a/tools/splat/segtypes/gc/relheader.py +++ /dev/null @@ -1,114 +0,0 @@ -from util import options - -from segtypes.common.header import CommonSegHeader - - -class RelSegHeader(CommonSegHeader): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - if isinstance(self.yaml, dict): - self.version: int = self.yaml.get("version", 0) - - def parse_header(self, rel_bytes): - header_lines = [] - header_lines.append(".section .data\n") - - # Module ID - header_lines.append(self.get_line("word", rel_bytes[0x00:0x04], "Module ID")) - - # Next module (filled at runtime) - header_lines.append(self.get_line("word", rel_bytes[0x04:0x08], "Next Module")) - # Last module (filled at runtime) - header_lines.append(self.get_line("word", rel_bytes[0x08:0x0C], "Last Module")) - - # Section count - header_lines.append( - self.get_line("word", rel_bytes[0x0C:0x10], "Section Count") - ) - # Section table offset - header_lines.append( - self.get_line("word", rel_bytes[0x10:0x14], "Section Table Offset") - ) - - # Module name offset (might be null) - header_lines.append( - self.get_line("word", rel_bytes[0x14:0x18], "Module Name Offset") - ) - # Module name length - header_lines.append( - self.get_line("word", rel_bytes[0x18:0x1C], "Module Name Length") - ) - - # REL format version - header_lines.append( - self.get_line("word", rel_bytes[0x1C:0x20], "REL Format Version") - ) - - # BSS size - header_lines.append(self.get_line("word", rel_bytes[0x20:0x24], "BSS Size")) - - # Relocation table offset - header_lines.append( - self.get_line("word", rel_bytes[0x24:0x28], "Relocation Table Offset") - ) - # Import table offset - header_lines.append( - self.get_line("word", rel_bytes[0x28:0x2C], "Import Table Offset") - ) - # Import table size - header_lines.append( - self.get_line("word", rel_bytes[0x2C:0x30], "Import Table Size") - ) - - # Prolog section index - header_lines.append( - self.get_line("byte", rel_bytes[0x30:0x31], "Prolog Section Index") - ) - # Epilog section index - header_lines.append( - self.get_line("byte", rel_bytes[0x31:0x32], "Epilog Section Index") - ) - # Unresolved section index - header_lines.append( - self.get_line("byte", rel_bytes[0x32:0x33], "Unresolved Section Index") - ) - # BSS section index (filled at runtime) - header_lines.append( - self.get_line("byte", rel_bytes[0x33:0x34], "BSS Section Index") - ) - - # Prolog function offset - header_lines.append( - self.get_line("word", rel_bytes[0x34:0x38], "Prolog Function Offset") - ) - # Epilog function offset - header_lines.append( - self.get_line("word", rel_bytes[0x38:0x3C], "Epilog Function Offset") - ) - # Unresolved function offset - header_lines.append( - self.get_line("word", rel_bytes[0x3C:0x40], "Unresolved Function Offset") - ) - - # Version 1 is only 0x40 bytes long - if self.version <= 1: - return header_lines - - # Alignment constraint - header_lines.append( - self.get_line("word", rel_bytes[0x40:0x44], "Alignment Constraint") - ) - # BSS alignment constraint - header_lines.append( - self.get_line("word", rel_bytes[0x44:0x48], "BSS Alignment Constraint") - ) - - # Version 2 is only 0x48 bytes long - if self.version <= 2: - return header_lines - - # Fix size - header_lines.append(self.get_line("word", rel_bytes[0x48:0x4C], "Fix Size")) - - return header_lines diff --git a/tools/splat/segtypes/gc/segment.py b/tools/splat/segtypes/gc/segment.py deleted file mode 100644 index 8e4d9bca25..0000000000 --- a/tools/splat/segtypes/gc/segment.py +++ /dev/null @@ -1,5 +0,0 @@ -from segtypes.segment import Segment - - -class GCSegment(Segment): - pass diff --git a/tools/splat/segtypes/linker_entry.py b/tools/splat/segtypes/linker_entry.py deleted file mode 100644 index fc9b873326..0000000000 --- a/tools/splat/segtypes/linker_entry.py +++ /dev/null @@ -1,641 +0,0 @@ -import os -import re -from functools import lru_cache -from pathlib import Path -from typing import Dict, List, OrderedDict, Set, Tuple, Union - -from util import options - -from segtypes.segment import Segment -from util.symbols import to_cname - - -# clean 'foo/../bar' to 'bar' -@lru_cache(maxsize=None) -def clean_up_path(path: Path) -> Path: - path_resolved = path.resolve() - base_resolved = options.opts.base_path.resolve() - try: - return path_resolved.relative_to(base_resolved) - except ValueError: - pass - - # If the path wasn't relative to the splat file, use the working directory instead - cwd = Path(os.getcwd()) - try: - return path_resolved.relative_to(cwd) - except ValueError: - pass - - # If it wasn't relative to that too, then just return the path as-is - return path - - -def path_to_object_path(path: Path) -> Path: - path = clean_up_path(path) - if options.opts.use_o_as_suffix: - full_suffix = ".o" - else: - full_suffix = path.suffix + ".o" - - if not str(path).startswith(str(options.opts.build_path)): - path = options.opts.build_path / path - return clean_up_path(path.with_suffix(full_suffix)) - - -def write_file_if_different(path: Path, new_content: str): - if path.exists(): - old_content = path.read_text() - else: - old_content = "" - - if old_content != new_content: - path.parent.mkdir(parents=True, exist_ok=True) - with path.open("w") as f: - f.write(new_content) - - -def get_segment_rom_start(cname: str) -> str: - if options.opts.segment_symbols_style == "makerom": - return f"_{cname}SegmentRomStart" - return f"{cname}_ROM_START" - - -def get_segment_rom_end(cname: str) -> str: - if options.opts.segment_symbols_style == "makerom": - return f"_{cname}SegmentRomEnd" - return f"{cname}_ROM_END" - - -def get_segment_vram_start(cname: str) -> str: - if options.opts.segment_symbols_style == "makerom": - return f"_{cname}SegmentStart" - return f"{cname}_VRAM" - - -def get_segment_vram_end(cname: str) -> str: - if options.opts.segment_symbols_style == "makerom": - return f"_{cname}SegmentEnd" - return f"{cname}_VRAM_END" - - -def convert_section_name_to_linker_format(section_type: str) -> str: - assert section_type.startswith(".") - if options.opts.segment_symbols_style == "makerom": - if section_type == ".rodata": - return "RoData" - return section_type[1:].capitalize() - - return to_cname(section_type.upper()) - - -def get_segment_section_start(segment_name: str, section_type: str) -> str: - sec = convert_section_name_to_linker_format(section_type) - if options.opts.segment_symbols_style == "makerom": - return f"_{segment_name}Segment{sec}Start" - return f"{segment_name}{sec}_START" - - -def get_segment_section_end(segment_name: str, section_type: str) -> str: - sec = convert_section_name_to_linker_format(section_type) - if options.opts.segment_symbols_style == "makerom": - return f"_{segment_name}Segment{sec}End" - return f"{segment_name}{sec}_END" - - -def get_segment_section_size(segment_name: str, section_type: str) -> str: - sec = convert_section_name_to_linker_format(section_type) - if options.opts.segment_symbols_style == "makerom": - return f"_{segment_name}Segment{sec}Size" - return f"{segment_name}{sec}_SIZE" - - -def get_segment_vram_end_symbol_name(segment: Segment) -> str: - return get_segment_vram_end(segment.get_cname()) - - -class LinkerEntry: - def __init__( - self, - segment: Segment, - src_paths: List[Path], - object_path: Path, - section_order: str, - section_link: str, - noload: bool = False, - ): - self.segment = segment - self.src_paths = [clean_up_path(p) for p in src_paths] - self.section_order = section_order - self.section_link = section_link - self.noload = noload - self.bss_contains_common = segment.bss_contains_common - if self.section_link == "linker" or self.section_link == "linker_offset": - self.object_path = None - elif self.segment.type == "lib": - self.object_path = object_path - else: - self.object_path = path_to_object_path(object_path) - - @property - def section_order_type(self) -> str: - if self.section_order == ".rdata": - return ".rodata" - else: - return self.section_order - - @property - def section_link_type(self) -> str: - if self.section_link == ".rdata": - return ".rodata" - else: - return self.section_link - - -class LinkerWriter: - def __init__(self, is_partial: bool = False): - self.linker_discard_section: bool = options.opts.ld_discard_section - self.sections_allowlist: List[str] = options.opts.ld_sections_allowlist - self.sections_denylist: List[str] = options.opts.ld_sections_denylist - # Used to store all the linker entries - build tools may want this information - self.entries: List[LinkerEntry] = [] - self.dependencies_entries: List[LinkerEntry] = [] - - self.buffer: List[str] = [] - self.header_symbols: Set[str] = set() - - self.is_partial: bool = is_partial - - self._indent_level = 0 - - self._writeln("SECTIONS") - self._begin_block() - - if not self.is_partial: - self._writeln(f"__romPos = {options.opts.ld_rom_start};") - - if options.opts.gp is not None: - self._writeln("_gp = " + f"0x{options.opts.gp:X};") - - # Write a series of statements which compute a symbol that represents the highest address among a list of segments' end addresses - def write_max_vram_end_sym(self, symbol: str, overlays: List[Segment]): - for segment in overlays: - if segment == overlays[0]: - self._writeln( - f"{symbol} = {get_segment_vram_end_symbol_name(segment)};" - ) - else: - self._writeln( - f"{symbol} = MAX({symbol}, {get_segment_vram_end_symbol_name(segment)});" - ) - - # Adds all the entries of a segment to the linker script buffer - def add(self, segment: Segment, max_vram_syms: List[Tuple[str, List[Segment]]]): - entries = segment.get_linker_entries() - self.entries.extend(entries) - self.dependencies_entries.extend(entries) - - seg_name = segment.get_cname() - - for sym, segs in max_vram_syms: - self.write_max_vram_end_sym(sym, segs) - - if options.opts.ld_legacy_generation: - self.add_legacy(segment, entries) - return - - section_entries: OrderedDict[str, List[LinkerEntry]] = OrderedDict() - for l in segment.section_order: - if l in options.opts.ld_section_labels: - section_entries[l] = [] - - # Add all entries to section_entries - prev_entry = None - for entry in entries: - if entry.section_order_type in section_entries: - # Search for the very first section type - # This is required in case the very first entry is a type that's not listed on ld_section_labels (like linker_offset) because it would be dropped - prev_entry = entry.section_order_type - break - - any_load = False - any_noload = False - for entry in entries: - if entry.section_order_type in section_entries: - section_entries[entry.section_order_type].append(entry) - elif prev_entry is not None: - # If this section is not present in section_order or ld_section_labels then pretend it is part of the last seen section, mainly for handling linker_offset - section_entries[prev_entry].append(entry) - any_load = any_load or not entry.noload - any_noload = any_noload or entry.noload - prev_entry = entry.section_order_type - - seg_rom_start = get_segment_rom_start(seg_name) - self._write_symbol(seg_rom_start, "__romPos") - - is_first = True - if any_load: - # Only emit normal segment if there's at least one normal entry - self._write_segment_sections( - segment, seg_name, section_entries, noload=False, is_first=is_first - ) - is_first = False - - if any_noload: - # Only emit NOLOAD segment if there is at least one noload entry - self._write_segment_sections( - segment, seg_name, section_entries, noload=True, is_first=is_first - ) - is_first = False - - self._end_segment(segment, all_bss=not any_load) - - def add_legacy(self, segment: Segment, entries: List[LinkerEntry]): - seg_name = segment.get_cname() - - # To keep track which sections has been started - started_sections: Dict[str, bool] = { - l: False for l in options.opts.ld_section_labels - } - - # Find where sections are last seen - last_seen_sections: Dict[LinkerEntry, str] = {} - for entry in reversed(entries): - if ( - entry.section_order_type in options.opts.ld_section_labels - and entry.section_order_type not in last_seen_sections.values() - ): - last_seen_sections[entry] = entry.section_order_type - - seg_rom_start = get_segment_rom_start(seg_name) - self._write_symbol(seg_rom_start, "__romPos") - - self._begin_segment(segment, seg_name, noload=False, is_first=True) - - i = 0 - for entry in entries: - if entry.noload: - break - - started = started_sections.get(entry.section_order_type, True) - if not started: - self._begin_section(seg_name, entry.section_order_type) - started_sections[entry.section_order_type] = True - - self._write_linker_entry(entry) - - if entry in last_seen_sections: - self._end_section(seg_name, entry.section_order_type) - - i += 1 - - if any(entry.noload for entry in entries): - self._end_block() - - self._begin_segment(segment, seg_name, noload=True, is_first=False) - - for entry in entries[i:]: - started = started_sections.get(entry.section_order_type, True) - if not started: - self._begin_section(seg_name, entry.section_order_type) - started_sections[entry.section_order_type] = True - - self._write_linker_entry(entry) - - if entry in last_seen_sections: - self._end_section(seg_name, entry.section_order_type) - - self._end_segment(segment, all_bss=False) - - def add_referenced_partial_segment( - self, segment: Segment, max_vram_syms: List[Tuple[str, List[Segment]]] - ): - entries = segment.get_linker_entries() - self.entries.extend(entries) - - segments_path = options.opts.ld_partial_build_segments_path - assert segments_path is not None - - seg_name = segment.get_cname() - - for sym, segs in max_vram_syms: - self.write_max_vram_end_sym(sym, segs) - - seg_rom_start = get_segment_rom_start(seg_name) - self._write_symbol(seg_rom_start, "__romPos") - - any_load = any(not e.noload for e in entries) - is_first = True - if any_load: - # Only emit normal segment if there's at least one normal entry - - self._begin_segment(segment, seg_name, noload=False, is_first=is_first) - - for l in segment.section_order: - if l not in options.opts.ld_section_labels: - continue - if l == ".bss": - continue - - entry = LinkerEntry( - segment, [], segments_path / f"{seg_name}.o", l, l, noload=False - ) - self.dependencies_entries.append(entry) - self._write_linker_entry(entry) - is_first = False - - if any(e.noload for e in entries): - # Only emit NOLOAD segment if there is at least one noload entry - - if not is_first: - self._end_block() - - self._begin_segment(segment, seg_name, noload=True, is_first=is_first) - - # Check if any section has the bss_contains_common option - bss_contains_common = False - for entry in entries: - if entry.segment.bss_contains_common: - bss_contains_common = True - break - - entry = LinkerEntry( - segment, - [], - segments_path / f"{seg_name}.o", - ".bss", - ".bss", - noload=True, - ) - entry.bss_contains_common = bss_contains_common - self.dependencies_entries.append(entry) - self._write_linker_entry(entry) - - self._end_segment(segment, all_bss=not any_load) - - def add_partial_segment(self, segment: Segment): - entries = segment.get_linker_entries() - self.entries.extend(entries) - self.dependencies_entries.extend(entries) - - seg_name = segment.get_cname() - - section_entries: OrderedDict[str, List[LinkerEntry]] = OrderedDict() - for l in segment.section_order: - if l in options.opts.ld_section_labels: - section_entries[l] = [] - - # Add all entries to section_entries - prev_entry = None - for entry in entries: - if entry.section_order_type in section_entries: - section_entries[entry.section_order_type].append(entry) - elif prev_entry is not None: - # If this section is not present in section_order or ld_section_labels then pretend it is part of the last seen section, mainly for handling linker_offset - section_entries[prev_entry].append(entry) - prev_entry = entry.section_order_type - - for section_name, entries in section_entries.items(): - if len(entries) == 0: - continue - first_entry = entries[0] - - self._begin_partial_segment(section_name, segment, first_entry.noload) - - self._begin_section(seg_name, section_name) - - for entry in entries: - self._write_linker_entry(entry) - - self._end_section(seg_name, section_name) - - self._end_partial_segment(section_name) - - def save_linker_script(self, output_path: Path): - if len(self.sections_allowlist) > 0: - address = " 0" - if self.is_partial: - address = "" - for sect in self.sections_allowlist: - self._writeln(f"{sect}{address} :") - self._begin_block() - self._writeln(f"*({sect});") - self._end_block() - - self._writeln("") - - if self.linker_discard_section or len(self.sections_denylist) > 0: - self._writeln("/DISCARD/ :") - self._begin_block() - for sect in self.sections_denylist: - self._writeln(f"*({sect});") - if self.linker_discard_section: - self._writeln("*(*);") - self._end_block() - - self._end_block() # SECTIONS - - assert self._indent_level == 0 - - write_file_if_different(output_path, "\n".join(self.buffer) + "\n") - - def save_symbol_header(self): - path = options.opts.ld_symbol_header_path - - if path: - write_file_if_different( - path, - "#ifndef _HEADER_SYMBOLS_H_\n" - "#define _HEADER_SYMBOLS_H_\n" - "\n" - '#include "common.h"\n' - "\n" - + "".join( - f"extern Addr {symbol};\n" for symbol in sorted(self.header_symbols) - ) - + "\n" - "#endif\n", - ) - - def save_dependencies_file(self, output_path: Path, target_elf_path: Path): - output = f"{target_elf_path}:" - - for entry in self.dependencies_entries: - if entry.object_path is None: - continue - output += f" \\\n {entry.object_path}" - - output += "\n" - for entry in self.dependencies_entries: - if entry.object_path is None: - continue - output += f"{entry.object_path}:\n" - write_file_if_different(output_path, output) - - def _writeln(self, line: str): - if len(line) == 0: - self.buffer.append(line) - else: - self.buffer.append(" " * self._indent_level + line) - - def _begin_block(self): - self._writeln("{") - self._indent_level += 1 - - def _end_block(self): - self._indent_level -= 1 - self._writeln("}") - - def _write_symbol(self, symbol: str, value: Union[str, int]): - symbol = to_cname(symbol) - - if isinstance(value, int): - value = f"0x{value:X}" - - self._writeln(f"{symbol} = {value};") - - self.header_symbols.add(symbol) - - def _begin_segment( - self, segment: Segment, seg_name: str, noload: bool, is_first: bool - ): - if ( - options.opts.ld_use_symbolic_vram_addresses - and segment.vram_symbol is not None - ): - vram_str = segment.vram_symbol + " " - else: - vram_str = ( - f"0x{segment.vram_start:X} " - if isinstance(segment.vram_start, int) - else "" - ) - - addr_str = " " - if is_first: - addr_str += f"{vram_str}" - if noload: - seg_name += "_bss" - addr_str += "(NOLOAD) " - - seg_vram_start = get_segment_vram_start(seg_name) - self._write_symbol(seg_vram_start, f"ADDR(.{seg_name})") - - line = f".{seg_name}{addr_str}:" - if not noload: - seg_rom_start = get_segment_rom_start(seg_name) - line += f" AT({seg_rom_start})" - if segment.subalign != None: - line += f" SUBALIGN({segment.subalign})" - - self._writeln(line) - self._begin_block() - - if segment.ld_fill_value is not None: - self._writeln(f"FILL(0x{segment.ld_fill_value:08X});") - - def _end_segment(self, segment: Segment, all_bss=False): - self._end_block() - - name = segment.get_cname() - - if not all_bss: - self._writeln(f"__romPos += SIZEOF(.{name});") - - # Align directive - if not options.opts.segment_end_before_align: - if segment.align: - self._writeln(f"__romPos = ALIGN(__romPos, {segment.align});") - - seg_rom_end = get_segment_rom_end(name) - self._write_symbol(seg_rom_end, "__romPos") - self._write_symbol(get_segment_vram_end_symbol_name(segment), ".") - - # Align directive - if options.opts.segment_end_before_align: - if segment.align: - self._writeln(f"__romPos = ALIGN(__romPos, {segment.align});") - - self._writeln("") - - def _begin_partial_segment(self, section_name: str, segment: Segment, noload: bool): - line = f"{section_name}" - if noload: - line += " (NOLOAD)" - line += " :" - if segment.subalign != None: - line += f" SUBALIGN({segment.subalign})" - - self._writeln(line) - self._begin_block() - - def _end_partial_segment(self, section_name: str, all_bss=False): - self._end_block() - - self._writeln("") - - def _begin_section(self, seg_name: str, cur_section: str) -> None: - section_start = get_segment_section_start(seg_name, cur_section) - self._write_symbol(section_start, ".") - - def _end_section(self, seg_name: str, cur_section: str) -> None: - section_start = get_segment_section_start(seg_name, cur_section) - section_end = get_segment_section_end(seg_name, cur_section) - section_size = get_segment_section_size(seg_name, cur_section) - self._write_symbol(section_end, ".") - self._write_symbol( - section_size, - f"ABSOLUTE({section_end} - {section_start})", - ) - - def _write_linker_entry(self, entry: LinkerEntry): - if entry.section_link_type == "linker_offset": - self._write_symbol(f"{entry.segment.get_cname()}_OFFSET", ".") - return - - # TODO: option to turn this off? - if ( - entry.object_path - and entry.section_link_type == ".data" - and entry.segment.type != "lib" - ): - path_cname = re.sub( - r"[^0-9a-zA-Z_]", - "_", - str(entry.segment.dir / entry.segment.name) - + ".".join(entry.object_path.suffixes[:-1]), - ) - self._write_symbol(path_cname, ".") - - if entry.noload and entry.bss_contains_common: - self._writeln(f"{entry.object_path}(.bss COMMON .scommon);") - else: - wildcard = "*" if options.opts.ld_wildcard_sections else "" - - self._writeln(f"{entry.object_path}({entry.section_link}{wildcard});") - - def _write_segment_sections( - self, - segment: Segment, - seg_name: str, - section_entries: OrderedDict[str, List[LinkerEntry]], - noload: bool, - is_first: bool, - ): - if not is_first: - self._end_block() - - self._begin_segment(segment, seg_name, noload=noload, is_first=is_first) - - for section_name, entries in section_entries.items(): - if len(entries) == 0: - continue - - first_entry = entries[0] - if first_entry.noload != noload: - continue - - self._begin_section(seg_name, section_name) - for entry in entries: - self._write_linker_entry(entry) - self._end_section(seg_name, section_name) diff --git a/tools/splat/segtypes/n64/__init__.py b/tools/splat/segtypes/n64/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/splat/segtypes/n64/asm.py b/tools/splat/segtypes/n64/asm.py deleted file mode 100644 index 4b93b2c3b0..0000000000 --- a/tools/splat/segtypes/n64/asm.py +++ /dev/null @@ -1,28 +0,0 @@ -from util import options - -from segtypes.common.asm import CommonSegAsm - - -class N64SegAsm(CommonSegAsm): - @staticmethod - def get_file_header(): - ret = [] - - ret.append('.include "macro.inc"') - ret.append("") - ret.append("/* assembler directives */") - ret.append(".set noat /* allow manual use of $at */") - ret.append(".set noreorder /* don't insert nops after branches */") - if options.opts.add_set_gp_64: - ret.append( - ".set gp=64 /* allow use of 64-bit general purpose registers */" - ) - ret.append("") - preamble = options.opts.generated_s_preamble - if preamble: - ret.append(preamble) - ret.append("") - ret.append('.section .text, "ax"') - ret.append("") - - return ret diff --git a/tools/splat/segtypes/n64/ci.py b/tools/splat/segtypes/n64/ci.py deleted file mode 100644 index 16756bf8d9..0000000000 --- a/tools/splat/segtypes/n64/ci.py +++ /dev/null @@ -1,42 +0,0 @@ -from typing import Optional, TYPE_CHECKING - -from util import log - -from segtypes.n64.img import N64SegImg - -if TYPE_CHECKING: - from segtypes.n64.palette import N64SegPalette - - -# Base class for CI4/CI8 -class N64SegCi(N64SegImg): - def parse_palette_name(self, yaml, args) -> str: - ret = self.name - if isinstance(yaml, dict): - if "palette" in yaml: - ret = yaml["palette"] - elif len(args) > 2: - ret = args[2] - - return ret - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.palette: "Optional[N64SegPalette]" = None - self.palette_name = self.parse_palette_name(self.yaml, self.args) - - def scan(self, rom_bytes: bytes) -> None: - self.n64img.data = rom_bytes[self.rom_start : self.rom_end] - - def split(self, rom_bytes): - if self.palette is None: - # TODO: output with blank palette - log.error( - f"no palette sibling segment exists\n(hint: add a segment with type 'palette' and name '{self.name}')" - ) - assert self.palette is not None - self.palette.extract = False - self.n64img.palette = self.palette.parse_palette(rom_bytes) - - super().split(rom_bytes) diff --git a/tools/splat/segtypes/n64/ci4.py b/tools/splat/segtypes/n64/ci4.py deleted file mode 100644 index 83258816e4..0000000000 --- a/tools/splat/segtypes/n64/ci4.py +++ /dev/null @@ -1,8 +0,0 @@ -import n64img.image - -from segtypes.n64.ci import N64SegCi - - -class N64SegCi4(N64SegCi): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs, img_cls=n64img.image.CI4) diff --git a/tools/splat/segtypes/n64/ci8.py b/tools/splat/segtypes/n64/ci8.py deleted file mode 100644 index dbf66cdff5..0000000000 --- a/tools/splat/segtypes/n64/ci8.py +++ /dev/null @@ -1,8 +0,0 @@ -import n64img.image - -from segtypes.n64.ci import N64SegCi - - -class N64SegCi8(N64SegCi): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs, img_cls=n64img.image.CI8) diff --git a/tools/splat/segtypes/n64/gfx.py b/tools/splat/segtypes/n64/gfx.py deleted file mode 100644 index 45f4a21cac..0000000000 --- a/tools/splat/segtypes/n64/gfx.py +++ /dev/null @@ -1,277 +0,0 @@ -""" -N64 f3dex display list splitter -Dumps out Gfx[] as a .inc.c file. -""" - -import re -from typing import Optional - -from pathlib import Path - -from pygfxd import ( - gfxd_buffer_to_string, - gfxd_cimg_callback, - gfxd_dl_callback, - gfxd_endian, - gfxd_execute, - gfxd_input_buffer, - gfxd_light_callback, - gfxd_lookat_callback, - gfxd_macro_dflt, - gfxd_macro_fn, - gfxd_mtx_callback, - gfxd_output_buffer, - gfxd_printf, - gfxd_puts, - gfxd_target, - gfxd_timg_callback, - gfxd_tlut_callback, - gfxd_vp_callback, - gfxd_vtx_callback, - gfxd_zimg_callback, - GfxdEndian, - gfxd_f3d, - gfxd_f3db, - gfxd_f3dex, - gfxd_f3dexb, - gfxd_f3dex2, -) -from segtypes.segment import Segment - -from util import log, options -from util.log import error - -from segtypes.common.codesubsegment import CommonSegCodeSubsegment - -from util import symbols - -LIGHTS_RE = re.compile(r"\*\(Lightsn \*\)0x[0-9A-F]{8}") - - -class N64SegGfx(CommonSegCodeSubsegment): - def __init__( - self, - rom_start: Optional[int], - rom_end: Optional[int], - type: str, - name: str, - vram_start: Optional[int], - args: list, - yaml, - ): - super().__init__( - rom_start, - rom_end, - type, - name, - vram_start, - args=args, - yaml=yaml, - ) - self.file_text = None - self.data_only = isinstance(yaml, dict) and yaml.get("data_only", False) - self.in_segment = not isinstance(yaml, dict) or yaml.get("in_segment", True) - - def format_sym_name(self, sym) -> str: - return sym.name - - def get_linker_section(self) -> str: - return ".data" - - def out_path(self) -> Path: - return options.opts.asset_path / self.dir / f"{self.name}.gfx.inc.c" - - def scan(self, rom_bytes: bytes): - self.file_text = self.disassemble_data(rom_bytes) - - def get_gfxd_target(self): - opt = options.opts.gfx_ucode - - if opt == "f3d": - return gfxd_f3d - elif opt == "f3db": - return gfxd_f3db - elif opt == "f3dex": - return gfxd_f3dex - elif opt == "f3dexb": - return gfxd_f3dexb - elif opt == "f3dex2": - return gfxd_f3dex2 - else: - log.error(f"Unknown target {opt}") - - def tlut_handler(self, addr, idx, count): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(self.format_sym_name(sym)) - return 1 - - def timg_handler(self, addr, fmt, size, width, height, pal): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(self.format_sym_name(sym)) - return 1 - - def cimg_handler(self, addr, fmt, size, width): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(self.format_sym_name(sym)) - return 1 - - def zimg_handler(self, addr): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(self.format_sym_name(sym)) - return 1 - - def dl_handler(self, addr): - # Look for 'Gfx'-typed symbols first - sym = self.retrieve_sym_type(symbols.all_symbols_dict, addr, "Gfx") - - if not sym: - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(self.format_sym_name(sym)) - return 1 - - def mtx_handler(self, addr): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(f"&{self.format_sym_name(sym)}") - return 1 - - def lookat_handler(self, addr, count): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(self.format_sym_name(sym)) - return 1 - - def light_handler(self, addr, count): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(self.format_sym_name(sym)) - return 1 - - def vtx_handler(self, addr, count): - # Look for 'Vtx'-typed symbols first - sym = self.retrieve_sym_type(symbols.all_symbols_dict, addr, "Vtx") - - if not sym: - sym = self.create_symbol( - addr=addr, - in_segment=self.in_segment, - type="Vtx", - reference=True, - search_ranges=True, - ) - - index = int((addr - sym.vram_start) / 0x10) - gfxd_printf(f"&{self.format_sym_name(sym)}[{index}]") - return 1 - - def vp_handler(self, addr): - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - gfxd_printf(f"&{self.format_sym_name(sym)}") - return 1 - - def macro_fn(self): - gfxd_puts(" ") - gfxd_macro_dflt() - gfxd_puts(",\n") - return 0 - - def disassemble_data(self, rom_bytes): - assert isinstance(self.rom_start, int) - assert isinstance(self.rom_end, int) - assert isinstance(self.vram_start, int) - - gfx_data = rom_bytes[self.rom_start : self.rom_end] - segment_length = len(gfx_data) - if (segment_length) % 8 != 0: - error( - f"Error: gfx segment {self.name} length ({segment_length}) is not a multiple of 8!" - ) - - out_str = "" if self.data_only else options.opts.generated_c_preamble + "\n\n" - - sym = self.create_symbol( - addr=self.vram_start, in_segment=True, type="data", define=True - ) - - gfxd_input_buffer(gfx_data) - - # TODO terrible guess at the size we'll need - improve this - outb = bytes([0] * segment_length * 100) - outbuf = gfxd_output_buffer(outb, len(outb)) - - gfxd_target(self.get_gfxd_target()) - gfxd_endian( - GfxdEndian.big if options.opts.endianness == "big" else GfxdEndian.little, 4 - ) - - # Callbacks - gfxd_macro_fn(self.macro_fn) - - gfxd_tlut_callback(self.tlut_handler) - gfxd_timg_callback(self.timg_handler) - gfxd_cimg_callback(self.cimg_handler) - gfxd_zimg_callback(self.zimg_handler) - gfxd_dl_callback(self.dl_handler) - gfxd_mtx_callback(self.mtx_handler) - gfxd_lookat_callback(self.lookat_handler) - gfxd_light_callback(self.light_handler) - # gfxd_seg_callback ? - gfxd_vtx_callback(self.vtx_handler) - gfxd_vp_callback(self.vp_handler) - # gfxd_uctext_callback ? - # gfxd_ucdata_callback ? - # gfxd_dram_callback ? - - gfxd_execute() - - if self.data_only: - out_str += gfxd_buffer_to_string(outbuf) - else: - out_str += "Gfx " + self.format_sym_name(sym) + "[] = {\n" - out_str += gfxd_buffer_to_string(outbuf) - out_str += "};\n" - - # Poor man's light fix until we get my libgfxd PR merged - def light_sub_func(match): - light = match.group(0) - addr = int(light[12:], 0) - sym = self.create_symbol( - addr=addr, in_segment=self.in_segment, type="data", reference=True - ) - return self.format_sym_name(sym) - - out_str = re.sub(LIGHTS_RE, light_sub_func, out_str) - - return out_str - - def split(self, rom_bytes: bytes): - if self.file_text and self.out_path(): - self.out_path().parent.mkdir(parents=True, exist_ok=True) - - with open(self.out_path(), "w", newline="\n") as f: - f.write(self.file_text) - - def should_scan(self) -> bool: - return ( - options.opts.is_mode_active("gfx") - and self.rom_start is not None - and self.rom_end is not None - ) - - def should_split(self) -> bool: - return self.extract and options.opts.is_mode_active("gfx") diff --git a/tools/splat/segtypes/n64/hasm.py b/tools/splat/segtypes/n64/hasm.py deleted file mode 100644 index 18def923fd..0000000000 --- a/tools/splat/segtypes/n64/hasm.py +++ /dev/null @@ -1,28 +0,0 @@ -from util import options - -from segtypes.common.hasm import CommonSegHasm - - -class N64SegHasm(CommonSegHasm): - @staticmethod - def get_file_header(): - ret = [] - - ret.append('.include "macro.inc"') - ret.append("") - ret.append("/* assembler directives */") - ret.append(".set noat /* allow manual use of $at */") - ret.append(".set noreorder /* don't insert nops after branches */") - if options.opts.add_set_gp_64: - ret.append( - ".set gp=64 /* allow use of 64-bit general purpose registers */" - ) - ret.append("") - preamble = options.opts.generated_s_preamble - if preamble: - ret.append(preamble) - ret.append("") - ret.append('.section .text, "ax"') - ret.append("") - - return ret diff --git a/tools/splat/segtypes/n64/header.py b/tools/splat/segtypes/n64/header.py deleted file mode 100644 index 96c59d5d4c..0000000000 --- a/tools/splat/segtypes/n64/header.py +++ /dev/null @@ -1,50 +0,0 @@ -from util import options - -from segtypes.common.header import CommonSegHeader - - -class N64SegHeader(CommonSegHeader): - def parse_header(self, rom_bytes): - encoding = options.opts.header_encoding - - header_lines = [] - header_lines.append(".section .data\n") - header_lines.append( - self.get_line("word", rom_bytes[0x00:0x04], "PI BSB Domain 1 register") - ) - header_lines.append( - self.get_line("word", rom_bytes[0x04:0x08], "Clockrate setting") - ) - header_lines.append( - self.get_line("word", rom_bytes[0x08:0x0C], "Entrypoint address") - ) - header_lines.append(self.get_line("word", rom_bytes[0x0C:0x10], "Revision")) - header_lines.append(self.get_line("word", rom_bytes[0x10:0x14], "Checksum 1")) - header_lines.append(self.get_line("word", rom_bytes[0x14:0x18], "Checksum 2")) - header_lines.append(self.get_line("word", rom_bytes[0x18:0x1C], "Unknown 1")) - header_lines.append(self.get_line("word", rom_bytes[0x1C:0x20], "Unknown 2")) - - if encoding != "word": - header_lines.append( - '.ascii "' - + rom_bytes[0x20:0x34].decode(encoding).strip().ljust(20) - + '" /* Internal name */' - ) - else: - for i in range(0x20, 0x34, 4): - header_lines.append( - self.get_line("word", rom_bytes[i : i + 4], "Internal name") - ) - - header_lines.append(self.get_line("word", rom_bytes[0x34:0x38], "Unknown 3")) - header_lines.append(self.get_line("word", rom_bytes[0x38:0x3C], "Cartridge")) - header_lines.append( - self.get_line("ascii", rom_bytes[0x3C:0x3E], "Cartridge ID") - ) - header_lines.append( - self.get_line("ascii", rom_bytes[0x3E:0x3F], "Country code") - ) - header_lines.append(self.get_line("byte", rom_bytes[0x3F:0x40], "Version")) - header_lines.append("") - - return header_lines diff --git a/tools/splat/segtypes/n64/i1.py b/tools/splat/segtypes/n64/i1.py deleted file mode 100644 index c293099938..0000000000 --- a/tools/splat/segtypes/n64/i1.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegI1(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.I1 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/i4.py b/tools/splat/segtypes/n64/i4.py deleted file mode 100644 index 16a68c5928..0000000000 --- a/tools/splat/segtypes/n64/i4.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegI4(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.I4 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/i8.py b/tools/splat/segtypes/n64/i8.py deleted file mode 100644 index c2364f2b7e..0000000000 --- a/tools/splat/segtypes/n64/i8.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegI8(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.I8 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/ia16.py b/tools/splat/segtypes/n64/ia16.py deleted file mode 100644 index f2884728f7..0000000000 --- a/tools/splat/segtypes/n64/ia16.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegIa16(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.IA16 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/ia4.py b/tools/splat/segtypes/n64/ia4.py deleted file mode 100644 index 0eef15b3ee..0000000000 --- a/tools/splat/segtypes/n64/ia4.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegIa4(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.IA4 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/ia8.py b/tools/splat/segtypes/n64/ia8.py deleted file mode 100644 index 186be5e115..0000000000 --- a/tools/splat/segtypes/n64/ia8.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegIa8(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.IA8 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/img.py b/tools/splat/segtypes/n64/img.py deleted file mode 100644 index dd4e32b6cb..0000000000 --- a/tools/splat/segtypes/n64/img.py +++ /dev/null @@ -1,99 +0,0 @@ -from pathlib import Path -from typing import Dict, List, Tuple, Type, Optional, Union - -from n64img.image import Image -from util import log, options - -from segtypes.n64.segment import N64Segment - - -class N64SegImg(N64Segment): - @staticmethod - def parse_dimensions(yaml: Union[Dict, List]) -> Tuple[int, int]: - if isinstance(yaml, dict): - return yaml["width"], yaml["height"] - else: - if len(yaml) < 5: - log.error(f"Error: {yaml} is missing width and height parameters") - return yaml[3], yaml[4] - - def __init__( - self, - rom_start: Optional[int], - rom_end: Optional[int], - type: str, - name: str, - vram_start: Optional[int], - args: list, - yaml, - img_cls: Type[Image], - ): - super().__init__( - rom_start, - rom_end, - type, - name, - vram_start, - args=args, - yaml=yaml, - ) - - if rom_start is None: - log.error(f"Error: {type} segment {name} rom start could not be determined") - - self.n64img: Image = img_cls(b"", 0, 0) - - if isinstance(yaml, dict): - self.n64img.flip_h = bool(yaml.get("flip_x", False)) - self.n64img.flip_v = bool(yaml.get("flip_y", False)) - - self.width, self.height = self.parse_dimensions(yaml) - - self.n64img.width = self.width - self.n64img.height = self.height - - self.check_len() - - def check_len(self) -> None: - expected_len = int(self.n64img.size()) - assert isinstance(self.rom_start, int) - assert isinstance(self.rom_end, int) - assert isinstance(self.subalign, int) - actual_len = self.rom_end - self.rom_start - if actual_len > expected_len and actual_len - expected_len > self.subalign: - log.error( - f"Error: {self.name} should end at 0x{self.rom_start + expected_len:X}, but it ends at 0x{self.rom_end:X}\n(hint: add a 'bin' segment after it)" - ) - - def out_path(self) -> Path: - return options.opts.asset_path / self.dir / f"{self.name}.png" - - def should_split(self) -> bool: - return options.opts.is_mode_active("img") - - def split(self, rom_bytes): - path = self.out_path() - path.parent.mkdir(parents=True, exist_ok=True) - - assert isinstance(self.rom_start, int) - assert isinstance(self.rom_end, int) - - if self.n64img.data == b"": - self.n64img.data = rom_bytes[self.rom_start : self.rom_end] - self.n64img.write(path) - - self.log(f"Wrote {self.name} to {path}") - - @staticmethod - def estimate_size(yaml: Union[Dict, List]) -> int: - width, height = N64SegImg.parse_dimensions(yaml) - typ = N64Segment.parse_segment_type(yaml) - - if typ == "ci4" or typ == "i4" or typ == "ia4": - return width * height // 2 - elif typ in ("ia16", "rgba16"): - return width * height * 2 - elif typ == "rgba32": - return width * height * 4 - else: - return width * height diff --git a/tools/splat/segtypes/n64/ipl3.py b/tools/splat/segtypes/n64/ipl3.py deleted file mode 100644 index 6a76cb34eb..0000000000 --- a/tools/splat/segtypes/n64/ipl3.py +++ /dev/null @@ -1,9 +0,0 @@ -from segtypes.common.code import CommonSegCode -from segtypes.common.hasm import CommonSegHasm - - -class N64SegIpl3(CommonSegHasm): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.special_vram_segment = True diff --git a/tools/splat/segtypes/n64/linker_offset.py b/tools/splat/segtypes/n64/linker_offset.py deleted file mode 100644 index b13a190d1c..0000000000 --- a/tools/splat/segtypes/n64/linker_offset.py +++ /dev/null @@ -1,14 +0,0 @@ -from pathlib import Path - -from segtypes.n64.segment import N64Segment - - -class N64SegLinker_offset(N64Segment): - def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry - - return [ - LinkerEntry( - self, [], Path(self.name), "linker_offset", "linker_offset", False - ) - ] diff --git a/tools/splat/segtypes/n64/mio0.py b/tools/splat/segtypes/n64/mio0.py deleted file mode 100644 index 5ad4635502..0000000000 --- a/tools/splat/segtypes/n64/mio0.py +++ /dev/null @@ -1,10 +0,0 @@ -from util.n64.Mio0decompress import Mio0Decompressor - -from segtypes.common.decompressor import CommonSegDecompressor - - -class N64SegMio0(CommonSegDecompressor): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.decompressor = Mio0Decompressor() - self.compression_type = "Mio0" diff --git a/tools/splat/segtypes/n64/palette.py b/tools/splat/segtypes/n64/palette.py deleted file mode 100644 index 7ede8d0d1f..0000000000 --- a/tools/splat/segtypes/n64/palette.py +++ /dev/null @@ -1,112 +0,0 @@ -from itertools import zip_longest -from pathlib import Path -from typing import Dict, List, Optional, Tuple, TYPE_CHECKING, Union - -from util import log, options -from util.color import unpack_color - -from segtypes.n64.segment import N64Segment -from util.symbols import to_cname - -if TYPE_CHECKING: - from segtypes.n64.ci import N64SegCi as Raster - - -def iter_in_groups(iterable, n, fillvalue=None): - args = [iter(iterable)] * n - return zip_longest(*args, fillvalue=fillvalue) - - -VALID_SIZES = [0x20, 0x40, 0x80, 0x100, 0x200] - - -class N64SegPalette(N64Segment): - require_unique_name = False - - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.raster: "Optional[Raster]" = None - - # palette segments must be named as one of the following: - # 1) same as the relevant raster segment name (max. 1 palette) - # 2) relevant raster segment name + "." + unique palette name - # 3) unique, referencing the relevant raster segment using `raster_name` - self.raster_name = ( - self.yaml.get("raster_name", self.name.split(".")[0]) - if isinstance(self.yaml, dict) - else self.name.split(".")[0] - ) - - if self.extract: - if self.rom_end is None: - log.error( - f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it" - ) - - if not isinstance(self.yaml, dict) or "size" not in self.yaml: - assert self.rom_end is not None - assert self.rom_start is not None - actual_len = self.rom_end - self.rom_start - - hint_msg = "(hint: add a 'bin' segment after it or specify the size in the segment)" - - if actual_len > VALID_SIZES[-1]: - log.error( - f"Error: {self.name} (0x{actual_len:X} bytes) is too long, max 0x{VALID_SIZES[-1]:X})\n{hint_msg}" - ) - - if actual_len not in VALID_SIZES: - log.error( - f"Error: {self.name} (0x{actual_len:X} bytes) is not a valid palette size ({', '.join(hex(s) for s in VALID_SIZES)})\n{hint_msg}" - ) - - def get_cname(self) -> str: - return super().get_cname() + "_pal" - - def split(self, rom_bytes): - if self.raster is None: - # TODO: output with no raster - log.error(f"orphaned palette segment: {self.name} lacks ci4/ci8 sibling") - - assert self.raster is not None - self.raster.n64img.palette = self.parse_palette(rom_bytes) # type: ignore - - self.raster.n64img.write(self.out_path()) - self.raster.extract = False - - def parse_palette(self, rom_bytes) -> List[Tuple[int, int, int, int]]: - data = rom_bytes[self.rom_start : self.rom_end] - palette = [] - - for a, b in iter_in_groups(data, 2): - palette.append(unpack_color([a, b])) - - return palette - - def out_path(self) -> Path: - return options.opts.asset_path / self.dir / f"{self.name}.png" - - def should_split(self) -> bool: - return self.extract and options.opts.is_mode_active("img") - - def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry - - return [ - LinkerEntry( - self, - [options.opts.asset_path / self.dir / f"{self.name}.png"], - options.opts.asset_path / self.dir / f"{self.name}.pal", - self.get_linker_section_order(), - self.get_linker_section_linksection(), - self.is_noload(), - ) - ] - - @staticmethod - def estimate_size(yaml: Union[Dict, List]) -> int: - if isinstance(yaml, dict): - if "size" in yaml: - return int(yaml["size"]) - return 0x20 diff --git a/tools/splat/segtypes/n64/rgba16.py b/tools/splat/segtypes/n64/rgba16.py deleted file mode 100644 index 4b6f4fdea7..0000000000 --- a/tools/splat/segtypes/n64/rgba16.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegRgba16(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.RGBA16 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/rgba32.py b/tools/splat/segtypes/n64/rgba32.py deleted file mode 100644 index ae3d5383e4..0000000000 --- a/tools/splat/segtypes/n64/rgba32.py +++ /dev/null @@ -1,9 +0,0 @@ -import n64img.image - -from segtypes.n64.img import N64SegImg - - -class N64SegRgba32(N64SegImg): - def __init__(self, *args, **kwargs): - kwargs["img_cls"] = n64img.image.RGBA32 - super().__init__(*args, **kwargs) diff --git a/tools/splat/segtypes/n64/rsp.py b/tools/splat/segtypes/n64/rsp.py deleted file mode 100644 index f69123a457..0000000000 --- a/tools/splat/segtypes/n64/rsp.py +++ /dev/null @@ -1,10 +0,0 @@ -import rabbitizer - -from segtypes.common.hasm import CommonSegHasm - - -class N64SegRsp(CommonSegHasm): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - - self.instr_category = rabbitizer.InstrCategory.RSP diff --git a/tools/splat/segtypes/n64/segment.py b/tools/splat/segtypes/n64/segment.py deleted file mode 100644 index 8f9c3cccd3..0000000000 --- a/tools/splat/segtypes/n64/segment.py +++ /dev/null @@ -1,5 +0,0 @@ -from segtypes.segment import Segment - - -class N64Segment(Segment): - pass diff --git a/tools/splat/segtypes/n64/vtx.py b/tools/splat/segtypes/n64/vtx.py deleted file mode 100644 index e0d1bc92f0..0000000000 --- a/tools/splat/segtypes/n64/vtx.py +++ /dev/null @@ -1,102 +0,0 @@ -""" -N64 Vtx struct splitter -Dumps out Vtx as a .inc.c file. - -Originally written by Mark Street (https://github.com/mkst) -""" - -import struct -from pathlib import Path -from typing import Optional - -from util import options, log - -from segtypes.common.codesubsegment import CommonSegCodeSubsegment - - -class N64SegVtx(CommonSegCodeSubsegment): - def __init__( - self, - rom_start: Optional[int], - rom_end: Optional[int], - type: str, - name: str, - vram_start: Optional[int], - args: list, - yaml, - ): - super().__init__( - rom_start, - rom_end, - type, - name, - vram_start, - args=args, - yaml=yaml, - ) - self.file_text: Optional[str] = None - self.data_only = isinstance(yaml, dict) and yaml.get("data_only", False) - - def format_sym_name(self, sym) -> str: - return sym.name - - def get_linker_section(self) -> str: - return ".data" - - def out_path(self) -> Path: - return options.opts.asset_path / self.dir / f"{self.name}.vtx.inc.c" - - def scan(self, rom_bytes: bytes): - self.file_text = self.disassemble_data(rom_bytes) - - def disassemble_data(self, rom_bytes) -> str: - assert isinstance(self.rom_start, int) - assert isinstance(self.rom_end, int) - assert isinstance(self.vram_start, int) - - vertex_data = rom_bytes[self.rom_start : self.rom_end] - segment_length = len(vertex_data) - if (segment_length) % 16 != 0: - log.error( - f"Error: Vtx segment {self.name} length ({segment_length}) is not a multiple of 16!" - ) - - lines = [] - if not self.data_only: - lines.append(options.opts.generated_c_preamble) - lines.append("") - - vertex_count = segment_length // 16 - sym = self.create_symbol( - addr=self.vram_start, in_segment=True, type="data", define=True - ) - - if not self.data_only: - lines.append(f"Vtx {self.format_sym_name(sym)}[{vertex_count}] = {{") - - for vtx in struct.iter_unpack(">hhhHhhBBBB", vertex_data): - x, y, z, flg, t, c, r, g, b, a = vtx - vtx_string = f" {{{{{{ {x:5}, {y:5}, {z:5} }}, {flg}, {{ {t:5}, {c:5} }}, {{ {r:3}, {g:3}, {b:3}, {a:3} }}}}}}," - if flg != 0: - self.warn(f"Non-zero flag found in vertex data {self.name}!") - lines.append(vtx_string) - - if not self.data_only: - lines.append("};") - - # enforce newline at end of file - lines.append("") - return "\n".join(lines) - - def split(self, rom_bytes: bytes): - if self.file_text and self.out_path(): - self.out_path().parent.mkdir(parents=True, exist_ok=True) - - with open(self.out_path(), "w", newline="\n") as f: - f.write(self.file_text) - - def should_scan(self) -> bool: - return options.opts.is_mode_active("vtx") - - def should_split(self) -> bool: - return self.extract and options.opts.is_mode_active("vtx") diff --git a/tools/splat/segtypes/n64/yay0.py b/tools/splat/segtypes/n64/yay0.py deleted file mode 100644 index 210f7611a9..0000000000 --- a/tools/splat/segtypes/n64/yay0.py +++ /dev/null @@ -1,10 +0,0 @@ -from util.n64.Yay0decompress import Yay0Decompressor - -from segtypes.common.decompressor import CommonSegDecompressor - - -class N64SegYay0(CommonSegDecompressor): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.decompressor = Yay0Decompressor() - self.compression_type = "Yay0" diff --git a/tools/splat/segtypes/ps2/__init__.py b/tools/splat/segtypes/ps2/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/splat/segtypes/ps2/asm.py b/tools/splat/segtypes/ps2/asm.py deleted file mode 100644 index 53e4a44920..0000000000 --- a/tools/splat/segtypes/ps2/asm.py +++ /dev/null @@ -1,23 +0,0 @@ -from util import options - -from segtypes.common.asm import CommonSegAsm - - -class Ps2SegAsm(CommonSegAsm): - @staticmethod - def get_file_header(): - ret = [] - - ret.append('.include "macro.inc"') - ret.append("") - ret.append(".set noat") - ret.append(".set noreorder") - ret.append("") - preamble = options.opts.generated_s_preamble - if preamble: - ret.append(preamble) - ret.append("") - ret.append('.section .text, "ax"') - ret.append("") - - return ret diff --git a/tools/splat/segtypes/psx/__init__.py b/tools/splat/segtypes/psx/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/splat/segtypes/psx/asm.py b/tools/splat/segtypes/psx/asm.py deleted file mode 100644 index 2e9ff5ea50..0000000000 --- a/tools/splat/segtypes/psx/asm.py +++ /dev/null @@ -1,23 +0,0 @@ -from util import options - -from segtypes.common.asm import CommonSegAsm - - -class PsxSegAsm(CommonSegAsm): - @staticmethod - def get_file_header(): - ret = [] - - ret.append('.include "macro.inc"') - ret.append("") - ret.append(".set noat") - ret.append(".set noreorder") - ret.append("") - preamble = options.opts.generated_s_preamble - if preamble: - ret.append(preamble) - ret.append("") - ret.append('.section .text, "ax"') - ret.append("") - - return ret diff --git a/tools/splat/segtypes/psx/header.py b/tools/splat/segtypes/psx/header.py deleted file mode 100644 index aeda3ee2b2..0000000000 --- a/tools/splat/segtypes/psx/header.py +++ /dev/null @@ -1,61 +0,0 @@ -from segtypes.common.header import CommonSegHeader - - -class PsxSegHeader(CommonSegHeader): - # little endian so reverse words, TODO: use struct.unpack(" Optional[int]: - if isinstance(segment, dict) and "vram" in segment: - return int(segment["vram"]) - else: - return None - - -def parse_segment_vram_symbol(segment: Union[dict, list]) -> Optional[str]: - if isinstance(segment, dict) and "vram_symbol" in segment: - return str(segment["vram_symbol"]) - else: - return None - - -def parse_segment_vram_class(segment: Union[dict, list]) -> Optional[VramClass]: - if isinstance(segment, dict) and "vram_class" in segment: - return vram_classes.resolve(segment["vram_class"]) - return None - - -def parse_segment_follows_vram(segment: Union[dict, list]) -> Optional[str]: - if isinstance(segment, dict): - return segment.get("follows_vram", None) - return None - - -def parse_segment_align(segment: Union[dict, list]) -> Optional[int]: - if isinstance(segment, dict) and "align" in segment: - return int(segment["align"]) - return None - - -def parse_segment_subalign(segment: Union[dict, list]) -> int: - default = options.opts.subalign - if isinstance(segment, dict): - subalign = segment.get("subalign", default) - if subalign != None: - subalign = int(subalign) - return subalign - return default - - -def parse_segment_section_order(segment: Union[dict, list]) -> List[str]: - default = options.opts.section_order - if isinstance(segment, dict): - return segment.get("section_order", default) - return default - - -class Segment: - require_unique_name = True - - @staticmethod - def get_class_for_type(seg_type) -> Type["Segment"]: - # so .data loads SegData, for example - if seg_type.startswith("."): - seg_type = seg_type[1:] - - segment_class = Segment.get_base_segment_class(seg_type) - if segment_class == None: - # Look in extensions - segment_class = Segment.get_extension_segment_class(seg_type) - return segment_class - - @staticmethod - def get_base_segment_class(seg_type): - platform = options.opts.platform - is_platform_seg = False - - # heirarchy is platform -> common -> fail - try: - segmodule = importlib.import_module(f"segtypes.{platform}.{seg_type}") - is_platform_seg = True - except ModuleNotFoundError: - try: - segmodule = importlib.import_module(f"segtypes.common.{seg_type}") - except ModuleNotFoundError: - return None - - seg_prefix = platform.capitalize() if is_platform_seg else "Common" - return getattr(segmodule, f"{seg_prefix}Seg{seg_type.capitalize()}") - - @staticmethod - def get_extension_segment_class(seg_type): - platform = options.opts.platform - - ext_path = options.opts.extensions_path - if not ext_path: - log.error( - f"could not load presumed extended segment type '{seg_type}' because no extensions path is configured" - ) - assert ext_path is not None - - try: - ext_spec = importlib.util.spec_from_file_location( - f"{platform}.segtypes.{seg_type}", ext_path / f"{seg_type}.py" - ) - assert ext_spec is not None - ext_mod = importlib.util.module_from_spec(ext_spec) - assert ext_spec.loader is not None - ext_spec.loader.exec_module(ext_mod) - except Exception as err: - log.write(err, status="error") - log.error( - f"could not load segment type '{seg_type}'\n(hint: confirm your extension directory is configured correctly)" - ) - - return getattr( - ext_mod, f"{platform.upper()}Seg{seg_type[0].upper()}{seg_type[1:]}" - ) - - @staticmethod - def parse_segment_start(segment: Union[dict, list]) -> Optional[int]: - if isinstance(segment, dict): - s = segment.get("start", "auto") - else: - s = segment[0] - - if s == "auto": - return None - elif s == "...": - return None - else: - return int(s) - - @staticmethod - def parse_segment_type(segment: Union[dict, list]) -> str: - if isinstance(segment, dict): - return str(segment["type"]) - else: - return str(segment[1]) - - @staticmethod - def parse_segment_name(cls, rom_start, segment: Union[dict, list]) -> str: - if isinstance(segment, dict) and "name" in segment: - return str(segment["name"]) - elif isinstance(segment, dict) and "dir" in segment: - return str(segment["dir"]) - elif isinstance(segment, list) and len(segment) >= 3: - return str(segment[2]) - else: - return str(cls.get_default_name(rom_start)) - - @staticmethod - def parse_segment_symbol_name_format(segment: Union[dict, list]) -> str: - if isinstance(segment, dict) and "symbol_name_format" in segment: - return str(segment["symbol_name_format"]) - else: - return options.opts.symbol_name_format - - @staticmethod - def parse_segment_symbol_name_format_no_rom(segment: Union[dict, list]) -> str: - if isinstance(segment, dict) and "symbol_name_format_no_rom" in segment: - return str(segment["symbol_name_format_no_rom"]) - else: - return options.opts.symbol_name_format_no_rom - - @staticmethod - def parse_segment_file_path(segment: Union[dict, list]) -> Optional[Path]: - if isinstance(segment, dict) and "path" in segment: - return Path(segment["path"]) - return None - - @staticmethod - def parse_segment_bss_contains_common(segment: Union[dict, list]) -> bool: - if isinstance(segment, dict) and "bss_contains_common" in segment: - return bool(segment["bss_contains_common"]) - else: - return False - - @staticmethod - def parse_linker_section_order(yaml: Union[dict, list]) -> Optional[str]: - if isinstance(yaml, dict) and "linker_section_order" in yaml: - return str(yaml["linker_section_order"]) - return None - - @staticmethod - def parse_linker_section(yaml: Union[dict, list]) -> Optional[str]: - if isinstance(yaml, dict) and "linker_section" in yaml: - return str(yaml["linker_section"]) - return None - - @staticmethod - def parse_ld_fill_value( - yaml: Union[dict, list], default: Optional[int] - ) -> Optional[int]: - if isinstance(yaml, dict) and "ld_fill_value" in yaml: - return yaml["ld_fill_value"] - return default - - def __init__( - self, - rom_start: Optional[int], - rom_end: Optional[int], - type: str, - name: str, - vram_start: Optional[int], - args: list, - yaml, - ): - self.rom_start = rom_start - self.rom_end = rom_end - self.type = type - self.name = name - self.vram_start: Optional[int] = vram_start - - self.align: Optional[int] = None - self.given_subalign: int = options.opts.subalign - self.exclusive_ram_id: Optional[str] = None - self.given_dir: Path = Path() - - # Default to global options. - self.given_find_file_boundaries: Optional[bool] = None - - # Symbols known to be in this segment - self.given_seg_symbols: Dict[int, List[Symbol]] = {} - - # Ranges for faster symbol lookup - self.symbol_ranges_ram: IntervalTree = IntervalTree() - self.symbol_ranges_rom: IntervalTree = IntervalTree() - - self.given_section_order: List[str] = options.opts.section_order - - self.vram_class: Optional[VramClass] = None - self.given_follows_vram: Optional[str] = None - self.given_vram_symbol: Optional[str] = None - - self.given_symbol_name_format: str = options.opts.symbol_name_format - self.given_symbol_name_format_no_rom: str = ( - options.opts.symbol_name_format_no_rom - ) - - self.parent: Optional[Segment] = None - self.sibling: Optional[Segment] = None - self.data_sibling: Optional[Segment] = None - self.rodata_sibling: Optional[Segment] = None - self.file_path: Optional[Path] = None - - self.args: List[str] = args - self.yaml = yaml - - self.extract: bool = True - self.has_linker_entry: bool = True - if self.rom_start is None: - self.extract = False - elif self.type.startswith("."): - self.extract = False - - self.warnings: List[str] = [] - self.did_run = False - self.bss_contains_common = Segment.parse_segment_bss_contains_common(yaml) - - # For segments which are not in the usual VRAM segment space, like N64's IPL3 which lives in 0xA4... - self.special_vram_segment: bool = False - - self.linker_section_order: Optional[str] = self.parse_linker_section_order(yaml) - self.linker_section: Optional[str] = self.parse_linker_section(yaml) - - # If not defined on the segment then default to the global option - self.ld_fill_value: Optional[int] = self.parse_ld_fill_value( - yaml, options.opts.ld_fill_value - ) - - if self.rom_start is not None and self.rom_end is not None: - if self.rom_start > self.rom_end: - log.error( - f"Error: segments out of order - ({self.name} starts at 0x{self.rom_start:X}, but next segment starts at 0x{self.rom_end:X})" - ) - - @staticmethod - def from_yaml( - cls: Type["Segment"], - yaml: Union[dict, list], - rom_start: Optional[int], - rom_end: Optional[int], - vram=None, - ): - type = Segment.parse_segment_type(yaml) - name = Segment.parse_segment_name(cls, rom_start, yaml) - - vram_class = parse_segment_vram_class(yaml) - - if vram is not None: - vram_start = vram - elif vram_class: - vram_start = vram_class.vram - else: - vram_start = parse_segment_vram(yaml) - - args: List[str] = [] if isinstance(yaml, dict) else yaml[3:] - - ret = cls( - rom_start=rom_start, - rom_end=rom_end, - type=type, - name=name, - vram_start=vram_start, - args=args, - yaml=yaml, - ) - ret.given_section_order = parse_segment_section_order(yaml) - ret.given_subalign = parse_segment_subalign(yaml) - - if isinstance(yaml, dict): - ret.extract = bool(yaml.get("extract", ret.extract)) - ret.exclusive_ram_id = yaml.get("exclusive_ram_id") - ret.given_dir = Path(yaml.get("dir", "")) - ret.has_linker_entry = bool(yaml.get("linker_entry", True)) - ret.given_find_file_boundaries = yaml.get("find_file_boundaries", None) - - ret.given_symbol_name_format = Segment.parse_segment_symbol_name_format(yaml) - ret.given_symbol_name_format_no_rom = ( - Segment.parse_segment_symbol_name_format_no_rom(yaml) - ) - ret.file_path = Segment.parse_segment_file_path(yaml) - - ret.bss_contains_common = Segment.parse_segment_bss_contains_common(yaml) - - ret.given_follows_vram = parse_segment_follows_vram(yaml) - ret.given_vram_symbol = parse_segment_vram_symbol(yaml) - - if vram_class: - ret.vram_class = vram_class - if ret.given_follows_vram: - log.error( - f"Error: segment {ret.name} has both a vram class and a follows_vram property" - ) - if ret.given_vram_symbol: - log.error( - f"Error: segment {ret.name} has both a vram class and a vram_symbol property" - ) - - if not ret.align: - ret.align = parse_segment_align(yaml) - return ret - - # For executable segments (.text); like c, asm or hasm - @staticmethod - def is_text() -> bool: - return False - - # For read-write segments (.data); like data - @staticmethod - def is_data() -> bool: - return False - - # For readonly segments (.rodata); like rodata or rdata - @staticmethod - def is_rodata() -> bool: - return False - - # For segments which does not take space in ROM; like bss - @staticmethod - def is_noload() -> bool: - return False - - @staticmethod - def estimate_size(yaml: Union[Dict, List]) -> Optional[int]: - return None - - @property - def needs_symbols(self) -> bool: - return False - - @property - def dir(self) -> Path: - if self.parent: - return self.parent.dir / self.given_dir - else: - return self.given_dir - - @property - def show_file_boundaries(self) -> bool: - # If the segment has explicitly set `find_file_boundaries`, use it. - if self.given_find_file_boundaries is not None: - return self.given_find_file_boundaries - - # If the segment has no parent, use options as default. - if not self.parent: - return options.opts.find_file_boundaries - - return self.parent.show_file_boundaries - - @property - def symbol_name_format(self) -> str: - return self.given_symbol_name_format - - @property - def symbol_name_format_no_rom(self) -> str: - return self.given_symbol_name_format_no_rom - - @property - def subalign(self) -> int: - if self.parent: - return self.parent.subalign - else: - return self.given_subalign - - @property - def vram_symbol(self) -> Optional[str]: - if self.vram_class and self.vram_class.vram_symbol: - return self.vram_class.vram_symbol - elif self.given_vram_symbol: - return self.given_vram_symbol - else: - return None - - def get_exclusive_ram_id(self) -> Optional[str]: - if self.parent: - return self.parent.get_exclusive_ram_id() - return self.exclusive_ram_id - - def add_symbol(self, symbol: Symbol): - if symbol.vram_start not in self.given_seg_symbols: - self.given_seg_symbols[symbol.vram_start] = [] - self.given_seg_symbols[symbol.vram_start].append(symbol) - - # For larger symbols, add their ranges to interval trees for faster lookup - if symbol.size > 4: - self.symbol_ranges_ram.addi(symbol.vram_start, symbol.vram_end, symbol) - if symbol.rom is not None: - self.symbol_ranges_rom.addi(symbol.rom, symbol.rom_end, symbol) - - @property - def seg_symbols(self) -> Dict[int, List[Symbol]]: - if self.parent: - return self.parent.seg_symbols - else: - return self.given_seg_symbols - - @property - def size(self) -> Optional[int]: - if self.rom_start is not None and self.rom_end is not None: - return self.rom_end - self.rom_start - else: - return None - - @property - def vram_end(self) -> Optional[int]: - if self.vram_start is not None and self.size is not None: - return self.vram_start + self.size - else: - return None - - @property - def section_order(self) -> List[str]: - return self.given_section_order - - @property - def rodata_follows_data(self) -> bool: - if ".rodata" not in self.section_order or ".data" not in self.section_order: - return False - return ( - self.section_order.index(".rodata") - self.section_order.index(".data") == 1 - ) - - def get_cname(self) -> str: - name = self.name - if self.parent: - name = self.parent.name + "_" + name - - return to_cname(name) - - def contains_vram(self, vram: int) -> bool: - if self.vram_start is not None and self.vram_end is not None: - return vram >= self.vram_start and vram < self.vram_end - else: - return False - - def contains_rom(self, rom: int) -> bool: - if self.rom_start is not None and self.rom_end is not None: - return rom >= self.rom_start and rom < self.rom_end - else: - return False - - def rom_to_ram(self, rom_addr: int) -> Optional[int]: - if self.vram_start is not None and self.rom_start is not None: - return self.vram_start + rom_addr - self.rom_start - else: - return None - - def ram_to_rom(self, ram_addr: int) -> Optional[int]: - if not self.contains_vram(ram_addr) and ram_addr != self.vram_end: - return None - - if self.vram_start is not None and self.rom_start is not None: - return self.rom_start + ram_addr - self.vram_start - else: - return None - - def should_scan(self) -> bool: - return self.should_split() - - def should_split(self) -> bool: - return self.extract and options.opts.is_mode_active(self.type) - - def scan(self, rom_bytes: bytes): - pass - - def split(self, rom_bytes: bytes): - pass - - def cache(self): - return (self.yaml, self.rom_end) - - def get_linker_section(self) -> str: - return ".data" - - def get_linker_section_order(self) -> str: - """ - Used to override the linking _order_ of a specific section - - Useful for files that may have non-conventional orderings (like putting .data with the other .rodata sections) - """ - if self.linker_section_order is not None: - return self.linker_section_order - return self.get_linker_section() - - def get_linker_section_linksection(self) -> str: - """ - The actual section that will be used when linking - """ - if self.linker_section is not None: - return self.linker_section - return self.get_linker_section() - - def get_section_flags(self) -> Optional[str]: - """ - Allows specifying flags for a section. - - This can be useful when creating a custom section, since sections not recognized by the linker will not be linked properly. - - GNU as docs about the section directive and flags: https://sourceware.org/binutils/docs/as/Section.html#ELF-Version - - Example: - - ``` - def get_section_flags(self) -> Optional[str]: - # Tells the linker to allocate this section - return "a" - ``` - """ - return None - - def out_path(self) -> Optional[Path]: - return None - - def get_most_parent(self) -> "Segment": - seg = self - - while seg.parent: - seg = seg.parent - - return seg - - def get_linker_entries(self) -> "List[LinkerEntry]": - from segtypes.linker_entry import LinkerEntry - - if not self.has_linker_entry: - return [] - - path = self.out_path() - - if path: - return [ - LinkerEntry( - self, - [path], - path, - self.get_linker_section_order(), - self.get_linker_section_linksection(), - self.is_noload(), - ) - ] - else: - return [] - - def log(self, msg): - if options.opts.verbose: - log.write(f"{self.type} {self.name}: {msg}") - - def warn(self, msg: str): - self.warnings.append(msg) - - @staticmethod - def get_default_name(addr) -> str: - return f"{addr:X}" - - def is_name_default(self): - return self.name == self.get_default_name(self.rom_start) - - def unique_id(self): - if self.parent: - s = self.parent.unique_id() + "_" - else: - s = "" - - return s + self.type + "_" + self.name - - @staticmethod - def visible_ram(seg1: "Segment", seg2: "Segment") -> bool: - if seg1.get_most_parent() == seg2.get_most_parent(): - return True - if seg1.get_exclusive_ram_id() is None or seg2.get_exclusive_ram_id() is None: - return True - return seg1.get_exclusive_ram_id() != seg2.get_exclusive_ram_id() - - def retrieve_symbol( - self, syms: Dict[int, List[Symbol]], addr: int - ) -> Optional[Symbol]: - if addr not in syms: - return None - - items = syms[addr] - - # Filter out symbols that are in different top-level segments with the same unique_ram_id - items = [ - i - for i in items - if i.segment is None or Segment.visible_ram(self, i.segment) - ] - - if len(items) > 1: - # print(f"Trying to retrieve {addr:X} from symbol dict but there are {len(items)} entries to pick from - picking the first") - pass - if len(items) == 0: - return None - return items[0] - - def retrieve_sym_type( - self, syms: Dict[int, List[Symbol]], addr: int, type: str - ) -> Optional[symbols.Symbol]: - if addr not in syms: - return None - - items = syms[addr] - - items = [ - i - for i in items - if i.segment is None - or Segment.visible_ram(self, i.segment) - and (type == i.type) - ] - - if len(items) == 0: - return None - - return items[0] - - def get_symbol( - self, - addr: int, - in_segment: bool = False, - type: Optional[str] = None, - create: bool = False, - define: bool = False, - reference: bool = False, - search_ranges: bool = False, - local_only: bool = False, - ) -> Optional[Symbol]: - ret: Optional[Symbol] = None - rom: Optional[int] = None - - most_parent = self.get_most_parent() - - if in_segment: - # If the vram address is within this segment, we can calculate the symbol's rom address - rom = most_parent.ram_to_rom(addr) - ret = most_parent.retrieve_symbol(most_parent.seg_symbols, addr) - - if not ret and search_ranges: - # Search ranges first, starting with rom - if rom is not None: - cands: Set[Interval] = most_parent.symbol_ranges_rom[rom] - if cands: - ret = cands.pop().data - # and then vram if we can't find a rom match - if not ret: - cands = most_parent.symbol_ranges_ram[addr] - if cands: - ret = cands.pop().data - elif not local_only: - ret = most_parent.retrieve_symbol(symbols.all_symbols_dict, addr) - - if not ret and search_ranges: - cands = symbols.all_symbols_ranges[addr] - if cands: - ret = cands.pop().data - - # Create the symbol if it doesn't exist - if not ret and create: - ret = Symbol(addr, rom=rom, type=type) - symbols.add_symbol(ret) - - if in_segment: - ret.segment = most_parent - if addr not in most_parent.seg_symbols: - most_parent.seg_symbols[addr] = [] - most_parent.seg_symbols[addr].append(ret) - - if ret: - if define: - ret.defined = True - if reference: - ret.referenced = True - if ret.type is None: - ret.type = type - if ret.rom is None: - ret.rom = rom - if in_segment: - if ret.segment is None: - ret.segment = most_parent - - return ret - - def create_symbol( - self, - addr: int, - in_segment: bool, - type: Optional[str] = None, - define: bool = False, - reference: bool = False, - search_ranges: bool = False, - local_only: bool = False, - ) -> Symbol: - ret = self.get_symbol( - addr, - in_segment=in_segment, - type=type, - create=True, - define=define, - reference=reference, - search_ranges=search_ranges, - local_only=local_only, - ) - assert ret is not None - - return ret - - def get_func_for_addr(self, addr) -> Optional[Symbol]: - for syms in self.seg_symbols.values(): - for sym in syms: - if sym.type == "func" and sym.contains_vram(addr): - return sym - - return None diff --git a/tools/splat/split.py b/tools/splat/split.py deleted file mode 100755 index 2e0bbbc812..0000000000 --- a/tools/splat/split.py +++ /dev/null @@ -1,572 +0,0 @@ -#! /usr/bin/env python3 - -import argparse -import hashlib -import importlib -import pickle -from typing import Any, Dict, List, Optional, Set, Tuple, Union -from pathlib import Path -from disassembler import disassembler_instance -from util import progress_bar, vram_classes - -# This unused import makes the yaml library faster. don't remove -import pylibyaml # pyright: ignore -import yaml - -from colorama import Fore, Style -from intervaltree import Interval, IntervalTree -import sys - -from segtypes.linker_entry import ( - LinkerWriter, - get_segment_vram_end_symbol_name, -) -from segtypes.segment import Segment -from util import log, options, palettes, symbols, relocs - -VERSION = "0.19.0" - -parser = argparse.ArgumentParser( - description="Split a rom given a rom, a config, and output directory" -) -parser.add_argument("config", help="path to a compatible config .yaml file", nargs="+") -parser.add_argument("--modes", nargs="+", default="all") -parser.add_argument("--verbose", action="store_true", help="Enable debug logging") -parser.add_argument( - "--use-cache", action="store_true", help="Only split changed segments in config" -) -parser.add_argument( - "--skip-version-check", - action="store_true", - help="Skips the disassembler's version check", -) -parser.add_argument( - "--stdout-only", help="Print all output to stdout", action="store_true" -) -parser.add_argument( - "--disassemble-all", - help="Disasemble matched functions and migrated data", - action="store_true", -) - -linker_writer: LinkerWriter -config: Dict[str, Any] - -segment_roms: IntervalTree = IntervalTree() -segment_rams: IntervalTree = IntervalTree() - - -def fmt_size(size): - if size > 1000000: - return str(size // 1000000) + " MB" - elif size > 1000: - return str(size // 1000) + " KB" - else: - return str(size) + " B" - - -def initialize_segments(config_segments: Union[dict, list]) -> List[Segment]: - global segment_roms - global segment_rams - - segment_roms = IntervalTree() - segment_rams = IntervalTree() - - segments_by_name: Dict[str, Segment] = {} - ret = [] - - last_rom_end = 0 - - for i, seg_yaml in enumerate(config_segments): - # end marker - if isinstance(seg_yaml, list) and len(seg_yaml) == 1: - continue - - seg_type = Segment.parse_segment_type(seg_yaml) - - segment_class = Segment.get_class_for_type(seg_type) - - this_start = Segment.parse_segment_start(seg_yaml) - - if i == len(config_segments) - 1 and Segment.parse_segment_file_path(seg_yaml): - next_start: Optional[int] = 0 - else: - next_start = Segment.parse_segment_start(config_segments[i + 1]) - - if segment_class.is_noload(): - # Pretend bss's rom address is after the last actual rom segment - this_start = last_rom_end - # and it has a rom size of zero - next_start = last_rom_end - - segment: Segment = Segment.from_yaml( - segment_class, seg_yaml, this_start, next_start - ) - - if segment.require_unique_name: - if segment.name in segments_by_name: - log.error(f"segment name '{segment.name}' is not unique") - - segments_by_name[segment.name] = segment - - ret.append(segment) - if ( - isinstance(segment.rom_start, int) - and isinstance(segment.rom_end, int) - and segment.rom_start != segment.rom_end - ): - segment_roms.addi(segment.rom_start, segment.rom_end, segment) - if ( - isinstance(segment.vram_start, int) - and isinstance(segment.vram_end, int) - and segment.vram_start != segment.vram_end - ): - segment_rams.addi(segment.vram_start, segment.vram_end, segment) - - if next_start is not None: - last_rom_end = next_start - - for segment in ret: - if segment.given_follows_vram: - if segment.given_follows_vram not in segments_by_name: - log.error( - f"segment '{segment.given_follows_vram}', the 'follows_vram' value for segment '{segment.name}', does not exist" - ) - segment.given_vram_symbol = get_segment_vram_end_symbol_name( - segments_by_name[segment.given_follows_vram] - ) - - return ret - - -def assign_symbols_to_segments(): - for symbol in symbols.all_symbols: - if symbol.segment: - continue - - if symbol.rom: - cands: Set[Interval] = segment_roms[symbol.rom] - if len(cands) > 1: - log.error("multiple segments rom overlap symbol", symbol) - elif len(cands) == 0: - log.error("no segment rom overlaps symbol", symbol) - else: - cand: Interval = cands.pop() - seg: Segment = cand.data - seg.add_symbol(symbol) - else: - cands = segment_rams[symbol.vram_start] - segs: List[Segment] = [cand.data for cand in cands] - for seg in segs: - if not seg.get_exclusive_ram_id(): - seg.add_symbol(symbol) - - -def do_statistics(seg_sizes, rom_bytes, seg_split, seg_cached): - unk_size = seg_sizes.get("unk", 0) - rest_size = 0 - total_size = len(rom_bytes) - - for typ in seg_sizes: - if typ != "unk": - rest_size += seg_sizes[typ] - - known_ratio = rest_size / total_size - unk_ratio = unk_size / total_size - - log.write(f"Split {fmt_size(rest_size)} ({known_ratio:.2%}) in defined segments") - for typ in seg_sizes: - if typ != "unk": - tmp_size = seg_sizes[typ] - tmp_ratio = tmp_size / total_size - log.write( - f"{typ:>20}: {fmt_size(tmp_size):>8} ({tmp_ratio:.2%}) {Fore.GREEN}{seg_split[typ]} split{Style.RESET_ALL}, {Style.DIM}{seg_cached[typ]} cached" - ) - log.write( - f"{'unknown':>20}: {fmt_size(unk_size):>8} ({unk_ratio:.2%}) from unknown bin files" - ) - - -def merge_configs(main_config, additional_config): - # Merge rules are simple - # For each key in the dictionary - # - If list then append to list - # - If a dictionary then repeat merge on sub dictionary entries - # - Else assume string or number and replace entry - - for curkey in additional_config: - if curkey not in main_config: - main_config[curkey] = additional_config[curkey] - elif type(main_config[curkey]) != type(additional_config[curkey]): - log.error(f"Type for key {curkey} in configs does not match") - else: - # keys exist and match, see if a list to append - if type(main_config[curkey]) == list: - main_config[curkey] += additional_config[curkey] - elif type(main_config[curkey]) == dict: - # need to merge sub areas - main_config[curkey] = merge_configs( - main_config[curkey], additional_config[curkey] - ) - else: - # not a list or dictionary, must be a number or string, overwrite - main_config[curkey] = additional_config[curkey] - - return main_config - - -def brief_seg_name(seg: Segment, limit: int, ellipsis="…") -> str: - s = seg.name.strip() - if len(s) > limit: - return s[:limit].strip() + ellipsis - return s - - -# Return a mapping of vram classes to segments that need to be part of their vram symbol's calculation -def calc_segment_dependences( - all_segments: List[Segment], -) -> Dict[vram_classes.VramClass, List[Segment]]: - # Map vram class names to segments that have that vram class - vram_class_to_segments: Dict[str, List[Segment]] = {} - for seg in all_segments: - if seg.vram_class is not None: - if seg.vram_class.name not in vram_class_to_segments: - vram_class_to_segments[seg.vram_class.name] = [] - vram_class_to_segments[seg.vram_class.name].append(seg) - - # Map vram class names to segments that the vram class follows - vram_class_to_follows_segments: Dict[vram_classes.VramClass, List[Segment]] = {} - for vram_class in vram_classes._vram_classes.values(): - if vram_class.follows_classes: - vram_class_to_follows_segments[vram_class] = [] - - for follows_class in vram_class.follows_classes: - if follows_class in vram_class_to_segments: - vram_class_to_follows_segments[ - vram_class - ] += vram_class_to_segments[follows_class] - return vram_class_to_follows_segments - - -def main( - config_path, - modes, - verbose, - use_cache=True, - skip_version_check=False, - stdout_only=False, - disassemble_all=False, -): - global config - - if stdout_only: - progress_bar.out_file = sys.stdout - - # Load config - config = {} - for entry in config_path: - with open(entry) as f: - additional_config = yaml.load(f.read(), Loader=yaml.SafeLoader) - config = merge_configs(config, additional_config) - - vram_classes.initialize(config.get("vram_classes")) - - options.initialize(config, config_path, modes, verbose, disassemble_all) - - disassembler_instance.create_disassembler_instance(options.opts.platform) - disassembler_instance.get_instance().check_version(skip_version_check, VERSION) - - with options.opts.target_path.open("rb") as f2: - rom_bytes = f2.read() - - if "sha1" in config: - sha1 = hashlib.sha1(rom_bytes).hexdigest() - e_sha1 = config["sha1"].lower() - if e_sha1 != sha1: - log.error(f"sha1 mismatch: expected {e_sha1}, was {sha1}") - else: - log.write("Warning: no sha1 in config") - - # Create main output dir - options.opts.base_path.mkdir(parents=True, exist_ok=True) - - processed_segments: List[Segment] = [] - - seg_sizes: Dict[str, int] = {} - seg_split: Dict[str, int] = {} - seg_cached: Dict[str, int] = {} - - # Load cache - if use_cache: - try: - with options.opts.cache_path.open("rb") as f3: - cache = pickle.load(f3) - - if verbose: - log.write(f"Loaded cache ({len(cache.keys())} items)") - except Exception: - cache = {} - else: - cache = {} - - # invalidate entire cache if options change - if use_cache and cache.get("__options__") != config.get("options"): - if verbose: - log.write("Options changed, invalidating cache") - - cache = { - "__options__": config.get("options"), - } - - disassembler_instance.get_instance().configure(options.opts) - - platform_module = importlib.import_module(f"platforms.{options.opts.platform}") - platform_init = getattr(platform_module, "init") - platform_init(rom_bytes) - - # Initialize segments - all_segments = initialize_segments(config["segments"]) - - # Load and process symbols - symbols.initialize(all_segments) - relocs.initialize() - - # Assign symbols to segments - assign_symbols_to_segments() - - if options.opts.is_mode_active("code"): - symbols.initialize_spim_context(all_segments) - relocs.initialize_spim_context() - - # Resolve raster/palette siblings - if options.opts.is_mode_active("img"): - palettes.initialize(all_segments) - - # Scan - scan_bar = progress_bar.get_progress_bar(all_segments) - for segment in scan_bar: - assert isinstance(segment, Segment) - scan_bar.set_description(f"Scanning {brief_seg_name(segment, 20)}") - typ = segment.type - if segment.type == "bin" and segment.is_name_default(): - typ = "unk" - - if typ not in seg_sizes: - seg_sizes[typ] = 0 - seg_split[typ] = 0 - seg_cached[typ] = 0 - seg_sizes[typ] += 0 if segment.size is None else segment.size - - if segment.should_scan(): - # Check cache but don't write anything - if use_cache: - if segment.cache() == cache.get(segment.unique_id()): - continue - - segment.did_run = True - segment.scan(rom_bytes) - - processed_segments.append(segment) - - seg_split[typ] += 1 - - symbols.mark_c_funcs_as_defined() - - # Split - split_bar = progress_bar.get_progress_bar(all_segments) - for segment in split_bar: - split_bar.set_description(f"Splitting {brief_seg_name(segment, 20)}") - - if use_cache: - cached = segment.cache() - - if cached == cache.get(segment.unique_id()): - # Cache hit - if segment.type not in seg_cached: - seg_cached[segment.type] = 0 - seg_cached[segment.type] += 1 - continue - else: - # Cache miss; split - cache[segment.unique_id()] = cached - - if segment.should_split(): - segment_bytes = rom_bytes - if segment.file_path: - with open(segment.file_path, "rb") as segment_input_file: - segment_bytes = segment_input_file.read() - segment.split(segment_bytes) - - if ( - options.opts.is_mode_active("ld") and options.opts.platform != "gc" - ): # TODO move this to platform initialization when it gets implemented - vram_class_dependencies = calc_segment_dependences(all_segments) - vram_classes_to_search = set(vram_class_dependencies.keys()) - - max_vram_end_insertion_points: Dict[ - Segment, List[Tuple[str, List[Segment]]] - ] = {} - for seg in reversed(all_segments): - if seg.vram_class in vram_classes_to_search: - assert seg.vram_class.vram_symbol is not None - if seg not in max_vram_end_insertion_points: - max_vram_end_insertion_points[seg] = [] - max_vram_end_insertion_points[seg].append( - ( - seg.vram_class.vram_symbol, - vram_class_dependencies[seg.vram_class], - ) - ) - vram_classes_to_search.remove(seg.vram_class) - - global linker_writer - linker_writer = LinkerWriter() - linker_bar = progress_bar.get_progress_bar(all_segments) - - partial_linking = options.opts.ld_partial_linking - partial_scripts_path = options.opts.ld_partial_scripts_path - segments_path = options.opts.ld_partial_build_segments_path - if partial_linking: - if partial_scripts_path is None: - log.error( - "Partial linking is enabled but `ld_partial_scripts_path` has not been set" - ) - if options.opts.ld_partial_build_segments_path is None: - log.error( - "Partial linking is enabled but `ld_partial_build_segments_path` has not been set" - ) - - for segment in linker_bar: - assert isinstance(segment, Segment) - linker_bar.set_description(f"Linker script {brief_seg_name(segment, 20)}") - max_vram_syms = max_vram_end_insertion_points.get(segment, []) - - if options.opts.ld_partial_linking: - linker_writer.add_referenced_partial_segment(segment, max_vram_syms) - - # Create linker script for segment - sub_linker_writer = LinkerWriter(is_partial=True) - sub_linker_writer.add_partial_segment(segment) - - assert partial_scripts_path is not None - assert segments_path is not None - - seg_name = segment.get_cname() - - sub_linker_writer.save_linker_script( - partial_scripts_path / f"{seg_name}.ld" - ) - if options.opts.ld_dependencies: - sub_linker_writer.save_dependencies_file( - partial_scripts_path / f"{seg_name}.d", - segments_path / f"{seg_name}.o", - ) - else: - linker_writer.add(segment, max_vram_syms) - - linker_writer.save_linker_script(options.opts.ld_script_path) - linker_writer.save_symbol_header() - if options.opts.ld_dependencies: - elf_path = options.opts.elf_path - if elf_path is None: - log.error( - "Generation of dependency file for linker script requested but `elf_path` was not provided in the yaml options" - ) - linker_writer.save_dependencies_file( - options.opts.ld_script_path.with_suffix(".d"), elf_path - ) - - # write elf_sections.txt - this only lists the generated sections in the elf, not subsections - # that the elf combines into one section - if options.opts.elf_section_list_path: - section_list = "" - for segment in all_segments: - section_list += "." + segment.get_cname() + "\n" - with open(options.opts.elf_section_list_path, "w", newline="\n") as f: - f.write(section_list) - - # Write undefined_funcs_auto.txt - if options.opts.create_undefined_funcs_auto: - to_write = [ - s - for s in symbols.all_symbols - if s.referenced and not s.defined and s.type == "func" - ] - to_write.sort(key=lambda x: x.vram_start) - - with open(options.opts.undefined_funcs_auto_path, "w", newline="\n") as f: - for symbol in to_write: - f.write(f"{symbol.name} = 0x{symbol.vram_start:X};\n") - - # write undefined_syms_auto.txt - if options.opts.create_undefined_syms_auto: - to_write = [ - s - for s in symbols.all_symbols - if s.referenced - and not s.defined - and s.type not in {"func", "label", "jtbl_label"} - ] - to_write.sort(key=lambda x: x.vram_start) - - with open(options.opts.undefined_syms_auto_path, "w", newline="\n") as f: - for symbol in to_write: - f.write(f"{symbol.name} = 0x{symbol.vram_start:X};\n") - - # print warnings during split - for segment in all_segments: - if len(segment.warnings) > 0: - log.write( - f"{Style.DIM}0x{segment.rom_start:06X}{Style.RESET_ALL} {segment.type} {Style.BRIGHT}{segment.name}{Style.RESET_ALL}:" - ) - - for warn in segment.warnings: - log.write("warning: " + warn, status="warn") - - log.write("") # empty line - - # Statistics - do_statistics(seg_sizes, rom_bytes, seg_split, seg_cached) - - # Save cache - if cache != {} and use_cache: - if verbose: - log.write("Writing cache") - with open(options.opts.cache_path, "wb") as f4: - pickle.dump(cache, f4) - - if options.opts.dump_symbols and options.opts.is_mode_active("code"): - splat_hidden_folder = Path(".splat/") - splat_hidden_folder.mkdir(exist_ok=True) - - with open(splat_hidden_folder / "splat_symbols.csv", "w") as f: - f.write( - "vram_start,given_name,name,type,given_size,size,rom,defined,user_declared,referenced,extract\n" - ) - for s in sorted(symbols.all_symbols, key=lambda x: x.vram_start): - f.write(f"{s.vram_start:X},{s.given_name},{s.name},{s.type},") - if s.given_size is not None: - f.write(f"0x{s.given_size:X},") - else: - f.write("None,") - f.write(f"{s.size},") - if s.rom is not None: - f.write(f"0x{s.rom:X},") - else: - f.write("None,") - f.write(f"{s.defined},{s.user_declared},{s.referenced},{s.extract}\n") - - symbols.spim_context.saveContextToFile(splat_hidden_folder / "spim_context.csv") - - -if __name__ == "__main__": - args = parser.parse_args() - main( - args.config, - args.modes, - args.verbose, - args.use_cache, - args.skip_version_check, - args.stdout_only, - args.disassemble_all, - ) diff --git a/tools/splat/stubs/colorama.pyi b/tools/splat/stubs/colorama.pyi deleted file mode 100644 index 5d13e507a8..0000000000 --- a/tools/splat/stubs/colorama.pyi +++ /dev/null @@ -1,8 +0,0 @@ -from typing import Any - -def init(**kwargs): ... - -Fore: Any -Back: Any -Style: Any -Cursor: Any diff --git a/tools/splat/test.py b/tools/splat/test.py deleted file mode 100644 index 549c112106..0000000000 --- a/tools/splat/test.py +++ /dev/null @@ -1,513 +0,0 @@ -from spimdisasm.common import FileSectionType - -from split import * -import unittest -import io -import filecmp -from util import symbols, options -import spimdisasm -from segtypes.common.rodata import CommonSegRodata -from segtypes.common.code import CommonSegCode -from segtypes.common.c import CommonSegC -from segtypes.common.bss import CommonSegBss -import difflib - - -class Testing(unittest.TestCase): - def compare_files(self, test_path, ref_path): - with io.open(test_path) as test_f, io.open(ref_path) as ref_f: - self.assertListEqual(list(test_f), list(ref_f)) - - def get_same_files(self, dcmp, out): - for name in dcmp.same_files: - out.append((name, dcmp.left, dcmp.right)) - - for sub_dcmp in dcmp.subdirs.values(): - self.get_same_files(sub_dcmp, out) - - def get_diff_files(self, dcmp, out): - for name in dcmp.diff_files: - out.append((name, dcmp.left, dcmp.right)) - - for sub_dcmp in dcmp.subdirs.values(): - self.get_diff_files(sub_dcmp, out) - - def get_left_only_files(self, dcmp, out): - for name in dcmp.left_only: - out.append((name, dcmp.left, dcmp.right)) - - for sub_dcmp in dcmp.subdirs.values(): - self.get_left_only_files(sub_dcmp, out) - - def get_right_only_files(self, dcmp, out): - for name in dcmp.right_only: - out.append((name, dcmp.left, dcmp.right)) - - for sub_dcmp in dcmp.subdirs.values(): - self.get_right_only_files(sub_dcmp, out) - - def test_basic_app(self): - spimdisasm.common.GlobalConfig.ASM_GENERATED_BY = False - main(["test/basic_app/splat.yaml"], None, None) - - comparison = filecmp.dircmp("test/basic_app/split", "test/basic_app/expected") - - diff_files: List[Tuple[str, str, str]] = [] - self.get_diff_files(comparison, diff_files) - - same_files: List[Tuple[str, str, str]] = [] - self.get_same_files(comparison, same_files) - - left_only_files: List[Tuple[str, str, str]] = [] - self.get_left_only_files(comparison, left_only_files) - - right_only_files: List[Tuple[str, str, str]] = [] - self.get_right_only_files(comparison, right_only_files) - - print("same_files", same_files) - print("diff_files", diff_files) - print("left_only_files", left_only_files) - print("right_only_files", right_only_files) - - # if the files are different print out the difference - for file in diff_files: - # can't diff binary - if file[0] == ".splache": - continue - with open(f"{file[1]}/{file[0]}") as file1: - file1_lines = file1.readlines() - with open(f"{file[2]}/{file[0]}") as file2: - file2_lines = file2.readlines() - - for line in difflib.unified_diff( - file1_lines, file2_lines, fromfile="file1", tofile="file2", lineterm="" - ): - print(line) - - assert len(diff_files) == 0 - assert len(left_only_files) == 0 - assert len(right_only_files) == 0 - - -def test_init(): - options_dict = { - "options": { - "platform": "n64", - "basename": "basic_app", - "base_path": ".", - "build_path": "build", - "target_path": "build/main.bin", - "asm_path": "split/asm", - "src_path": "split/src", - "ld_script_path": "split/basic_app.ld", - "cache_path": "split/.splache", - "symbol_addrs_path": "split/generated.symbols.txt", - "undefined_funcs_auto_path": "split/undefined_funcs_auto.txt", - "undefined_syms_auto_path": "split/undefined_syms_auto.txt", - }, - "segments": [ - { - "name": "basic_app", - "type": "code", - "start": 0, - "vram": 0x400000, - "subalign": 4, - "subsegments": [[0, "data"], [0x1DC, "c", "main"], [0x1FC, "data"]], - }, - [0x1290], - ], - } - options.initialize(options_dict, ["./test/basic_app/splat.yaml"], [], False) - - -class Symbols(unittest.TestCase): - def test_check_valid_type(self): - disassembler_instance.create_disassembler_instance("n64") - - # first char is uppercase - assert symbols.check_valid_type("Symbol") - - splat_sym_types = {"func", "jtbl", "jtbl_label", "label"} - - for type in splat_sym_types: - assert symbols.check_valid_type(type) - - spim_types = [ - "char*", - "u32", - "Vec3f", - "u8", - "char", - "u16", - "f32", - "u64", - "asciz", - "s8", - "s64", - "f64", - "s16", - "s32", - ] - - for type in spim_types: - assert symbols.check_valid_type(type) - - def test_add_symbol_to_spim_segment(self): - segment = spimdisasm.common.SymbolsSegment( - context=spimdisasm.common.Context(), - vromStart=0x0, - vromEnd=0x10, - vramStart=0x40000000 + 0x0, - vramEnd=0x40000000 + 0x10, - ) - sym = symbols.Symbol(0x40000000) - sym.user_declared = False - sym.defined = True - sym.rom = 0x0 - sym.type = "func" - result = symbols.add_symbol_to_spim_segment(segment, sym) - assert result.type == spimdisasm.common.SymbolSpecialType.function - assert sym.user_declared == result.isUserDeclared - assert sym.defined == result.isDefined - - def test_add_symbol_to_spim_section(self): - section = spimdisasm.mips.sections.SectionBase( - context=spimdisasm.common.Context(), - vromStart=0x0, - vromEnd=0x10, - vram=0x40000000, - filename="test", - words=[], - sectionType=FileSectionType.Text, - segmentVromStart=0x0, - overlayCategory=None, - ) - sym = symbols.Symbol(0x100) - sym.type = "func" - sym.user_declared = False - sym.defined = True - result = symbols.add_symbol_to_spim_section(section, sym) - assert result.type == spimdisasm.common.SymbolSpecialType.function - assert sym.user_declared == result.isUserDeclared - assert sym.defined == result.isDefined - - def test_create_symbol_from_spim_symbol(self): - # need to init otherwise options.opts isn't defined. - # used in initializing a Segment - test_init() - - segment = Segment( - rom_start=0x0, - rom_end=0x100, - type="func", - name="MyFunc", - vram_start=0x40000000, - args=[], - yaml=None, - ) - context_sym = spimdisasm.common.ContextSymbol(address=0) - result = symbols.create_symbol_from_spim_symbol(segment, context_sym) - assert result.referenced - assert result.extract - assert result.name == "D_0" - - -def get_yaml(): - return { - "name": "basic_app", - "type": "code", - "start": 0, - "vram": 0x400000, - "subalign": 4, - "subsegments": [[0, "data"], [0x1DC, "c", "main"], [0x1FC, "data"]], - } - - -class Rodata(unittest.TestCase): - def test_disassemble_data(self): - test_init() - common_seg_rodata = CommonSegRodata( - rom_start=0x0, - rom_end=0x100, - type=".rodata", - name="MyRodata", - vram_start=0x400, - args=None, - yaml=None, - ) - rom_data = [] - for i in range(0x100): - rom_data.append(i) - common_seg_rodata.disassemble_data(bytes(rom_data)) - assert common_seg_rodata.spim_section is not None - assert common_seg_rodata.spim_section.get_section().words[0] == 0x0010203 - assert symbols.get_all_symbols()[0].vram_start == 0x400 - assert symbols.get_all_symbols()[0].segment == common_seg_rodata - assert symbols.get_all_symbols()[0].linker_section == ".rodata" - - def test_get_possible_text_subsegment_for_symbol(self): - context = spimdisasm.common.Context() - - result_symbol_addr = 0x2DC - - # use SymbolRodata to test migration - rodata_sym = spimdisasm.mips.symbols.SymbolRodata( - context=context, - vromStart=0x100, - vromEnd=0x200, - inFileOffset=0, - vram=0x100, - words=[0, 1, 2, 3, 4, 5, 6, 7], - segmentVromStart=0, - overlayCategory=None, - ) - rodata_sym.contextSym.forceMigration = True - - context_sym = spimdisasm.common.ContextSymbol(address=0) - context_sym.address = result_symbol_addr - - rodata_sym.contextSym.referenceFunctions = {context_sym} - # Segment __init__ requires opts to be initialized - test_init() - - common_seg_rodata = CommonSegRodata( - rom_start=0x0, - rom_end=0x100, - type=".rodata", - name="MyRodata", - vram_start=0x400, - args=None, - yaml=None, - ) - - common_seg_rodata.parent = CommonSegCode( - rom_start=0x0, - rom_end=0x200, - type="code", - name="MyCode", - vram_start=0x100, - args=[], - yaml=get_yaml(), - ) - - result = common_seg_rodata.get_possible_text_subsegment_for_symbol(rodata_sym) - assert result is not None - assert type(result[0]) == CommonSegC - assert result[1].address == result_symbol_addr - - -class Bss(unittest.TestCase): - def test_disassemble_data(self): - # Segment __init__ requires opts to be initialized - test_init() - - bss = CommonSegBss( - rom_start=0x0, - rom_end=0x100, - type=".bss", - name=None, - vram_start=0x40000000, - args=None, - yaml=None, - ) - - bss.parent = CommonSegCode( - rom_start=0x0, - rom_end=0x200, - type="code", - name="MyCode", - vram_start=0x100, - args=[], - yaml=get_yaml(), - ) - - rom_bytes = bytes([0, 1, 2, 3, 4, 5, 6, 7]) - bss.disassemble_data(rom_bytes) - - assert bss.spim_section is not None - - assert isinstance( - bss.spim_section.get_section(), spimdisasm.mips.sections.SectionBss - ) - assert bss.spim_section.get_section().bssVramStart == 0x40000000 - assert bss.spim_section.get_section().bssVramEnd == 0x300 - - -class SymbolsInitialize(unittest.TestCase): - def test_attrs(self): - import pathlib - - symbols.reset_symbols() - test_init() - - sym_addrs_lines = [ - "func_1 = 0x100; // type:func size:10 rom:100 segment:test_segment name_end:the_name_end " - ] - - all_segments = [ - Segment( - rom_start=0x100, - rom_end=0x200, - type="func", - name="test_segment", - vram_start=0x300, - args=[], - yaml={}, - ) - ] - - symbols.handle_sym_addrs( - pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments - ) - assert symbols.all_symbols[0].given_name == "func_1" - assert symbols.all_symbols[0].type == "func" - assert symbols.all_symbols[0].given_size == 10 - assert symbols.all_symbols[0].rom == 100 - assert symbols.all_symbols[0].segment == all_segments[0] - assert symbols.all_symbols[0].given_name_end == "the_name_end" - - def test_boolean_attrs(self): - import pathlib - - symbols.reset_symbols() - test_init() - - sym_addrs_lines = [ - "func_1 = 0x100; // defined:True extract:True force_migration:True force_not_migration:True " - "allow_addend:True dont_allow_addend:True" - ] - - all_segments = [ - Segment( - rom_start=0x100, - rom_end=0x200, - type="func", - name="test_segment", - vram_start=0x300, - args=[], - yaml={}, - ) - ] - - symbols.handle_sym_addrs( - pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments - ) - assert symbols.all_symbols[0].defined == True - assert symbols.all_symbols[0].force_migration == True - assert symbols.all_symbols[0].force_not_migration == True - assert symbols.all_symbols[0].allow_addend == True - assert symbols.all_symbols[0].dont_allow_addend == True - - # test spim ban range - def test_ignore(self): - import pathlib - - symbols.reset_symbols() - test_init() - - sym_addrs_lines = ["func_1 = 0x100; // ignore:True size:4"] - - all_segments = [ - Segment( - rom_start=0x100, - rom_end=0x200, - type="func", - name="test_segment", - vram_start=0x300, - args=[], - yaml={}, - ) - ] - - symbols.handle_sym_addrs( - pathlib.Path("/tmp/thing"), sym_addrs_lines, all_segments - ) - assert symbols.spim_context.bannedRangedSymbols[0].start == 0x100 - assert symbols.spim_context.bannedRangedSymbols[0].end == 0x100 + 4 - - -class InitializeSpimContext(unittest.TestCase): - def test_overlay(self): - symbols.reset_symbols() - test_init() - - yaml = { - "name": "boot", - "type": "code", - "start": 4096, - "vram": 2147484672, - "bss_size": 128, - "exclusive_ram_id": "overlay", - "subsegments": [ - [4096, "c", "main"], - [4336, "hasm", "handwritten"], - [4352, "data", "main"], - [4368, "rodata", "main"], - {"type": "bss", "vram": 2147484992, "name": "main"}, - ], - } - - all_segments: List["Segment"] = [ - CommonSegCode( - rom_start=0x0, - rom_end=0x200, - type="code", - name="main", - vram_start=0x100, - args=[], - yaml=yaml, - ) - ] - - # force this since it's hard to set up - all_segments[0].exclusive_ram_id = "overlay" - - symbols.initialize_spim_context(all_segments) - # spim should have added something to overlaySegments - assert ( - type(symbols.spim_context.overlaySegments["overlay"][0]) - == spimdisasm.common.SymbolsSegment - ) - - # test globalSegment settings - def test_global(self): - symbols.reset_symbols() - test_init() - - yaml = { - "name": "boot", - "type": "code", - "start": 4096, - "vram": 2147484672, - "bss_size": 128, - "exclusive_ram_id": "overlay", - "subsegments": [ - [4096, "c", "main"], - [4336, "hasm", "handwritten"], - [4352, "data", "main"], - [4368, "rodata", "main"], - {"type": "bss", "vram": 2147484992, "name": "main"}, - ], - } - - all_segments: List["Segment"] = [ - CommonSegCode( - rom_start=0x0, - rom_end=0x200, - type="code", - name="main", - vram_start=0x100, - args=[], - yaml=yaml, - ) - ] - - assert symbols.spim_context.globalSegment.vramStart == 2147483648 - assert symbols.spim_context.globalSegment.vramEnd == 2147487744 - symbols.initialize_spim_context(all_segments) - assert symbols.spim_context.globalSegment.vramStart == 256 - assert symbols.spim_context.globalSegment.vramEnd == 896 - - -if __name__ == "__main__": - unittest.main() diff --git a/tools/splat/test/basic_app/.gitignore b/tools/splat/test/basic_app/.gitignore deleted file mode 100644 index df3d40d83c..0000000000 --- a/tools/splat/test/basic_app/.gitignore +++ /dev/null @@ -1,3 +0,0 @@ -build/ -split/ -gcc-2.7.2/ diff --git a/tools/splat/test/basic_app/basic_app.ld b/tools/splat/test/basic_app/basic_app.ld deleted file mode 100644 index 7a94e16d39..0000000000 --- a/tools/splat/test/basic_app/basic_app.ld +++ /dev/null @@ -1,78 +0,0 @@ -SECTIONS -{ - __romPos = 0; - _gp = 0x0; - header_ROM_START = __romPos; - header_VRAM = ADDR(.header); - .header : AT(header_ROM_START) SUBALIGN(16) - { - header_DATA_START = .; - build/header.o(.data); - header_DATA_END = .; - header_DATA_SIZE = ABSOLUTE(header_DATA_END - header_DATA_START); - } - __romPos += SIZEOF(.header); - header_ROM_END = __romPos; - header_VRAM_END = .; - - dummy_ipl3_ROM_START = __romPos; - dummy_ipl3_VRAM = ADDR(.dummy_ipl3); - .dummy_ipl3 0xA4000040 : AT(dummy_ipl3_ROM_START) SUBALIGN(16) - { - dummy_ipl3_DATA_START = .; - build/dummy_ipl3.o(.data); - dummy_ipl3_DATA_END = .; - dummy_ipl3_DATA_SIZE = ABSOLUTE(dummy_ipl3_DATA_END - dummy_ipl3_DATA_START); - } - __romPos += SIZEOF(.dummy_ipl3); - __romPos = ALIGN(__romPos, 16); - dummy_ipl3_ROM_END = __romPos; - dummy_ipl3_VRAM_END = .; - - boot_ROM_START = __romPos; - boot_VRAM = ADDR(.boot); - .boot 0x80000400 : AT(boot_ROM_START) SUBALIGN(16) - { - boot_TEXT_START = .; - build/main.o(.text); - build/handwritten.o(.text); - boot_TEXT_END = .; - boot_TEXT_SIZE = ABSOLUTE(boot_TEXT_END - boot_TEXT_START); - - boot_DATA_START = .; - build/main.o(.data); - build/handwritten.o(.data); - boot_DATA_END = .; - boot_DATA_SIZE = ABSOLUTE(boot_DATA_END - boot_DATA_START); - - boot_RODATA_START = .; - build/main.o(.rodata); - build/handwritten.o(.rodata); - boot_RODATA_END = .; - boot_RODATA_SIZE = ABSOLUTE(boot_RODATA_END - boot_RODATA_START); - } - - boot_bss_VRAM = ADDR(.boot_bss); - .boot_bss (NOLOAD) : SUBALIGN(16) - { - boot_BSS_START = .; - build/main.o(.bss); - build/main.o(.scommon); - build/main.o(COMMON); - build/handwritten.o(.bss); - build/handwritten.o(.scommon); - build/handwritten.o(COMMON); - boot_BSS_END = .; - boot_BSS_SIZE = ABSOLUTE(boot_BSS_END - boot_BSS_START); - } - - __romPos += SIZEOF(.boot); - __romPos = ALIGN(__romPos, 16); - boot_ROM_END = __romPos; - boot_VRAM_END = .; - - /DISCARD/ : - { - *(*); - } -} diff --git a/tools/splat/test/basic_app/build.sh b/tools/splat/test/basic_app/build.sh deleted file mode 100755 index 0eb0ff12cc..0000000000 --- a/tools/splat/test/basic_app/build.sh +++ /dev/null @@ -1,16 +0,0 @@ -#!/bin/bash - -set -e - -mkdir -p build -echo "Building..." -export PATH=/gcc-2.7.2:$PATH -COMPILER_PATH=/gcc-2.7.2 /gcc-2.7.2/gcc -v -G0 -mgp32 -mfp32 -mips3 main.c -c -o build/main.o -COMPILER_PATH=/gcc-2.7.2 /gcc-2.7.2/gcc -G0 -mgp32 -mfp32 -mips3 header.s -c -o build/header.o -COMPILER_PATH=/gcc-2.7.2 /gcc-2.7.2/gcc -G0 -mgp32 -mfp32 -mips3 handwritten.s -c -o build/handwritten.o -COMPILER_PATH=/gcc-2.7.2 /gcc-2.7.2/gcc -G0 -mgp32 -mfp32 -mips3 dummy_ipl3.s -c -o build/dummy_ipl3.o -mips-linux-gnu-strip -N dummy_symbol_name build/*.o -echo "Linking..." -mips-linux-gnu-ld -Map build/basic_app.map -T basic_app.ld -o build/basic_app.elf -echo "Dumping bin..." -mips-linux-gnu-objcopy build/basic_app.elf -O binary build/basic_app.bin --pad-to=0x1140 --gap-fill=0xFF diff --git a/tools/splat/test/basic_app/clean.sh b/tools/splat/test/basic_app/clean.sh deleted file mode 100644 index 2fcd367d6a..0000000000 --- a/tools/splat/test/basic_app/clean.sh +++ /dev/null @@ -1,2 +0,0 @@ -rm -rf ./build -rm -rf ./split diff --git a/tools/splat/test/basic_app/dummy_ipl3.s b/tools/splat/test/basic_app/dummy_ipl3.s deleted file mode 100644 index fc223a1475..0000000000 --- a/tools/splat/test/basic_app/dummy_ipl3.s +++ /dev/null @@ -1,3 +0,0 @@ -.section .data -dummy_ipl3: -.zero 0x1000 - 0x40 diff --git a/tools/splat/test/basic_app/expected/.splache b/tools/splat/test/basic_app/expected/.splache deleted file mode 100644 index 8f17e1346d..0000000000 Binary files a/tools/splat/test/basic_app/expected/.splache and /dev/null differ diff --git a/tools/splat/test/basic_app/expected/asm/data/main.bss.s b/tools/splat/test/basic_app/expected/asm/data/main.bss.s deleted file mode 100644 index 96a20c23f8..0000000000 --- a/tools/splat/test/basic_app/expected/asm/data/main.bss.s +++ /dev/null @@ -1,6 +0,0 @@ -.include "macro.inc" - -.section .bss - -glabel D_80000540 -/* 1110 80000540 */ .space 0x78 diff --git a/tools/splat/test/basic_app/expected/asm/data/main.data.s b/tools/splat/test/basic_app/expected/asm/data/main.data.s deleted file mode 100644 index 1ed582a025..0000000000 --- a/tools/splat/test/basic_app/expected/asm/data/main.data.s +++ /dev/null @@ -1,13 +0,0 @@ -.include "macro.inc" - -.section .data - -glabel D_80000500 -/* 1100 80000500 */ .word 0x00000001 -.size D_80000500, . - D_80000500 - -glabel D_80000504 -/* 1104 80000504 */ .word 0x00000000 -/* 1108 80000508 */ .word 0x00000000 -/* 110C 8000050C */ .word 0x00000000 -.size D_80000504, . - D_80000504 diff --git a/tools/splat/test/basic_app/expected/asm/handwritten.s b/tools/splat/test/basic_app/expected/asm/handwritten.s deleted file mode 100644 index 539d92ac11..0000000000 --- a/tools/splat/test/basic_app/expected/asm/handwritten.s +++ /dev/null @@ -1,16 +0,0 @@ -.include "macro.inc" - -/* assembler directives */ -.set noat /* allow manual use of $at */ -.set noreorder /* don't insert nops after branches */ -.set gp=64 /* allow use of 64-bit general purpose registers */ - -.section .text, "ax" - -# Handwritten function -glabel func_800004F0 -/* 10F0 800004F0 00851020 */ add $v0, $a0, $a1 # handwritten instruction -/* 10F4 800004F4 03E00008 */ jr $ra -/* 10F8 800004F8 00000000 */ nop -/* 10FC 800004FC 00000000 */ nop -.size func_800004F0, . - func_800004F0 diff --git a/tools/splat/test/basic_app/expected/asm/header.s b/tools/splat/test/basic_app/expected/asm/header.s deleted file mode 100644 index be176b7e5d..0000000000 --- a/tools/splat/test/basic_app/expected/asm/header.s +++ /dev/null @@ -1,16 +0,0 @@ -.section .data - -.word 0x80371240 /* PI BSB Domain 1 register */ -.word 0x0000000F /* Clockrate setting */ -.word 0x800004A0 /* Entrypoint address */ -.word 0x00001444 /* Revision */ -.word 0x00000000 /* Checksum 1 */ -.word 0x00000000 /* Checksum 2 */ -.word 0x00000000 /* Unknown 1 */ -.word 0x00000000 /* Unknown 2 */ -.ascii "SPLAT TEST ROM " /* Internal name */ -.word 0x00000000 /* Unknown 3 */ -.word 0x0000004E /* Cartridge */ -.ascii "SP" /* Cartridge ID */ -.ascii "E" /* Country code */ -.byte 0x00 /* Version */ diff --git a/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s b/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s deleted file mode 100644 index 21d0010cbd..0000000000 --- a/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_80000400.s +++ /dev/null @@ -1,55 +0,0 @@ -.set noat /* allow manual use of $at */ -.set noreorder /* don't insert nops after branches */ - -glabel func_80000400 -/* 1000 80000400 27BDFFF8 */ addiu $sp, $sp, -0x8 -/* 1004 80000404 AFBE0000 */ sw $fp, 0x0($sp) -/* 1008 80000408 03A0F021 */ addu $fp, $sp, $zero -/* 100C 8000040C 3C028000 */ lui $v0, %hi(D_80000504) -/* 1010 80000410 8C420504 */ lw $v0, %lo(D_80000504)($v0) -/* 1014 80000414 2C430008 */ sltiu $v1, $v0, 0x8 -/* 1018 80000418 1060001B */ beqz $v1, .L80000488 -/* 101C 8000041C 00000000 */ nop -/* 1020 80000420 3C028000 */ lui $v0, %hi(D_80000504) -/* 1024 80000424 8C420504 */ lw $v0, %lo(D_80000504)($v0) -/* 1028 80000428 00401821 */ addu $v1, $v0, $zero -/* 102C 8000042C 00031080 */ sll $v0, $v1, 2 -/* 1030 80000430 3C038000 */ lui $v1, %hi(jtbl_80000518) -/* 1034 80000434 24630518 */ addiu $v1, $v1, %lo(jtbl_80000518) -/* 1038 80000438 00431021 */ addu $v0, $v0, $v1 -/* 103C 8000043C 8C430000 */ lw $v1, 0x0($v0) -/* 1040 80000440 00600008 */ jr $v1 -/* 1044 80000444 00000000 */ nop -glabel .L80000448 -/* 1048 80000448 08000124 */ j .L80000490 -/* 104C 8000044C 24020007 */ addiu $v0, $zero, 0x7 -glabel .L80000450 -/* 1050 80000450 08000124 */ j .L80000490 -/* 1054 80000454 24020006 */ addiu $v0, $zero, 0x6 -glabel .L80000458 -/* 1058 80000458 08000124 */ j .L80000490 -/* 105C 8000045C 24020005 */ addiu $v0, $zero, 0x5 -glabel .L80000460 -/* 1060 80000460 08000124 */ j .L80000490 -/* 1064 80000464 24020004 */ addiu $v0, $zero, 0x4 -glabel .L80000468 -/* 1068 80000468 08000124 */ j .L80000490 -/* 106C 8000046C 24020003 */ addiu $v0, $zero, 0x3 -glabel .L80000470 -/* 1070 80000470 08000124 */ j .L80000490 -/* 1074 80000474 24020002 */ addiu $v0, $zero, 0x2 -glabel .L80000478 -/* 1078 80000478 08000124 */ j .L80000490 -/* 107C 8000047C 24020001 */ addiu $v0, $zero, 0x1 -glabel .L80000480 -/* 1080 80000480 08000124 */ j .L80000490 -/* 1084 80000484 00001021 */ addu $v0, $zero, $zero -.L80000488: -/* 1088 80000488 08000124 */ j .L80000490 -/* 108C 8000048C 00001021 */ addu $v0, $zero, $zero -.L80000490: -/* 1090 80000490 03C0E821 */ addu $sp, $fp, $zero -/* 1094 80000494 8FBE0000 */ lw $fp, 0x0($sp) -/* 1098 80000498 03E00008 */ jr $ra -/* 109C 8000049C 27BD0008 */ addiu $sp, $sp, 0x8 -.size func_80000400, . - func_80000400 diff --git a/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s b/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s deleted file mode 100644 index 112eb68090..0000000000 --- a/tools/splat/test/basic_app/expected/asm/nonmatchings/main/func_800004A0.s +++ /dev/null @@ -1,26 +0,0 @@ -.set noat /* allow manual use of $at */ -.set noreorder /* don't insert nops after branches */ - -glabel func_800004A0 -/* 10A0 800004A0 27BDFFE8 */ addiu $sp, $sp, -0x18 -/* 10A4 800004A4 AFBF0014 */ sw $ra, 0x14($sp) -/* 10A8 800004A8 AFBE0010 */ sw $fp, 0x10($sp) -/* 10AC 800004AC 0C000100 */ jal func_80000400 -/* 10B0 800004B0 03A0F021 */ addu $fp, $sp, $zero -.L800004B4: -/* 10B4 800004B4 3C028000 */ lui $v0, %hi(D_80000500) -/* 10B8 800004B8 8C420500 */ lw $v0, %lo(D_80000500)($v0) -/* 10BC 800004BC 24430001 */ addiu $v1, $v0, 0x1 -/* 10C0 800004C0 3C018000 */ lui $at, %hi(D_80000500) -/* 10C4 800004C4 AC230500 */ sw $v1, %lo(D_80000500)($at) -/* 10C8 800004C8 3C028000 */ lui $v0, %hi(D_80000500) -/* 10CC 800004CC 8C420500 */ lw $v0, %lo(D_80000500)($v0) -/* 10D0 800004D0 0800012D */ j .L800004B4 -/* 10D4 800004D4 00000000 */ nop -/* 10D8 800004D8 03C0E821 */ addu $sp, $fp, $zero -/* 10DC 800004DC 8FBF0014 */ lw $ra, 0x14($sp) -/* 10E0 800004E0 8FBE0010 */ lw $fp, 0x10($sp) -/* 10E4 800004E4 03E00008 */ jr $ra -/* 10E8 800004E8 27BD0018 */ addiu $sp, $sp, 0x18 -/* 10EC 800004EC 00000000 */ nop -.size func_800004A0, . - func_800004A0 diff --git a/tools/splat/test/basic_app/expected/assets/dummy_ipl3.bin b/tools/splat/test/basic_app/expected/assets/dummy_ipl3.bin deleted file mode 100644 index 26460b77b4..0000000000 Binary files a/tools/splat/test/basic_app/expected/assets/dummy_ipl3.bin and /dev/null differ diff --git a/tools/splat/test/basic_app/expected/basic_app.ld b/tools/splat/test/basic_app/expected/basic_app.ld deleted file mode 100644 index bd3d10bced..0000000000 --- a/tools/splat/test/basic_app/expected/basic_app.ld +++ /dev/null @@ -1,74 +0,0 @@ -SECTIONS -{ - __romPos = 0; - _gp = 0x0; - header_ROM_START = __romPos; - header_VRAM = ADDR(.header); - .header : AT(header_ROM_START) SUBALIGN(16) - { - FILL(0x00000000); - header_DATA_START = .; - header_s = .; - build/split/asm/header.s.o(.data); - header_DATA_END = .; - header_DATA_SIZE = ABSOLUTE(header_DATA_END - header_DATA_START); - } - __romPos += SIZEOF(.header); - header_ROM_END = __romPos; - header_VRAM_END = .; - - dummy_ipl3_ROM_START = __romPos; - dummy_ipl3_VRAM = ADDR(.dummy_ipl3); - .dummy_ipl3 0xA4000040 : AT(dummy_ipl3_ROM_START) SUBALIGN(16) - { - FILL(0x00000000); - dummy_ipl3_DATA_START = .; - dummy_ipl3_bin = .; - build/split/assets/dummy_ipl3.bin.o(.data); - dummy_ipl3_DATA_END = .; - dummy_ipl3_DATA_SIZE = ABSOLUTE(dummy_ipl3_DATA_END - dummy_ipl3_DATA_START); - } - __romPos += SIZEOF(.dummy_ipl3); - __romPos = ALIGN(__romPos, 16); - dummy_ipl3_ROM_END = __romPos; - dummy_ipl3_VRAM_END = .; - - boot_ROM_START = __romPos; - boot_VRAM = ADDR(.boot); - .boot 0x80000400 : AT(boot_ROM_START) SUBALIGN(16) - { - FILL(0x00000000); - boot_TEXT_START = .; - build/split/src/main.c.o(.text); - build/split/asm/handwritten.s.o(.text); - boot_TEXT_END = .; - boot_TEXT_SIZE = ABSOLUTE(boot_TEXT_END - boot_TEXT_START); - boot_DATA_START = .; - main_data__s = .; - build/split/asm/data/main.data.s.o(.data); - boot_DATA_END = .; - boot_DATA_SIZE = ABSOLUTE(boot_DATA_END - boot_DATA_START); - boot_RODATA_START = .; - build/split/asm/data/main.rodata.s.o(.rodata); - boot_RODATA_END = .; - boot_RODATA_SIZE = ABSOLUTE(boot_RODATA_END - boot_RODATA_START); - } - boot_bss_VRAM = ADDR(.boot_bss); - .boot_bss (NOLOAD) : SUBALIGN(16) - { - FILL(0x00000000); - boot_BSS_START = .; - build/split/asm/data/main.bss.s.o(.bss); - boot_BSS_END = .; - boot_BSS_SIZE = ABSOLUTE(boot_BSS_END - boot_BSS_START); - } - __romPos += SIZEOF(.boot); - __romPos = ALIGN(__romPos, 16); - boot_ROM_END = __romPos; - boot_VRAM_END = .; - - /DISCARD/ : - { - *(*); - } -} diff --git a/tools/splat/test/basic_app/expected/src/main.c b/tools/splat/test/basic_app/expected/src/main.c deleted file mode 100644 index 41b2e705ef..0000000000 --- a/tools/splat/test/basic_app/expected/src/main.c +++ /dev/null @@ -1,5 +0,0 @@ -#include "common.h" - -INCLUDE_ASM(const s32, "main", func_80000400); - -INCLUDE_ASM(const s32, "main", func_800004A0); diff --git a/tools/splat/test/basic_app/expected/undefined_funcs_auto.txt b/tools/splat/test/basic_app/expected/undefined_funcs_auto.txt deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/splat/test/basic_app/expected/undefined_syms_auto.txt b/tools/splat/test/basic_app/expected/undefined_syms_auto.txt deleted file mode 100644 index 8b96b0cd95..0000000000 --- a/tools/splat/test/basic_app/expected/undefined_syms_auto.txt +++ /dev/null @@ -1 +0,0 @@ -jtbl_80000518 = 0x80000518; diff --git a/tools/splat/test/basic_app/handwritten.s b/tools/splat/test/basic_app/handwritten.s deleted file mode 100644 index 1a5f482712..0000000000 --- a/tools/splat/test/basic_app/handwritten.s +++ /dev/null @@ -1,12 +0,0 @@ -.section .text -.balign 16 -.set noreorder - -.global asm_func -.type asm_func, @function -.ent asm_func -asm_func: - add $v0, $a0, $a1 # Instruction that a compiler will never emit - jr $ra - nop -.end asm_func diff --git a/tools/splat/test/basic_app/header.s b/tools/splat/test/basic_app/header.s deleted file mode 100644 index 1e19f59bea..0000000000 --- a/tools/splat/test/basic_app/header.s +++ /dev/null @@ -1,16 +0,0 @@ -.section .data - -.word 0x80371240 /* PI BSB Domain 1 register */ -.word 0x0000000F /* Clockrate setting */ -.word bootproc /* Entrypoint address */ -.word 0x00001444 /* Revision */ -.word 0x00000000 /* Checksum 1 */ -.word 0x00000000 /* Checksum 2 */ -.word 0x00000000 /* Unknown 1 */ -.word 0x00000000 /* Unknown 2 */ -.ascii "SPLAT TEST ROM " /* Internal name */ -.word 0x00000000 /* Unknown 3 */ -.word 0x0000004E /* Cartridge */ -.ascii "SP" /* Cartridge ID */ -.ascii "E" /* Country code */ -.byte 0x00 /* Version */ diff --git a/tools/splat/test/basic_app/main.c b/tools/splat/test/basic_app/main.c deleted file mode 100644 index cf119f7c19..0000000000 --- a/tools/splat/test/basic_app/main.c +++ /dev/null @@ -1,45 +0,0 @@ -// .data -volatile int test = 1; - -// bin (.rodata) -const char bin_data[] = {0,1,2,3,4,5,6,7}; - -// .bss -unsigned long long bss_data[0x10]; - -// .data -volatile int switch_arg = 0; - -int do_switch() -{ - switch(switch_arg) - { - case 0: - return 7; - case 1: - return 6; - case 2: - return 5; - case 3: - return 4; - case 4: - return 3; - case 5: - return 2; - case 6: - return 1; - case 7: - return 0; - } - return 0; -} - -// c -void bootproc() -{ - do_switch(); - for (;;) - { - test++; - } -} diff --git a/tools/splat/test/basic_app/splat.yaml b/tools/splat/test/basic_app/splat.yaml deleted file mode 100644 index d4d4a41c14..0000000000 --- a/tools/splat/test/basic_app/splat.yaml +++ /dev/null @@ -1,37 +0,0 @@ -options: - platform: n64 - basename: basic_app - base_path: . - build_path: build - target_path: build/basic_app.bin - asm_path: split/asm - src_path: split/src - ld_script_path: split/basic_app.ld - cache_path: split/.splache - symbol_addrs_path: split/generated.symbols.txt - undefined_funcs_auto_path: split/undefined_funcs_auto.txt - undefined_syms_auto_path: split/undefined_syms_auto.txt - asset_path: split/assets - compiler: GCC -segments: - - name: header - type: header - start: 0x00 - - name: dummy_ipl3 - type: code - start: 0x40 - vram: 0xA4000040 - subsegments: - - [0x0040, bin, dummy_ipl3] - - name: boot - type: code - start: 0x1000 - vram: 0x80000400 - bss_size: 0x80 - subsegments: - - [0x1000, c, main] - - [0x10F0, hasm, handwritten] - - [0x1100, data, main] - - [0x1110, rodata, main] - - { type: bss, vram: 0x80000540, name: main } - - [0x1138] diff --git a/tools/splat/test_gen_expected.sh b/tools/splat/test_gen_expected.sh deleted file mode 100644 index 1b4a629abc..0000000000 --- a/tools/splat/test_gen_expected.sh +++ /dev/null @@ -1,3 +0,0 @@ -docker run --rm -it -v $(pwd):/splat -w /splat/test/basic_app splat-build sh build.sh && \ -rm -rf $(pwd)/test/basic_app/expected && \ -cp -r $(pwd)/test/basic_app/split $(pwd)/test/basic_app/expected diff --git a/tools/splat/util/__init__.py b/tools/splat/util/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/splat/util/color.py b/tools/splat/util/color.py deleted file mode 100644 index 6d75cbd7cd..0000000000 --- a/tools/splat/util/color.py +++ /dev/null @@ -1,19 +0,0 @@ -from math import ceil - -from util import options - - -# RRRRRGGG GGBBBBBA -def unpack_color(data): - s = int.from_bytes(data[0:2], byteorder=options.opts.endianness) - - r = (s >> 11) & 0x1F - g = (s >> 6) & 0x1F - b = (s >> 1) & 0x1F - a = (s & 1) * 0xFF - - r = ceil(0xFF * (r / 31)) - g = ceil(0xFF * (g / 31)) - b = ceil(0xFF * (b / 31)) - - return r, g, b, a diff --git a/tools/splat/util/compiler.py b/tools/splat/util/compiler.py deleted file mode 100644 index 3c67e5989d..0000000000 --- a/tools/splat/util/compiler.py +++ /dev/null @@ -1,44 +0,0 @@ -from dataclasses import dataclass -from typing import Optional - - -@dataclass -class Compiler: - name: str - asm_function_macro: str = "glabel" - asm_function_alt_macro: str = "glabel" - asm_jtbl_label_macro: str = "glabel" - asm_data_macro: str = "glabel" - asm_end_label: str = "" - c_newline: str = "\n" - asm_inc_header: str = "" - include_macro_inc: bool = True - asm_emit_size_directive: Optional[bool] = None - - -GCC = Compiler( - "GCC", - asm_inc_header=".set noat /* allow manual use of $at */\n.set noreorder /* don't insert nops after branches */\n\n", -) - -SN64 = Compiler( - "SN64", - asm_function_macro=".globl", - asm_jtbl_label_macro=".globl", - asm_data_macro=".globl", - asm_end_label=".end", - c_newline="\r\n", - include_macro_inc=False, - asm_emit_size_directive=False, -) - -IDO = Compiler("IDO", include_macro_inc=False, asm_emit_size_directive=False) - -compiler_for_name = {"GCC": GCC, "SN64": SN64, "IDO": IDO} - - -def for_name(name: str) -> Compiler: - name = name.upper() - if name in compiler_for_name: - return compiler_for_name[name] - return Compiler(name) diff --git a/tools/splat/util/floats.py b/tools/splat/util/floats.py deleted file mode 100644 index 009725e7ff..0000000000 --- a/tools/splat/util/floats.py +++ /dev/null @@ -1,63 +0,0 @@ -import math -import struct - - -# From mips_to_c: https://github.com/matt-kempster/mips_to_c/blob/d208400cca045113dada3e16c0d59c50cdac4529/src/translate.py#L2085 -def format_f32_imm(num: int) -> str: - packed = struct.pack(">I", num & (2**32 - 1)) - value = struct.unpack(">f", packed)[0] - - if not value or value == 4294967296.0: - # Zero, negative zero, nan, or INT_MAX. - return str(value) - - # Write values smaller than 1e-7 / greater than 1e7 using scientific notation, - # and values in between using fixed point. - if abs(math.log10(abs(value))) > 6.9: - fmt_char = "e" - elif abs(value) < 1: - fmt_char = "f" - else: - fmt_char = "g" - - def fmt(prec: int) -> str: - """Format 'value' with 'prec' significant digits/decimals, in either scientific - or regular notation depending on 'fmt_char'.""" - ret = ("{:." + str(prec) + fmt_char + "}").format(value) - if fmt_char == "e": - return ret.replace("e+", "e").replace("e0", "e").replace("e-0", "e-") - if "e" in ret: - # The "g" format character can sometimes introduce scientific notation if - # formatting with too few decimals. If this happens, return an incorrect - # value to prevent the result from being used. - # - # Since the value we are formatting is within (1e-7, 1e7) in absolute - # value, it will at least be possible to format with 7 decimals, which is - # less than float precision. Thus, this annoying Python limitation won't - # lead to us outputting numbers with more precision than we really have. - return "0" - return ret - - # 20 decimals is more than enough for a float. Start there, then try to shrink it. - prec = 20 - while prec > 0: - prec -= 1 - value2 = float(fmt(prec)) - if struct.pack(">f", value2) != packed: - prec += 1 - break - - if prec == 20: - # Uh oh, even the original value didn't format correctly. Fall back to str(), - # which ought to work. - return str(value) - - ret = fmt(prec) - if "." not in ret and "e" not in ret: - ret += ".0" - return ret - - -def format_f64_imm(num: int) -> str: - (value,) = struct.unpack(">d", struct.pack(">Q", num & (2**64 - 1))) - return str(value) diff --git a/tools/splat/util/gc/gcfst.py b/tools/splat/util/gc/gcfst.py deleted file mode 100644 index e50420f71c..0000000000 --- a/tools/splat/util/gc/gcfst.py +++ /dev/null @@ -1,181 +0,0 @@ -import struct -from pathlib import Path -from typing import List, Optional - -from segtypes.gc.segment import GCSegment - -from util import options -from util.gc.gcutil import read_string_from_bytes - - -# Represents the info for either a directory or a file within a GameCube disc image's file system. -class GCFSTEntry: - def __init__(self, flags: bool, name_offset, offset, length): - self.flags = flags - self.name_offset = name_offset - self.offset = offset - self.length = length - - self.name = "" - self.parent: Optional[GCFSTEntry] = None - self.children: List[GCFSTEntry] = [] - - def populate_children_recursive( - self, root_dir: "GCFSTEntry", current_node_offset, fst_bytes, string_table_bytes - ): - self.parent = root_dir - self.name = read_string_from_bytes(self.name_offset, string_table_bytes) - - # This node is a file, so we don't do anything but return that we read 1 node. - if self.flags == False: - return 1 - - nodes_read = 1 - next_child_offset = current_node_offset + 0x0C - - # Directory nodes contain the index of the next node that is NOT its child, meaning the index of their next sibling node. - # We can figure out when we're done reading child nodes by comparing the offset of the next node to read to the - # offset of the next sibling node. We stop reading when the next node offset is >= the offset of the next sibling node. - while next_child_offset < self.length * 0x0C: - new_entry = GCFSTEntry( - bool(fst_bytes[next_child_offset + 0x0000]), - struct.unpack_from( - ">I", fst_bytes[next_child_offset : next_child_offset + 0x0004] - )[0] - & 0x00FFFFFF, - struct.unpack_from(">I", fst_bytes, next_child_offset + 0x0004)[0], - struct.unpack_from(">I", fst_bytes, next_child_offset + 0x0008)[0], - ) - - self.children.append(new_entry) - nodes_read += new_entry.populate_children_recursive( - self, next_child_offset, fst_bytes, string_table_bytes - ) - - next_child_offset = current_node_offset + nodes_read * 0x0C - - return nodes_read - - # Builds this entry's full path within the filesystem from its parents' names. - def get_full_name(self): - path_components: List[str] = [] - - entry = self - while entry.parent != None: - path_components.insert(0, entry.name) - - if entry.parent is None: - break - entry = entry.parent - - return Path(*path_components) - - # Emits this entry to the filesystem. - def emit(self, filesystem_dir: Path, iso_bytes): - full_path = filesystem_dir / self.get_full_name() - - # If this is a directory, we just need to make the directory on disk. - if self.flags == True: - full_path.mkdir(parents=True, exist_ok=True) - return - - file_bytes = iso_bytes[self.offset : self.offset + self.length] - with open(full_path, "wb") as f: - f.write(file_bytes) - - def emit_recursive(self, filesystem_dir: Path, iso_bytes): - # Don't emit if this is the root directory. - if self.parent != None: - self.emit(filesystem_dir, iso_bytes) - - for e in self.children: - e.emit_recursive(filesystem_dir, iso_bytes) - - -# Splits the ISO into its component parts - header info, apploader, DOL, FST metadata, and the individual files in the filesystem. -def split_iso(iso_bytes): - split_sys_info(iso_bytes) - split_content(iso_bytes) - - -# Splits the header info, apploader, DOL, and FST metadata from the ISO. -def split_sys_info(iso_bytes): - assert options.opts.filesystem_path is not None - - sys_path = options.opts.filesystem_path / "sys" - sys_path.mkdir(parents=True, exist_ok=True) - - # Split boot.info. Always at 0x0000 and 0x0440 bytes long. - with open(sys_path / "boot.bin", "wb") as f: - f.write(iso_bytes[0x0000:0x0440]) - - # Split bi2.info. Always at 0x0440 and 0x2000 bytes long. - with open(sys_path / "bi2.bin", "wb") as f: - f.write(iso_bytes[0x0440:0x2440]) - - # Split apploader.img. Always at 0x2440 and size is listed at 0x0400. - apploader_size = struct.unpack_from(">I", iso_bytes, 0x0400)[0] - with open(sys_path / "apploader.img", "wb") as f: - f.write(iso_bytes[0x2440 : 0x2440 + apploader_size]) - - # Split main.dol. Offset specified explicitly at 0x0420, but size must be calculated. - dol_offset = struct.unpack_from(">I", iso_bytes, 0x0420)[0] - fst_offset = struct.unpack_from(">I", iso_bytes, 0x0424)[0] - - dol_size = fst_offset - dol_offset - with open(sys_path / "main.dol", "wb") as f: - f.write(iso_bytes[dol_offset : dol_offset + dol_size]) - - # Split fst.bin. Offset specified at 0x0424 and size specified at 0x402C. - fst_size = struct.unpack_from(">I", iso_bytes, 0x0428)[0] - with open(sys_path / "fst.bin", "wb") as f: - f.write(iso_bytes[fst_offset : fst_offset + fst_size]) - - -# Splits the ISO's filesystem into individual files. -def split_content(iso_bytes): - assert options.opts.filesystem_path is not None - - fst_path = options.opts.filesystem_path / "sys" / "fst.bin" - assert fst_path.is_file() - - fst_bytes = fst_path.read_bytes() - fst_root_entry = populate_filesystem(fst_bytes) - - files_path = options.opts.filesystem_path / "files" - files_path.mkdir(parents=True, exist_ok=True) - fst_root_entry.emit_recursive(files_path, iso_bytes) - - -# Loads the FST data needed to split the filesystem. -def populate_filesystem(fst_bytes): - root_dir = GCFSTEntry( - bool(fst_bytes[0x0000]), - struct.unpack_from(">I", fst_bytes, 0x0000)[0] & 0x00FFFFFF, - struct.unpack_from(">I", fst_bytes, 0x0004)[0], - struct.unpack_from(">I", fst_bytes, 0x0008)[0], - ) - - string_table_bytes = fst_bytes[root_dir.length * 0x0C : len(fst_bytes)] - - # Parsing the filesystem is a bit tricky. The root directory's length property is the total number of nodes in the FST. - # So, we initialize nodes_read to 1, since the root is included in the number of nodes. - # We will rely on each directory and file on the root directory to tell us how many nodes were read while parsing them. - # We can stop reading the FST when our total number of nodes read is >= the number of nodes in the FST. - nodes_read = 1 - while nodes_read < root_dir.length: - current_offset = nodes_read * 0x0C - - new_entry = GCFSTEntry( - bool(fst_bytes[current_offset + 0x0000]), - struct.unpack_from(">I", fst_bytes, current_offset)[0] & 0x00FFFFFF, - struct.unpack_from(">I", fst_bytes, current_offset + 0x0004)[0], - struct.unpack_from(">I", fst_bytes, current_offset + 0x0008)[0], - ) - - root_dir.children.append(new_entry) - nodes_read += new_entry.populate_children_recursive( - root_dir, current_offset, fst_bytes, string_table_bytes - ) - - return root_dir diff --git a/tools/splat/util/gc/gcinfo.py b/tools/splat/util/gc/gcinfo.py deleted file mode 100644 index 78130df8bc..0000000000 --- a/tools/splat/util/gc/gcinfo.py +++ /dev/null @@ -1,88 +0,0 @@ -#! /usr/bin/env python3 - - -import argparse - -import hashlib - -from pathlib import Path -from typing import Optional - -parser = argparse.ArgumentParser( - description="Gives information on GameCube disc images" -) -parser.add_argument("iso", help="path to a GameCube disc image") - -system_codes = { - "D": "GameCube Demo", - "G": "GameCube", - "P": "GameCube Promotional", - "R": "Early Wii", - "S": "Later Wii", -} - -region_codes = {"E": "NTSC-U", "J": "NTSC-J", "P": "PAL"} - -publisher_codes = {"01": "Nintendo", "08": "Capcom", "8P": "Sega", "E9": "Natsume"} - - -def get_info(iso_path: Path, iso_bytes: Optional[bytes] = None): - if iso_bytes is None: - iso_bytes = iso_path.read_bytes() - - return get_info_bytes(iso_bytes) - - -def get_info_bytes(iso_bytes: bytes): - system_code = chr(iso_bytes[0x00]) - game_code = iso_bytes[0x01:0x03].decode("utf-8") - region_code = chr(iso_bytes[0x03]) - publisher_code = iso_bytes[0x04:0x06].decode("utf-8") - - name = str(iso_bytes[0x20:0x400], "utf-8").strip("\x00") - root = "filesystem" - - compiler = "mwcc" - sha1 = hashlib.sha1(iso_bytes).hexdigest() - - return GCIso( - name, - root, - system_code, - game_code, - region_code, - publisher_code, - compiler, - sha1, - ) - - -class GCIso: - def __init__( - self, - name: str, - root: str, - system_code, - game_code, - region_code, - publisher_code, - compiler, - sha1, - ): - self.name = name - self.root = root - self.system_code = system_code - self.game_code = game_code - self.region_code = region_code - self.publisher_code = publisher_code - self.compiler = compiler - self.sha1 = sha1 - - def get_system_name(self): - return system_codes[self.system_code] - - def get_publisher_name(self): - return publisher_codes[self.publisher_code] - - def get_region_name(self): - return region_codes[self.region_code] diff --git a/tools/splat/util/gc/gcutil.py b/tools/splat/util/gc/gcutil.py deleted file mode 100644 index ba78bc2f05..0000000000 --- a/tools/splat/util/gc/gcutil.py +++ /dev/null @@ -1,11 +0,0 @@ -def read_string_from_bytes(name_offset, string_table_bytes): - bytes = bytearray() - - for offset in range(len(string_table_bytes) - name_offset): - cur_byte = string_table_bytes[name_offset + offset] - if cur_byte == 0x00: - break - - bytes.append(cur_byte) - - return bytes.decode("shift-jis") diff --git a/tools/splat/util/log.py b/tools/splat/util/log.py deleted file mode 100644 index 4f296da1bd..0000000000 --- a/tools/splat/util/log.py +++ /dev/null @@ -1,46 +0,0 @@ -import sys -from typing import NoReturn, Optional -from pathlib import Path - -from colorama import Fore, init, Style - -init(autoreset=True) - -newline = True - -Status = Optional[str] - - -def write(*args, status=None, **kwargs): - global newline - - if not newline: - print("") - newline = True - - print(status_to_ansi(status) + str(args[0]), *args[1:], **kwargs) - - -def error(*args, **kwargs) -> NoReturn: - write(*args, **kwargs, status="error") - sys.exit(2) - - -# The line_num is expected to be zero-indexed -def parsing_error_preamble(path: Path, line_num: int, line: str): - write("") - write(f"error reading {path}, line {line_num + 1}:", status="error") - write(f"\t{line}") - - -def status_to_ansi(status: Status): - if status == "ok": - return Fore.GREEN - elif status == "warn": - return Fore.YELLOW + Style.BRIGHT - elif status == "error": - return Fore.RED + Style.BRIGHT - elif status == "skip": - return Style.DIM - else: - return "" diff --git a/tools/splat/util/n64/Mio0decompress.py b/tools/splat/util/n64/Mio0decompress.py deleted file mode 100644 index 64a059e7cf..0000000000 --- a/tools/splat/util/n64/Mio0decompress.py +++ /dev/null @@ -1,108 +0,0 @@ -import argparse -import struct -import sys - -try: - from .. import log - from .decompressor import Decompressor -except ImportError: - print(f"Run as python3 -m util.n64.Miodecompress") - sys.exit(1) - - -class GenericMio0Decompressor(Decompressor): - def __init__( - self, unpacked_offset, compressed_offset, uncompressed_offset, header_length - ): - self.unpacked_offset = unpacked_offset - self.compressed_offset = compressed_offset - self.uncompressed_offset = uncompressed_offset - self.header_length = header_length - - @staticmethod - def read_word(data, offset): - (res,) = struct.unpack(">I", data[offset : offset + 4]) - return res - - @staticmethod - def read_short(data, offset): - (res,) = struct.unpack(">H", data[offset : offset + 2]) - return res - - def decompress(self, in_bytes, byte_order="big") -> bytearray: - magic = in_bytes[0:4] - if magic != b"MIO0": - log.error(f"MIO0 magic is incorrect: {magic}") - - unpacked_size = self.read_word(in_bytes, self.unpacked_offset) - comp_offset = self.read_word(in_bytes, self.compressed_offset) - uncomp_offset = self.read_word(in_bytes, self.uncompressed_offset) - - layout_data = struct.iter_unpack(">I", in_bytes[self.header_length :]) - uncompressed_data = struct.iter_unpack(">B", in_bytes[uncomp_offset:]) - compressed_data = struct.iter_unpack(">H", in_bytes[comp_offset:]) - - idx = 0 - ret = bytearray(unpacked_size) - - mask_bit_counter = 0 - while idx < unpacked_size: - if mask_bit_counter == 0: - (current_mask,) = next(layout_data) - mask_bit_counter = 32 - - if current_mask & 0x80000000: - (ud,) = next(uncompressed_data) - ret[idx] = ud - idx += 1 - else: - (length_offset,) = next(compressed_data) - - length = (length_offset >> 12) + 3 - index = (length_offset & 0xFFF) + 1 - offset = idx - index - - if not (3 <= length <= 18): - log.error(f"Invalid length: {length}, corrupt data?") - - if not (1 <= index <= 4096): - log.error(f"Invalid index: {index}, corrupt data?") - - for i in range(length): - ret[idx] = ret[offset + i] - idx += 1 - - current_mask <<= 1 - mask_bit_counter -= 1 - - return ret - - -class Mio0Decompressor(GenericMio0Decompressor): - def __init__(self): - super().__init__( - unpacked_offset=4, - compressed_offset=8, - uncompressed_offset=12, - header_length=16, - ) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("infile") - parser.add_argument("outfile") - args = parser.parse_args() - - with open(args.infile, "rb") as f: - raw_bytes = f.read() - - miodecompress = Mio0Decompressor() - decompressed = miodecompress.decompress(raw_bytes) - - with open(args.outfile, "wb") as f: - f.write(decompressed) - - -if __name__ == "__main__": - main() diff --git a/tools/splat/util/n64/Yay0decompress.c b/tools/splat/util/n64/Yay0decompress.c deleted file mode 100644 index 250f3476bf..0000000000 --- a/tools/splat/util/n64/Yay0decompress.c +++ /dev/null @@ -1,44 +0,0 @@ -#include -#include -#include -#include - -typedef struct { - uint32_t magic; - uint32_t uncompressedLength; - uint32_t opPtr; - uint32_t dataPtr; -} Yay0Header; - -void decompress(Yay0Header* hdr, uint8_t* srcPtr, uint8_t* dstPtr, bool isBigEndian) { - uint8_t byte = 0, mask = 0; - uint8_t* ctrl, * ops, * data; - uint16_t copy, op; - uint32_t written = 0; - - ctrl = srcPtr + sizeof(Yay0Header); - ops = srcPtr + hdr->opPtr; - data = srcPtr + hdr->dataPtr; - - while (written < hdr->uncompressedLength) { - if ((mask >>= 1) == 0) { - byte = *ctrl++; - mask = 0x80; - } - - if (byte & mask) { - *dstPtr++ = *data++; - written++; - } else { - op = isBigEndian ? (ops[0] << 8) | ops[1] : (ops[1] << 8) | ops[0]; - ops += 2; - - written += copy = (op >> 12) ? (2 + (op >> 12)) : (18 + *data++); - - while (copy--) { - *dstPtr = dstPtr[-(op & 0xfff) - 1]; - dstPtr++; - } - } - } -} diff --git a/tools/splat/util/n64/Yay0decompress.py b/tools/splat/util/n64/Yay0decompress.py deleted file mode 100644 index 780d1cb199..0000000000 --- a/tools/splat/util/n64/Yay0decompress.py +++ /dev/null @@ -1,162 +0,0 @@ -import argparse -import os -import sys -from ctypes import CDLL, Structure, byref, c_bool, c_uint32, c_uint8, cdll -from struct import pack, unpack_from -from typing import Literal, Optional - -try: - from .. import log - from .decompressor import Decompressor -except ImportError: - print(f"Run as python3 -m util.n64.Yay0decompress") - sys.exit(1) - -tried_loading = False -lib: Optional[CDLL] = None - - -def setup_lib(): - global lib - global tried_loading - if lib: - return True - if tried_loading: - return False - tried_loading = True - try: - lib = cdll.LoadLibrary( - os.path.dirname(os.path.realpath(__file__)) + "/Yay0decompress" - ) - return True - except Exception: - log.write( - "Failed to load Yay0 C library; falling back to Python method", - status="warn", - ) - tried_loading = True - return False - - -class Yay0Decompressor(Decompressor): - @staticmethod - def decompress(in_bytes, byte_order: Literal["little", "big"] = "big") -> bytearray: - # attempt to load the library only once per execution - global lib - if not setup_lib(): - return Yay0Decompressor.decompress_python(in_bytes, byte_order) - assert lib is not None - - class Yay0(Structure): - _fields_ = [ - ("magic", c_uint32), - ("uncompressedLength", c_uint32), - ("opPtr", c_uint32), - ("dataPtr", c_uint32), - ] - - # read the file header - bigEndian = byte_order == "big" - if bigEndian: - # the struct is only a view, so when passed to C it will keep - # its BigEndian values and crash. Explicitly convert them here to little - hdr = Yay0.from_buffer_copy( - pack("IIII", in_bytes, 0)) - ) - else: - hdr = Yay0.from_buffer_copy(in_bytes, 0) - - magic = getattr(hdr, hdr._fields_[0][0]) - if magic != int.from_bytes(str.encode("Yay0"), byteorder="big"): - log.error(f"Yay0 magic is incorrect: {magic}") - - # create the input/output buffers, copying data to in - src = (c_uint8 * len(in_bytes)).from_buffer_copy(in_bytes, 0) - dst = (c_uint8 * hdr.uncompressedLength)() - - # call decompress, equivilant to, in C: - # decompress(&hdr, &src, &dst, bigEndian) - lib.decompress(byref(hdr), byref(src), byref(dst), c_bool(bigEndian)) - - # other functions want the results back as a non-ctypes type - return bytearray(dst) - - @staticmethod - def decompress_python(in_bytes, byte_order: Literal["little", "big"] = "big"): - if in_bytes[:4] != b"Yay0": - log.error("Input file is not Yay0") - - decompressed_size = int.from_bytes(in_bytes[4:8], byteorder=byte_order) - link_table_offset = int.from_bytes(in_bytes[8:12], byteorder=byte_order) - chunk_offset = int.from_bytes(in_bytes[12:16], byteorder=byte_order) - - link_table_idx = link_table_offset - chunk_idx = chunk_offset - other_idx = 16 - - mask_bit_counter = 0 - current_mask = 0 - - # preallocate result and index into it - idx = 0 - ret = bytearray(decompressed_size) - - while idx < decompressed_size: - # If we're out of bits, get the next mask - if mask_bit_counter == 0: - current_mask = int.from_bytes( - in_bytes[other_idx : other_idx + 4], byteorder=byte_order - ) - other_idx += 4 - mask_bit_counter = 32 - - if current_mask & 0x80000000: - ret[idx] = in_bytes[chunk_idx] - idx += 1 - chunk_idx += 1 - else: - link = int.from_bytes( - in_bytes[link_table_idx : link_table_idx + 2], byteorder=byte_order - ) - link_table_idx += 2 - - offset = idx - (link & 0xFFF) - - count = link >> 12 - - if count == 0: - count_modifier = in_bytes[chunk_idx] - chunk_idx += 1 - count = count_modifier + 18 - else: - count += 2 - - # Copy the block - for i in range(count): - ret[idx] = ret[offset + i - 1] - idx += 1 - - current_mask <<= 1 - mask_bit_counter -= 1 - - return ret - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument("infile") - parser.add_argument("outfile") - parser.add_argument("--byte-order", default="big", choices=["big", "little"]) - args = parser.parse_args() - - with open(args.infile, "rb") as f: - raw_bytes = f.read() - - decompressed = Yay0Decompressor.decompress(raw_bytes, args.byte_order) - - with open(args.outfile, "wb") as f: - f.write(decompressed) - - -if __name__ == "__main__": - main() diff --git a/tools/splat/util/n64/__init__.py b/tools/splat/util/n64/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/splat/util/n64/decompressor.py b/tools/splat/util/n64/decompressor.py deleted file mode 100644 index 7df0de05ab..0000000000 --- a/tools/splat/util/n64/decompressor.py +++ /dev/null @@ -1,10 +0,0 @@ -from abc import ABC, abstractmethod -from typing import Literal - - -class Decompressor(ABC): - @abstractmethod - def decompress( - self, in_bytes, byte_order: Literal["little", "big"] = "big" - ) -> bytearray: - pass diff --git a/tools/splat/util/n64/find_code_length.py b/tools/splat/util/n64/find_code_length.py deleted file mode 100755 index 1f9053cab4..0000000000 --- a/tools/splat/util/n64/find_code_length.py +++ /dev/null @@ -1,63 +0,0 @@ -#! /usr/bin/env python3 - -import argparse - -import rabbitizer -import spimdisasm - - -def int_any_base(x): - return int(x, 0) - - -parser = argparse.ArgumentParser( - description="Given a rom and start offset, find where the code ends" -) -parser.add_argument("rom", help="path to a .z64 rom") -parser.add_argument("start", help="start offset", type=int_any_base) -parser.add_argument("--end", help="end offset", default=None, type=int_any_base) -parser.add_argument( - "--vram", - help="vram address to start disassembly at", - default="0x80000000", - type=int_any_base, -) - - -def run(rom_bytes, start_offset, vram, end_offset=None): - rom_addr = start_offset - last_return = rom_addr - - wordList = spimdisasm.common.Utils.bytesToBEWords(rom_bytes[start_offset:]) - for word in wordList: - insn = rabbitizer.Instruction(word) - if not insn.isImplemented(): - break - - if insn.isJrRa(): - last_return = rom_addr - rom_addr += 4 - if end_offset and rom_addr >= end_offset: - break - - # align to next 0x10 boundary - end = last_return + 0x10 - end -= end % 0x10 - return end - - -def main(): - args = parser.parse_args() - - with open(args.rom, "rb") as f: - rom_bytes = f.read() - - start = args.start - end = args.end - vram = args.vram - - print(f"0x{run(rom_bytes, start, vram, end):X}") - - -if __name__ == "__main__": - main() diff --git a/tools/splat/util/n64/rominfo.py b/tools/splat/util/n64/rominfo.py deleted file mode 100755 index cede4debf0..0000000000 --- a/tools/splat/util/n64/rominfo.py +++ /dev/null @@ -1,322 +0,0 @@ -#! /usr/bin/env python3 - -import argparse - -import hashlib -import itertools -import struct - -import sys -import zlib -from dataclasses import dataclass - -from pathlib import Path -from typing import Optional - -import rabbitizer -import spimdisasm - -parser = argparse.ArgumentParser(description="Gives information on N64 roms") -parser.add_argument("rom", help="path to an N64 rom") -parser.add_argument( - "--header-encoding", - dest="header_encoding", - help=( - "Text encoding the game header is using;" - " see docs.python.org/3/library/codecs.html#standard-encodings for valid encodings" - ), -) - -country_codes = { - 0x00: "Unknown", - 0x37: "Beta", - 0x41: "Asian (NTSC)", - 0x42: "Brazilian", - 0x43: "Chinese", - 0x44: "German", - 0x45: "North America", - 0x46: "French", - 0x47: "Gateway 64 (NTSC)", - 0x48: "Dutch", - 0x49: "Italian", - 0x4A: "Japanese", - 0x4B: "Korean", - 0x4C: "Gateway 64 (PAL)", - 0x4E: "Canadian", - 0x50: "European (basic spec.)", - 0x53: "Spanish", - 0x55: "Australian", - 0x57: "Scandinavian", - 0x58: "European", - 0x59: "European", -} - - -@dataclass -class CIC: - ntsc_name: str - pal_name: str - offset: int - - -crc_to_cic = { - 0x6170A4A1: CIC("6101", "7102", 0x000000), - 0x90BB6CB5: CIC("6102", "7101", 0x000000), - 0x0B050EE0: CIC("6103", "7103", 0x100000), - 0x98BC2C86: CIC("6105", "7105", 0x000000), - 0xACC8580A: CIC("6106", "7106", 0x200000), -} -unknown_cic = CIC("unknown", "unknown", 0x0000000) - - -@dataclass -class N64EntrypointInfo: - entry_size: int - bss_start_address: Optional[int] - bss_size: Optional[int] - main_address: Optional[int] - stack_top: int - - @staticmethod - def parse_rom_bytes( - rom_bytes, offset: int = 0x1000, size: int = 0x60 - ) -> "N64EntrypointInfo": - word_list = spimdisasm.common.Utils.bytesToWords( - rom_bytes, offset, offset + size - ) - nops_count = 0 - - register_values = [0 for _ in range(32)] - - register_bss_address: Optional[int] = None - register_bss_size: Optional[int] = None - register_main_address: Optional[int] = None - - size = 0 - for word in word_list: - insn = rabbitizer.Instruction(word) - if not insn.isImplemented(): - break - - if insn.isNop(): - nops_count += 1 - elif nops_count >= 3: - break - elif insn.canBeHi(): - register_values[insn.rt.value] = insn.getProcessedImmediate() << 16 - elif insn.canBeLo(): - if insn.isLikelyHandwritten(): - # Try to skip these instructions: - # addi $t0, $t0, 0x8 - # addi $t1, $t1, -0x8 - pass - elif insn.modifiesRt(): - register_values[insn.rt.value] = ( - register_values[insn.rs.value] + insn.getProcessedImmediate() - ) - elif insn.doesStore(): - if insn.rt == rabbitizer.RegGprO32.zero: - # Try to detect the zero-ing bss algorithm - # sw $zero, 0x0($t0) - register_bss_address = insn.rs.value - elif insn.isBranch(): - # lui $t1, 0x2 - # addiu $t1, $t1, -0x7220 - # ... - # addi $t1, $t1, -0x8 - # ... - # bnez $t1, label - register_bss_size = insn.rs.value - - elif insn.isJumptableJump() or insn.isReturn(): - # lui $t2, 0x8000 - # addiu $t2, $t2, 0x494 - # ... - # jr $t2 - register_main_address = insn.rs.value - - # print(f"{word:08X}", insn) - size += 4 - - # for i, val in enumerate(register_values): - # print(i, f"{val:08X}") - - bss_address = ( - register_values[register_bss_address] - if register_bss_address is not None - else None - ) - bss_size = ( - register_values[register_bss_size] - if register_bss_size is not None - else None - ) - main_address = ( - register_values[register_main_address] - if register_main_address is not None - else None - ) - stack_top = register_values[rabbitizer.RegGprO32.sp.value] - return N64EntrypointInfo(size, bss_address, bss_size, main_address, stack_top) - - -@dataclass -class N64Rom: - name: str - header_encoding: str - country_code: int - libultra_version: str - checksum: str - cic: CIC - entry_point: int - size: int - compiler: str - sha1: str - entrypoint_info: N64EntrypointInfo - - def get_country_name(self) -> str: - return country_codes[self.country_code] - - -def swap_bytes(data): - return bytes( - itertools.chain.from_iterable( - struct.pack(">H", x) for (x,) in struct.iter_unpack(" N64Rom: - if rom_bytes is None: - rom_bytes = read_rom(rom_path) - - if header_encoding is None: - header_encoding = guess_header_encoding(rom_bytes) - - return get_info_bytes(rom_bytes, header_encoding) - - -def get_info_bytes(rom_bytes: bytes, header_encoding: str) -> N64Rom: - (program_counter,) = struct.unpack(">I", rom_bytes[0x8:0xC]) - libultra_version = chr(rom_bytes[0xF]) - checksum = rom_bytes[0x10:0x18].hex().upper() - - try: - name = rom_bytes[0x20:0x34].decode(header_encoding).rstrip(" \0") or "empty" - except: - sys.exit( - "splat could not decode the game name;" - " try using a different encoding by passing the --header-encoding argument" - " (see docs.python.org/3/library/codecs.html#standard-encodings for valid encodings)" - ) - - country_code = rom_bytes[0x3E] - - cic = get_cic(rom_bytes) - entry_point = get_entry_point(program_counter, cic) - - compiler = get_compiler_info(rom_bytes, entry_point, print_result=False) - - sha1 = hashlib.sha1(rom_bytes).hexdigest() - - entrypoint_info = N64EntrypointInfo.parse_rom_bytes(rom_bytes) - - return N64Rom( - name, - header_encoding, - country_code, - libultra_version, - checksum, - cic, - entry_point, - len(rom_bytes), - compiler, - sha1, - entrypoint_info, - ) - - -def get_compiler_info(rom_bytes, entry_point, print_result=True): - jumps = 0 - branches = 0 - - word_list = spimdisasm.common.Utils.bytesToWords(rom_bytes[0x1000:]) - for word in word_list: - insn = rabbitizer.Instruction(word) - if not insn.isImplemented(): - break - - if insn.uniqueId == rabbitizer.InstrId.cpu_j: - jumps += 1 - elif insn.uniqueId == rabbitizer.InstrId.cpu_b: - branches += 1 - - compiler = "IDO" if branches > jumps else "GCC" - if print_result: - print( - f"{branches} branches and {jumps} jumps detected in the first code segment." - f" Compiler is most likely {compiler}" - ) - return compiler - - -def main(): - rabbitizer.config.pseudos_pseudoB = True - - args = parser.parse_args() - rom_bytes = read_rom(Path(args.rom)) - rom = get_info(Path(args.rom), rom_bytes, args.header_encoding) - - print("Image name: " + rom.name) - print("Country code: " + chr(rom.country_code) + " - " + rom.get_country_name()) - print("Libultra version: " + rom.libultra_version) - print("Checksum: " + rom.checksum) - print("CIC: " + rom.cic.ntsc_name + " / " + rom.cic.pal_name) - print("RAM entry point: " + hex(rom.entry_point)) - print("Header encoding: " + rom.header_encoding) - print("") - - get_compiler_info(rom_bytes, rom.entry_point) - - -if __name__ == "__main__": - main() diff --git a/tools/splat/util/options.py b/tools/splat/util/options.py deleted file mode 100644 index c1c1b19336..0000000000 --- a/tools/splat/util/options.py +++ /dev/null @@ -1,520 +0,0 @@ -from dataclasses import dataclass -import os -from pathlib import Path -from typing import cast, Dict, List, Literal, Mapping, Optional, Set, Type, TypeVar - -from util import compiler -from util.compiler import Compiler - - -@dataclass -class SplatOpts: - # Debug / logging - verbose: bool - dump_symbols: bool - modes: List[str] - - # Project configuration - - # Determines the base path of the project. Everything is relative to this path - base_path: Path - # Determines the path to the target binary - target_path: Path - # Path to the final elf target - elf_path: Optional[Path] - # Determines the platform of the target binary - platform: str - # Determines the compiler used to compile the target binary - compiler: Compiler - # Determines the endianness of the target binary - endianness: Literal["big", "little"] - # Determines the default section order of the target binary - # this can be overridden per-segment - section_order: List[str] - # Determines the code that is inserted by default in generated .c files - generated_c_preamble: str - # Determines the code that is inserted by default in generated .s files - generated_s_preamble: str - # Determines whether to use .o as the suffix for all binary files?... TODO document - use_o_as_suffix: bool - # the value of the $gp register to correctly calculate offset to %gp_rel relocs - gp: Optional[int] - # Checks and errors if there are any non consecutive segment types - check_consecutive_segment_types: bool - - # Paths - asset_path: Path - # Determines the path to the symbol addresses file(s) - # A symbol_addrs file is to be updated/curated manually and contains addresses of symbols - # as well as optional metadata such as rom address, type, and more - # - # It's possible to use more than one file by supplying a list instead of a string - symbol_addrs_paths: List[Path] - reloc_addrs_paths: List[Path] - # Determines the path to the project build directory - build_path: Path - # Determines the path to the source code directory - src_path: Path - # Determines the path to the asm code directory - asm_path: Path - # Determines the path to the asm data directory - data_path: Path - # Determines the path to the asm nonmatchings directory - nonmatchings_path: Path - # Determines the path to the cache file (used when supplied --use-cache via the CLI) - cache_path: Path - - # Determines whether to create an automatically-generated undefined functions file - # this file stores all functions that are referenced in the code but are not defined as seen by splat - create_undefined_funcs_auto: bool - # Determines the path to the undefined_funcs_auto file - undefined_funcs_auto_path: Path - - # Determines whether to create an automatically-generated undefined symbols file - # this file stores all symbols that are referenced in the code but are not defined as seen by splat - create_undefined_syms_auto: bool - # Determines the path to the undefined_symbols_auto file - undefined_syms_auto_path: Path - - # Determines the path in which to search for custom splat extensions - extensions_path: Optional[Path] - - # Determines the path to library files that are to be linked into the target binary - lib_path: Path - - # TODO document - elf_section_list_path: Optional[Path] - - # Linker script - # Determines the default subalign value to be specified in the generated linker script - subalign: int - # The following option determines whether to automatically configure the linker script to link against - # specified sections for all "base" (asm/c) files when the yaml doesn't have manual configurations - # for these sections. - auto_all_sections: List[str] - # Determines the desired path to the linker script that splat will generate - ld_script_path: Path - # Determines the desired path to the linker symbol header, - # which exposes externed definitions for all segment ram/rom start/end locations - ld_symbol_header_path: Optional[Path] - # Determines whether to add a discard section with a wildcard to the linker script - ld_discard_section: bool - # A list of sections to preserve during link time. It can be useful to preserve debugging sections - ld_sections_allowlist: List[str] - # A list of sections to discard during link time. It can be useful to avoid using the wildcard discard. Note that this option does not turn off `ld_discard_section` - ld_sections_denylist: List[str] - # Determines the list of section labels that are to be added to the linker script - ld_section_labels: List[str] - # Determines whether to add wildcards for section linking in the linker script (.rodata* for example) - ld_wildcard_sections: bool - # Determines whether to use `follows_vram` (segment option) and - # `vram_symbol` / `follows_classes` (vram_class options) to calculate vram addresses in the linker script. - # If disabled, this uses the plain integer values for vram addresses defined in the yaml. - ld_use_symbolic_vram_addresses: bool - # Change linker script generation to allow partially linking segments. Requires both `ld_partial_scripts_path` and `ld_partial_build_segments_path` to be set. - ld_partial_linking: bool - # Folder were each intermediary linker script will be written to. - ld_partial_scripts_path: Optional[Path] - # Folder where the built partially linked segments will be placed by the build system. - ld_partial_build_segments_path: Optional[Path] - # Generate a dependency file for every linker script generated. Dependency files will have the same path and name as the corresponding linker script, but changing the extension to `.d`. Requires `elf_path` to be set. - ld_dependencies: bool - # Legacy linker script generation does not impose the section_order specified in the yaml options or per-segment options. - ld_legacy_generation: bool - # If enabled, the end symbol for each segment will be placed before the alignment directive for the segment - segment_end_before_align: bool - # Controls the style of the auto-generated segment symbols in the linker script. Possible values: splat, makerom - segment_symbols_style: str - # Specifies the starting offset for rom address symbols in the linker script. - ld_rom_start: int - # The value passed to the FILL statement on each segment. `None` disables using FILL statements on the linker script. Defaults to a fill value of 0. - ld_fill_value: Optional[int] - - ################################################################################ - # C file options - ################################################################################ - # Determines whether to create new c files if they don't exist - create_c_files: bool - # Determines whether to "auto-decompile" empty functions - auto_decompile_empty_functions: bool - # Determines whether to detect matched/unmatched functions in existing c files - # so we can avoid creating .s files for already-decompiled functions - do_c_func_detection: bool - # Determines the newline char(s) to be used in c files - c_newline: str - - ################################################################################ - # (Dis)assembly-related options - ################################################################################ - # The following options determine the format that symbols should be named by default - symbol_name_format: str - # Same as above but for symbols with no rom address - symbol_name_format_no_rom: str - # Determines whether to detect and hint to the user about likely file splits when disassembling - find_file_boundaries: bool - # Determines whether to detect and hint to the user about possible rodata sections corresponding to a text section - pair_rodata_to_text: bool - # Determines whether to attempt to automatically migrate rodata into functions - # (only works in certain circumstances) - migrate_rodata_to_functions: bool - # Determines the header to be used in every asm file that's included from c files - asm_inc_header: str - # Determines the macro used to declare functions in asm files - asm_function_macro: str - # Determines the macro used to declare symbols in the middle of functions in asm files (which may be alternative entries) - asm_function_alt_macro: str - # Determines the macro used to declare jumptable labels in asm files - asm_jtbl_label_macro: str - # Determines the macro used to declare data symbols in asm files - asm_data_macro: str - # Determines the macro used at the end of a function, such as endlabel or .end - asm_end_label: str - # Toggles the .size directive emitted by the disassembler - asm_emit_size_directive: Optional[bool] - # Determines including the macro.inc file on non-migrated rodata variables - include_macro_inc: bool - # Determines the number of characters to left align before the TODO finish documenting - mnemonic_ljust: int - # Determines whether to pad the rom address - rom_address_padding: bool - # Determines which ABI names to use for general purpose registers - mips_abi_gpr: str - # Determines which ABI names to use for floating point registers - # Valid values: 'numeric', 'o32', 'n32', 'n64' - # o32 is highly recommended, as it provides logically named registers for floating point instructions - # For more info, see https://gist.github.com/EllipticEllipsis/27eef11205c7a59d8ea85632bc49224d - mips_abi_float_regs: str - # Determines whether to add ".set gp=64" to asm/hasm files - add_set_gp_64: bool - # Generate .asmproc.d dependency files for each C file which still reference functions in assembly files - create_asm_dependencies: bool - # Global option for rodata string encoding. This can be overriden per segment - string_encoding: Optional[str] - # Global option for data string encoding. This can be overriden per segment - data_string_encoding: Optional[str] - # Global option for the rodata string guesser. 0 disables the guesser completely. - rodata_string_guesser_level: Optional[int] - # Global option for the data string guesser. 0 disables the guesser completely. - data_string_guesser_level: Optional[int] - # Global option for allowing data symbols using addends on symbol references. It can be overriden per symbol - allow_data_addends: bool - # Tells the disassembler to try disassembling functions with unknown instructions instead of falling back to disassembling as raw data - disasm_unknown: bool - # Tries to detect redundant and unreferenced functions ends and merge them together. This option is ignored if the compiler is not set to IDO. - detect_redundant_function_end: bool - # Don't skip disassembling already matched functions and migrated sections - disassemble_all: bool - - ################################################################################ - # N64-specific options - ################################################################################ - # Determines the encoding of the header - header_encoding: str - # Determines the type gfx ucode (used by gfx segments) - # Valid options are ['f3d', 'f3db', 'f3dex', 'f3dexb', 'f3dex2'] - gfx_ucode: str - # Use named libultra symbols by default. Those will need to be added to a linker script manually by the user - libultra_symbols: bool - # Use named libultra symbols by default. Those will need to be added to a linker script manually by the user - ique_symbols: bool - # Use named hardware register symbols by default. Those will need to be added to a linker script manually by the user - hardware_regs: bool - - ################################################################################ - # Gamecube-specific options - ################################################################################ - # Path where the iso's filesystem will be extracted to - filesystem_path: Optional[Path] - - ################################################################################ - # Compiler-specific options - ################################################################################ - # Determines whether to use a legacy INCLUDE_ASM macro format in c files - # only applies to GCC/SN64 - use_legacy_include_asm: bool - - # Returns whether the given mode is currently enabled - def is_mode_active(self, mode: str) -> bool: - return mode in self.modes or "all" in self.modes - - -opts: SplatOpts - - -T = TypeVar("T") - - -class OptParser: - _read_opts: Set[str] - - def __init__(self, yaml: Mapping[str, object]) -> None: - self._yaml = yaml - self._read_opts = set() - - def parse_opt(self, opt: str, t: Type[T], default: Optional[T] = None) -> T: - if opt not in self._yaml: - if default is not None: - return default - raise ValueError(f"Missing required option {opt}") - self._read_opts.add(opt) - value = self._yaml[opt] - if isinstance(value, t): - return value - if t is float and isinstance(value, int): - return cast(T, float(value)) - raise ValueError(f"Expected {opt} to have type {t}, got {type(value)}") - - def parse_optional_opt(self, opt: str, t: Type[T]) -> Optional[T]: - if opt not in self._yaml: - return None - return self.parse_opt(opt, t) - - def parse_opt_within( - self, opt: str, t: Type[T], within: List[T], default: Optional[T] = None - ) -> T: - value = self.parse_opt(opt, t, default) - if value not in within: - raise ValueError(f"Invalid value for {opt}: {value}") - return value - - def parse_path( - self, base_path: Path, opt: str, default: Optional[str] = None - ) -> Path: - return Path(os.path.normpath(base_path / self.parse_opt(opt, str, default))) - - def parse_optional_path(self, base_path: Path, opt: str) -> Optional[Path]: - if opt not in self._yaml: - return None - return self.parse_path(base_path, opt) - - def parse_path_list(self, base_path: Path, opt: str, default: str) -> List[Path]: - paths = self.parse_opt(opt, object, default) - - if isinstance(paths, str): - return [base_path / paths] - elif isinstance(paths, list): - return [base_path / path for path in paths] - else: - raise ValueError(f"Expected str or list for '{opt}', got {type(paths)}") - - def check_no_unread_opts(self) -> None: - opts = [opt for opt in self._yaml if opt not in self._read_opts] - if opts: - raise ValueError(f"Unrecognized YAML option(s): {', '.join(opts)}") - - -def _parse_yaml( - yaml: Dict, - config_paths: List[str], - modes: List[str], - verbose: bool = False, - disasm_all: bool = False, -) -> SplatOpts: - p = OptParser(yaml) - - basename = p.parse_opt("basename", str) - platform = p.parse_opt_within("platform", str, ["n64", "psx", "gc", "ps2"]) - comp = compiler.for_name(p.parse_opt("compiler", str, "IDO")) - - base_path = Path( - os.path.normpath(Path(config_paths[0]).parent / p.parse_opt("base_path", str)) - ) - asm_path: Path = p.parse_path(base_path, "asm_path", "asm") - - asm_emit_size_directive = p.parse_optional_opt("asm_emit_size_directive", bool) - # If option not provided then use the compiler default - if asm_emit_size_directive is None: - asm_emit_size_directive = comp.asm_emit_size_directive - - def parse_endianness() -> Literal["big", "little"]: - endianness = p.parse_opt_within( - "endianness", - str, - ["big", "little"], - "little" if platform in ["psx", "ps2"] else "big", - ) - - if endianness == "big": - return "big" - elif endianness == "little": - return "little" - else: - raise ValueError(f"Invalid endianness: {endianness}") - - ret = SplatOpts( - verbose=verbose, - dump_symbols=p.parse_opt("dump_symbols", bool, False), - modes=modes, - base_path=base_path, - target_path=p.parse_path(base_path, "target_path"), - elf_path=p.parse_optional_path(base_path, "elf_path"), - platform=platform, - compiler=comp, - endianness=parse_endianness(), - section_order=p.parse_opt( - "section_order", list, [".text", ".data", ".rodata", ".bss"] - ), - generated_c_preamble=p.parse_opt( - "generated_c_preamble", str, '#include "common.h"' - ), - generated_s_preamble=p.parse_opt("generated_s_preamble", str, ""), - use_o_as_suffix=p.parse_opt("o_as_suffix", bool, False), - gp=p.parse_opt("gp_value", int, 0), - check_consecutive_segment_types=p.parse_opt( - "check_consecutive_segment_types", bool, True - ), - asset_path=p.parse_path(base_path, "asset_path", "assets"), - symbol_addrs_paths=p.parse_path_list( - base_path, "symbol_addrs_path", "symbol_addrs.txt" - ), - reloc_addrs_paths=p.parse_path_list( - base_path, "reloc_addrs_path", "reloc_addrs.txt" - ), - build_path=p.parse_path(base_path, "build_path", "build"), - src_path=p.parse_path(base_path, "src_path", "src"), - asm_path=asm_path, - data_path=p.parse_path(asm_path, "data_path", "data"), - nonmatchings_path=p.parse_path(asm_path, "nonmatchings_path", "nonmatchings"), - cache_path=p.parse_path(base_path, "cache_path", ".splache"), - create_undefined_funcs_auto=p.parse_opt( - "create_undefined_funcs_auto", bool, True - ), - undefined_funcs_auto_path=p.parse_path( - base_path, "undefined_funcs_auto_path", "undefined_funcs_auto.txt" - ), - create_undefined_syms_auto=p.parse_opt( - "create_undefined_syms_auto", bool, True - ), - undefined_syms_auto_path=p.parse_path( - base_path, "undefined_syms_auto_path", "undefined_syms_auto.txt" - ), - extensions_path=p.parse_optional_path(base_path, "extensions_path"), - lib_path=p.parse_path(base_path, "lib_path", "lib"), - elf_section_list_path=p.parse_optional_path(base_path, "elf_section_list_path"), - subalign=p.parse_opt("subalign", int, 16), - auto_all_sections=p.parse_opt( - "auto_all_sections", list, [".data", ".rodata", ".bss"] - ), - ld_script_path=p.parse_path(base_path, "ld_script_path", f"{basename}.ld"), - ld_symbol_header_path=p.parse_optional_path(base_path, "ld_symbol_header_path"), - ld_discard_section=p.parse_opt("ld_discard_section", bool, True), - ld_sections_allowlist=p.parse_opt("ld_sections_allowlist", list, []), - ld_sections_denylist=p.parse_opt("ld_sections_denylist", list, []), - ld_section_labels=p.parse_opt( - "ld_section_labels", - list, - [".text", ".data", ".rodata", ".bss"], - ), - ld_wildcard_sections=p.parse_opt("ld_wildcard_sections", bool, False), - ld_use_symbolic_vram_addresses=p.parse_opt( - "ld_use_symbolic_vram_addresses", bool, True - ), - ld_partial_linking=p.parse_opt("ld_partial_linking", bool, False), - ld_partial_scripts_path=p.parse_optional_path( - base_path, "ld_partial_scripts_path" - ), - ld_partial_build_segments_path=p.parse_optional_path( - base_path, "ld_partial_build_segments_path" - ), - ld_dependencies=p.parse_opt("ld_dependencies", bool, False), - ld_legacy_generation=p.parse_opt("ld_legacy_generation", bool, False), - segment_end_before_align=p.parse_opt("segment_end_before_align", bool, False), - segment_symbols_style=p.parse_opt_within( - "segment_symbols_style", str, ["splat", "makerom"], "splat" - ), - ld_rom_start=p.parse_opt("ld_rom_start", int, 0), - ld_fill_value=p.parse_opt("ld_fill_value", int, 0), - create_c_files=p.parse_opt("create_c_files", bool, True), - auto_decompile_empty_functions=p.parse_opt( - "auto_decompile_empty_functions", bool, True - ), - do_c_func_detection=p.parse_opt("do_c_func_detection", bool, True), - c_newline=p.parse_opt("c_newline", str, comp.c_newline), - symbol_name_format=p.parse_opt("symbol_name_format", str, "$VRAM"), - symbol_name_format_no_rom=p.parse_opt( - "symbol_name_format_no_rom", str, "$VRAM_$SEG" - ), - find_file_boundaries=p.parse_opt("find_file_boundaries", bool, True), - pair_rodata_to_text=p.parse_opt("pair_rodata_to_text", bool, True), - migrate_rodata_to_functions=p.parse_opt( - "migrate_rodata_to_functions", bool, True - ), - asm_inc_header=p.parse_opt("asm_inc_header", str, comp.asm_inc_header), - asm_function_macro=p.parse_opt( - "asm_function_macro", str, comp.asm_function_macro - ), - asm_function_alt_macro=p.parse_opt( - "asm_function_alt_macro", str, comp.asm_function_alt_macro - ), - asm_jtbl_label_macro=p.parse_opt( - "asm_jtbl_label_macro", str, comp.asm_jtbl_label_macro - ), - asm_data_macro=p.parse_opt("asm_data_macro", str, comp.asm_data_macro), - asm_end_label=p.parse_opt("asm_end_label", str, comp.asm_end_label), - asm_emit_size_directive=asm_emit_size_directive, - include_macro_inc=p.parse_opt( - "include_macro_inc", bool, comp.include_macro_inc - ), - mnemonic_ljust=p.parse_opt("mnemonic_ljust", int, 11), - rom_address_padding=p.parse_opt("rom_address_padding", bool, False), - mips_abi_gpr=p.parse_opt_within( - "mips_abi_gpr", - str, - ["numeric", "o32", "n32", "n64"], - "o32", - ), - mips_abi_float_regs=p.parse_opt_within( - "mips_abi_float_regs", - str, - ["numeric", "o32", "n32", "n64"], - "numeric", - ), - add_set_gp_64=p.parse_opt("add_set_gp_64", bool, True), - create_asm_dependencies=p.parse_opt("create_asm_dependencies", bool, False), - string_encoding=p.parse_optional_opt("string_encoding", str), - data_string_encoding=p.parse_optional_opt("data_string_encoding", str), - rodata_string_guesser_level=p.parse_optional_opt( - "rodata_string_guesser_level", int - ), - data_string_guesser_level=p.parse_optional_opt( - "data_string_guesser_level", int - ), - allow_data_addends=p.parse_opt("allow_data_addends", bool, True), - header_encoding=p.parse_opt("header_encoding", str, "ASCII"), - gfx_ucode=p.parse_opt_within( - "gfx_ucode", - str, - ["f3d", "f3db", "f3dex", "f3dexb", "f3dex2"], - "f3dex2", - ), - libultra_symbols=p.parse_opt("libultra_symbols", bool, False), - ique_symbols=p.parse_opt("ique_symbols", bool, False), - hardware_regs=p.parse_opt("hardware_regs", bool, False), - use_legacy_include_asm=p.parse_opt("use_legacy_include_asm", bool, True), - filesystem_path=p.parse_optional_path(base_path, "filesystem_path"), - disasm_unknown=p.parse_opt("disasm_unknown", bool, False), - detect_redundant_function_end=p.parse_opt( - "detect_redundant_function_end", bool, True - ), - # Command line argument takes precedence over yaml option - disassemble_all=disasm_all - if disasm_all - else p.parse_opt("disassemble_all", bool, False), - ) - p.check_no_unread_opts() - return ret - - -def initialize( - config: Dict, - config_paths: List[str], - modes: Optional[List[str]] = None, - verbose=False, - disasm_all=False, -): - global opts - - if not modes: - modes = ["all"] - - opts = _parse_yaml(config["options"], config_paths, modes, verbose, disasm_all) diff --git a/tools/splat/util/palettes.py b/tools/splat/util/palettes.py deleted file mode 100644 index 8ee1e9a014..0000000000 --- a/tools/splat/util/palettes.py +++ /dev/null @@ -1,34 +0,0 @@ -from typing import Dict - -from segtypes.common.group import CommonSegGroup -from segtypes.n64.ci import N64SegCi -from segtypes.n64.palette import N64SegPalette as Palette - - -# Resolve Raster#palette and Palette#raster links -def initialize(all_segments): - def process(segments): - raster_map: Dict[str, N64SegCi] = {} - palette_map: Dict[str, Palette] = {} - - for segment in segments: - if isinstance(segment, Palette): - palette_map[segment.name] = segment - - if isinstance(segment, N64SegCi): - raster_map[segment.name] = segment - - if isinstance(segment, CommonSegGroup): - process(segment.subsegments) - - for raster_name in raster_map: - raster = raster_map[raster_name] - # print(f"{raster_name} -> {raster.palette_name}") - raster.palette = palette_map.get(raster.palette_name) - - for palette_name in palette_map: - palette = palette_map[palette_name] - # print(f"{palette_name} -> {palette.raster_name}") - palette.raster = raster_map.get(palette.raster_name) - - process(all_segments) diff --git a/tools/splat/util/progress_bar.py b/tools/splat/util/progress_bar.py deleted file mode 100644 index caec2cee8c..0000000000 --- a/tools/splat/util/progress_bar.py +++ /dev/null @@ -1,8 +0,0 @@ -import tqdm -import sys - -out_file = sys.stderr - - -def get_progress_bar(elements: list) -> tqdm.tqdm: - return tqdm.tqdm(elements, total=len(elements), file=out_file) diff --git a/tools/splat/util/psx/__init__.py b/tools/splat/util/psx/__init__.py deleted file mode 100644 index e69de29bb2..0000000000 diff --git a/tools/splat/util/psx/psxexeinfo.py b/tools/splat/util/psx/psxexeinfo.py deleted file mode 100755 index 490bb6d819..0000000000 --- a/tools/splat/util/psx/psxexeinfo.py +++ /dev/null @@ -1,117 +0,0 @@ -#! /usr/bin/env python3 - -from __future__ import annotations - -import argparse - -import hashlib -import struct - -import dataclasses - -from pathlib import Path - - -@dataclasses.dataclass -class PsxExe: - # Based on https://psx-spx.consoledev.net/cdromdrive/#filenameexe-general-purpose-executable - initial_pc: int # offset: 0x10 - initial_gp: int # offset: 0x14 - destination_vram: int # offset: 0x18 - file_size: int # offset: 0x1C - data_vram: int # offset: 0x20 - data_size: int # offset: 0x24 - bss_vram: int # offset: 0x28 - bss_size: int # offset: 0x2C - initial_sp_base: int # offset: 0x30 - initial_sp_offset: int # offset: 0x34 - - size: int - sha1: str - - @property - def text_offset(self) -> int: - return self.initial_pc - self.destination_vram + 0x800 - - @property - def data_offset(self) -> int: - if self.data_vram == 0 or self.data_size == 0: - return 0 - return self.data_vram - self.destination_vram + 0x800 - - @staticmethod - def get_info(exe_path: Path, exe_bytes: bytes) -> PsxExe: - initial_pc = struct.unpack(" 1: - log.parsing_error_preamble(path, line_num, line) - log.write(f"Too many ':'s in '{info}'") - log.error("") - - attr_name, attr_val = info.split(":") - if attr_name == "": - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Missing attribute name in '{info}', is there extra whitespace?" - ) - log.error("") - if attr_val == "": - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Missing attribute value in '{info}', is there extra whitespace?" - ) - log.error("") - - # Non-Boolean attributes - try: - if attr_name == "rom": - rom_addr = int(attr_val, 0) - continue - if attr_name == "reloc": - reloc_type = attr_val - continue - if attr_name == "symbol": - symbol_name = attr_val - continue - if attr_name == "addend": - addend = int(attr_val, 0) - continue - except: - log.parsing_error_preamble(path, line_num, line) - log.write(f"value of attribute '{attr_name}' could not be read:") - log.write("") - raise - - if rom_addr is None: - log.parsing_error_preamble(path, line_num, line) - log.error(f"Missing required 'rom' attribute for reloc") - if reloc_type is None: - log.parsing_error_preamble(path, line_num, line) - log.error(f"Missing required 'reloc' attribute for reloc") - if symbol_name is None: - log.parsing_error_preamble(path, line_num, line) - log.error(f"Missing required 'symbol' attribute for reloc") - - reloc = Reloc(rom_addr, reloc_type, symbol_name) - if addend is not None: - reloc.addend = addend - - if reloc.rom_address in all_relocs: - log.parsing_error_preamble(path, line_num, line) - log.error( - f"Duplicated 'rom' address for reloc: 0x{reloc.rom_address:X}" - ) - add_reloc(reloc) - - -def initialize_spim_context(): - for rom_address, reloc in all_relocs.items(): - reloc_type = spimdisasm.common.RelocType.fromStr(reloc.reloc_type) - - if reloc_type is None: - log.error( - f"Reloc type '{reloc.reloc_type}' is not valid. Rom address: 0x{rom_address:X}" - ) - - symbols.spim_context.addGlobalReloc( - rom_address, reloc_type, reloc.symbol_name, addend=reloc.addend - ) diff --git a/tools/splat/util/symbols.py b/tools/splat/util/symbols.py deleted file mode 100644 index 762418ef8b..0000000000 --- a/tools/splat/util/symbols.py +++ /dev/null @@ -1,681 +0,0 @@ -from dataclasses import dataclass -import re -from typing import Dict, List, Optional, Set, TYPE_CHECKING - -import spimdisasm - -from intervaltree import IntervalTree -from disassembler import disassembler_instance -from pathlib import Path - -# circular import -if TYPE_CHECKING: - from segtypes.segment import Segment - -from util import log, options, progress_bar - -all_symbols: List["Symbol"] = [] -all_symbols_dict: Dict[int, List["Symbol"]] = {} -all_symbols_ranges = IntervalTree() -ignored_addresses: Set[int] = set() -to_mark_as_defined: Set[str] = set() - -# Initialize a spimdisasm context, used to store symbols and functions -spim_context = spimdisasm.common.Context() - -TRUEY_VALS = ["true", "on", "yes", "y"] -FALSEY_VALS = ["false", "off", "no", "n"] - -splat_sym_types = {"func", "jtbl", "jtbl_label", "label"} - - -def check_valid_type(typename: str) -> bool: - if typename[0].isupper(): - return True - - if typename in splat_sym_types: - return True - - if typename in disassembler_instance.get_instance().known_types(): - return True - - return False - - -def is_truey(str: str) -> bool: - return str.lower() in TRUEY_VALS - - -def is_falsey(str: str) -> bool: - return str.lower() in FALSEY_VALS - - -def add_symbol(sym: "Symbol"): - all_symbols.append(sym) - if sym.vram_start is not None: - if sym.vram_start not in all_symbols_dict: - all_symbols_dict[sym.vram_start] = [] - all_symbols_dict[sym.vram_start].append(sym) - - # For larger symbols, add their ranges to interval trees for faster lookup - if sym.size > 4: - all_symbols_ranges.addi(sym.vram_start, sym.vram_end, sym) - - -def to_cname(symbol_name: str) -> str: - symbol_name = re.sub(r"[^0-9a-zA-Z_]", "_", symbol_name) - - if symbol_name[0] in ["0", "1", "2", "3", "4", "5", "6", "7", "8", "9"]: - symbol_name = "_" + symbol_name - - return symbol_name - - -def handle_sym_addrs( - path: Path, sym_addrs_lines: List[str], all_segments: "List[Segment]" -): - def get_seg_for_name(name: str) -> Optional["Segment"]: - for segment in all_segments: - if segment.name == name: - return segment - return None - - def get_seg_for_rom(rom: int) -> Optional["Segment"]: - for segment in all_segments: - if segment.contains_rom(rom): - return segment - return None - - seen_symbols: Dict[str, "Symbol"] = dict() - prog_bar = progress_bar.get_progress_bar(sym_addrs_lines) - prog_bar.set_description(f"Loading symbols ({path.stem})") - line: str - for line_num, line in enumerate(prog_bar): - line = line.strip() - if not line == "" and not line.startswith("//"): - comment_loc = line.find("//") - line_main = line - line_ext = "" - - if comment_loc != -1: - line_ext = line[comment_loc + 2 :].strip() - line_main = line[:comment_loc].strip() - - try: - assert line.count(";") == 1, "Line must contain a single semi-colon" - line_split = line_main.split("=") - name = line_split[0].strip() - addr = int(line_split[1].strip()[:-1], 0) - except: - log.parsing_error_preamble(path, line_num, line) - log.write("Line must be of the form") - log.write(" =
; // attr0:val0 attr1:val1 [...]") - log.write("with
in hex preceded by 0x, or dec") - log.write("") - raise - - sym = Symbol(addr, given_name=name) - - ignore_sym = False - if line_ext: - for info in line_ext.split(" "): - if ":" in info: - if info.count(":") > 1: - log.parsing_error_preamble(path, line_num, line) - log.write(f"Too many ':'s in '{info}'") - log.error("") - - attr_name, attr_val = info.split(":") - if attr_name == "": - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Missing attribute name in '{info}', is there extra whitespace?" - ) - log.error("") - if attr_val == "": - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Missing attribute value in '{info}', is there extra whitespace?" - ) - log.error("") - - # Non-Boolean attributes - try: - if attr_name == "type": - if not check_valid_type(attr_val): - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Unrecognized symbol type in '{info}', it should be one of" - ) - log.write( - [ - *splat_sym_types, - *spimdisasm.common.gKnownTypes, - ] - ) - log.write( - "You may use a custom type that starts with a capital letter" - ) - log.error("") - type = attr_val - sym.type = type - continue - if attr_name == "size": - size = int(attr_val, 0) - sym.given_size = size - continue - if attr_name == "rom": - rom_addr = int(attr_val, 0) - sym.rom = rom_addr - continue - if attr_name == "segment": - seg = get_seg_for_name(attr_val) - if seg is None: - log.parsing_error_preamble(path, line_num, line) - log.write(f"Cannot find segment '{attr_val}'") - log.error("") - else: - # Add segment to symbol - sym.segment = seg - continue - if attr_name == "name_end": - sym.given_name_end = attr_val - continue - except: - log.parsing_error_preamble(path, line_num, line) - log.write( - f"value of attribute '{attr_name}' could not be read:" - ) - log.write("") - raise - - # Boolean attributes - tf_val = ( - True - if is_truey(attr_val) - else False - if is_falsey(attr_val) - else None - ) - if tf_val is None: - log.parsing_error_preamble(path, line_num, line) - log.write( - f"Invalid Boolean value '{attr_val}' for attribute '{attr_name}', should be one of" - ) - log.write([*TRUEY_VALS, *FALSEY_VALS]) - log.error("") - else: - if attr_name == "defined": - sym.defined = tf_val - continue - if attr_name == "extract": - sym.extract = tf_val - continue - if attr_name == "ignore": - ignore_sym = tf_val - continue - if attr_name == "force_migration": - sym.force_migration = tf_val - continue - if attr_name == "force_not_migration": - sym.force_not_migration = tf_val - continue - if attr_name == "allow_addend": - sym.allow_addend = tf_val - continue - if attr_name == "dont_allow_addend": - sym.dont_allow_addend = tf_val - continue - - if ignore_sym: - if sym.given_size is None or sym.given_size == 0: - ignored_addresses.add(sym.vram_start) - else: - spim_context.addBannedSymbolRangeBySize( - sym.vram_start, sym.given_size - ) - - continue - - if sym.segment is None and sym.rom is not None: - sym.segment = get_seg_for_rom(sym.rom) - - if sym.segment: - sym.segment.add_symbol(sym) - - sym.user_declared = True - - if sym.name in seen_symbols: - log.parsing_error_preamble(path, line_num, line) - log.error( - f"Duplicate symbol detected! {sym.name} has already been defined at 0x{seen_symbols[sym.name].vram_start:X}" - ) - - if addr in all_symbols_dict: - items = all_symbols_dict[addr] - for item in items: - if ( - sym.rom == item.rom - or None in (sym.rom, item.rom) - or sym.segment == item.segment - or None in (sym.segment, item.rom) - ): - log.parsing_error_preamble(path, line_num, line) - log.error( - f"Duplicate symbol detected! {sym.name} clashes with {item.name} defined at 0x{addr:X}" - ) - - seen_symbols[sym.name] = sym - - add_symbol(sym) - - -def initialize(all_segments: "List[Segment]"): - global all_symbols - global all_symbols_dict - global all_symbols_ranges - - all_symbols = [] - all_symbols_dict = {} - all_symbols_ranges = IntervalTree() - - # Manual list of func name / addrs - for path in options.opts.symbol_addrs_paths: - if path.exists(): - with open(path) as f: - sym_addrs_lines = f.readlines() - handle_sym_addrs(path, sym_addrs_lines, all_segments) - - -def initialize_spim_context(all_segments: "List[Segment]") -> None: - global_vrom_start = None - global_vrom_end = None - global_vram_start = None - global_vram_end = None - overlay_segments: Set[spimdisasm.common.SymbolsSegment] = set() - - spim_context.bannedSymbols |= ignored_addresses - - from segtypes.common.code import CommonSegCode - - for segment in all_segments: - if not isinstance(segment, CommonSegCode): - # We only care about the VRAMs of code segments - continue - - if segment.special_vram_segment: - # Special segments which should not be accounted in the global VRAM calculation, like N64's IPL3 - continue - - if ( - not isinstance(segment.vram_start, int) - or not isinstance(segment.vram_end, int) - or not isinstance(segment.rom_start, int) - or not isinstance(segment.rom_end, int) - ): - continue - - ram_id = segment.get_exclusive_ram_id() - - if ram_id is None: - if global_vram_start is None: - global_vram_start = segment.vram_start - elif segment.vram_start < global_vram_start: - global_vram_start = segment.vram_start - - if global_vram_end is None: - global_vram_end = segment.vram_end - elif global_vram_end < segment.vram_end: - global_vram_end = segment.vram_end - - if global_vrom_start is None: - global_vrom_start = segment.rom_start - elif segment.rom_start < global_vrom_start: - global_vrom_start = segment.rom_start - - if global_vrom_end is None: - global_vrom_end = segment.rom_end - elif global_vrom_end < segment.rom_end: - global_vrom_end = segment.rom_end - - else: - spim_segment = spim_context.addOverlaySegment( - ram_id, - segment.rom_start, - segment.rom_end, - segment.vram_start, - segment.vram_end, - ) - # Add the segment-specific symbols first - for symbols_list in segment.seg_symbols.values(): - for sym in symbols_list: - add_symbol_to_spim_segment(spim_segment, sym) - - overlay_segments.add(spim_segment) - - if ( - global_vram_start is not None - and global_vram_end is not None - and global_vrom_start is not None - and global_vrom_end is not None - ): - spim_context.changeGlobalSegmentRanges( - global_vrom_start, global_vrom_end, global_vram_start, global_vram_end - ) - - # Check the vram range of the global segment does not overlap with any overlay segment - for ovl_segment in overlay_segments: - assert ( - ovl_segment.vramStart <= ovl_segment.vramEnd - ), f"{ovl_segment.vramStart:X} {ovl_segment.vramEnd:X}" - if ( - ovl_segment.vramEnd > global_vram_start - and global_vram_end > ovl_segment.vramStart - ): - log.write( - f"Warning: the vram range ([0x{ovl_segment.vramStart:X}, 0x{ovl_segment.vramEnd:X}]) of the non-global segment at rom address 0x{ovl_segment.vromStart:X} overlaps with the global vram range ([0x{global_vram_start:X}, 0x{global_vram_end:X}])", - status="warn", - ) - - # pass the global symbols to spimdisasm - for segment in all_segments: - if not isinstance(segment, CommonSegCode): - # We only care about the VRAMs of code segments - continue - - ram_id = segment.get_exclusive_ram_id() - if ram_id is not None: - continue - - for symbols_list in segment.seg_symbols.values(): - for sym in symbols_list: - add_symbol_to_spim_segment(spim_context.globalSegment, sym) - - -def add_symbol_to_spim_segment( - segment: spimdisasm.common.SymbolsSegment, sym: "Symbol" -) -> spimdisasm.common.ContextSymbol: - if sym.type == "func": - context_sym = segment.addFunction( - sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom - ) - elif sym.type == "jtbl": - context_sym = segment.addJumpTable( - sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom - ) - elif sym.type == "jtbl_label": - context_sym = segment.addJumpTableLabel( - sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom - ) - elif sym.type == "label": - context_sym = segment.addBranchLabel( - sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom - ) - else: - context_sym = segment.addSymbol( - sym.vram_start, isAutogenerated=not sym.user_declared, vromAddress=sym.rom - ) - if sym.type is not None: - context_sym.type = sym.type - - if sym.user_declared: - context_sym.isUserDeclared = True - if sym.defined: - context_sym.isDefined = True - if sym.rom is not None: - context_sym.vromAddress = sym.rom - if sym.given_size is not None: - context_sym.size = sym.size - if sym.force_migration: - context_sym.forceMigration = True - if sym.force_not_migration: - context_sym.forceNotMigration = True - if sym.allow_addend: - context_sym.allowedToReferenceAddends = True - if sym.dont_allow_addend: - context_sym.notAllowedToReferenceAddends = True - context_sym.setNameGetCallbackIfUnset(lambda _: sym.name) - if sym.given_name_end: - context_sym.nameEnd = sym.given_name_end - - return context_sym - - -def add_symbol_to_spim_section( - section: spimdisasm.mips.sections.SectionBase, sym: "Symbol" -) -> spimdisasm.common.ContextSymbol: - if sym.type == "func": - context_sym = section.addFunction( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - elif sym.type == "jtbl": - context_sym = section.addJumpTable( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - elif sym.type == "jtbl_label": - context_sym = section.addJumpTableLabel( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - elif sym.type == "label": - context_sym = section.addBranchLabel( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - else: - context_sym = section.addSymbol( - sym.vram_start, isAutogenerated=not sym.user_declared, symbolVrom=sym.rom - ) - if sym.type is not None: - context_sym.type = sym.type - - if sym.user_declared: - context_sym.isUserDeclared = True - if sym.defined: - context_sym.isDefined = True - if sym.rom is not None: - context_sym.vromAddress = sym.rom - if sym.given_size is not None: - context_sym.size = sym.size - if sym.force_migration: - context_sym.forceMigration = True - if sym.force_not_migration: - context_sym.forceNotMigration = True - context_sym.setNameGetCallbackIfUnset(lambda _: sym.name) - if sym.given_name_end: - context_sym.nameEnd = sym.given_name_end - - return context_sym - - -def create_symbol_from_spim_symbol( - segment: "Segment", context_sym: spimdisasm.common.ContextSymbol -) -> "Symbol": - in_segment = False - - sym_type = None - if context_sym.type == spimdisasm.common.SymbolSpecialType.jumptable: - in_segment = True - sym_type = "jtbl" - elif context_sym.type == spimdisasm.common.SymbolSpecialType.function: - sym_type = "func" - elif context_sym.type == spimdisasm.common.SymbolSpecialType.branchlabel: - in_segment = True - sym_type = "label" - elif context_sym.type == spimdisasm.common.SymbolSpecialType.jumptablelabel: - in_segment = True - sym_type = "jtbl_label" - - if not in_segment: - if ( - context_sym.overlayCategory is None - and segment.get_exclusive_ram_id() is None - ): - in_segment = segment.contains_vram(context_sym.vram) - elif context_sym.overlayCategory == segment.get_exclusive_ram_id(): - if context_sym.vromAddress is not None: - in_segment = segment.contains_rom(context_sym.vromAddress) - else: - in_segment = segment.contains_vram(context_sym.vram) - - sym = segment.create_symbol( - context_sym.vram, in_segment, type=sym_type, reference=True - ) - - if sym.given_name is None and context_sym.name is not None: - sym.given_name = context_sym.name - - # To keep the symbol name in sync between splat and spimdisasm - context_sym.setNameGetCallback(lambda _: sym.name) - - if context_sym.size is not None: - sym.given_size = context_sym.getSize() - if context_sym.vromAddress is not None: - sym.rom = context_sym.getVrom() - if context_sym.isDefined: - sym.defined = True - if context_sym.referenceCounter > 0: - sym.referenced = True - - return sym - - -def mark_c_funcs_as_defined(): - for symbol in all_symbols: - if len(to_mark_as_defined) == 0: - return - sym_name = symbol.name - if sym_name in to_mark_as_defined: - symbol.defined = True - to_mark_as_defined.remove(sym_name) - - -@dataclass -class Symbol: - vram_start: int - - given_name: Optional[str] = None - given_name_end: Optional[str] = None - rom: Optional[int] = None - type: Optional[str] = None - given_size: Optional[int] = None - segment: Optional["Segment"] = None - - defined: bool = False - referenced: bool = False - extract: bool = True - user_declared: bool = False - - force_migration: bool = False - force_not_migration: bool = False - - allow_addend: bool = False - dont_allow_addend: bool = False - - linker_section: Optional[str] = None - - _generated_default_name: Optional[str] = None - _last_type: Optional[str] = None - - def __str__(self): - return self.name - - def __eq__(self, other: object) -> bool: - if not isinstance(other, Symbol): - return False - return self.vram_start == other.vram_start and self.segment == other.segment - - # https://stackoverflow.com/a/56915493/6292472 - def __hash__(self): - return hash((self.vram_start, self.segment)) - - def format_name(self, format: str) -> str: - ret = format - - ret = ret.replace("$VRAM", f"{self.vram_start:08X}") - - if "$ROM" in ret: - if not isinstance(self.rom, int): - log.error( - f"Attempting to rom-name a symbol with no ROM address: {self.vram_start:X} typed {self.type}" - ) - ret = ret.replace("$ROM", f"{self.rom:X}") - - if "$SEG" in ret: - if self.segment is None: - # This probably is fine - we can't expect every symbol to have a segment. Fall back to just the ram address - return f"{self.vram_start:X}" - assert self.segment is not None - ret = ret.replace("$SEG", self.segment.name) - - return ret - - @property - def default_name(self) -> str: - if self._generated_default_name is not None: - if self.type == self._last_type: - return self._generated_default_name - - if self.segment: - if isinstance(self.rom, int): - suffix = self.format_name(self.segment.symbol_name_format) - else: - suffix = self.format_name(self.segment.symbol_name_format_no_rom) - else: - if isinstance(self.rom, int): - suffix = self.format_name(options.opts.symbol_name_format) - else: - suffix = self.format_name(options.opts.symbol_name_format_no_rom) - - if self.type == "func": - prefix = "func" - elif self.type == "jtbl": - prefix = "jtbl" - elif self.type in {"jtbl_label", "label"}: - return f".L{suffix}" - else: - prefix = "D" - - self._last_type = self.type - self._generated_default_name = f"{prefix}_{suffix}" - return self._generated_default_name - - @property - def rom_end(self): - return None if not self.rom else self.rom + self.size - - @property - def vram_end(self): - return self.vram_start + self.size - - @property - def name(self) -> str: - return self.given_name if self.given_name else self.default_name - - @property - def size(self) -> int: - if self.given_size is not None: - return self.given_size - return 4 - - def contains_vram(self, offset): - return offset >= self.vram_start and offset < self.vram_end - - def contains_rom(self, offset): - return offset >= self.rom and offset < self.rom_end - - -def get_all_symbols(): - global all_symbols - return all_symbols - - -def reset_symbols(): - global all_symbols - global all_symbols_dict - global all_symbols_ranges - global ignored_addresses - global to_mark_as_defined - all_symbols = [] - all_symbols_dict = {} - all_symbols_ranges = IntervalTree() - ignored_addresses = set() - to_mark_as_defined = set() diff --git a/tools/splat/util/vram_classes.py b/tools/splat/util/vram_classes.py deleted file mode 100644 index c296bbbd59..0000000000 --- a/tools/splat/util/vram_classes.py +++ /dev/null @@ -1,102 +0,0 @@ -from dataclasses import dataclass, field -from typing import Any, Dict, List, Optional - -from util import log - - -@dataclass(frozen=True) -class VramClass: - name: str - vram: int - given_vram_symbol: Optional[str] = None - follows_classes: List[str] = field(default_factory=list, compare=False) - - @property - def vram_symbol(self) -> Optional[str]: - if self.given_vram_symbol is not None: - return self.given_vram_symbol - elif self.follows_classes: - return self.name + "_CLASS_VRAM" - else: - return None - - -_vram_classes: Dict[str, VramClass] = {} - - -def initialize(yaml: Any): - global _vram_classes - - _vram_classes = {} - - if yaml is None: - return - - if not isinstance(yaml, list): - log.error("vram_classes must be a list") - - class_names = set() - for vram_class in yaml: - if isinstance(vram_class, dict): - if "name" not in vram_class: - log.error(f"vram_class ({vram_class}) must have a name") - class_names.add(vram_class["name"]) - elif isinstance(vram_class, list): - class_names.add(vram_class[0]) - - for vram_class in yaml: - name: str - vram: int - vram_symbol: Optional[str] = None - follows_classes: List[str] = [] - - if isinstance(vram_class, dict): - if "name" not in vram_class: - log.error(f"vram_class ({vram_class}) must have a name") - name = vram_class["name"] - - if "vram" not in vram_class: - log.error(f"vram_class ({vram_class}) must have a vram") - vram = vram_class["vram"] - - if "vram_symbol" in vram_class: - vram_symbol = vram_class["vram_symbol"] - if not isinstance(vram_symbol, str): - log.error( - f"vram_symbol ({vram_symbol})must be a string, got {type(vram_symbol)}" - ) - - if "follows_classes" in vram_class: - follows_classes = vram_class["follows_classes"] - if not isinstance(follows_classes, list): - log.error( - f"vram_symbol ({follows_classes})must be a list, got {type(follows_classes)}" - ) - for follows_class in follows_classes: - if follows_class not in class_names: - log.error( - f"follows_class ({follows_class}) not found in vram_classes" - ) - elif isinstance(vram_class, list): - if len(vram_class) != 2: - log.error( - f"vram_class ({vram_class}) must have 2 elements, got {len(vram_class)}" - ) - name = vram_class[0] - vram = vram_class[1] - else: - log.error(f"vram_class must be a dict or list, got {type(vram_class)}") - - if not isinstance(name, str): - log.error(f"vram_class name ({name}) must be a string, got {type(name)}") - if not isinstance(vram, int): - log.error(f"vram_class vram ({vram}) must be an int, got {type(vram)}") - if name in _vram_classes: - log.error(f"Duplicate vram class name '{name}'") - _vram_classes[name] = VramClass(name, vram, vram_symbol, follows_classes) - - -def resolve(name: str) -> VramClass: - if name not in _vram_classes: - log.error(f"Unknown vram class '{name}'") - return _vram_classes[name] diff --git a/tools/splat_ext/gfx_common.py b/tools/splat_ext/gfx_common.py index ce70d2c7fc..9e96bb4f8e 100644 --- a/tools/splat_ext/gfx_common.py +++ b/tools/splat_ext/gfx_common.py @@ -1,4 +1,4 @@ -from segtypes.n64.gfx import N64SegGfx +from splat.segtypes.n64.gfx import N64SegGfx class N64SegGfx_common(N64SegGfx): diff --git a/tools/splat_ext/pm_charset.py b/tools/splat_ext/pm_charset.py index 012883d057..ae6fff39ee 100644 --- a/tools/splat_ext/pm_charset.py +++ b/tools/splat_ext/pm_charset.py @@ -1,5 +1,5 @@ -from segtypes.n64.segment import N64Segment -from util import options +from splat.segtypes.n64.segment import N64Segment +from splat.util import options import png # type: ignore @@ -78,7 +78,7 @@ class N64SegPm_charset(N64Segment): w.write_array(f, raster) def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry + from splat.segtypes.linker_entry import LinkerEntry # start, type, name, WIDTH, HEIGHT self.width = self.yaml[3] diff --git a/tools/splat_ext/pm_charset_palettes.py b/tools/splat_ext/pm_charset_palettes.py index c15ea30653..208b801479 100644 --- a/tools/splat_ext/pm_charset_palettes.py +++ b/tools/splat_ext/pm_charset_palettes.py @@ -1,7 +1,7 @@ -from segtypes.n64.segment import N64Segment -from segtypes.n64.palette import iter_in_groups -from util.color import unpack_color -from util import options +from splat.segtypes.n64.segment import N64Segment +from splat.segtypes.n64.palette import iter_in_groups +from splat.util.color import unpack_color +from splat.util import options import png # type: ignore @@ -46,7 +46,7 @@ class N64SegPm_charset_palettes(N64Segment): w.write_array(f, raster) def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry + from splat.segtypes.linker_entry import LinkerEntry fs_dir = options.opts.asset_path / self.dir / self.name / "palette" diff --git a/tools/splat_ext/pm_effect_loads.py b/tools/splat_ext/pm_effect_loads.py index 293b0282d3..811231ef18 100644 --- a/tools/splat_ext/pm_effect_loads.py +++ b/tools/splat_ext/pm_effect_loads.py @@ -1,8 +1,8 @@ from dataclasses import dataclass from pathlib import Path from typing import List -from segtypes.n64.segment import N64Segment -from util import options +from splat.segtypes.n64.segment import N64Segment +from splat.util import options import yaml as yaml_loader @@ -120,7 +120,7 @@ glabel fx_{name} f.write(effect_asm) def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry + from splat.segtypes.linker_entry import LinkerEntry ret = [] diff --git a/tools/splat_ext/pm_effect_shims.py b/tools/splat_ext/pm_effect_shims.py index 9f39475eaf..ae5d35c320 100644 --- a/tools/splat_ext/pm_effect_shims.py +++ b/tools/splat_ext/pm_effect_shims.py @@ -1,7 +1,7 @@ from typing import List from yaml.loader import Loader -from segtypes.n64.segment import N64Segment -from util import options +from splat.segtypes.n64.segment import N64Segment +from splat.util import options import yaml as yaml_loader @@ -62,7 +62,7 @@ glabel {name} f.write(shim_asm) def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry + from splat.segtypes.linker_entry import LinkerEntry ret = [] diff --git a/tools/splat_ext/pm_icons.py b/tools/splat_ext/pm_icons.py index 8425a45a62..c48370b6c8 100644 --- a/tools/splat_ext/pm_icons.py +++ b/tools/splat_ext/pm_icons.py @@ -1,11 +1,11 @@ import os import re from pathlib import Path -from segtypes.n64.segment import N64Segment +from splat.segtypes.n64.segment import N64Segment import n64img.image -from util.color import unpack_color +from splat.util.color import unpack_color from common import iter_in_groups -from util import options +from splat.util import options import yaml as yaml_loader import xml.etree.ElementTree as ET @@ -111,7 +111,7 @@ class N64SegPm_icons(N64Segment): pretty_print_xml(xml, self.out_dir / "Icons.xml") def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry + from splat.segtypes.linker_entry import LinkerEntry return [ LinkerEntry( diff --git a/tools/splat_ext/pm_imgfx_data.py b/tools/splat_ext/pm_imgfx_data.py index c79ab29330..fa1de6f29b 100644 --- a/tools/splat_ext/pm_imgfx_data.py +++ b/tools/splat_ext/pm_imgfx_data.py @@ -2,15 +2,14 @@ import os import struct import sys from pathlib import Path -import yaml as yaml_loader from typing import List, Tuple TOOLS_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) sys.path.append(str(Path(TOOLS_DIR) / "build/imgfx")) from imgfx_data import Anim, Triangle, Vertex -from segtypes.n64.segment import N64Segment -from util import log, options +from splat.segtypes.n64.segment import N64Segment +from splat.util import log, options class N64SegPm_imgfx_data(N64Segment): @@ -114,7 +113,7 @@ class N64SegPm_imgfx_data(N64Segment): f.write(anim.toJSON()) def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry + from splat.segtypes.linker_entry import LinkerEntry return [ LinkerEntry( diff --git a/tools/splat_ext/pm_map_data.py b/tools/splat_ext/pm_map_data.py index 0f45163ac6..3174e1a006 100644 --- a/tools/splat_ext/pm_map_data.py +++ b/tools/splat_ext/pm_map_data.py @@ -1,16 +1,14 @@ from math import ceil import os, sys from pathlib import Path -from segtypes.n64.segment import N64Segment -from util.n64.Yay0decompress import Yay0Decompressor -from segtypes.n64.palette import iter_in_groups -from util import options + +import crunch64 +from splat.segtypes.n64.segment import N64Segment +from common import iter_in_groups +from splat.util import options import png # type: ignore import yaml as yaml_loader import n64img.image - -SPLAT_EXT_DIR = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(str(Path(SPLAT_EXT_DIR))) from tex_archives import TexArchive script_dir = Path(os.path.dirname(os.path.realpath(__file__))) @@ -126,7 +124,7 @@ class N64SegPm_map_data(N64Segment): bytes = rom_bytes[bytes_start : bytes_start + size] if is_compressed: - bytes = Yay0Decompressor.decompress_python(bytes) + bytes = crunch64.yay0.decompress(bytes) if name.startswith("party_"): assert path is not None @@ -201,7 +199,7 @@ class N64SegPm_map_data(N64Segment): asset_idx += 1 def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry + from splat.segtypes.linker_entry import LinkerEntry fs_dir = options.opts.asset_path / self.dir / self.name diff --git a/tools/splat_ext/pm_msg.py b/tools/splat_ext/pm_msg.py index 1c98086210..fdabe96d15 100644 --- a/tools/splat_ext/pm_msg.py +++ b/tools/splat_ext/pm_msg.py @@ -1,7 +1,7 @@ import shutil -from segtypes.n64.segment import N64Segment +from splat.segtypes.n64.segment import N64Segment from pathlib import Path -from util import options +from splat.util import options import re import pylibyaml @@ -894,7 +894,7 @@ class N64SegPm_msg(N64Segment): self.f.write("\n}\n") def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry + from splat.segtypes.linker_entry import LinkerEntry base_path = options.opts.asset_path / f"{self.name}" out_paths = [base_path / Path(f + ".msg") for f in self.files] diff --git a/tools/splat_ext/pm_sbn.py b/tools/splat_ext/pm_sbn.py index 1c3d0c0297..5c20c091e8 100644 --- a/tools/splat_ext/pm_sbn.py +++ b/tools/splat_ext/pm_sbn.py @@ -8,9 +8,9 @@ from typing import List # splat imports; will fail if script run directly try: - from segtypes.n64.segment import N64Segment - from segtypes.linker_entry import LinkerEntry - from util import options + from splat.segtypes.n64.segment import N64Segment + from splat.segtypes.linker_entry import LinkerEntry + from splat.util import options splat_loaded = True except ImportError: diff --git a/tools/splat_ext/pm_sprite_shading_profiles.py b/tools/splat_ext/pm_sprite_shading_profiles.py index 278af18afc..1b0b96b18c 100644 --- a/tools/splat_ext/pm_sprite_shading_profiles.py +++ b/tools/splat_ext/pm_sprite_shading_profiles.py @@ -10,8 +10,8 @@ import json import struct from typing import Literal -from segtypes.n64.segment import N64Segment -from util import options +from splat.segtypes.n64.segment import N64Segment +from splat.util import options GROUPS = [ SpriteShadingGroup("TIK"), @@ -211,7 +211,7 @@ class N64SegPm_sprite_shading_profiles(N64Segment): return self.OUT_DIR / f"{self.name}.json" def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry + from splat.segtypes.linker_entry import LinkerEntry return [ LinkerEntry( diff --git a/tools/splat_ext/pm_sprites.py b/tools/splat_ext/pm_sprites.py index b251255cd2..5b25dc793b 100644 --- a/tools/splat_ext/pm_sprites.py +++ b/tools/splat_ext/pm_sprites.py @@ -7,14 +7,14 @@ import xml.etree.ElementTree as ET from dataclasses import dataclass, field from pathlib import Path from typing import Any, Dict, List, Optional, Set, Tuple +import crunch64 import png # type: ignore import yaml as yaml_loader from n64img.image import CI4 -from segtypes.n64.segment import N64Segment -from util import options -from util.color import unpack_color -from util.n64.Yay0decompress import Yay0Decompressor +from splat.segtypes.n64.segment import N64Segment +from splat.util import options +from splat.util.color import unpack_color sys.path.insert(0, str(Path(__file__).parent)) from sprite_common import AnimComponent, iter_in_groups, read_offset_list @@ -302,7 +302,7 @@ def extract_sprites(yay0_data: bytes, raster_sets: List[PlayerSpriteRasterSet]) ret: List[PlayerSprite] = [] for i, yay0_piece in enumerate(yay0_sprite_data): - sprite_data = Yay0Decompressor.decompress(yay0_piece, "big") + sprite_data = crunch64.yay0.decompress(yay0_piece) sprite = PlayerSprite.from_bytes(sprite_data, raster_sets[i]) ret.append(sprite) @@ -805,7 +805,7 @@ class N64SegPm_sprites(N64Segment): start = int.from_bytes(data[i * 4 : (i + 1) * 4], byteorder="big") end = int.from_bytes(data[(i + 1) * 4 : (i + 2) * 4], byteorder="big") - sprite_data = Yay0Decompressor.decompress(data[start:end], "big") + sprite_data = crunch64.yay0.decompress(data[start:end]) sprite = NpcSprite.from_bytes(sprite_data) sprite.image_names = self.npc_cfg[sprite_name].get("frames", []) @@ -831,7 +831,7 @@ class N64SegPm_sprites(N64Segment): self.split_npc(npc_yay0_data) def get_linker_entries(self): - from segtypes.linker_entry import LinkerEntry + from splat.segtypes.linker_entry import LinkerEntry src_paths = [options.opts.asset_path / "sprite"] diff --git a/tools/splat_ext/tex_archives.py b/tools/splat_ext/tex_archives.py index 3531d97146..10f84071b4 100644 --- a/tools/splat_ext/tex_archives.py +++ b/tools/splat_ext/tex_archives.py @@ -6,7 +6,7 @@ from pathlib import Path import png import n64img.image -from segtypes.n64.palette import iter_in_groups +from common import iter_in_groups from sys import path diff --git a/tools/splat_ext/vtx_common.py b/tools/splat_ext/vtx_common.py index bdfa05815f..1cf5593a57 100644 --- a/tools/splat_ext/vtx_common.py +++ b/tools/splat_ext/vtx_common.py @@ -1,4 +1,4 @@ -from segtypes.n64.vtx import N64SegVtx +from splat.segtypes.n64.vtx import N64SegVtx class N64SegVtx_common(N64SegVtx):