pmret/papermario
1
0
Fork 0
mirror of https://github.com/pmret/papermario.git synced 2024-11-08 12:02:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

Splat update (#1136)

Browse Source 
* Remove splat from repo

* refactor scraps

* requirements update
This commit is contained in:
Ethan Roseman 2024-01-03 02:16:18 +09:00 committed by GitHub
parent 9b96f91dc6
commit 262428c68a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
166 changed files with 69 additions and 12638 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.vscode/settings.json vendored
View File

@ -36,8 +36,8 @@
"./tools", "./tools",
"./tools/build", "./tools/build",
"./tools/build/sprite", "./tools/build/sprite",
"./tools/splat", "./tools/build/imgfx",
"./tools/build/imgfx" "./tools/splat/src"
], ],
"[c]": { "[c]": {
"editor.wordSeparators": "`~!@#%^&*()-=+[{]}\\|;:'\",.<>/?", // no $, for scripts "editor.wordSeparators": "`~!@#%^&*()-=+[{]}\\|;:'\",.<>/?", // no $, for scripts

2
requirements.txt
View File

@ -12,3 +12,5 @@ intervaltree
rabbitizer rabbitizer
n64img n64img
python-githooks python-githooks
crunch64>=0.2.0
splat64>=0.21.0

47
tools/build/configure.py
View File

@ -376,7 +376,7 @@ class Configure:
self.linker_entries = None self.linker_entries = None
def split(self, assets: bool, code: bool, shift: bool, debug: bool): def split(self, assets: bool, code: bool, shift: bool, debug: bool):
import split import splat.scripts.split as split
modes = ["ld"] modes = ["ld"]
if assets: if assets:
@ -500,19 +500,6 @@ class Configure:
modern_gcc: bool, modern_gcc: bool,
c_maps: bool = False, c_maps: bool = False,
): ):
import segtypes
import segtypes.common.asm
import segtypes.common.bin
import segtypes.common.c
import segtypes.common.data
import segtypes.common.group
import segtypes.common.rodatabin
import segtypes.common.textbin
import segtypes.n64.header
import segtypes.n64.img
import segtypes.n64.palette
import segtypes.n64.yay0
assert self.linker_entries is not None assert self.linker_entries is not None
built_objects = set() built_objects = set()
@ -632,6 +619,8 @@ class Configure:
"actor_types", "actor_types",
) )
import splat
# Build objects # Build objects
for entry in self.linker_entries: for entry in self.linker_entries:
seg = entry.segment seg = entry.segment
@ -641,16 +630,16 @@ class Configure:
assert entry.object_path is not None assert entry.object_path is not None
if isinstance(seg, segtypes.n64.header.N64SegHeader): if isinstance(seg, splat.segtypes.n64.header.N64SegHeader):
build(entry.object_path, entry.src_paths, "as") build(entry.object_path, entry.src_paths, "as")
elif isinstance(seg, segtypes.common.asm.CommonSegAsm) or ( elif isinstance(seg, splat.segtypes.common.asm.CommonSegAsm) or (
isinstance(seg, segtypes.common.data.CommonSegData) and not seg.type[0] == "." isinstance(seg, splat.segtypes.common.data.CommonSegData) and not seg.type[0] == "."
): ):
build(entry.object_path, entry.src_paths, "as") build(entry.object_path, entry.src_paths, "as")
elif seg.type in ["pm_effect_loads", "pm_effect_shims"]: elif seg.type in ["pm_effect_loads", "pm_effect_shims"]:
build(entry.object_path, entry.src_paths, "as") build(entry.object_path, entry.src_paths, "as")
elif isinstance(seg, segtypes.common.c.CommonSegC) or ( elif isinstance(seg, splat.segtypes.common.c.CommonSegC) or (
isinstance(seg, segtypes.common.data.CommonSegData) and seg.type[0] == "." isinstance(seg, splat.segtypes.common.data.CommonSegData) and seg.type[0] == "."
): ):
cflags = None cflags = None
if isinstance(seg.yaml, dict): if isinstance(seg.yaml, dict):
@ -733,9 +722,9 @@ class Configure:
) )
# images embedded inside data aren't linked, but they do need to be built into .inc.c files # images embedded inside data aren't linked, but they do need to be built into .inc.c files
if isinstance(seg, segtypes.common.group.CommonSegGroup): if isinstance(seg, splat.segtypes.common.group.CommonSegGroup):
for seg in seg.subsegments: for seg in seg.subsegments:
if isinstance(seg, segtypes.n64.img.N64SegImg): if isinstance(seg, splat.segtypes.n64.img.N64SegImg):
flags = "" flags = ""
if seg.n64img.flip_h: if seg.n64img.flip_h:
flags += "--flip-x " flags += "--flip-x "
@ -780,7 +769,7 @@ class Configure:
"bin_inc_c", "bin_inc_c",
vars, vars,
) )
elif isinstance(seg, segtypes.n64.palette.N64SegPalette): elif isinstance(seg, splat.segtypes.n64.palette.N64SegPalette):
src_paths = [seg.out_path().relative_to(ROOT)] src_paths = [seg.out_path().relative_to(ROOT)]
inc_dir = self.build_path() / "include" / seg.dir inc_dir = self.build_path() / "include" / seg.dir
bin_path = self.build_path() / seg.dir / (seg.name + ".pal.bin") bin_path = self.build_path() / seg.dir / (seg.name + ".pal.bin")
@ -810,16 +799,16 @@ class Configure:
vars, vars,
) )
elif ( elif (
isinstance(seg, segtypes.common.bin.CommonSegBin) isinstance(seg, splat.segtypes.common.bin.CommonSegBin)
or isinstance(seg, segtypes.common.textbin.CommonSegTextbin) or isinstance(seg, splat.segtypes.common.textbin.CommonSegTextbin)
or isinstance(seg, segtypes.common.rodatabin.CommonSegRodatabin) or isinstance(seg, splat.segtypes.common.rodatabin.CommonSegRodatabin)
): ):
build(entry.object_path, entry.src_paths, "bin") build(entry.object_path, entry.src_paths, "bin")
elif isinstance(seg, segtypes.n64.yay0.N64SegYay0): elif isinstance(seg, splat.segtypes.n64.yay0.N64SegYay0):
compressed_path = entry.object_path.with_suffix("") # remove .o compressed_path = entry.object_path.with_suffix("") # remove .o
build(compressed_path, entry.src_paths, "yay0") build(compressed_path, entry.src_paths, "yay0")
build(entry.object_path, [compressed_path], "bin") build(entry.object_path, [compressed_path], "bin")
elif isinstance(seg, segtypes.n64.img.N64SegImg): elif isinstance(seg, splat.segtypes.n64.img.N64SegImg):
flags = "" flags = ""
if seg.n64img.flip_h: if seg.n64img.flip_h:
flags += "--flip-x " flags += "--flip-x "
@ -845,7 +834,7 @@ class Configure:
# ) # )
# vars = {"c_name": c_sym.name} # vars = {"c_name": c_sym.name}
build(inc_dir / (seg.name + ".png.h"), entry.src_paths, "img_header") build(inc_dir / (seg.name + ".png.h"), entry.src_paths, "img_header")
elif isinstance(seg, segtypes.n64.palette.N64SegPalette): elif isinstance(seg, splat.segtypes.n64.palette.N64SegPalette):
bin_path = entry.object_path.with_suffix(".bin") bin_path = entry.object_path.with_suffix(".bin")
build( build(
@ -1382,7 +1371,7 @@ if __name__ == "__main__":
extra_cflags += " -Wmissing-braces -Wimplicit -Wredundant-decls -Wstrict-prototypes -Wno-redundant-decls" extra_cflags += " -Wmissing-braces -Wimplicit -Wredundant-decls -Wstrict-prototypes -Wno-redundant-decls"
# add splat to python import path # add splat to python import path
sys.path.insert(0, str((ROOT / args.splat).resolve())) sys.path.insert(0, str((ROOT / args.splat / "src").resolve()))
ninja = ninja_syntax.Writer(open(str(ROOT / "build.ninja"), "w"), width=9999) ninja = ninja_syntax.Writer(open(str(ROOT / "build.ninja"), "w"), width=9999)

23
tools/splat/.github/workflows/black.yml vendored
View File

@ -1,23 +0,0 @@
name: black
on:
push:
pull_request:
jobs:
black_checks:
runs-on: ubuntu-latest
name: black
steps:
- uses: actions/checkout@v1
- name: Set up Python 3.8
uses: actions/setup-python@v1
with:
python-version: 3.8
- name: Install Dependencies
run: |
pip install black
pip install -r requirements.txt
pip install types-PyYAML
- name: black
run: black --check .

23
tools/splat/.github/workflows/mypy.yml vendored
View File

@ -1,23 +0,0 @@
name: mypy
on:
push:
pull_request:
jobs:
mypy_checks:
runs-on: ubuntu-latest
name: mypy
steps:
- uses: actions/checkout@v1
- name: Set up Python 3.8
uses: actions/setup-python@v1
with:
python-version: 3.8
- name: Install Dependencies
run: |
pip install mypy
pip install -r requirements.txt
pip install types-PyYAML
- name: mypy
run: mypy --show-column-numbers --hide-error-context .

54
tools/splat/.github/workflows/publish_docs_to_wiki.yml vendored
View File

@ -1,54 +0,0 @@
# Based on script from https://github.com/orgs/community/discussions/25929
name: Publish docs to Wiki
# Trigger this action only if there are changes pushed to the docs/** directory under the main branch
on:
push:
paths:
- docs/** # This includes all sub folders
branches:
- main # This can be changed to any branch of your preference
jobs:
publish_docs_to_wiki:
name: Publish docs to Wiki
runs-on: ubuntu-latest
steps:
# Clone the wiki repository
- name: Checkout Wiki repository
uses: actions/checkout@v4
with:
repository: ${{ github.event.repository.owner.name }}/${{ github.event.repository.name }}.wiki
path: wiki_repo
# Clone the main repository
- name: Checkout main repository
uses: actions/checkout@v4
with:
repository: ${{ github.event.repository.owner.name }}/${{ github.event.repository.name }}
path: splat_repo
- name: Get the new Wiki files
run: |
cd wiki_repo
rm *.md
cp ../splat_repo/docs/* .
# `git log -1 --pretty=%aN` prints the current commit's author name
# `git log -1 --pretty=%aE` prints the current commit's author mail
- name: Stage new files
run: |
cd wiki_repo
git config user.name $(git log -1 --pretty=%aN)
git config user.email $(git log -1 --pretty=%aE)
git add .
# `git diff-index --quiet HEAD` returns non-zero if there are any changes.
# This allows to avoid making a commit/push if there are no changes to the Wiki files
# `git log -1 --pretty=%B` prints the current commit's message
- name: Push new files to the Wiki
run: |
cd wiki_repo
git diff-index --quiet HEAD || (git commit -m "$(git log -1 --pretty=%B)" && git push)

20
tools/splat/.github/workflows/unit_tests.yml vendored
View File

@ -1,20 +0,0 @@
name: unit_tests
on:
push:
pull_request:
jobs:
unit_tests:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v1
- name: Set up Python 3.8
uses: actions/setup-python@v1
with:
python-version: 3.8
- name: Install Dependencies
run: pip install -r requirements.txt
- name: Run unit tests
run: sh run_tests.sh

10
tools/splat/.gitignore vendored
View File

@ -1,10 +0,0 @@
.idea/
venv/
.vscode/
__pycache__/
.mypy_cache/
util/n64/Yay0decompress
*.ld
*.n64
*.yaml
*.z64

12
tools/splat/.gitrepo
View File

@ -1,12 +0,0 @@
; DO NOT EDIT (unless you know what you are doing)
;
; This subdirectory is a git "subrepo", and this file is maintained by the
; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme
;
[subrepo]
remote = https://github.com/ethteck/splat.git
branch = main
commit = d6490bb877f692f5e20a7846cabd5f793b314054
parent = c829fde77b3ccc15761582437966317d3d5b13d6
method = merge
cmdver = 0.4.5

565
tools/splat/CHANGELOG.md
View File

@ -1,565 +0,0 @@
# splat Release Notes
### 0.19.0: vram_classes
* New top-level yaml feature: `vram_classes`. This allows you to make common definitions for vram locations that can be applied to multiple segments. Please see the [documentation](docs/VramClasses.md) for more details!
* Renamed `ld_use_follows` to `ld_use_symbolic_vram_addresses` to more accurately describe what it's doing
* Renamed `vram_of_symbol` segment option to `vram_symbol` to provide consistency between the segment-level option and the vram class field.
* Removed `appears_after_overlays_addr` symbol_addrs option in favor of specifying this behavior with `vram_classes`
* Removed `dead` symbol_addrs option
* A warning is now emitted when the `sha1` top-level yaml option is not provided. Adding this is highly recommended, as it prevents errors using splat in which the wrong binary is provided.
### 0.18.3
* splat now will emit a `FILL(0)` statement on each segment of a linker script by default, to customize this behavior use the `ld_fill_value` yaml option or the per-segment `ld_fill_value` option.
* New yaml option: `ld_fill_value`
* Allows to specify the value of the `FILL` statement generated on every segment of the linker script.
* It must be either an integer, which will be used as the parameter for the `FILL` statement, or `null`, which tells splat to not emit `FILL` statements.
* This behavior can be customized per segment too.
* New per segment option: `ld_fill_value`
* Allows to specify the value of the `FILL` statement generated for this specific top-level segment of the linker script, ignoring the global configuration.
* If not set, then the global configuration is used.
### 0.18.2
* Fix rodata migration for `.rdata` sections (and other rodata sections that don't use the name `.rodata`)
* `spimdisasm` 1.18.0 or above is now required.
### 0.18.1
* New yaml options: `check_consecutive_segment_types`
* Allows to turn off checking for segment types not being in a consecutive order
* New option for segments: `linker_section_order` and `linker_section`
* `linker_section_order`: Allows overriding the section order used for linker script generation. Useful when a section of a file is not between the other sections of the same type in the ROM, for example a file having its data section between other files's rodata.
* `linker_section`: Allows to override the `.section` directive that will be used when generating the disassembly of the corresponding section, without needing to write an extension segment. This also affects the section name that will be used during link time. Useful for sections with special names, like an executable section named `.start`
### 0.18.0
* `symbol_addrs` parsing checks:
* Enforce lines contain a single `;`
* Enforce no duplicates (same vram, same rom)
### 0.17.3
* Move wiki to the `docs` folder
* Added the ability to specify `find_file_boundaries` on a per segment basis
* Fix `cpp` segment not symbolizing rodata symbols properly
### 0.17.2
* Added more support for PS2 elf files
### 0.17.1
* New yaml options: `ld_sections_allowlist` and `ld_sections_denylist`
* `ld_sections_allowlist`: A list of sections to preserve during link time. It can be useful to preserve debugging sections.
* `ld_sections_denylist`: A list of sections to discard during link time. It can be useful to avoid using the wildcard discard. Note that this option does not turn off `ld_discard_section`.
### 0.17.0
* BREAKING: Linker script generation now imposes the specified `section_order`, which may not completely reflect the yaml order.
* In case this new linker script generation can't be properly adapted to a repo, the old generation can be reenabled by using the `ld_legacy_generation` flag as a temporary solution. Keep in mind this option may be removed in the future.
* New yaml options related to linker script generation: `ld_partial_linking`, `ld_partial_scripts_path`, `ld_partial_build_segments_path`, `elf_path`, `ld_dependencies`
* `ld_partial_linking`: Changes how the linker script is generated, allowing partially linking each segment. This allows for faster linking times when making changes to files at the cost of a slower build time from a clean build and loosing filepaths in the mapfile. This is also known as "incremental linking". This option requires both `ld_partial_scripts_path` and `ld_partial_build_segments_path`.
* `ld_partial_scripts_path`: Folder were each intermediary linker script will be written to.
* `ld_partial_build_segments_path`: Folder where the built partially linked segments will be placed by the build system.
* `elf_path`: Path to the final elf target.
* `ld_dependencies`: Generate a dependency file for every linker script generated, including the main linker script and the ones for partial linking. Dependency files will have the same path and name as the corresponding linker script, but changing the extension to `.d`. Requires `elf_path` to be set.
* New misc yaml options: `asm_function_alt_macro` and `ique_symbols`
* `asm_function_alt_macro`: Allows to use a different label on symbols that are in the middle of functions (that are not branch targets of any kind) than the one used for the label for functions, allowing for alternative function entrypoints.
* `ique_symbols` Automatically fills libultra symbols that are exclusive for iQue. This option is ignored if platform is not N64.
* New "incbin" segments: `textbin`, `databin` and `rodatabin`
* Allows to specify binary blobs to be linked in a specific section instead of the data default.
* If a `textbin` section has a corresponding `databin` and/or `rodatabin` section with the same name then those will be included in the same generated assembly file.
* If a known symbol matches the vram of a incbin section then it will be emitted properly, allowing for better integration with the rest of splat's symbol system.
* `spimdisasm` 1.17.0 or above is now required.
### 0.16.10
* Produce an error if subsegments do not have an ascending vram order.
* This can happen because bss subsegments need their vram to be specified explicitly.
### 0.16.9
* Add command line argument `--disassemble-all`, which has the same effect as the `disassemble_all` yaml option so will disamble already matched functions as well as migrated data.
* Note: the command line argument takes precedence over the yaml, so will take effect even if the yaml option is set to false.
### 0.16.8
* Avoid ignoring the `align` defined in a segment for `code` segments
### 0.16.7
* Use `pylibyaml` to speed-up yaml parsing
### 0.16.6
* Add option `ld_rom_start`.
* Allows offsetting rom address linker symbols by some arbitrary value.
* Useful for SN64 games which often have rom addresses offset by 0xB0000000.
* Defaults to 0.
### 0.16.5
* Add option `segment_symbols_style`.
* Allows changing the style of the generated segment symbols in the linker script.
* Possible values:
* `splat`: The current style for segment symbols.
* `makerom`: Style that aims to be compatible with makerom generated symbols.
* Defaults to `splat`.
### 0.16.4
* Add `get_section_flags` method to the `Segment` class.
* Useful for providing linker section flags when creating a custom section when making splat extensions.
* This may be necessary for some custom section types, because sections unrecognized by the linker will not link its data properly.
* More info about section flags: <https://sourceware.org/binutils/docs/as/Section.html#ELF-Version>
### 0.16.3
* Add `--stdout-only` flag. Redirects the progress bar output to `stdout` instead of `stderr`.
* Add a check to prevent relocs with duplicated rom addresses.
* Check empty functions only have 2 instructions before autodecompiling them.
### 0.16.2
* Add option `disassemble_all`. If enabled then already matched functions and migrated data will be disassembled to files anyways.
### 0.16.1
* Various changes so that series of image and palette subsegments can have `auto` rom addresses (as long as the first can find its rom address from the parent segment or its own definition)
### 0.16.0
* Add option `detect_redundant_function_end`. It tries to detect redundant and unreferenced functions ends and merge them together.
* This option is ignored if the compiler is not set to IDO.
* This type of codegen is only affected by flags `-g`, `-g1` and `-g2`.
* This option can also be overriden per file.
* Disable `include_macro_inc` by default for IDO projects.
* Disable `asm_emit_size_directive` by default for SN64 projects.
* `spimdisasm` 1.16.0 or above is now required.
### 0.15.4
* Try to assign a segment to an user-declared symbol if the user declared the rom address.
* Helps to disambiguate symbols for same-address overlays.
### 0.15.3
* Disabled `asm_emit_size_directive` by default for IDO projects.
### 0.15.2
* Various cleanup and fixes to support more liberal use of `auto` for rom addresses
### 0.15.1
* Made some modifications such that linker object paths should be simpler in some circumstances
### 0.15.0
* New options:
* `data_string_encoding` can be set at the global level (or `str_encoding` at the segment level) to specify the encoding using when guessing and disassembling strings the the data section. In spimdisasm this value defaults to ASCII.
* `rodata_string_guesser_level` changes the behaviour of the rodata string guesser. A higher value means more agressive guessing, while 0 and negative means no guessing at all. Even if the guesser feature is disabled, symbols manually marked as strings in the symbol_addrs.txt file will still be disassembled as strings. In spimdisasm this value defaults to 1.
* level 0: Completely disable the guessing feature.
* level 1: The most conservative guessing level. Imposes the following restrictions:
* Do not try to guess if the user provided a type for the symbol.
* Do no try to guess if type information for the symbol can be inferred by other means.
* A string symbol must be referenced only once.
* Strings must not be empty.
* level 2: A string no longer needs to be referenced only once to be considered a possible string. This can happen because of a deduplication optimization.
* level 3: Empty strings are allowed.
* level 4: Symbols with autodetected type information but no user type information can still be guessed as strings.
* `data_string_guesser_level` is similar to `rodata_string_guesser_level`, but for the data section instead. In spimdisasm this value defaults to 2.
* `asm_emit_size_directive` toggles the size directived emitted by the disassembler. In spimdisasm this defaults to True.
### 0.14.1
* Fix bug, cod cleanup
### 0.14.0
* Add support for PSX's GTE instruction set
### 0.13.10
* New option `disasm_unknown` (False by default)
* If enabled it tells the disassembler to try disassembling functions with unknown instructions instead of falling back to disassembling as raw data
### 0.13.9
* New segment option `linker_entry` (true by default).
* If disabled, this segment will not produce entries in the linker script.
### 0.13.8
* New option `segment_end_before_align`.
* If enabled, the end symbol for each segment will be placed before the alignment directive for the segment
### 0.13.7
* Severely sped-up linker entry writing by using a dict instead of a list. Symbol headers will no longer be in any specific order (which shouldn't matter, because they're headers).
### 0.13.6
* Changed CI image processing so that their data is fetched during the scan phase, supporting palettes that come before CI images.
### 0.13.5
* An error will be produced if a symbol is declared with an unknown type in the symbol_addrs file.
* The current list of known symbols is `'func', 'label', 'jtbl', 'jtbl_label', 's8', 'u8', 's16', 'u16', 's32', 'u32', 's64', 'u64', 'f32', 'f64', 'Vec3f', 'asciz', 'char*', 'char'`.
* Custom types are allowed if they start with a capital letter.
### 0.13.4
* Renamed `follows_vram_symbol` segment option to `vram_of_symbol` to more accurately reflect what it's used for - to set the segment's vram based on a symbol.
* Refactored the `appears_after_overlays_addr` feature so that expressions are written at the latest possible moment in the linker script. This fixes errors and warnings regarding forward references to later symbols.
### 0.13.3
* Added a new symbol_addrs attribute `appears_after_overlays_addr:0x1234` which will modify the linker script such that the symbol's address is equal to the value of the end of the longest overlay starting with address 0x1234. It achieves this by writing a series of sym = MAX(sym, seg_vram_END) statements into the linker script. For some games, it's feasible to manually create such statements, but for games with hundreds of overlays at the same address, this is very tedious and prone to error. The new attribute allows you to have peace of mind that the symbol will end up after all of these overlays.
### 0.13.2
* Actually implemented `ld_use_follows`. Oopz
### 0.13.1
* Added `ld_wildcard_sections` option (disabled by default), which adds a wildcard to the linker script for section linking. This can be helpful for modern GCC, which creates additional rodata sections such as ".rodata.xyz".
* Added `ld_use_follows` option (enabled by default), which, if disabled, makes splat ignore follows_vram and follows_symbols. This helps for fixing matching builds while being able to add infrastructure to the yaml for non-matching builds by just re-enabling the option.
### 0.13.0
* Automatically generate `INCLUDE_RODATA`/`#pragma GLOBAL_ASM` directives for non migrated rodata symbols when creating new C files.
* Non migrated rodata symbols will now only be produced if the C file has a corresponding rodata file with the same name and the C file has a `INCLUDE_RODATA`/`#pragma GLOBAL_ASM` directive referencing the symbol, similar to how functions are disassembled.
* Because of this, the `partial_migration` attribute has lost its purpose and has been removed.
* Rodata symbol files are now included in the autogenerated dependency files too.
### 0.12.14
* New option: `pair_rodata_to_text`.
* If enabled, splat will try to find to which text segment an unpaired rodata segment belongs, and it will hint it to the user.
### 0.12.13
* bss segments can now omit the rom offset.
### 0.12.12
* Try to detect and warn to the user if a gap between two migrated rodata symbols is detected and suggest possible solutions to the user.
### 0.12.11
* New disassembly option in the yaml: `allow_data_addends`.
* Allows enabling/disabling using addends on all `.data` symbols.
* Three new options for symbols: `name_end`, `allow_addend` and `dont_allow_addend`.
* `name_end`: allows to provide a closing name for any symbol. Useful for handwritten asm which usually have an "end" name.
* `allow_addend` and `dont_allow_addend`: Allow overriding the global `allow_data_addends` option for allowing addends on data symbols.
### 0.12.10
* Allows passing user-created relocs to the disassembler via the `reloc_addrs.txt` file, allowing to improve the automatic disassembly.
* Multiple reloc_addrs files can be specified in the yaml with the `reloc_addrs_path` option.
### 0.12.9
* Added `format_sym_name()` to the vtx segment so it, too, can be extended
### 0.12.8
* The gfx and vtx segments now have a `data_only` option, which, if enabled, will emit only the plain data for the type and omit the enclosing symbol definition. This mode is useful when you want to manually declare the symbol and then #include the extracted data within the declaration.
* The gfx segment has a method, `format_sym_name()`, which will allow custom overriding of the output of symbol names by extending the `gfx` segment. For example, this can be used to transform context-specific symbol names like mac_01_vtx into N(vtx), where N() is a macro that applies the current "namespace" to the symbol. Paper Mario plans to use this, so we can extract an asset once and then #include it in multiple places, while giving each inclusion unique symbol names for each component.
### 0.12.7
* Allow setting a different macro for jumptable labels with `asm_jtbl_label_macro`
* The currently recommended one is `jlabel` instead of `glabel`
* Two new options for symbols: `force_migration` and `force_not_migration`
* Useful for weird cases where the disassembler decided a rodata symbol must (or must not) be migrated when it really shouldn't (or should)
* Fix `str_encoding` defaulting to `False` instead of `None`
* Output empty rules in generated dependency files to avoid issues when the function file does not exist anymore (i.e. when it gets matched)
* Allow changing the `include_macro_inc` option in the yaml
### 0.12.6
* Adds two new N64-specific segments:
* IPL3: Allows setting its correct VRAM address without messing the global segment detection
* RSP: Allows disassembling using the RSP instruction set instead of the default one
* PS2 was added as a new platform option.
* When this is selected the R5900 instruction set will be used when disassembling instead of the default one.
### 0.12.5
* Update minimal spimdisasm version to 1.7.1.
* Fix spimdisasm>=1.7.0 non being able to see symbols which only are referenced by other data symbols.
* A check was added to prevent segments marked with `exclusive_ram_id` have a vram address range which overlaps with segments not marked with said tag. If this happens it will be warned to the user.
### 0.12.4
* Fixed a bug involving the order of attributes in symbol_addrs preventing proper range searching during calls to `get_symbol`
### 0.12.3: Initial Gamecube Support
Initial support for Gamecube disk images has been set up! Disassembly is not currently supported, and a more comprehensive explanation of Gamecube support will come once that is finished.
* The Symbol class is now hashable
* Added the ability for segments to specify a file path (`path`) to receive that file's contents as their split input
* The `generated_s_preamble` option now will be applied to data files created by spimdisasm
* Rewrote symbol range check code to be more efficient
* Fixed bug that allowed empty top-level segments of type `code`.
* Fixed progress bars to properly update their descriptions
* Fixed bug pertaining to symbols getting assigned to segments they shouldn't if their segment is given in symbol_addrs (`segment:`)
### 0.12.2
* Fixed bug where `given_dir` was possibly not a `Path`
### 0.12.1
* The constructor for `Segment` takes far fewer arguments now, which will affect (and hopefully simplify) any custom segments that are implemented.
* The new option `string_encoding` can be set at the global or segment level and will influence the encoding for strings in rodata during disassembly. The default encoding used is EUC-JP, as it was previously.
## 0.12.0: Performance Boost
In this release, we bring many performance improvements, making splat dramatically faster. We have observed speedups of 10-20x, though your results may vary.
* Linker script `_romPos` alignment statements now take a form that is friendlier to different assemblers.
* Fixed the default value of `use_legacy_include_asm` to be what it was before 0.11.2
### 0.11.2
* The way options are parsed and accessed has been completely refactored. The following option names have changed:
`linker_symbol_header_path` -> `ld_symbol_header_path`
`asm_endlabels` -> `asm_end_label`
Additionally, any custom segments or code that needs to read options will have to accommodate the new API for doing so. Options are now fields of an object named `opts` within the existing `options` namespace. Because the options are fields, `get_` is no longer necessary. To give an example:
Before: `options.get_asm_path()`
After: `options.opts.asm_path`
The clean_up_path function in linker_entry.py now uses a cache, offering a small performance improvement during the linker script writing phase.
### 0.11.1
* The linker script now includes a `_SIZE` symbol for each segment.
* The new `create_asm_dependencies`, if enabled, will cause splat to create `.asmproc.d` files that can inform a build system which asm files a c file depends upon. If your build system is configured correctly, this can allow triggering a rebuild of a C file when its included asm files are modified.
* Splat no longer depends directly on pypng and now instead uses [n64img](https://github.com/decompals/n64img). Currently, all image behavior uses the exact same code. Eventually, n64img will be implemented in C and support rebuilding images as well.
## 0.11.0: Spimdisasm Returns
Spimdisasm now handles data (data, rodata, bss) disassembly in splat! This includes a few changes in behavior:
* Rodata will be migrated to c files' asm function files when a .rodata subsegment is used that corresponds with an identically-named c file. Some symbols may not be automatically migrated to functions when it is not clear if they belong to the function itself (an example of which being const arrays). In this case, the `partial_migration` option can be enabled for the given .rodata subsegment and splat will create .s files for these unmigrated rodata symbols. These files can then be included in your c files, or you can go ahead and migrate these symbols to c and disable the `partial_migration` feature.
* BSS can now be disassembled as well, and the size of a code segment's bss section can be specified with the `bss_size` option. This option will tell splat how large the bss section is in bytes so BSS can properly be handled during disassembly. For bss subsegments, the rom address will of course not change, but the vram address should still be specified. This currently can only be done in the dict form of segment representation, rather than the list form.
Thanks again to [AngheloAlf](https://github.com/AngheloAlf) for adding this functionality and continuing to improve splat's disassembler.
## 0.10.0: The Linker Script Update
Linker scripts splat produces are now capable of being shift-friendly. Rom addresses will automatically shift, and ram addresses will still be hard-coded unless the new segment option `follows_vram` is specified. The value of this option should be the name of a segment (a) that this segment (b) should follow in memory. If a grows or shrinks, b's start address will also do so to accommodate it.
The `enable_ld_alignment_hack` option and corresponding behavior has been removed. This proved to add too much complexity to the linker script generation code and was becoming quite a burden to keep dealing with. Apologies for any inconvenience this may cause. But trust me: in the long run, it's good you won't be depending on that madness.
### 0.9.5
* Changes have been made to the linker script such that it is more shiftable. Rather than setting the rom position to hard-coded addresses, it increments the position by the size of the previous segment. Some projects may experience some alignment-related issues after this change. If specified, the new segment option `align: n` will add an `ALIGN(n)` directive for that section's linker segment.
### 0.9.4
* A new linker script section is now automatically created when the .bss section begins, using NOLOAD as opposed to the previous hacky rom rewinding we were previously doing. Additionally, `ld_section_labels` now includes `.rodata` by default.
### 0.9.3
* Added `add_set_gp_64` option (true by default), which allows controlling whether to add ".set gp=64" to asm/hasm files
### 0.9.2
* Added "palette" argument to ci4/ci8 segments so that segments' palettes can be manually specified
### 0.9.1
* Fixed a bug in which local labels and jump table labels could replace raw words in data blobs during data disassembly
## 0.9.0: The Big Update
### Introducing [spimdisasm](https://github.com/Decompollaborate/spimdisasm)!
* Thanks to [AngheloAlf](https://github.com/AngheloAlf), we now have a much better MIPS disassembler in splat! spimdisasm has much better hi/lo matching, much lower ram usage, and plenty of other goodies.
We plan to roll this out in phases. Currently, it only handles actual code disassembly. Later on, we will probably migrate our current data assembly code to use spimdisasm as well.
**NOTICE**: This integration has been tested on a variety of games and configurations. However, with any giant change to the platform like this, there are bound to be things we didn't catch. Please be patient with us as we handle these remaining issues. Though from what we've seen already, the slight bugs one may come across are totally worth the much improved disassembly.
### gfx segment type
* A new `gfx` segment type is available, which creates a c file containing a disassembled display list according to the segment's start and end offsets. Thanks to [Glank](https://github.com/glankk) and [Tharo](https://github.com/thar0/) for their work on [libgfxd](https://github.com/glankk/libgfxd) and [pygfxd](https://github.com/thar0/pygfxd/), respectively, for helping make this a possibility in splat.
### API breaking changes
* Some `Segment()` arguments have changed, which may cause extensions to break. Please see the `__init__` function for `Segment` for more details.
### symbol_addrs.txt changes
* symbol_addrs now supports the `segment:` attribute, which allows specifying the symbol's top-level segment. This can be helpful for symbol resolution when overlays use overlapping vram ranges. See `exclusive_ram_id` below for more information.
### Global options changes
The new `symbol_name_format` option allows specification of how symbols will be named. This can be set as a global option and also changed per-segment. `symbol_name_format_no_rom` is used when the symbol does not have a rom address (BSS).
The following substitutions are allowed:
`$ROM` - the rom address of the symbol, hex-formatted and padded to 6 characters (ABCF10, 000030, 123456) (note: only for `symbol_name_format`, usage in `symbol_name_format_no_rom` will cause an error)
`$VRAM` - the vram address of the symbol, hex-formatted and padded to 8 characters (00030010, 00020015, ABCDEF10)
`$SEG` - the name of the top-level segment in which the symbol resides
The default values for these options are as follows
`symbol_name_format` : `$VRAM`
`symbol_name_format_no_rom` : `$VRAM_$SEG`
The appropriate prefix string will still automatically be applied depending on the type of the symbol: `D_` for data, `jtbl_` for jump tables, and `func_` for functions. This functionality may be customizable in the future.
----
The `auto_all_sections` option now should be a list of section names (`[".data", ".rodata", ".bss"]` by default) indicating the sections that should be linked from .o files built from source files (.c or asm/hasm .s files), when no subsegment explicitly indicates linking this type of section.
For example, if any subsegment of a code segment is of segment type `data` or `.data`, the `.data` section from all `c`/`asm`/`hasm` subsegments will not be linked unless explicitly indicated with a relevant `.data` subsegment.
Previously, this option was a bool, and it enabled this feature for all sections specified in `section_order`. Now, the desired sections must be specified manually. The default value for this option retains previous behavior.
----
The new `mips_abi_float_regs` option allows for changing the format of float registers for MIPS disassembly. The default value does not change any prior behavior, but `o32` is heavily encouraged and may become the default option in the future. For more information, see this [great writeup](https://gist.github.com/EllipticEllipsis/27eef11205c7a59d8ea85632bc49224d).
----
The new `gfx_ucode` option allows for specifying the target for the graphics macro format, which is used in the gfx segment type. The default is `f3dex2`.
### Segment options changes
The new `exclusive_ram_id` segment option allows specifying an identifer that will prevent the segment from seeing any symbols from other segments with the same identifer. This is useful when multiple segments are mapped to the same vram address at runtime and should never be able to refer to each other's symbols. Setting all of these segments to have the same value for this option will prevent their symbols from clashing / meshing unexpectedly.
----
The `overlay` setting on segments has been removed. Please see `symbol_name_format` above for info on how to influence the names of symbols, which can be applied at the segment level as well as the global level.
----
## 0.8.0: Arbitrary Section Order
* You can now use the option `section_order` to define the binary section order for your target binary. By default, this is `[".text", ".data", ".rodata", ".bss"]`. See options.py for more details
* Documented all options in options.py
* Support for SN64 games (thanks Wiseguy!)
* More consistent handling of paths (thanks Mkst!)
* Various other cleanup and fixes across the board
### 0.7.10: WIP PSX support
* WIP PSX support has been added, thanks to @mkst! (https://github.com/ethteck/splat/pull/99)
* Many segments have moved to a "common" package
* Endianness of the input binary is now a configurable option
* Linker hack restored but is now optional and off by default
### 0.7.9
* Finally removed the dumb linker section alignment hack
* Added version number to output on execution
### 0.7.8
* Fixed a bug relating to a linker section alignment hack (thanks Wiseguy!)
* Fixed a bug in linker_entry.py's clean_up_path that should make this function more versatile (thanks Wiseguy!)
### 0.7.7
* Disassembly now reads the `size` property of a function in symbol_addrs.txt to disassemble `size / 4` number of instructions. Feel free to specify the size of your functions in your symbol_addrs file if splat's disassembly is chopping a function too short or making a function too long.
### 0.7.6
* Fixed a bug involving detection of defined functions in c files for GLOBAL_ASM-using projects
* Added options to disable the creation of undefined_funcs/syms_auto.txt files
* Added a Vtx segment type for creating c files containg model vertex data in the n64 libultra Vtx format
* Added a `cpp` segment type which is identical to `c` but looks for a file with the extension ".cpp" instead of ".c".
### 0.7.5: all_ types and auto_all_sections
If you have a group segment with multiple c files and want splat to automatically create linker entries at a given position for each code object (c, asm, hasm) in the segment, you can use an `all_` type for that section. For example, you can add `[auto, all_bss]` as the last subsegment in a segment. This will direct splat to create a linker entry for each code object in the segment. This saves a lot of time when it comes to manually adding .bss subsegments for bss support, for example. The same thing can be done for data and rodata sections, but note this should probably be done later into a project when all data / rodata is migrated to c files, as the `all_` types lose the rom positioning information that's necessary for splat to do proper disassembly.
The `auto_all_sections` option, when set to true, will automatically add `all_` types into every group. This is only done for a section in a group if no other manual declarations for that section exist. For example, if you have 30 c files in a group and a .data later on for one of them, `auto_all_sections` will not interfere with your `.data` subsegment. If you remove this, however, splat will use `auto_all_sections` to implicitly `.data` subsegments for all of your code objects behind the scenes. This feature is again particualrly helpful for bss support, as it will create bss linker entries for every file in your project (assuming you don't have any manual .bss subsegments), which eliminates the need to create dummy .bss subsegments just for the sake of configuring the linker script.
### 0.7.2
* Data disassembly changes:
* String detection has been improved. Please send me false positives / negatives as you see them and I can try to improve it further!
* Symbols in a data segment pointed to by other symbols will now properly be split out as their own symbols
### 0.7.1
* Image segment changes:
* Added `flip_x` and `flip_y` boolean parameters to replace `flip`.
* `flip` is deprecated and will produce a warning when used.
* Fixed flipping of `ci4` and `ci8` images.
* Fixed `extract: false` (and `start: auto`) behaviour.
## 0.7.0: The Path Update
* Significantly better performance, especially when using the cache feature (`--use-cache` CLI arg).
* BREAKING: Some cli args for splat have been renamed. Please consult the usage output (-h or no args) for more information.
* `--new` has been renamed to `--use-cache`
* `--modes` arg changes:
* Image modes have been combined into the `img` mode
* Code and ASM modes have been combined into the `code` mode
* BREAKING: The `name` attribute of a segment now should no longer be a subdirectory but rather a meaningful name for the segment which will be used as the name of the linker section. If your `name` was previously a directory, please change it into a `dir`.
* BREAKING: `subsections` has been renamed to `subsegments`
* New `dir` segment attribute specifies a subdirectory into which files will be saved. You can combine `dir` ("foo") with a subsegment name containing a subdirectory ("bar/out"), and the paths will be joined (foo/bar/out.c)
* If the `dir` attribute is specified but the `name` isn't, the `name` becomes `dir` with directory separation slashes replaced with underscores (foo/bar/baz -> foo_bar_baz)
* BREAKING: Many configuration options have been renamed. `_dir` options have been changed to the suffix `_path`.
* BREAKING: Assets (non-code, like `bin` and images) are now placed in the directory `asset_path` (defaults to `assets`).
* Linker symbol header generation. Set the `linker_symbol_header_path` option to use.
* `typedef u8[] Addr;` is recommended in your `common.h` header.
* You can now provide `auto` as the `start` attribute for a segment, e.g. `[auto, c, my_file]`. This causes the segment to not be extracted, but linked. This feature is intended for modding.
* Providing just a ROM address but no type or name for a segment is now valid anywhere in `segments` or `subsegments` rather than just at the end of the ROM. It specifies the end of the previous segment for types that need it (`palette`, `bin`, `Yay0`) and causes the linker to simply write padding until that address.
* The linker script file is left untouched if the contents have not changed since the previous split.
* You can now group together segments with `type: group` (similar to `code`). Note that any ASM or C segments must live under a `type: code` segment, not a basic `group`.
### 0.6.5: Bugfixes, rodata migration, and made options static
If you wrote a custom extension, options should be imported and statically referenced
`from util import options`
see options.py for more info on how to now get and set options
BREAKING: vram can only be specified on a segment if the segment is defined as a dict in the config
### 0.6.3: More refactoring
**Breaking Change**: The command line args to split.py have changed. Currently, only the config path is now a required argument to splat. The old `rom` and `outdir` parameters are now optional (`--rom`, `--outdir`). Now, you can add rom and out directory paths in the yaml.
The `out_dir` option specifies a directory relative to the config file. If your config file is in a subdirectory of the main repo, you can set `out_dir: ../`, for example.
The `target_path` option spcifies a path to the binary file to split, relative to the `out_dir`. If your `baserom.z64` is in the top-level of the repo, you can set `target_path: baserom.z64`, for example.
### 0.6.2: subsegments
I've begun a refactor of the code "files" code, which makes everything cleaner and easier to extend.
There's also a new option, `create_new_c_files`, which disables the creation of nonexistent c files. This behavior is on by default, but if you want to disable it for any reason, you now have the option to do so.
I am also working on adding bss support as well. It should almost be all set, aside from the changes needed in the linker script.
**Breaking change**: The `files` field in `code` segments should now be renamed to `subsegments`.
### 0.6.1: `assets_dir` option
This release adds a new `assets_dir` option in `splat.yaml`s that allows you to override the default `img`, `bin`, and other directories that segments output to.
Want to interdisperse split assets with your sourcecode? `assets_dir: src`!
Want to have all assets live in a single directory? `assets_dir: assets`!
## 0.6: The Symbol Update
Internally, there's a new Symbol class which stores information about a symbol and is stored in a couple places during disassembly. Many things should be improved, such as reconciling symbols within overlays, things being named functions vs data symbols, and more.
**Breaking change**: The format to symbol_addrs.txt has been updated. After specifying the name and address of a symbol (`symbol = addr;`), optional properties of symbols can be set via inline comment, space delimited, in any order. The properties are of the format `name:value`
* `type:` supports `func` mostly right now but will support `label` and `data` later on. Internally, `jtbl` is used as well, for jump tables. Splat uses type information during disassembly to disambiguate symbols with the same addresses.
* `rom:` is for the hex rom address of the symbol, beginning with `0x`. If available, this information is extremely valuable for use in disambiguating symbols.
* `size:` specifies the size of the symbol, which splat will use to generate offsets during disassembly. Uses the same format as `rom:`
**function example**: `FuncNameHere = 0x80023423; // type:func rom:0x10023`
**data example**: `gSomeDataVar = 0x80024233; // type:data size:0x100`
## 0.5 The Rename Update
* n64splat name changed to splat
* Some refactoring was done to support other platforms besides n64 in the future
* New `platform` option, which defaults to `n64`
* This will cause breaking changes in custom segments, so please refer to one of the changes in one of the n64 base segments for details
* Support for custom artifact paths
* New `undefined_syms_auto_path` option
* New `undefined_funcs_auto_path` option
* New `cache_path` option
* (All path-like options' names now end with `_path`)

8
tools/splat/Dockerfile
View File

@ -1,8 +0,0 @@
FROM ubuntu:22.04
RUN apt-get update
RUN apt install -y binutils-mips-linux-gnu
ADD https://github.com/decompals/mips-binutils-2.6/releases/download/main/binutils-2.6-linux.tar.gz .
ADD https://github.com/decompals/mips-gcc-2.7.2/releases/download/main/gcc-2.7.2-linux.tar.gz .
RUN mkdir -p ./gcc-2.7.2 && \
tar -xvf gcc-2.7.2-linux.tar.gz -C ./gcc-2.7.2 && \
tar -xvf binutils-2.6-linux.tar.gz -C ./gcc-2.7.2

21
tools/splat/LICENSE
View File

@ -1,21 +0,0 @@
MIT License
Copyright (c) 2021 Ethan Roseman
Permission is hereby granted, free of charge, to any person obtaining a copy
of this software and associated documentation files (the "Software"), to deal
in the Software without restriction, including without limitation the rights
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
copies of the Software, and to permit persons to whom the Software is
furnished to do so, subject to the following conditions:
The above copyright notice and this permission notice shall be included in all
copies or substantial portions of the Software.
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
SOFTWARE.

11
tools/splat/Makefile
View File

@ -1,11 +0,0 @@
UTIL_DIR := util
default: all
all: Yay0decompress
Yay0decompress:
gcc $(UTIL_DIR)/n64/Yay0decompress.c -fPIC -shared -O3 -Wall -Wextra -o $(UTIL_DIR)/n64/Yay0decompress
clean:
rm -f $(UTIL_DIR)/n64/Yay0decompress

9
tools/splat/README.md
View File

@ -1,9 +0,0 @@
# splat
A binary splitting tool to assist with decompilation and modding projects
Currently, only N64, PSX, and PS2 binaries are supported.
Please check out the [wiki](https://github.com/ethteck/splat/wiki) for more information including [examples](https://github.com/ethteck/splat/wiki/Examples) of projects that use splat.
### Requirements
splat requires Python 3.8+. Package requirements can be installed via `pip3 install -U -r requirements.txt`

320
tools/splat/create_config.py
View File

@ -1,320 +0,0 @@
#! /usr/bin/env python3
import argparse
import sys
from pathlib import Path
from util.gc import gcinfo
from util.n64 import find_code_length, rominfo
from util.psx import psxexeinfo
parser = argparse.ArgumentParser(
description="Create a splat config from an N64 ROM, PSX executable, or a GameCube disc image."
)
parser.add_argument(
"file", help="Path to a .z64/.n64 ROM, PSX executable, or .iso/.gcm GameCube image"
)
def main(file_path: Path):
if not file_path.exists():
sys.exit(f"File {file_path} does not exist ({file_path.absolute()})")
if file_path.is_dir():
sys.exit(f"Path {file_path} is a directory ({file_path.absolute()})")
# Check for N64 ROM
if file_path.suffix.lower() == ".n64" or file_path.suffix.lower() == ".z64":
create_n64_config(file_path)
return
file_bytes = file_path.read_bytes()
# Check for GC disc image
if int.from_bytes(file_bytes[0x1C:0x20], byteorder="big") == 0xC2339F3D:
create_gc_config(file_path, file_bytes)
return
# Check for PSX executable
if file_bytes[0:8] == b"PS-X EXE":
create_psx_config(file_path, file_bytes)
return
def create_n64_config(rom_path: Path):
rom_bytes = rominfo.read_rom(rom_path)
rom = rominfo.get_info(rom_path, rom_bytes)
basename = rom.name.replace(" ", "").lower()
header = f"""\
name: {rom.name.title()} ({rom.get_country_name()})
sha1: {rom.sha1}
options:
basename: {basename}
target_path: {rom_path.with_suffix(".z64")}
elf_path: build/{basename}.elf
base_path: .
platform: n64
compiler: {rom.compiler}
# asm_path: asm
# src_path: src
# build_path: build
# create_asm_dependencies: True
ld_script_path: {basename}.ld
ld_dependencies: True
find_file_boundaries: True
header_encoding: {rom.header_encoding}
o_as_suffix: True
use_legacy_include_asm: False
mips_abi_float_regs: o32
asm_function_macro: glabel
asm_jtbl_label_macro: jlabel
asm_data_macro: dlabel
# section_order: [".text", ".data", ".rodata", ".bss"]
# auto_all_sections: [".data", ".rodata", ".bss"]
symbol_addrs_path:
- symbol_addrs.txt
reloc_addrs_path:
- reloc_addrs.txt
# undefined_funcs_auto_path: undefined_funcs_auto.txt
# undefined_syms_auto_path: undefined_syms_auto.txt
extensions_path: tools/splat_ext
# string_encoding: ASCII
# data_string_encoding: ASCII
rodata_string_guesser_level: 2
data_string_guesser_level: 2
# libultra_symbols: True
# hardware_regs: True
# gfx_ucode: # one of [f3d, f3db, f3dex, f3dexb, f3dex2]
"""
first_section_end = find_code_length.run(rom_bytes, 0x1000, rom.entry_point)
segments = f"""\
segments:
- name: header
type: header
start: 0x0
- name: boot
type: bin
start: 0x40
- name: entry
type: code
start: 0x1000
vram: 0x{rom.entry_point:X}
subsegments:
- [0x1000, hasm]
- name: main
type: code
start: 0x{0x1000 + rom.entrypoint_info.entry_size:X}
vram: 0x{rom.entry_point + rom.entrypoint_info.entry_size:X}
follows_vram: entry
"""
if rom.entrypoint_info.bss_size is not None:
segments += f"""\
bss_size: 0x{rom.entrypoint_info.bss_size:X}
"""
segments += f"""\
subsegments:
- [0x{0x1000 + rom.entrypoint_info.entry_size:X}, asm]
"""
if (
rom.entrypoint_info.bss_size is not None
and rom.entrypoint_info.bss_start_address is not None
):
bss_start = rom.entrypoint_info.bss_start_address - rom.entry_point + 0x1000
# first_section_end points to the start of data
segments += f"""\
- [0x{first_section_end:X}, data]
- {{ start: 0x{bss_start:X}, type: bss, vram: 0x{rom.entrypoint_info.bss_start_address:08X} }}
"""
# Point next segment to the detected end of the main one
first_section_end = bss_start
segments += f"""\
- type: bin
start: 0x{first_section_end:X}
follows_vram: main
- [0x{rom.size:X}]
"""
out_file = f"{basename}.yaml"
with open(out_file, "w", newline="\n") as f:
print(f"Writing config to {out_file}")
f.write(header)
f.write(segments)
def create_gc_config(iso_path: Path, iso_bytes: bytes):
gc = gcinfo.get_info(iso_path, iso_bytes)
basename = gc.system_code + gc.game_code + gc.region_code + gc.publisher_code
header = f"""\
name: \"{gc.name.title()} ({gc.get_region_name()})\"
system_code: {gc.system_code}
game_code: {gc.game_code}
region_code: {gc.region_code}
publisher_code: {gc.publisher_code}
sha1: {gc.sha1}
options:
filesystem_path: filesystem
basename: {basename}
target_path: {iso_path.with_suffix(".iso")}
base_path: .
compiler: {gc.compiler}
platform: gc
# undefined_funcs_auto: True
# undefined_funcs_auto_path: undefined_funcs_auto.txt
# undefined_syms_auto: True
# undefined_syms_auto_path: undefined_syms_auto.txt
# symbol_addrs_path: symbol_addrs.txt
# asm_path: asm
# src_path: src
# build_path: build
# extensions_path: tools/splat_ext
# section_order: [".text", ".data", ".rodata", ".bss"]
# auto_all_sections: [".data", ".rodata", ".bss"]
"""
segments = f"""\
segments:
- name: filesystem
type: fst
path: filesystem/sys/fst.bin
- name: bootinfo
type: bootinfo
path: filesystem/sys/boot.bin
- name: bi2
type: bi2
path: filesystem/sys/bi2.bin
- name: apploader
type: apploader
path: filesystem/sys/apploader.img
- name: main
type: dol
path: filesystem/sys/main.dol
"""
out_file = f"{basename}.yaml"
with open(out_file, "w", newline="\n") as f:
print(f"Writing config to {out_file}")
f.write(header)
f.write(segments)
def create_psx_config(exe_path: Path, exe_bytes: bytes):
exe = psxexeinfo.PsxExe.get_info(exe_path, exe_bytes)
basename = exe_path.name.replace(" ", "").lower()
header = f"""\
name: {exe_path.name}
sha1: {exe.sha1}
options:
basename: {basename}
target_path: {exe_path}
base_path: .
platform: psx
compiler: GCC
# asm_path: asm
# src_path: src
# build_path: build
# create_asm_dependencies: True
ld_script_path: {basename}.ld
find_file_boundaries: False
gp_value: 0x{exe.initial_gp:08X}
o_as_suffix: True
use_legacy_include_asm: False
asm_function_macro: glabel
asm_jtbl_label_macro: jlabel
asm_data_macro: dlabel
section_order: [".rodata", ".text", ".data", ".bss"]
# auto_all_sections: [".data", ".rodata", ".bss"]
symbol_addrs_path:
- symbol_addrs.txt
reloc_addrs_path:
- reloc_addrs.txt
# undefined_funcs_auto_path: undefined_funcs_auto.txt
# undefined_syms_auto_path: undefined_syms_auto.txt
extensions_path: tools/splat_ext
subalign: 2
string_encoding: ASCII
data_string_encoding: ASCII
rodata_string_guesser_level: 2
data_string_guesser_level: 2
"""
segments = f"""\
segments:
- name: header
type: header
start: 0x0
- name: main
type: code
start: 0x800
vram: 0x{exe.destination_vram:X}
bss_size: 0x{exe.bss_size:X}
subsegments:
"""
text_offset = exe.text_offset
if text_offset != 0x800:
segments += f"""\
- [0x800, rodata, 800]
"""
segments += f"""\
- [0x{text_offset:X}, asm, {text_offset:X}]
"""
if exe.data_vram != 0 and exe.data_size != 0:
data_offset = exe.data_offset
segments += f"""\
- [0x{data_offset:X}, data, {data_offset:X}]
"""
if exe.bss_size != 0:
segments += f"""\
- {{ start: 0x{exe.size:X}, type: bss, name: {exe.bss_vram:X}, vram: 0x{exe.bss_vram:X} }}
"""
segments += f"""\
- [0x{exe.size:X}]
"""
out_file = f"{basename}.yaml"
with open(out_file, "w", newline="\n") as f:
print(f"Writing config to {out_file}")
f.write(header)
f.write(segments)
if __name__ == "__main__":
args = parser.parse_args()
main(Path(args.file))

2
tools/splat/disassembler/__init__.py
View File

@ -1,2 +0,0 @@
from .disassembler import Disassembler
from .spimdisasm_disassembler import SpimdisasmDisassembler

17
tools/splat/disassembler/disassembler.py
View File

@ -1,17 +0,0 @@
from abc import ABC, abstractmethod
from util.options import SplatOpts
from typing import Set
class Disassembler(ABC):
@abstractmethod
def configure(self, options: SplatOpts):
raise NotImplementedError("configure")
@abstractmethod
def check_version(self, skip_version_check: bool, splat_version: str):
raise NotImplementedError("check_version")
@abstractmethod
def known_types(self) -> Set[str]:
raise NotImplementedError("known_types")

26
tools/splat/disassembler/disassembler_instance.py
View File

@ -1,26 +0,0 @@
from .disassembler import Disassembler
from .spimdisasm_disassembler import SpimdisasmDisassembler
from .null_disassembler import NullDisassembler
__instance: Disassembler = NullDisassembler()
__initialized = False
def create_disassembler_instance(platform: str):
global __instance
global __initialized
if platform in ["n64", "psx", "ps2"]:
__instance = SpimdisasmDisassembler()
__initialized = True
return
raise NotImplementedError("No disassembler for requested platform")
def get_instance() -> Disassembler:
global __instance
global __initialized
if not __initialized:
raise Exception("Disassembler instance not initialized")
return None
return __instance

14
tools/splat/disassembler/null_disassembler.py
View File

@ -1,14 +0,0 @@
from disassembler import disassembler
from util.options import SplatOpts
from typing import Set
class NullDisassembler(disassembler.Disassembler):
def configure(self, opts: SplatOpts):
pass
def check_version(self, skip_version_check: bool, splat_version: str):
pass
def known_types(self) -> Set[str]:
return set()

124
tools/splat/disassembler/spimdisasm_disassembler.py
View File

@ -1,124 +0,0 @@
from disassembler import disassembler
import spimdisasm
import rabbitizer
from util import log, compiler
from util.options import SplatOpts
from typing import Set
class SpimdisasmDisassembler(disassembler.Disassembler):
# This value should be kept in sync with the version listed on requirements.txt
SPIMDISASM_MIN = (1, 18, 0)
def configure(self, opts: SplatOpts):
# Configure spimdisasm
spimdisasm.common.GlobalConfig.PRODUCE_SYMBOLS_PLUS_OFFSET = True
spimdisasm.common.GlobalConfig.TRUST_USER_FUNCTIONS = True
spimdisasm.common.GlobalConfig.TRUST_JAL_FUNCTIONS = True
spimdisasm.common.GlobalConfig.GLABEL_ASM_COUNT = False
if opts.rom_address_padding:
spimdisasm.common.GlobalConfig.ASM_COMMENT_OFFSET_WIDTH = 6
else:
spimdisasm.common.GlobalConfig.ASM_COMMENT_OFFSET_WIDTH = 0
# spimdisasm is not performing any analyzis on non-text sections so enabling this options is pointless
spimdisasm.common.GlobalConfig.AUTOGENERATED_NAMES_BASED_ON_SECTION_TYPE = False
spimdisasm.common.GlobalConfig.AUTOGENERATED_NAMES_BASED_ON_DATA_TYPE = False
spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO = False
if opts.rodata_string_guesser_level is not None:
spimdisasm.common.GlobalConfig.RODATA_STRING_GUESSER_LEVEL = (
opts.rodata_string_guesser_level
)
if opts.data_string_guesser_level is not None:
spimdisasm.common.GlobalConfig.DATA_STRING_GUESSER_LEVEL = (
opts.data_string_guesser_level
)
rabbitizer.config.regNames_userFpcCsr = False
rabbitizer.config.regNames_vr4300Cop0NamedRegisters = False
rabbitizer.config.misc_opcodeLJust = opts.mnemonic_ljust - 1
rabbitizer.config.regNames_gprAbiNames = rabbitizer.Abi.fromStr(
opts.mips_abi_gpr
)
rabbitizer.config.regNames_fprAbiNames = rabbitizer.Abi.fromStr(
opts.mips_abi_float_regs
)
if opts.endianness == "big":
spimdisasm.common.GlobalConfig.ENDIAN = spimdisasm.common.InputEndian.BIG
else:
spimdisasm.common.GlobalConfig.ENDIAN = spimdisasm.common.InputEndian.LITTLE
rabbitizer.config.pseudos_pseudoMove = False
selected_compiler = opts.compiler
if selected_compiler == compiler.SN64:
rabbitizer.config.regNames_namedRegisters = False
rabbitizer.config.toolchainTweaks_sn64DivFix = True
rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch = True
spimdisasm.common.GlobalConfig.ASM_COMMENT = False
spimdisasm.common.GlobalConfig.SYMBOL_FINDER_FILTERED_ADDRESSES_AS_HILO = (
False
)
spimdisasm.common.GlobalConfig.COMPILER = spimdisasm.common.Compiler.SN64
elif selected_compiler == compiler.GCC:
rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch = True
spimdisasm.common.GlobalConfig.COMPILER = spimdisasm.common.Compiler.GCC
elif selected_compiler == compiler.IDO:
spimdisasm.common.GlobalConfig.COMPILER = spimdisasm.common.Compiler.IDO
spimdisasm.common.GlobalConfig.DETECT_REDUNDANT_FUNCTION_END = (
opts.detect_redundant_function_end
)
spimdisasm.common.GlobalConfig.GP_VALUE = opts.gp
spimdisasm.common.GlobalConfig.ASM_TEXT_LABEL = opts.asm_function_macro
spimdisasm.common.GlobalConfig.ASM_TEXT_ALT_LABEL = opts.asm_function_alt_macro
spimdisasm.common.GlobalConfig.ASM_JTBL_LABEL = opts.asm_jtbl_label_macro
spimdisasm.common.GlobalConfig.ASM_DATA_LABEL = opts.asm_data_macro
spimdisasm.common.GlobalConfig.ASM_TEXT_END_LABEL = opts.asm_end_label
if opts.asm_emit_size_directive is not None:
spimdisasm.common.GlobalConfig.ASM_EMIT_SIZE_DIRECTIVE = (
opts.asm_emit_size_directive
)
if spimdisasm.common.GlobalConfig.ASM_TEXT_LABEL == ".globl":
spimdisasm.common.GlobalConfig.ASM_TEXT_ENT_LABEL = ".ent"
spimdisasm.common.GlobalConfig.ASM_TEXT_FUNC_AS_LABEL = True
if spimdisasm.common.GlobalConfig.ASM_DATA_LABEL == ".globl":
spimdisasm.common.GlobalConfig.ASM_DATA_SYM_AS_LABEL = True
spimdisasm.common.GlobalConfig.LINE_ENDS = opts.c_newline
spimdisasm.common.GlobalConfig.ALLOW_ALL_ADDENDS_ON_DATA = (
opts.allow_data_addends
)
spimdisasm.common.GlobalConfig.DISASSEMBLE_UNKNOWN_INSTRUCTIONS = (
opts.disasm_unknown
)
if opts.compiler == compiler.GCC and opts.platform == "ps2":
rabbitizer.config.toolchainTweaks_treatJAsUnconditionalBranch = False
def check_version(self, skip_version_check: bool, splat_version: str):
if not skip_version_check and spimdisasm.__version_info__ < self.SPIMDISASM_MIN:
log.error(
f"splat {splat_version} requires as minimum spimdisasm {self.SPIMDISASM_MIN}, but the installed version is {spimdisasm.__version_info__}"
)
log.write(
f"splat {splat_version} (powered by spimdisasm {spimdisasm.__version__})"
)
def known_types(self) -> Set[str]:
return spimdisasm.common.gKnownTypes

280
tools/splat/disassembler_section.py
View File

@ -1,280 +0,0 @@
from abc import ABC, abstractmethod
from typing import Optional
import spimdisasm
from util import options, symbols
class DisassemblerSection(ABC):
@abstractmethod
def disassemble(self):
raise NotImplementedError("disassemble")
@abstractmethod
def analyze(self):
raise NotImplementedError("analyze")
@abstractmethod
def set_comment_offset(self, rom_start: int):
raise NotImplementedError("set_comment_offset")
@abstractmethod
def make_bss_section(
self,
rom_start,
rom_end,
vram_start,
bss_end,
name,
segment_rom_start,
exclusive_ram_id,
):
raise NotImplementedError("make_bss_section")
@abstractmethod
def make_data_section(
self,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
):
raise NotImplementedError("make_data_section")
@abstractmethod
def get_section(self):
raise NotImplementedError("get_section")
@abstractmethod
def make_rodata_section(
self,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
):
raise NotImplementedError("make_rodata_section")
@abstractmethod
def make_text_section(
self,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
):
raise NotImplementedError("make_text_section")
class SpimdisasmDisassemberSection(DisassemblerSection):
def __init__(self):
self.spim_section: Optional[spimdisasm.mips.sections.SectionBase] = None
def disassemble(self) -> str:
assert self.spim_section is not None
return self.spim_section.disassemble()
def analyze(self):
assert self.spim_section is not None
self.spim_section.analyze()
def set_comment_offset(self, rom_start: int):
assert self.spim_section is not None
self.spim_section.setCommentOffset(rom_start)
def make_bss_section(
self,
rom_start: int,
rom_end: int,
vram_start: int,
bss_end: int,
name: str,
segment_rom_start: int,
exclusive_ram_id,
):
self.spim_section = spimdisasm.mips.sections.SectionBss(
symbols.spim_context,
rom_start,
rom_end,
vram_start,
bss_end,
name,
segment_rom_start,
exclusive_ram_id,
)
def make_data_section(
self,
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
):
self.spim_section = spimdisasm.mips.sections.SectionData(
symbols.spim_context,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
def get_section(self) -> Optional[spimdisasm.mips.sections.SectionBase]:
return self.spim_section
def make_rodata_section(
self,
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
):
self.spim_section = spimdisasm.mips.sections.SectionRodata(
symbols.spim_context,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
def make_text_section(
self,
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
):
self.spim_section = spimdisasm.mips.sections.SectionText(
symbols.spim_context,
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
def make_disassembler_section() -> Optional[SpimdisasmDisassemberSection]:
if options.opts.platform in ["n64", "psx", "ps2"]:
return SpimdisasmDisassemberSection()
raise NotImplementedError("No disassembler section for requested platform")
return None
def make_text_section(
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
) -> DisassemblerSection:
section = make_disassembler_section()
assert section is not None
section.make_text_section(
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
return section
def make_data_section(
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
) -> DisassemblerSection:
section = make_disassembler_section()
assert section is not None
section.make_data_section(
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
return section
def make_rodata_section(
rom_start: int,
rom_end: int,
vram_start: int,
name: str,
rom_bytes: bytes,
segment_rom_start: int,
exclusive_ram_id,
) -> DisassemblerSection:
section = make_disassembler_section()
assert section is not None
section.make_rodata_section(
rom_start,
rom_end,
vram_start,
name,
rom_bytes,
segment_rom_start,
exclusive_ram_id,
)
return section
def make_bss_section(
rom_start: int,
rom_end: int,
vram_start: int,
bss_end: int,
name: str,
segment_rom_start: int,
exclusive_ram_id,
) -> DisassemblerSection:
section = make_disassembler_section()
assert section is not None
section.make_bss_section(
rom_start,
rom_end,
vram_start,
bss_end,
name,
segment_rom_start,
exclusive_ram_id,
)
return section

136
tools/splat/docs/Adding-Symbols.md
View File

@ -1,136 +0,0 @@
Symbols (i.e. labelling a function or variable) are controlled by the `symbols_addrs.txt` file.
The format for defining symbols is:
```ini
symbol = address; // option1:value1 option2:value2
```
e.g.
```ini
osInitialize = 0x801378C0; // type:func
```
:information_source: The file used can be overridden via the `symbol_addrs_path` setting in the global `options` section of the splat yaml. This option can also accept a list of paths, allowing for symbols to be organized in multiple files.
## symbol
This is the name of the symbol and can be any valid C variable name, e.g. `myCoolFunction` or `gReticulatedSplineCounter`
## address
This is the VRAM address expressed in hexadecimal, e.g. `0x80001050`
## options
An optional `key:pair` list of settings, note that each option should be separated by whitespace, but there should be no whitespace between the key:value pairs themselves.
### `type`
Override splat's automatic type detection, possible values are:
- `func`: Functions
- `jtbl`: Jumptables
- `jtbl_label`: Jumptables labels (inside functions)
- `label`: Branch labels (inside functions)
- `s8`, `u8`: To specify data/rodata to be disassembled as `.byte`s
- `s16`, `u16`: To specify data/rodata to be disassembled as `.short`s
- `s32`, `u32`: To specify data/rodata to be disassembled as `.word`s (the default)
- `s64`, `u64`: :man_shrugging:
- `f32`, `Vec3f`: To specify data/rodata to be disassembled as `.float`s
- `f64`: To specify data/rodata to be disassembled as `.double`s
- `asciz`, `char*`, `char`: C strings (disassembled as `.asciz`)
- Any custom type starting with a capital letter (will default to `.word`s)
Any other type will produce an error.
**Example:**
```ini
minFrameSize = 0x80241D08; // type:s32
```
### `size`
The size of the function or the size of the data depending on the type of the symbol. It specifies a size in bytes. e.g. `size:0x10`.
**Example:**
```ini
RawHuffmanTable = 0x8022E0E0; // type:symbol size:0x100
```
### `rom`
The ROM offset for the symbol, useful (potentially mandatory) for symbols in overlays where multiple symbols could share the same VRAM address.
**Example:**
```ini
create_particle_effect = 0x802D5F4C; // type:func rom:0x6E75FC
```
### `segment`
Allows specifying to which specific segment this symbol belongs to, useful to disambiguate symbols from segments that share the same VRAM address. This name must be the same as the name of a segment listed in the yaml.
**Example:**
```ini
sMenuTexture = 0x06004040; // segment:menu_assets
```
### `name_end`
Emits a symbol after the end of the data of the current symbol. Useful to reference the end of an assembly symbol, like RSP data.
**Example:**
```ini
rspbootTextStart = 0x80084690; // name_end:rspbootTextEnd
```
### `defined`
Forces the symbol to be defined - i.e. prevent it from appearing in `undefined_syms_auto.txt` should splat not encounter the symbol during the symbol detection phase.
**Example:**
```ini
__osDpDeviceBusy = 0x8014B3D0; // defined:true
```
### `extract`
TBD
### `ignore`
Prevents an address from being symbolized and referenced. Useful to get a finer control over the disassembled output.
**Example:**
```ini
D_A0000000 = 0xA0000000; // ignore:true
```
It can also be combined with the `size` attribute to avoid a range of addresses of being symbolized.
**Example:**
```ini
D_80000000 = 0x80000000; // ignore:true size:0x10
```
### `force_migration` and `force_not_migration`
Grants a finer control over the automatic rodata migration to functions. This may be required because of the migration heuristic failing to migrate (or to not migrate) a symbol, producing a disordered rodata section. Forcing the migration of a rodata symbol to a function will only work if that function references said rodata symbol. Forcing the not-migration of a rodata symbol always works.
This attribute is ignored if the `migrate_rodata_to_functions` option is disabled.
**Example:**
```ini
jtbl_800B13D0 = 0x800B13D0; // type:jtbl force_migration:True
STR_800B32A8 = 0x800C9520; // type:asciz force_not_migration:True
```
### `allow_addend` and `dont_allow_addend`
Allows this symbol to reference (or not reference) other symbols with an addend.
This attribute overrides the global `allow_data_addends` option.
**Example:**
```ini
aspMainTextStart = 0x80084760; // dont_allow_addend:True
```

7
tools/splat/docs/Advanced.md
View File

@ -1,7 +0,0 @@
## Writing custom segment handler
The following list contains examples of custom segments:
- [RNC](https://github.com/mkst/sssv/blob/master/tools/splat_ext/rnc.py)
- [Vtx](https://github.com/mkst/sssv/blob/master/tools/splat_ext/sssv_vtx.py)
- [Multiple](https://github.com/pmret/papermario/tree/main/tools/splat_ext)

612
tools/splat/docs/Configuration.md
View File

@ -1,612 +0,0 @@
Splat has various options for configuration, all of which are listed under the `options` section of the yaml file.
## Project configuration
### base_path
Path that all other configured paths are relative to.
#### Usage
```yaml
base_path: path/to/base/folder
```
#### Default
`.` *(Current directory)*
### target_path
Path to target binary.
#### Usage
```yaml
target_path: path/to/target/binary
```
### elf_path
Path to the final elf target
#### Default
Path to the binary that was used as the input to `create_config.py`
### platform
The target platform for the binary. Options are
- `n64` (Nintendo 64)
- `psx` (PlayStation 1)
- `ps2` (PlayStation 2)
- `gc` (GameCube)
#### Usage
```yaml
platform: psx
```
### compiler
Compiler used to build the binary.
splat recognizes the following compilers, and it will adapt it behavior accordingly for them, but unknown compilers can be passed as well:
- GCC
- SN64
- IDO
#### Usage
```yaml
compiler: IDO
```
#### Default
`ido`
### endianness
Determines the endianness of the target binary. If not set, the endiannesss will be guessed from the selected platform.
Valid values:
- big
- little
### section_order
Determines the default section order of the target binary. This can be overridden per-segment.
Expects a list of strings.
### generated_c_preamble
String that is placed before the contents of newly-generated `.c` files.
#### Usage
```yaml
generated_c_preamble: #include "header.h"
```
#### Default
`#include "common.h"`
### generated_s_preamble
String that is placed before the contents of newly-generated assembly (`.s`) files.
#### Usage
```yaml
generated_s_preamble: .set fp=64
```
### o_as_suffix
Determines whether to replace the suffix of the file to `.o` or to append `.o` to the suffix of the file.
### gp_value
The value of the `$gp` register to correctly calculate offset to `%gp_rel` relocs.
### check_consecutive_segment_types
By default splat will check and error if there are any non consecutive segment types.
This option disables said feature.
#### Usage
```yaml
# Disable checking for non-consecutive segments
check_consecutive_segment_types: False
```
## Paths
### asset_path
Path to output split asset files.
#### Usage
```yaml
asset_path: path/to/assets/folder
```
#### Default
`assets`
### symbol_addrs_path
Determines the path to the symbol addresses file(s). A `symbol_addrs` file is to be updated/curated manually and contains addresses of symbols as well as optional metadata such as rom address, type, and more
It's possible to use more than one file by supplying a list instead of a string
#### Usage
```yaml
symbol_addrs_path: path/to/symbol_addrs
```
#### Default
`symbol_addrs.txt`
### reloc_addrs_paths
### build_path
Path that built files will be found. Used for generation of the linker script.
#### Usage
```yaml
build_path: path/to/build/folder
```
#### Default
`build`
### src_path
Path to split `.c` files.
#### Usage
```yaml
src_path: path/to/src/folder
```
#### Default
`src`
### asm_path
Path to output split assembly files.
#### Usage
```yaml
asm_path: path/to/asm/folder
```
#### Default
`asm`
### data_path
Determines the path to the asm data directory
### nonmatchings_path
Determines the path to the asm nonmatchings directory
### cache_path
Path to splat cache
#### Usage
```yaml
cache_path: path/to/splat/cache
```
#### Default
`.splat_cache`
### create_undefined_funcs_auto
If `True`, splat will generate an `undefined_funcs_auto.txt` file.
#### Usage
```yaml
create_undefined_funcs_auto: False
```
#### Default
`True`
### undefined_funcs_auto_path
Path to file containing automatically defined functions.
#### Usage
```yaml
undefined_funcs_auto_path: path/to/undefined_funcs_auto.txt
```
#### Default
`undefined_funcs_auto.txt`
### create_undefined_syms_auto
If `True`, splat will generate an `undefined_syms_auto.txt` file.
#### Usage
```yaml
create_undefined_syms_auto: False
```
#### Default
`True`
### undefined_syms_auto_path
Path to file containing automatically defined symbols.
#### Usage
```yaml
undefined_syms_auto_path: path/to/undefined_syms_auto.txt
```
#### Default
`undefined_syms_auto.txt`
### extensions_path
If you are using splat extension(s), this is the path they will be loaded from.
#### Usage
```yaml
extensions_path: path/to/extensions/folder
```
#### Default
`tools/splat_ext`
### lib_path
Determines the path to library files that are to be linked into the target binary
### elf_section_list_path
Path to file containing elf section list.
#### Usage
```yaml
elf_section_list_path: path/to/elf_sections
```
#### Default
`elf_sections.txt`
## Linker script
### subalign
Sub-alignment (in bytes) of sections.
#### Usage
```yaml
subalign: 4
```
#### Default
`16`
### auto_all_sections
TODO
### ld_script_path
Path to output ld script.
#### Usage
```yaml
ld_script_path: path/to/ld/script.ld
```
#### Default
`{basename}.ld`
### ld_symbol_header_path
Path to output a header containing linker symbols.
#### Usage
```yaml
ld_symbol_header_path: path/to/linker_symbol_header
```
### ld_discard_section
Determines whether to add a discard section to the linker script
### ld_section_labels
Determines the list of section labels that are to be added to the linker script
### ld_wildcard_sections
Determines whether to add wildcards for section linking in the linker script (.rodata* for example)
### ld_use_symbolic_vram_addreses
Determines whether to use `follows_vram` (segment option) and `vram_symbol` / `follows_classes` (vram_class options) to calculate vram addresses in the linker script.
Enabled by default. If disabled, this uses the plain integer values for vram addresses defined in the yaml.
### ld_partial_linking
Change linker script generation to allow partially linking segments. Requires both `ld_partial_scripts_path` and `ld_partial_build_segments_path` to be set.
### ld_partial_scripts_path
Folder were each intermediary linker script will be written to.
### ld_partial_build_segments_path
Folder where the built partially linked segments will be placed by the build system.
### ld_dependencies
Generate a dependency file for every linker script generated. Dependency files will have the same path and name as the corresponding linker script, but changing the extension to `.d`. Requires `elf_path` to be set.
### ld_legacy_generation
Legacy linker script generation does not impose the section_order specified in the yaml options or per-segment options.
### segment_end_before_align
If enabled, the end symbol for each segment will be placed before the alignment directive for the segment
### segment_symbols_style
Controls the style of the auto-generated segment symbols in the linker script.
Possible values:
- splat
- makerom
### ld_rom_start
Specifies the starting offset for rom address symbols in the linker script.
### ld_fill_value
Allows to specify the value of the `FILL` statement generated on every segment of the linker script.
It must be either an integer, which will be used as the parameter for the `FILL` statement, or `null`, which tells splat to not emit `FILL` statements.
This behavior can be customized per segment too. See [ld_fill_value](Segments.md#ld_fill_value) on the Segments section.
Defaults to 0.
## C file options
### create_c_files
Determines whether to create new c files if they don't exist
### auto_decompile_empty_functions
Determines whether to "auto-decompile" empty functions
### do_c_func_detection
Determines whether to detect matched/unmatched functions in existing c files so we can avoid creating `.s` files for already-decompiled functions.
### c_newline
Determines the newline char(s) to be used in c files
## (Dis)assembly-related options
### symbol_name_format
Determine the format that symbols should be named by default
### symbol_name_format_no_rom
Same as `symbol_name_format` but for symbols with no rom address
### find_file_boundaries
Determines whether to detect and hint to the user about likely file splits when disassembling.
This setting can also be set on a per segment basis, if you'd like to enable or disable detection for specific segments. This could be useful when you are confident you identified all subsegments in a segment, yet `splat` still hints that subsegments could be split.
### pair_rodata_to_text
Determines whether to detect and hint to the user about possible rodata sections corresponding to a text section
### migrate_rodata_to_functions
Determines whether to attempt to automatically migrate rodata into functions
### asm_inc_header
Determines the header to be used in every asm file that's included from c files
### asm_function_macro
Determines the macro used to declare functions in asm files
### asm_function_alt_macro
Determines the macro used to declare symbols in the middle of functions in asm files (which may be alternative entries)
### asm_jtbl_label_macro
Determines the macro used to declare jumptable labels in asm files
### asm_data_macro
Determines the macro used to declare data symbols in asm files
### asm_end_label
Determines the macro used at the end of a function, such as endlabel or .end
### asm_emit_size_directive
Toggles the .size directive emitted by the disassembler
### include_macro_inc
Determines including the macro.inc file on non-migrated rodata variables
### mnemonic_ljust
Determines the number of characters to left align before the instruction
### rom_address_padding
Determines whether to pad the rom address
### mips_abi_gpr
Determines which ABI names to use for general purpose registers
### mips_abi_float_regs
Determines which ABI names to use for floating point registers.
Valid values:
- numeric
- o32
- n32
- n64
`o32`` is highly recommended, as it provides logically named registers for floating point instructions.
For more info, see https://gist.github.com/EllipticEllipsis/27eef11205c7a59d8ea85632bc49224d
### add_set_gp_64
Determines whether to add ".set gp=64" to asm/hasm files
### create_asm_dependencies
Generate `.asmproc.d` dependency files for each C file which still reference functions in assembly files
### string_encoding
Global option for rodata string encoding. This can be overriden per segment
### data_string_encoding
Global option for data string encoding. This can be overriden per segment
### rodata_string_guesser_level
Global option for the rodata string guesser. 0 disables the guesser completely.
### data_string_guesser_level
Global option for the data string guesser. 0 disables the guesser completely.
### allow_data_addends
Global option for allowing data symbols using addends on symbol references. It can be overriden per symbol
### disasm_unknown
Tells the disassembler to try disassembling functions with unknown instructions instead of falling back to disassembling as raw data
### detect_redundant_function_end
Tries to detect redundant and unreferenced functions ends and merge them together. This option is ignored if the compiler is not set to IDO.
### disassemble_all
Don't skip disassembling already matched functions and migrated sections
## N64-specific options
### header_encoding
Used to specify what encoding should be used used when parsing the N64 ROM header.
#### Default
`ASCII`
### gfx_ucode
Determines the type gfx ucode (used by gfx segments)
Valid options are:
- f3d
- f3db
- f3dex
- f3dexb
- f3dex2
### libultra_symbols
Use named libultra symbols by default. Those will need to be added to a linker script manually by the user
### ique_symbols
Use named libultra symbols by default. Those will need to be added to a linker script manually by the user
### hardware_regs
Use named hardware register symbols by default. Those will need to be added to a linker script manually by the user
## Gamecube-specific options
### filesystem_path
Path where the iso's filesystem will be extracted to
## Compiler-specific options
### use_legacy_include_asm
If `True`, generate c files using the longer `INCLUDE_ASM` macro. This is defaulted to `True` to by-default support projects using the longer macro.
#### Usage
```yaml
use_legacy_include_asm: False
```
#### Default
`True`

32
tools/splat/docs/Examples.md
View File

@ -1,32 +0,0 @@
The following is a list of projects known to be using **splat** along with the compilers used:
## N64 Projects
- [Aidyn Chronicles](https://github.com/blackgamma7/Aidyn) `unknown` (does not build source)
- [Animal Forest](https://github.com/zeldaret/af) `ido7.1`
- [Banjo Kazooie](https://gitlab.com/banjo.decomp/banjo-kazooie) `ido5.3`
- [Conker's Bad Fur Day](https://github.com/mkst/conker) `ido5.3`
- [Dinosaur Planet](https://github.com/zestydevy/dinosaur-planet) `ido5.3`
- [Dr. Mario 64](https://github.com/AngheloAlf/drmario64) `kmc gcc2.7.2` & `egcs gcc2.91.66`
- [Gauntlet Legends](https://github.com/Drahsid/gauntlet-legends) `kmc gcc2.7.2` (uses old KMC GCC wrapper - do not use for reference)
- [Mario Party 3](https://github.com/PartyPlanner64/mp3) `gcc2.7.2 TBD`
- [Mischief Makers](https://github.com/Drahsid/mischief-makers) `ido5.3`
- [Neon Genesis Evangelion 64](https://github.com/farisawan-2000/evangelion) `kmc gcc2.7.2`
- [Paper Mario](https://github.com/pmret/papermario) `gcc2.8.1`
- [Pokemon Snap](https://github.com/ethteck/pokemonsnap) `ido7.1`
- [Pokemon Stadium](https://github.com/ethteck/pokemonstadium) `ido7.1`
- [Pokémon Puzzle League](https://github.com/AngheloAlf/puzzleleague64) `kmc gcc2.7.2`
- [Rocket Robot on Wheels](https://github.com/RocketRet/Rocket-Robot-On-Wheels) `SN64 (build 970404)`
- [Space Station Silicon Valley](https://github.com/mkst/sssv) `ido5.3`
- [Turok 3](https://github.com/drahsid/turok3) `psyq gcc2.8.0`
- [Yoshi's Story](https://github.com/decompals/yoshis-story) `ido7.1`
## PS1 Projects
- [Evo's Space Adventures](https://github.com/mkst/esa) `psyq 4.6 (gcc2.95)`
- [Final Fantasy 7](https://github.com/Drahsid/ffvii) `psyq <= 4.1 (gcc2.7.2)`
- [Silent Hill](https://github.com/Vatuu/silent-hill-decomp) `psyq <= 4.1 (gcc2.7.2) TBD`
## PS2 Projects
- [Kingdom Hearts](https://github.com/ethteck/kh1) TBD

179
tools/splat/docs/General-Workflow.md
View File

@ -1,179 +0,0 @@
This describes an example of how to iteratively edit the splat segments config in order to maximise code and data migration from the binary.
# 1 Initial configuration
After successfully following the [Quickstart](https://github.com/ethteck/splat/wiki/Quickstart), you should have an initial configuration like the one below:
```yaml
- name: main
type: code
start: 0x1060
vram: 0x80070C60
follows_vram: entry
bss_size: 0x3AE70
subsegments:
- [0x1060, asm]
# ... a lot of additional `asm` sections
# This section is found out to contain __osViSwapContext
- [0x25C20, asm, energy_orb_wave]
# ... a lot of additional `asm` sections
- [0x2E450, data]
- [0x3E330, rodata]
# ... a lot of additional `rodata` sections
- { start: 0x3F1B0, type: bss, vram: 0x800E9C20 }
- [0x3F1B0, bin]
```
## 1.1 Match `rodata` to `asm` sections
It's good practice to start pairing `rodata` sections with `asm` sections _before_ changing the `asm` sections into `c` files. This is because rodata may need to be explicitly included within the `c` file (via `INCLUDE_RODATA` or `GLOBAL_ASM` macros).
`splat` provides hints about which `rodata` segments are referenced by which `asm` segments based on references to these symbols within the disassembled functions.
These messages are output when splitting and look like:
```
Rodata segment '3EE10' may belong to the text segment 'energy_orb_wave'
Based on the usage from the function func_0xXXXXXXXX to the symbol D_800AEA10
```
To pair these two sections, simply add the _name_ of the suggested text (i.e. `asm`) segment to the `rodata` segment:
```yaml
- [0x3EE10, rodata, energy_orb_wave] # segment will be paired with a text (i.e. asm or c) segment named "energy_orb_wave"
```
**NOTE:**
By default `migrate_rodata_to_functions` functionality is enabled. This causes splat to include paired rodata along with the disassembled assembly code, allowing it to be linked via `.rodata` segments from the get-go. This guide assumes that you will disable this functionality until you have successfully paired up the segments.
### Troubleshooting
#### Multiple `rodata` segments for a single text segment
Using the following configuration:
```yaml
# ...
- [0x3E900, rodata]
- [0x3E930, rodata]
# ...
```
`splat` outputs a hint that doesn't immediately seem to make sense:
```
Rodata segment '3E900' may belong to the text segment '16100'
Based on the usage from the function func_80085DA0 to the symbol jtbl_800AE500
Rodata segment '3E930' may belong to the text segment '16100'
Based on the usage from the function func_800862C0 to the symbol jtbl_800AE530
```
This hint tells you that `splat` believes one text segment references two `rodata` sections. This usually means that either the `rodata` should not be split at `0x3E930`, or that there is a missing split in the `asm` at `0x16100`, as a text segment can only have one `rodata` segment.
If we assume that the rodata split is incorrect, we can remove the extraneous split:
```yaml
# ...
- [0x3E900, rodata, "16100"]
# ...
```
**NOTE:** Splat uses heuristics to determine `rodata` and `asm` splits and is not perfect - false positives are possible and, if in doubt, double-check the assembly yourself before changing the splits.
### Multiple `asm` segments referring to the same `rodata` segment
Sometimes the opposite is true, and `splat` believes two `asm` segments belong to a single `rodata` segment. In this case, you can split the `asm` segment to make sure two files are not paired with the same `rodata`. Note that this too can be a false positive.
# 2 Disassemble text, data, rodata
Let's say you want to start decompiling the subsegment at `0x25C20` (`energy_orb_wave`). Start by replacing the `asm` type with `c`, and then re-run splat.
```yaml
- [0x25C20, c, energy_orb_wave]
# ...
- [0x3EE10, rodata, energy_orb_wave]
```
This will disassemble the ROM at `0x25C20` as code, creating individual `.s` files for each function found. The output will be located in `{asm_path}/nonmatchings/energy_orb_wave/<function_name>.s`.
Assuming `data` and `rodata` segments have been paired with the `c` segment, splat will generate `{asm_path}/energy_orb_wave.data.s` and `{asm_path}/energy_orb_wave.rodata.s` respectively.
Finally, splat will generate a C file, at `{src_path}/energy_orb_wave.c` containing macros that will be used to include all disassembled function assembly.
**NOTE:**
- the path for where assembly is written can be configured via `asm_path`, the default is `{base_dir}/asm`
- the source code path can be configured via `src_path`, the default is `{base_path}/src`
## Macros
The macros to include text/rodata assembly are different for GCC vs IDO compiler:
**GCC**: `INCLUDE_ASM` & `INCLUDE_RODATA` (text/rodata respectively)
**IDO**: `GLOBAL_ASM`
These macros must be defined in an included header, which splat currently does not produce.
For a GCC example, see the [include.h](https://github.com/AngheloAlf/drmario64/blob/master/include/include_asm.h) from the Dr. Mario project.
For IDO, you will need to use [asm-processor](https://github.com/simonlindholm/asm-processor) in order to include assembly code within the c files.
# 3 Decompile text
This involved back and forth between `.c` and `.s` files:
- editing the `data.s`, `rodata.s` files to add/fixup symbols at the proper locations
- decompiling functions, declaring symbols (`extern`s) in the `.c`
The linker script links
- `.text` (only) from the `.o` built from `energy_orb_wave.c`
- `.data` (only) from the `.o` built from `energy_orb_wave.data.s`
- `.rodata` (only) from the `.o` built from `energy_orb_wave.rodata.s`
# 4 Decompile (ro)data
Migrate data to the .c file, using raw values, lists or structs as appropriate code.
Once you have paired the rodata and text segments together, you can enabled `migrate_rodata_to_functions`. This will add the paired rodata into each individual function's assembly file, and therefore, the rodata will end up in the compiled .o file.
To link the .data/.rodata from the .o built from the .c file (instead of from the .s files), the subsegments must be changed from:
```yaml
- [0x42100, c, energy_orb_wave]
- [0x42200, data, energy_orb_wave] # extract data at this ROM address as energy_orb_wave.data.s
- [0x42300, rodata, energy_orb_wave] # extract rodata at this ROM address as energy_orb_wave.rodata.s
```
to:
```yaml
- [0x42100, c, energy_orb_wave]
- [0x42200, .data, energy_orb_wave] # take the .data section from the compiled c file named energy_orb_wave
- [0x42300, .rodata, energy_orb_wave] # take the .rodata section from the compiled c file named energy_orb_wave
```
**NOTE:**
If using `auto_all_section` and there are no other `data`/`.data`/`rodata`/`.rodata` in the subsegments in the code segment, the subsegments can also be changed to
```yaml
- [0x42100, c, energy_orb_wave]
- [0x42200]
```
# 5 Decompile bss
`bss` works in a similar way to data/rodata. However, `bss` is usually discarded from the final binary, which makes it somewhat tricker to migrate.
The `bss` segment will create assembly files that are full of `space`. The `.bss` segment will link the `.bss` section of the referenced `c` file.
# 6 Done!
`.text`, `.data`, `.rodata` and `.bss` are linked from the .o built from `energy_orb_wave.c` which now has everything to match when building
The assembly files (functions .s, data.s and rodata.s files) can be deleted

25
tools/splat/docs/Home.md
View File

@ -1,25 +0,0 @@
### What is splat?
**splat** is a binary splitting tool, written in Python. Its goal is to support the successful disassembly and then rebuilding of binary data.
It is the spiritual successor to [n64split](https://github.com/queueRAM/sm64tools/blob/master/n64split.c), originally written to handle N64 ROMs, it now has limited support for PSX and PS2 binaries.
MIPS code disassembly is handled via [spimdisasm](https://github.com/Decompollaborate/spimdisasm/).
There are a number of asset types built-in (e.g. various image formats, N64 Vtx data, etc), and it is designed to be simple to extend by writing your own custom types that can do anything you want as part of the **splat** pipeline.
### How does it work?
**splat** takes a [yaml](https://en.wikipedia.org/wiki/YAML) configuration file which tells it *where* and *how* to split a given file. Symbols can be mapped to addresses (and their types provided) via an optional "symbol_addrs" file.
**splat** runs two distinct phases: scan and split.
The _scan_ phase makes a first pass over the data and performs the initial disassembly of code and data. During the _split_ phase, information gathered during the _scan_ phase is used and files & data are written out to disk.
After scanning and splitting, **splat** will output a linker script that can be used as part of re-building the input file.
### Sounds great, how do I get started?
Have a look at the [Quickstart](https://github.com/ethteck/splat/wiki/Quickstart), or check out the [Examples](https://github.com/ethteck/splat/wiki/Examples) page to see projects that are using **splat**.

153
tools/splat/docs/Quickstart.md
View File

@ -1,153 +0,0 @@
> **Note**: This quickstart is written with N64 ROMs in mind, and the assumption that you are using Ubuntu 20.04 either natively, via WSL2 or via Docker.
For the purposes of this quickstart, we will assume that we are going to split a game called `mygame` and we have the ROM in `.z64` format named `baserom.z64`.
Create a directory for `~/mygame` and `cd` into it:
```sh
mkdir -p ${HOME}/mygame && cd ${HOME}/mygame
```
Copy the `baserom.z64` file into the `mygame` directory inside your home directory.
### System packages
#### Python 3.8
Ensure you are have **Python 3.8** or higher installed:
```sh
$ python3 --version
Python 3.8.10
```
If you get `bash: python3: command not found` install it with the following command:
```sh
sudo apt-get update && sudo apt-get install -y python3 python3-pip
```
#### Git
Ensure you have **git**:
```sh
$ git --version
```
If you get `bash: git: command not found`, install it with the following command:
```sh
sudo apt-get update && sudo apt-get install -y git
```
## Checkout the repository
We will clone **splat** into a `tools` directory to keep things organised:
```sh
git clone https://github.com/ethteck/splat.git tools/splat
```
## Python packages
Run the following to install the prerequisite Python packages:
```sh
python3 -m pip install -r ./tools/splat/requirements.txt
```
## Create a config file for your baserom
**splat** has a script that will generate a `yaml` file for your ROM.
```sh
python3 tools/splat/create_config.py baserom.z64
```
The `yaml` file generated will be named based upon the name of the ROM (taken from its header). The example below is for Super Mario 64:
```yaml
$ cat supermario64.yaml
name: Super Mario 64 (North America)
sha1: 9bef1128717f958171a4afac3ed78ee2bb4e86ce
options:
basename: supermario64
target_path: baserom.z64
base_path: .
compiler: IDO
find_file_boundaries: True
# platform: n64
# undefined_funcs_auto_path: undefined_funcs_auto.txt
# undefined_syms_auto_path: undefined_syms_auto.txt
# symbol_addrs_path: symbol_addrs.txt
# undefined_syms_path: undefined_syms.txt
# asm_path: asm
# src_path: src
# build_path: build
# extensions_path: tools/splat_ext
# auto_all_sections: True
segments:
- name: header
type: header
start: 0x0
- name: boot
type: bin
start: 0x40
- name: main
type: code
start: 0x1000
vram: 0x80246000
subsegments:
- [0x1000, asm]
- type: bin
start: 0xE6430
- [0x800000]
```
This is a bare-bones configuration and there is a lot of work required to map out the different sections of the ROM.
## Run splat with your configuration
```sh
python3 tools/splat/split.py supermario64.yaml
```
The output will look something like this:
```
splat 0.7.10.1
Loading and processing symbols
Starting scan
..Segment 1000, function at vram 80246DF8 ends with extra nops, indicating a likely file split.
File split suggestions for this segment will follow in config yaml format:
- [0x1E70, asm]
- [0x3C40, asm]
- [0x45E0, asm]
- [0x6FF0, asm]
# < -- snip -->
- [0xE6060, asm]
- [0xE61F0, asm]
- [0xE6200, asm]
- [0xE6260, asm]
..
Starting split
....
Split 943 KB (11.24%) in defined segments
header: 64 B (0.00%) 1 split, 0 cached
bin: 4 KB (0.05%) 1 split, 0 cached
code: 939 KB (11.19%) 1 split, 0 cached
unknown: 7 MB (88.76%) from unknown bin files
```
Notice that **splat** has found some potential file splits (function start/end with 16 byte alignment padded with nops).
It's up to you to figure out the layout of the ROM.
## Next Steps
The reassembly of the ROM is currently out of scope of this quickstart, as is switching out the `asm` segments for `c`.
You can find a general workflow for using `splat` at [General Workflow](https://github.com/ethteck/splat/wiki/General-Workflow)
Please feel free to improve this guide!

312
tools/splat/docs/Segments.md
View File

@ -1,312 +0,0 @@
# Segments
The configuration file for **splat** consists of a number of well-defined segments.
Most segments can be defined as a either a dictionary or a list, however the list syntax is only suitable for simple cases as it does not allow for specifying many of the options a segment type has to offer.
Splat segments' behavior generally falls under two categories: extraction and linking. Some segments will only do extraction, some will only do linking, some both, and some neither. Generally, segments will describe both extraction and linking behavior. Additionally, a segment type whose name starts with a dot (.) will only focus on linking.
## `asm`
**Description:**
Segments designated Assembly, `asm`, will be disassembled via [spimdisasm](https://github.com/Decompollaborate/spimdisasm) and enriched with Symbols based on the contents of the `symbol_addrs` configuration.
**Example:**
```yaml
# as list
- [0xABC, asm, filepath1]
- [0xABC, asm, dir1/filepath2] # this will create filepath2.s inside a directory named dir1
# as dictionary
- name: filepath
type: asm
start: 0xABC
```
### `hasm`
**Description:**
Hand-written Assembly, `hasm`, similar to `asm` except it will not overwrite any existing files. Useful when assembly has been manually edited.
**Example:**
```yaml
# as list
- [0xABC, hasm, filepath]
# as dictionary
- name: filepath
type: hasm
start: 0xABC
```
## `bin`
**Description:**
The `bin`(ary) segment type is for raw data, or data where the type is yet to be determined, data will be written out as raw `.bin` files.
**Example:**
```yaml
# as list
- [0xABC, bin, filepath]
# as dictionary
- name: filepath
type: bin
start: 0xABC
```
## `code`
**Description:**
The 'code' segment type, `code` is a group that can have many `subsegments`. Useful to group sections of code together (e.g. all files part of the same overlay).
**Example:**
```yaml
# must be a dictionary
- name: main
type: code
start: 0x00001000
vram: 0x80125900
subsegments:
- [0x1000, asm, entrypoint]
- [0x1050, c, main]
```
## `c`
**Description:**
The C code segments have two behaviors:
- If the target `.c` file does not exist, a new file will be generated with macros to include the original assembly (macros differ for IDO vs GCC compiler).
- Otherwise the target `.c` file is scanned to determine what assembly needs to be extracted from the ROM.
Assembly that is extracted due to a `c` segment will be written to a `nonmatchings` folder, with one function per file.
**Example:**
```yaml
# as list
- [0xABC, c, filepath]
# as dictionary
- name: filepath
type: c
start: 0xABC
```
## `header`
**Description:**
This is platform specific; parses the data and interprets as a header for e.g. N64 or PS1 elf.
**Example:**
```yaml
# as list
- [0xABC, header, filepath]
# as dictionary
- name: filepath
type: header
start: 0xABC
```
## `data`
**Description:**
Data located in the ROM. Extracted as assembly; integer, float and string types will be attempted to be inferred by the disassembler.
**Example:**
```yaml
# as list
- [0xABC, data, filepath]
# as dictionary
- name: filepath
type: data
start: 0xABC
```
This will created `filepath.data.s` within the `asm` folder.
## `.data`
**Description:**
Data located in the ROM that is linked from a C file. Use the `.data` segment to tell the linker to pull the `.data` section from the compiled object of corresponding `c` segment.
**Example:**
```yaml
# as list
- [0xABC, .data, filepath]
# as dictionary
- name: filepath
type: .data
start: 0xABC
```
**NOTE:** `splat` will not generate any `.data.s` files for these `.` (dot) sections.
## `rodata`
**Description:**
Read-only data located in the ROM, e.g. floats, strings and jump tables. Extracted as assembly; integer, float and string types will be attempted to be inferred by the disassembler.
**Example:**
```yaml
# as list
- [0xABC, rodata, filepath]
# as dictionary
- name: filepath
type: rodata
start: 0xABC
```
This will created `filepath.rodata.s` within the `asm` folder.
## `.rodata`
**Description:**
Read-only data located in the ROM, linked to a C file. Use the `.rodata` segment to tell the linker to pull the `.rodata` section from the compiled object of corresponding `c` segment.
**Example:**
```yaml
# as list
- [0xABC, .rodata, filepath]
# as dictionary
- name: filepath
type: .rodata
start: 0xABC
```
**NOTE:** `splat` will not generate any `.rodata.s` files for these `.` (dot) sections.
## `bss`
**Description:**
`bss` is where variables are placed that have been declared but are not given an initial value. These sections are usually discarded from the final binary (although PSX binaries seem to include them!).
Note that the `bss_size` option needs to be set at segment level for `bss` segments to work correctly.
**Example:**
```yaml
- { start: 0x7D1AD0, type: bss, name: filepath, vram: 0x803C0420 }
```
## `.bss`
**Description:**
Links the `.bss` section of the associated `c` file.
**Example:**
```yaml
- { start: 0x7D1AD0, type: .bss, name: filepath, vram: 0x803C0420 }
```
## Images
**Description:**
**splat** supports most of the [N64 image formats](https://n64squid.com/homebrew/n64-sdk/textures/image-formats/):
- `i`, i.e. `i4` and `i8`
- `ia`, i.e. `ia4`, `ia8`, and `ia16`
- `ci`, i.e. `ci4` and `ci8`
- `rgb`, i.e. `rgba32` and `rgba16`
These segments will parse the image data and dump out a `png` file.
**Note:** Using the dictionary syntax allows for richer configuration.
**Example:**
```yaml
# as list
- [0xABC, i4, filename, width, height]
# as a dictionary
- name: filename
type: i4
start: 0xABC
width: 64
height: 64
flip_x: yes
flip_y: no
```
## General segment options
All splat's segments can be passed extra options for finer configuration. Note that those extra options require to rewrite the entry using the dictionary yaml notation instead of the list one.
### `linker_section_order`
**Description:**
Allows overriding the section order used for linker script generation.
Useful when a section of a file is not between the other sections of the same type in the ROM, for example a file having its data section between other files's rodata.
Take in mind this option may need the [`check_consecutive_segment_types`](Configuration.md#check_consecutive_segment_types) yaml option to be turned off.
**Example:**
```yaml
- [0x400, data, file1]
# data ends
# rodata starts
- [0x800, rodata, file2]
- { start: 0xA00, type: data, name: file3, linker_section_order: .rodata }
- [0xC00, rodata, file4]
```
This will created `file3.data.s` within the `asm` folder, but won't be reordered in the generated linker script to be placed on the data section.
### `linker_section`
**Description:**
Allows to override the `.section` directive that will be used when generating the disassembly of the corresponding section, without needing to write an extension segment. This also affects the section name that will be used during link time.
Useful for sections with special names, like an executable section named `.start`
**Example:**
```yaml
- { start: 0x1000, type: asm, name: snmain, linker_section: .start }
- [0x1070, rdata, libc]
- [0x10A0, rdata, main_030]
```
### `ld_fill_value`
Allows to specify the value of the `FILL` statement generated for this specific top-level segment of the linker script, ignoring the global configuration.
It must be either an integer, which will be used as the parameter for the `FILL` statement, or `null`, which tells splat to not emit a `FILL` statement for this segment.
If not set, then the global configuration is used. See [ld_fill_value](Configuration.md#ld_fill_value) on the Configuration section.
Defaults to the value of the global option.

59
tools/splat/docs/VramClasses.md
View File

@ -1,59 +0,0 @@
# vram classes
Version 0.19.0 introduced `vram_classes`, a new top-level yaml section that can be used to help reduce duplicated data in your yaml and more clearly organize its memory layout.
## Introduction
Before vram classes, you might have had something like this in your yaml:
```yaml
- type: code
start: 0x4269D0
vram: 0x802A9000
vram_symbol: battle_move_end
subsegments: ...
- type: code
start: 0x4273B0
vram: 0x802A9000 # notice same `vram` and `vram_symbol` for both segments
vram_symbol: battle_move_end
subsegments: ...
```
Having to duplicate the vram address and vram_symbol properties for potentially dozens of hundreds of overlay segments is tedious and pollutes your yaml with repeated information that can become out of sync. Enter vram_classes!
```yaml
- type: code
start: 0x4269D0
vram_class: maps
subsegments: ...
- type: code
start: 0x4273B0
vram_class: maps
subsegments: ...
```
Here, we are telling splat that both of these segments use the `maps` vram class. We are now effectively pointing both segments to the same source of information. Now let's look at how vram classes are defined:
## Format
```yaml
options:
...
ld_use_symbolic_vram_addresses: True
...
vram_classes:
- { name: maps, vram: 0x802A9000, vram_symbol: battle_move_end }
```
`vram_classes` is a top-level yaml section that contains a list of vram classes. You can either define them in dict form (as seen above) or list form. However, for list form, only `name` and `vram` are supported (`[maps, 0x802A9000]`). If you want to specify other options, please use the dict form. The fields supported are as follows:
- `name` (required): The name of the class
- `vram` (required): The vram address to be used during disasembly. If `ld_use_symbolic_vram_addresses` is disabled or no `vram_symbol` or `follows_classes` properties are provided, this address will be used in the linker script.
The following properties are optional and only take effect if `ld_use_symbolic_vram_addresses` is enabled:
- `vram_symbol`: The name of the symbol to use in the linker script for this class.
- `follows_classes`: A list of vram class names that this class must come after in memory. If we added `follows_classes: [apples, bananas]` to our above vram_class, this would make all `maps` segments start at the end of all `apples` and `bananas` segments.
The internal linker script symbol name that is chosen for `follows_classes` is the name of the class followed by `_CLASS_VRAM`. You can override this by also specifying `vram_symbol`.

4
tools/splat/mypy.ini
View File

@ -1,4 +0,0 @@
[mypy]
ignore_missing_imports = True
check_untyped_defs = True
mypy_path = stubs

5
tools/splat/platforms/gc.py
View File

@ -1,5 +0,0 @@
from util.gc import gcfst
def init(target_bytes: bytes):
gcfst.split_iso(target_bytes)

12
tools/splat/platforms/n64.py
View File

@ -1,12 +0,0 @@
from util import options, symbols
def init(target_bytes: bytes):
symbols.spim_context.fillDefaultBannedSymbols()
if options.opts.libultra_symbols:
symbols.spim_context.globalSegment.fillLibultraSymbols()
if options.opts.ique_symbols:
symbols.spim_context.globalSegment.fillIQueSymbols()
if options.opts.hardware_regs:
symbols.spim_context.globalSegment.fillHardwareRegs(True)

2
tools/splat/platforms/ps2.py
View File

@ -1,2 +0,0 @@
def init(target_bytes: bytes):
pass

2
tools/splat/platforms/psx.py
View File

@ -1,2 +0,0 @@
def init(target_bytes: bytes):
pass

10
tools/splat/requirements.txt
View File

@ -1,10 +0,0 @@
PyYAML
pylibyaml
tqdm
intervaltree
colorama
# This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py
spimdisasm>=1.18.0
rabbitizer>=1.7.0
pygfxd
n64img>=0.1.4

11
tools/splat/run_tests.sh
View File

@ -1,11 +0,0 @@
#!/bin/bash
# docker build container
docker build --tag splat-build:latest . && \
# get compilers and tools
# clean
cd test/basic_app && sh clean.sh && cd ../../ && \
# build
docker run --rm -v $(pwd):/splat -w /splat/test/basic_app splat-build sh build.sh && \
# test
python3 test.py

0
tools/splat/segtypes/__init__.py
View File

10
tools/splat/segtypes/address_range.py
View File

@ -1,10 +0,0 @@
from dataclasses import dataclass
@dataclass
class AddressRange:
start: int
end: int
def contains(self, addr: int) -> bool:
return self.start <= addr < self.end

0
tools/splat/segtypes/common/__init__.py
View File

39
tools/splat/segtypes/common/asm.py
View File

@ -1,39 +0,0 @@
from pathlib import Path
from typing import Optional
from util import options
from segtypes.common.codesubsegment import CommonSegCodeSubsegment
class CommonSegAsm(CommonSegCodeSubsegment):
@staticmethod
def is_text() -> bool:
return True
def out_path(self) -> Optional[Path]:
return options.opts.asm_path / self.dir / f"{self.name}.s"
def scan(self, rom_bytes: bytes):
if (
self.rom_start is not None
and self.rom_end is not None
and self.rom_start != self.rom_end
):
self.scan_code(rom_bytes)
def get_file_header(self):
return []
def split(self, rom_bytes: bytes):
if not self.rom_start == self.rom_end and self.spim_section is not None:
out_path = self.out_path()
if out_path:
out_path.parent.mkdir(parents=True, exist_ok=True)
self.print_file_boundaries()
with open(out_path, "w", newline="\n") as f:
for line in self.get_file_header():
f.write(line + "\n")
f.write(self.spim_section.disassemble())

32
tools/splat/segtypes/common/bin.py
View File

@ -1,32 +0,0 @@
from pathlib import Path
from typing import Optional
from util import log, options
from segtypes.common.segment import CommonSegment
class CommonSegBin(CommonSegment):
@staticmethod
def is_data() -> bool:
return True
def out_path(self) -> Optional[Path]:
return options.opts.asset_path / self.dir / f"{self.name}.bin"
def split(self, rom_bytes):
path = self.out_path()
assert path is not None
path.parent.mkdir(parents=True, exist_ok=True)
if self.rom_end is None:
log.error(
f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it"
)
with open(path, "wb") as f:
assert isinstance(self.rom_start, int)
assert isinstance(self.rom_end, int)
f.write(rom_bytes[self.rom_start : self.rom_end])
self.log(f"Wrote {self.name} to {path}")

62
tools/splat/segtypes/common/bss.py
View File

@ -1,62 +0,0 @@
from util import options, symbols, log
from segtypes.common.data import CommonSegData
from disassembler_section import make_bss_section
class CommonSegBss(CommonSegData):
def get_linker_section(self) -> str:
return ".bss"
@staticmethod
def is_noload() -> bool:
return True
def disassemble_data(self, rom_bytes: bytes):
if not isinstance(self.rom_start, int):
log.error(
f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'"
)
# Supposedly logic error, not user error
assert isinstance(self.rom_end, int), f"{self.name} {self.rom_end}"
# Supposedly logic error, not user error
segment_rom_start = self.get_most_parent().rom_start
assert isinstance(segment_rom_start, int), f"{self.name} {segment_rom_start}"
if not isinstance(self.vram_start, int):
log.error(
f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'"
)
next_subsegment = self.parent.get_next_subsegment_for_ram(self.vram_start)
if next_subsegment is None:
bss_end = self.get_most_parent().vram_end
else:
bss_end = next_subsegment.vram_start
assert isinstance(bss_end, int), f"{self.name} {bss_end}"
self.spim_section = make_bss_section(
self.rom_start,
self.rom_end,
self.vram_start,
bss_end,
self.name,
segment_rom_start,
self.get_exclusive_ram_id(),
)
assert self.spim_section is not None
self.spim_section.analyze()
self.spim_section.set_comment_offset(self.rom_start)
for spim_sym in self.spim_section.get_section().symbolList:
symbols.create_symbol_from_spim_symbol(
self.get_most_parent(), spim_sym.contextSym
)
def should_scan(self) -> bool:
return options.opts.is_mode_active("code") and self.vram_start is not None

439
tools/splat/segtypes/common/c.py
View File

@ -1,439 +0,0 @@
import os
import re
from pathlib import Path
from typing import Optional, Set, List
import spimdisasm
from util import log, options, symbols
from util.compiler import GCC, SN64, IDO
from util.symbols import Symbol
from segtypes.common.codesubsegment import CommonSegCodeSubsegment
from segtypes.common.rodata import CommonSegRodata
STRIP_C_COMMENTS_RE = re.compile(
r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
re.DOTALL | re.MULTILINE,
)
C_FUNC_RE = re.compile(
r"^(?:static\s+)?[^\s]+\s+([^\s(]+)\(([^;)]*)\)[^;]+?{", re.MULTILINE
)
C_GLOBAL_ASM_IDO_RE = re.compile(r"GLOBAL_ASM\(\"(\w+\/)*(\w+)\.s\"\)", re.MULTILINE)
class CommonSegC(CommonSegCodeSubsegment):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.defined_funcs: Set[str] = set()
self.global_asm_funcs: Set[str] = set()
self.global_asm_rodata_syms: Set[str] = set()
self.file_extension = "c"
@staticmethod
def strip_c_comments(text):
def replacer(match):
s = match.group(0)
if s.startswith("/"):
return " "
else:
return s
return re.sub(STRIP_C_COMMENTS_RE, replacer, text)
@staticmethod
def get_funcs_defined_in_c(c_file: Path) -> Set[str]:
with open(c_file, "r") as f:
text = CommonSegC.strip_c_comments(f.read())
return set(m.group(1) for m in C_FUNC_RE.finditer(text))
@staticmethod
def find_all_instances(string: str, sub: str):
start = 0
while True:
start = string.find(sub, start)
if start == -1:
return
yield start
start += len(sub)
@staticmethod
def get_close_parenthesis(string: str, pos: int):
paren_count = 0
while True:
cur_char = string[pos]
if cur_char == "(":
paren_count += 1
elif cur_char == ")":
if paren_count == 0:
return pos + 1
else:
paren_count -= 1
pos += 1
@staticmethod
def find_include_macro(text: str, macro_name: str):
for pos in CommonSegC.find_all_instances(text, f"{macro_name}("):
close_paren_pos = CommonSegC.get_close_parenthesis(
text, pos + len(f"{macro_name}(")
)
macro_contents = text[pos:close_paren_pos]
macro_args = macro_contents.split(",")
if options.opts.use_legacy_include_asm:
if len(macro_args) >= 3:
yield macro_args[2].strip(" )")
else:
if len(macro_args) >= 2:
yield macro_args[1].strip(" )")
@staticmethod
def find_include_asm(text: str):
return CommonSegC.find_include_macro(text, "INCLUDE_ASM")
@staticmethod
def find_include_rodata(text: str):
return CommonSegC.find_include_macro(text, "INCLUDE_RODATA")
@staticmethod
def get_global_asm_funcs(c_file: Path) -> Set[str]:
with c_file.open() as f:
text = CommonSegC.strip_c_comments(f.read())
if options.opts.compiler in [GCC, SN64]:
return set(CommonSegC.find_include_asm(text))
else:
return set(m.group(2) for m in C_GLOBAL_ASM_IDO_RE.finditer(text))
@staticmethod
def get_global_asm_rodata_syms(c_file: Path) -> Set[str]:
with c_file.open() as f:
text = CommonSegC.strip_c_comments(f.read())
if options.opts.compiler in [GCC, SN64]:
return set(CommonSegC.find_include_rodata(text))
else:
return set(m.group(2) for m in C_GLOBAL_ASM_IDO_RE.finditer(text))
@staticmethod
def is_text() -> bool:
return True
def out_path(self) -> Optional[Path]:
return options.opts.src_path / self.dir / f"{self.name}.{self.file_extension}"
def scan(self, rom_bytes: bytes):
if (
self.rom_start is not None
and self.rom_end is not None
and self.rom_start != self.rom_end
):
path = self.out_path()
if path:
if options.opts.do_c_func_detection and os.path.exists(path):
# TODO run cpp?
self.defined_funcs = self.get_funcs_defined_in_c(path)
self.global_asm_funcs = self.get_global_asm_funcs(path)
self.global_asm_rodata_syms = self.get_global_asm_rodata_syms(path)
symbols.to_mark_as_defined.update(self.defined_funcs)
symbols.to_mark_as_defined.update(self.global_asm_funcs)
symbols.to_mark_as_defined.update(self.global_asm_rodata_syms)
self.scan_code(rom_bytes)
def split(self, rom_bytes: bytes):
if self.rom_start != self.rom_end:
asm_out_dir = options.opts.nonmatchings_path / self.dir
asm_out_dir.mkdir(parents=True, exist_ok=True)
self.print_file_boundaries()
assert self.spim_section is not None and isinstance(
self.spim_section.get_section(), spimdisasm.mips.sections.SectionText
), f"{self.name}, rom_start:{self.rom_start}, rom_end:{self.rom_end}"
rodata_section_type = ""
rodata_spim_segment: Optional[spimdisasm.mips.sections.SectionRodata] = None
if (
options.opts.migrate_rodata_to_functions
and self.rodata_sibling is not None
):
assert isinstance(
self.rodata_sibling, CommonSegRodata
), self.rodata_sibling.type
rodata_section_type = (
self.rodata_sibling.get_linker_section_linksection()
)
if self.rodata_sibling.spim_section is not None:
assert isinstance(
self.rodata_sibling.spim_section.get_section(),
spimdisasm.mips.sections.SectionRodata,
)
rodata_spim_segment = self.rodata_sibling.spim_section.get_section()
# Precompute function-rodata pairings
symbols_entries = (
spimdisasm.mips.FunctionRodataEntry.getAllEntriesFromSections(
self.spim_section.get_section(), rodata_spim_segment
)
)
is_new_c_file = False
# Check and create the C file
c_path = self.out_path()
if c_path:
if not c_path.exists() and options.opts.create_c_files:
self.create_c_file(asm_out_dir, c_path, symbols_entries)
is_new_c_file = True
self.create_asm_dependencies_file(
c_path, asm_out_dir, is_new_c_file, symbols_entries
)
# Produce the asm files for functions
for entry in symbols_entries:
entry.sectionText = self.get_linker_section_linksection()
entry.sectionRodata = rodata_section_type
if entry.function is not None:
if (
entry.function.getName() in self.global_asm_funcs
or is_new_c_file
or options.opts.disassemble_all
):
func_sym = self.get_symbol(
entry.function.vram,
in_segment=True,
type="func",
local_only=True,
)
assert func_sym is not None
self.create_c_asm_file(entry, asm_out_dir, func_sym)
else:
for spim_rodata_sym in entry.rodataSyms:
if (
spim_rodata_sym.getName() in self.global_asm_rodata_syms
or is_new_c_file
or options.opts.disassemble_all
):
rodata_sym = self.get_symbol(
spim_rodata_sym.vram, in_segment=True, local_only=True
)
assert rodata_sym is not None
self.create_unmigrated_rodata_file(
spim_rodata_sym, asm_out_dir, rodata_sym
)
def get_c_preamble(self):
ret = []
preamble = options.opts.generated_c_preamble
ret.append(preamble)
ret.append("")
return ret
def check_gaps_in_migrated_rodata(
self,
func: spimdisasm.mips.symbols.SymbolFunction,
rodata_list: List[spimdisasm.mips.symbols.SymbolBase],
):
for index in range(len(rodata_list) - 1):
rodata_sym = rodata_list[index]
next_rodata_sym = rodata_list[index + 1]
if rodata_sym.vramEnd != next_rodata_sym.vram:
log.write(
f"\nA gap was detected in migrated rodata symbols!", status="warn"
)
log.write(
f"\t In function '{func.getName()}' (0x{func.vram:08X}), gap detected between '{rodata_sym.getName()}' (0x{rodata_sym.vram:08X}) and '{next_rodata_sym.getName()}' (0x{next_rodata_sym.vram:08X})"
)
log.write(
f"\t The address of the missing rodata symbol is 0x{rodata_sym.vramEnd:08X}"
)
log.write(
f"\t Try to force the migration of that symbol with `force_migration:True` in the symbol_addrs.txt file; or avoid the migration of symbols around this address with `force_not_migration:True`"
)
def create_c_asm_file(
self,
func_rodata_entry: spimdisasm.mips.FunctionRodataEntry,
out_dir: Path,
func_sym: Symbol,
):
outpath = out_dir / self.name / (func_sym.name + ".s")
# Skip extraction if the file exists and the symbol is marked as extract=false
if outpath.exists() and not func_sym.extract:
return
outpath.parent.mkdir(parents=True, exist_ok=True)
with outpath.open("w", newline="\n") as f:
if options.opts.asm_inc_header:
f.write(
options.opts.c_newline.join(options.opts.asm_inc_header.split("\n"))
)
func_rodata_entry.writeToFile(f)
if func_rodata_entry.function is not None:
self.check_gaps_in_migrated_rodata(
func_rodata_entry.function, func_rodata_entry.rodataSyms
)
self.check_gaps_in_migrated_rodata(
func_rodata_entry.function, func_rodata_entry.lateRodataSyms
)
self.log(f"Disassembled {func_sym.name} to {outpath}")
def create_unmigrated_rodata_file(
self,
spim_rodata_sym: spimdisasm.mips.symbols.SymbolBase,
out_dir: Path,
rodata_sym: Symbol,
):
outpath = out_dir / self.name / (rodata_sym.name + ".s")
# Skip extraction if the file exists and the symbol is marked as extract=false
if outpath.exists() and not rodata_sym.extract:
return
outpath.parent.mkdir(parents=True, exist_ok=True)
with outpath.open("w", newline="\n") as f:
if options.opts.include_macro_inc:
f.write('.include "macro.inc"\n\n')
preamble = options.opts.generated_s_preamble
if preamble:
f.write(preamble + "\n")
assert rodata_sym.linker_section is not None, rodata_sym.name
f.write(f".section {rodata_sym.linker_section}\n\n")
f.write(spim_rodata_sym.disassemble())
self.log(f"Disassembled {rodata_sym.name} to {outpath}")
def get_c_line_include_macro(
self,
spim_sym: spimdisasm.mips.symbols.SymbolBase,
asm_out_dir: Path,
macro_name: str,
) -> str:
if options.opts.compiler == IDO:
# IDO uses the asm processor to embeed assembly, and it doesn't require a special directive to include symbols
asm_outpath = Path(
os.path.join(asm_out_dir, self.name, spim_sym.getName() + ".s")
)
rel_asm_outpath = os.path.relpath(asm_outpath, options.opts.base_path)
return f'#pragma GLOBAL_ASM("{rel_asm_outpath}")'
if options.opts.use_legacy_include_asm:
rel_asm_out_dir = asm_out_dir.relative_to(options.opts.nonmatchings_path)
return f'{macro_name}(const s32, "{rel_asm_out_dir / self.name}", {spim_sym.getName()});'
return f'{macro_name}("{asm_out_dir / self.name}", {spim_sym.getName()});'
def get_c_lines_for_function(
self, func: spimdisasm.mips.symbols.SymbolFunction, asm_out_dir: Path
) -> List[str]:
c_lines = []
# Terrible hack to "auto-decompile" empty functions
if (
options.opts.auto_decompile_empty_functions
and len(func.instructions) == 2
and func.instructions[0].isReturn()
and func.instructions[1].isNop()
):
c_lines.append("void " + func.getName() + "(void) {")
c_lines.append("}")
else:
c_lines.append(
self.get_c_line_include_macro(func, asm_out_dir, "INCLUDE_ASM")
)
c_lines.append("")
return c_lines
def get_c_lines_for_rodata_sym(
self, rodata_sym: spimdisasm.mips.symbols.SymbolBase, asm_out_dir: Path
):
c_lines = [
self.get_c_line_include_macro(rodata_sym, asm_out_dir, "INCLUDE_RODATA")
]
c_lines.append("")
return c_lines
def create_c_file(
self,
asm_out_dir: Path,
c_path: Path,
symbols_entries: List[spimdisasm.mips.FunctionRodataEntry],
):
c_lines = self.get_c_preamble()
for entry in symbols_entries:
if entry.function is not None:
c_lines += self.get_c_lines_for_function(entry.function, asm_out_dir)
else:
for rodata_sym in entry.rodataSyms:
c_lines += self.get_c_lines_for_rodata_sym(rodata_sym, asm_out_dir)
c_path.parent.mkdir(parents=True, exist_ok=True)
with c_path.open("w") as f:
f.write("\n".join(c_lines))
log.write(f"Wrote {self.name} to {c_path}")
def create_asm_dependencies_file(
self,
c_path: Path,
asm_out_dir: Path,
is_new_c_file: bool,
symbols_entries: List[spimdisasm.mips.FunctionRodataEntry],
):
if not options.opts.create_asm_dependencies:
return
if (
len(self.global_asm_funcs) + len(self.global_asm_rodata_syms)
) == 0 and not is_new_c_file:
return
assert self.spim_section is not None
build_path = options.opts.build_path
dep_path = build_path / c_path.with_suffix(".asmproc.d")
dep_path.parent.mkdir(parents=True, exist_ok=True)
with dep_path.open("w") as f:
if options.opts.use_o_as_suffix:
o_path = build_path / c_path.with_suffix(".o")
else:
o_path = build_path / c_path.with_suffix(c_path.suffix + ".o")
f.write(f"{o_path}:")
depend_list = []
for entry in symbols_entries:
if entry.function is not None:
func_name = entry.function.getName()
if func_name in self.global_asm_funcs or is_new_c_file:
outpath = asm_out_dir / self.name / (func_name + ".s")
depend_list.append(outpath)
f.write(f" \\\n {outpath}")
else:
for rodata_sym in entry.rodataSyms:
rodata_name = rodata_sym.getName()
if rodata_name in self.global_asm_rodata_syms or is_new_c_file:
outpath = asm_out_dir / self.name / (rodata_name + ".s")
depend_list.append(outpath)
f.write(f" \\\n {outpath}")
f.write("\n")
for depend_file in depend_list:
f.write(f"{depend_file}:\n")

419
tools/splat/segtypes/common/code.py
View File

@ -1,419 +0,0 @@
import typing
from collections import OrderedDict
from typing import Dict, List, Optional, Tuple, Set
from util import log, options
from util.range import Range
from util.symbols import Symbol
from segtypes.common.group import CommonSegGroup
from segtypes.segment import Segment, parse_segment_align
def dotless_type(type: str) -> str:
return type[1:] if type[0] == "." else type
# code group
class CommonSegCode(CommonSegGroup):
def __init__(
self,
rom_start: Optional[int],
rom_end: Optional[int],
type: str,
name: str,
vram_start: Optional[int],
args: list,
yaml,
):
self.bss_size: int = yaml.get("bss_size", 0) if isinstance(yaml, dict) else 0
super().__init__(
rom_start,
rom_end,
type,
name,
vram_start,
args=args,
yaml=yaml,
)
self.reported_file_split = False
self.jtbl_glabels_to_add: Set[int] = set()
self.jumptables: Dict[int, Tuple[int, int]] = {}
self.rodata_syms: Dict[int, List[Symbol]] = {}
self.align = parse_segment_align(yaml)
if self.align is None:
self.align = 0x10
@property
def needs_symbols(self) -> bool:
return True
@property
def vram_end(self) -> Optional[int]:
if self.vram_start is not None and self.size is not None:
return self.vram_start + self.size + self.bss_size
else:
return None
def check_rodata_sym_impl(self, func_addr: int, sym: Symbol, rodata_section: Range):
if rodata_section.is_complete():
assert rodata_section.start is not None
assert rodata_section.end is not None
rodata_start: int = rodata_section.start
rodata_end: int = rodata_section.end
if rodata_start <= sym.vram_start < rodata_end:
if func_addr not in self.rodata_syms:
self.rodata_syms[func_addr] = []
self.rodata_syms[func_addr].append(sym)
# Prepare symbol for migration to the function
def check_rodata_sym(self, func_addr: int, sym: Symbol):
rodata_section = self.section_boundaries.get(".rodata")
if rodata_section is not None:
self.check_rodata_sym_impl(func_addr, sym, rodata_section)
rodata_section = self.section_boundaries.get(".rdata")
if rodata_section is not None:
self.check_rodata_sym_impl(func_addr, sym, rodata_section)
def handle_alls(self, segs: List[Segment], base_segs) -> bool:
for i, elem in enumerate(segs):
if elem.type.startswith("all_"):
alls = []
rep_type = f"{elem.type[4:]}"
replace_class = Segment.get_class_for_type(rep_type)
for base in base_segs.items():
if isinstance(elem.rom_start, int) and isinstance(
self.rom_start, int
):
# Shoddy rom to ram
assert self.vram_start is not None, self.vram_start
vram_start = elem.rom_start - self.rom_start + self.vram_start
else:
vram_start = None
rep: Segment = replace_class(
rom_start=elem.rom_start,
rom_end=elem.rom_end,
type=rep_type,
name=base[0],
vram_start=vram_start,
args=[],
yaml={},
)
rep.extract = False
rep.given_subalign = self.given_subalign
rep.exclusive_ram_id = self.get_exclusive_ram_id()
rep.given_dir = self.given_dir
rep.given_symbol_name_format = self.symbol_name_format
rep.given_symbol_name_format_no_rom = self.symbol_name_format_no_rom
rep.sibling = base[1]
rep.parent = self
if rep.special_vram_segment:
self.special_vram_segment = True
alls.append(rep)
# Insert alls into segs at i
del segs[i]
segs[i:i] = alls
return True
return False
# Find places we should automatically add "all_data" / "all_rodata" / "all_bss"
def find_inserts(
self, found_sections: typing.OrderedDict[str, Range]
) -> "OrderedDict[str, int]":
inserts: OrderedDict[str, int] = OrderedDict()
section_order = self.section_order.copy()
section_order.remove(".text")
for i, section in enumerate(section_order):
if section not in options.opts.auto_all_sections:
continue
if not found_sections[section].has_start():
search_done = False
for j in range(i - 1, -1, -1):
end = found_sections[section_order[j]].end
if end is not None:
inserts[section] = end
search_done = True
break
if not search_done:
inserts[section] = -1
pass
return inserts
def parse_subsegments(self, segment_yaml) -> List[Segment]:
if "subsegments" not in segment_yaml:
if not self.parent:
raise Exception(
f"No subsegments provided in top-level code segment {self.name}"
)
return []
base_segments: OrderedDict[str, Segment] = OrderedDict()
ret = []
prev_start: Optional[int] = -1
prev_vram: Optional[int] = -1
inserts: OrderedDict[
str, int
] = (
OrderedDict()
) # Used to manually add "all_" types for sections not otherwise defined in the yaml
self.section_boundaries = OrderedDict(
(s_name, Range()) for s_name in options.opts.section_order
)
found_sections = OrderedDict(
(s_name, Range()) for s_name in self.section_boundaries
) # Stores yaml index where a section was first found
found_sections.pop(".text")
# Mark any manually added dot types
cur_section = None
for i, subsegment_yaml in enumerate(segment_yaml["subsegments"]):
# endpos marker
if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1:
continue
typ = Segment.parse_segment_type(subsegment_yaml)
if typ.startswith("all_"):
typ = typ[4:]
if not typ.startswith("."):
typ = f".{typ}"
if typ in found_sections:
if cur_section is None:
# Starting point
found_sections[typ].start = i
cur_section = typ
else:
if cur_section != typ:
# We're changing sections
if options.opts.check_consecutive_segment_types:
if found_sections[cur_section].has_end():
log.error(
f"Section {cur_section} end encountered but was already ended earlier!"
)
if found_sections[typ].has_start():
log.error(
f"Section {typ} start encounted but has already started earlier!"
)
# End the current section
found_sections[cur_section].end = i
# Start the next section
found_sections[typ].start = i
cur_section = typ
if cur_section is not None:
found_sections[cur_section].end = -1
inserts = self.find_inserts(found_sections)
last_rom_end = None
for i, subsegment_yaml in enumerate(segment_yaml["subsegments"]):
# endpos marker
if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1:
continue
typ = Segment.parse_segment_type(subsegment_yaml)
start = Segment.parse_segment_start(subsegment_yaml)
# Add dummy segments to be expanded later
if typ.startswith("all_"):
dummy_seg = Segment(
rom_start=start,
rom_end=None,
type=typ,
name="",
vram_start=None,
args=[],
yaml={},
)
dummy_seg.given_subalign = self.given_subalign
dummy_seg.exclusive_ram_id = self.exclusive_ram_id
dummy_seg.given_dir = self.given_dir
dummy_seg.given_symbol_name_format = self.symbol_name_format
dummy_seg.given_symbol_name_format_no_rom = (
self.symbol_name_format_no_rom
)
ret.append(dummy_seg)
continue
segment_class = Segment.get_class_for_type(typ)
end = self.get_next_seg_start(i, segment_yaml["subsegments"])
if start is None:
# Attempt to infer the start address
if i == 0:
# The start address of this segment is the start address of the group
start = self.rom_start
else:
# The start address is the end address of the previous segment
start = last_rom_end
if start is not None and end is None:
est_size = segment_class.estimate_size(subsegment_yaml)
if est_size is not None:
end = start + est_size
if start is not None and prev_start is not None and start < prev_start:
log.error(
f"Error: Group segment '{self.name}' contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})"
)
vram = None
if start is not None:
assert isinstance(start, int)
vram = self.get_most_parent().rom_to_ram(start)
if segment_class.is_noload():
# Pretend bss's rom address is after the last actual rom segment
start = last_rom_end
# and it has a rom size of zero
end = last_rom_end
segment: Segment = Segment.from_yaml(
segment_class, subsegment_yaml, start, end, vram
)
if (
segment.vram_start is not None
and prev_vram is not None
and segment.vram_start < prev_vram
):
log.error(
f"Error: Group segment '{self.name}' contains subsegments which are out of ascending vram order (0x{prev_vram:X} followed by 0x{segment.vram_start:X}).\n"
+ f"Detected when processing file '{segment.name}' of type '{segment.type}'"
)
segment.sibling = base_segments.get(segment.name, None)
if segment.sibling is not None:
if self.section_order.index(".text") < self.section_order.index(
".rodata"
):
if segment.is_rodata():
segment.sibling.rodata_sibling = segment
else:
if segment.is_text() and segment.sibling.is_rodata():
segment.rodata_sibling = segment.sibling
segment.sibling.sibling = segment
if self.section_order.index(".text") < self.section_order.index(
".data"
):
if segment.is_data():
segment.sibling.data_sibling = segment
else:
if segment.is_text() and segment.sibling.is_data():
segment.data_sibling = segment.sibling
segment.sibling.sibling = segment
segment.parent = self
if segment.special_vram_segment:
self.special_vram_segment = True
for i, section in enumerate(self.section_order):
if not self.section_boundaries[section].has_start() and dotless_type(
section
) == dotless_type(segment.type):
if i > 0:
prev_section = self.section_order[i - 1]
self.section_boundaries[prev_section].end = segment.vram_start
self.section_boundaries[section].start = segment.vram_start
segment.bss_contains_common = self.bss_contains_common
ret.append(segment)
if segment.is_text():
base_segments[segment.name] = segment
if self.section_order.index(".rodata") < self.section_order.index(".text"):
if segment.is_rodata() and segment.sibling is None:
base_segments[segment.name] = segment
prev_start = start
prev_vram = segment.vram_start
if end is not None:
last_rom_end = end
# Add the automatic all_ sections
orig_len = len(ret)
for section in reversed(inserts):
idx = inserts[section]
if idx == -1:
idx = orig_len
# bss hack TODO maybe rethink
if (
section == "bss"
and self.vram_start is not None
and self.rom_end is not None
and self.rom_start is not None
):
rom_start = self.rom_end
vram_start = self.vram_start + self.rom_end - self.rom_start
else:
rom_start = None
vram_start = None
new_seg = Segment(
rom_start=rom_start,
rom_end=None,
type="all_" + section,
name="",
vram_start=vram_start,
args=[],
yaml={},
)
new_seg.given_subalign = self.given_subalign
new_seg.exclusive_ram_id = self.exclusive_ram_id
new_seg.given_dir = self.given_dir
new_seg.given_symbol_name_format = self.symbol_name_format
new_seg.given_symbol_name_format_no_rom = self.symbol_name_format_no_rom
ret.insert(idx, new_seg)
check = True
while check:
check = self.handle_alls(ret, base_segments)
# TODO why is this necessary?
rodata_section = self.section_boundaries.get(
".rodata"
) or self.section_boundaries.get(".rdata")
if (
rodata_section is not None
and rodata_section.has_start()
and not rodata_section.has_end()
):
assert self.vram_end is not None
rodata_section.end = self.vram_end
return ret
def scan(self, rom_bytes):
# Always scan code first
for sub in self.subsegments:
if sub.is_text() and sub.should_scan():
sub.scan(rom_bytes)
# Scan everyone else
for sub in self.subsegments:
if not sub.is_text() and sub.should_scan():
sub.scan(rom_bytes)

214
tools/splat/segtypes/common/codesubsegment.py
View File

@ -1,214 +0,0 @@
from typing import Optional
import spimdisasm
import rabbitizer
from util import options, symbols, log
from segtypes import segment
from segtypes.common.code import CommonSegCode
from segtypes.segment import Segment
from disassembler_section import DisassemblerSection, make_text_section
# abstract class for c, asm, data, etc
class CommonSegCodeSubsegment(Segment):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
vram = segment.parse_segment_vram(self.yaml)
if vram is not None:
self.vram_start = vram
self.str_encoding: Optional[str] = (
self.yaml.get("str_encoding", None) if isinstance(self.yaml, dict) else None
)
self.spim_section: Optional[DisassemblerSection] = None
self.instr_category = rabbitizer.InstrCategory.CPU
if options.opts.platform == "ps2":
self.instr_category = rabbitizer.InstrCategory.R5900
elif options.opts.platform == "psx":
self.instr_category = rabbitizer.InstrCategory.R3000GTE
self.detect_redundant_function_end: Optional[bool] = (
self.yaml.get("detect_redundant_function_end", None)
if isinstance(self.yaml, dict)
else None
)
@property
def needs_symbols(self) -> bool:
return True
def get_linker_section(self) -> str:
return ".text"
def scan_code(self, rom_bytes, is_hasm=False):
if not isinstance(self.rom_start, int):
log.error(
f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'"
)
# Supposedly logic error, not user error
assert isinstance(self.rom_end, int), self.rom_end
# Supposedly logic error, not user error
segment_rom_start = self.get_most_parent().rom_start
assert isinstance(segment_rom_start, int), segment_rom_start
if not isinstance(self.vram_start, int):
log.error(
f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'"
)
self.spim_section = make_text_section(
self.rom_start,
self.rom_end,
self.vram_start,
self.name,
rom_bytes,
segment_rom_start,
self.get_exclusive_ram_id(),
)
assert self.spim_section is not None
self.spim_section.get_section().isHandwritten = is_hasm
self.spim_section.get_section().instrCat = self.instr_category
self.spim_section.get_section().detectRedundantFunctionEnd = (
self.detect_redundant_function_end
)
self.spim_section.analyze()
self.spim_section.set_comment_offset(self.rom_start)
for func in self.spim_section.get_section().symbolList:
assert isinstance(func, spimdisasm.mips.symbols.SymbolFunction)
self.process_insns(func)
# Process jumptable labels and pass them to spimdisasm
self.gather_jumptable_labels(rom_bytes)
for jtbl_label_vram in self.parent.jtbl_glabels_to_add:
sym = self.create_symbol(
jtbl_label_vram, True, type="jtbl_label", define=True
)
sym.type = "jtbl_label"
symbols.add_symbol_to_spim_section(self.spim_section.get_section(), sym)
def process_insns(
self,
func_spim: spimdisasm.mips.symbols.SymbolFunction,
):
assert isinstance(self.parent, CommonSegCode)
assert func_spim.vram is not None
assert func_spim.vramEnd is not None
assert self.spim_section is not None
self.parent: CommonSegCode = self.parent
symbols.create_symbol_from_spim_symbol(
self.get_most_parent(), func_spim.contextSym
)
# Gather symbols found by spimdisasm and create those symbols in splat's side
for referenced_vram in func_spim.instrAnalyzer.referencedVrams:
context_sym = self.spim_section.get_section().getSymbol(
referenced_vram, tryPlusOffset=False
)
if context_sym is not None:
if context_sym.type == spimdisasm.common.SymbolSpecialType.jumptable:
self.parent.jumptables[referenced_vram] = (
func_spim.vram,
func_spim.vramEnd,
)
symbols.create_symbol_from_spim_symbol(
self.get_most_parent(), context_sym
)
# Main loop
for i, insn in enumerate(func_spim.instructions):
instr_offset = i * 4
# update pointer accesses from this function
if instr_offset in func_spim.instrAnalyzer.symbolInstrOffset:
sym_address = func_spim.instrAnalyzer.symbolInstrOffset[instr_offset]
context_sym = self.spim_section.get_section().getSymbol(
sym_address, tryPlusOffset=False
)
if context_sym is not None:
sym = symbols.create_symbol_from_spim_symbol(
self.get_most_parent(), context_sym
)
if self.parent:
self.parent.check_rodata_sym(func_spim.vram, sym)
def print_file_boundaries(self):
if not self.show_file_boundaries or not self.spim_section:
return
assert isinstance(self.rom_start, int)
for in_file_offset in self.spim_section.get_section().fileBoundaries:
if (in_file_offset % 16) != 0:
continue
if not self.parent.reported_file_split:
self.parent.reported_file_split = True
# Look up for the last symbol in this boundary
sym_addr = 0
for sym in self.spim_section.get_section().symbolList:
symOffset = (
sym.inFileOffset - self.spim_section.get_section().inFileOffset
)
if in_file_offset == symOffset:
break
sym_addr = sym.vram
print(
f"\nSegment {self.name}, symbol at vram {sym_addr:X} ends with extra nops, indicating a likely file split."
)
print(
"File split suggestions for this segment will follow in config yaml format:"
)
print(f" - [0x{self.rom_start+in_file_offset:X}, {self.type}]")
def gather_jumptable_labels(self, rom_bytes):
assert isinstance(self.rom_start, int)
assert isinstance(self.vram_start, int)
# TODO: use the seg_symbols for this
# jumptables = [j.type == "jtbl" for j in self.seg_symbols]
for jumptable in self.parent.jumptables:
start, end = self.parent.jumptables[jumptable]
rom_offset = self.rom_start + jumptable - self.vram_start
if rom_offset <= 0:
return
while rom_offset:
word = rom_bytes[rom_offset : rom_offset + 4]
word_int = int.from_bytes(word, options.opts.endianness)
if word_int >= start and word_int <= end:
self.parent.jtbl_glabels_to_add.add(word_int)
else:
break
rom_offset += 4
def should_scan(self) -> bool:
return (
options.opts.is_mode_active("code")
and self.rom_start is not None
and self.rom_end is not None
)
def should_split(self) -> bool:
return (
self.extract and options.opts.is_mode_active("code") and self.should_scan()
) # only split if the segment was scanned first

8
tools/splat/segtypes/common/cpp.py
View File

@ -1,8 +0,0 @@
from segtypes.common.c import CommonSegC
class CommonSegCpp(CommonSegC):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.file_extension = "cpp"

151
tools/splat/segtypes/common/data.py
View File

@ -1,151 +0,0 @@
from pathlib import Path
from typing import Optional
from util import options, symbols, log
from segtypes.common.codesubsegment import CommonSegCodeSubsegment
from segtypes.common.group import CommonSegGroup
from disassembler_section import make_data_section
class CommonSegData(CommonSegCodeSubsegment, CommonSegGroup):
@staticmethod
def is_data() -> bool:
return True
def asm_out_path(self) -> Path:
typ = self.type
if typ.startswith("."):
typ = typ[1:]
return options.opts.data_path / self.dir / f"{self.name}.{typ}.s"
def out_path(self) -> Optional[Path]:
if self.type.startswith("."):
if self.sibling:
# C file
return self.sibling.out_path()
else:
# Implied C file
return options.opts.src_path / self.dir / f"{self.name}.c"
else:
# ASM
return self.asm_out_path()
def scan(self, rom_bytes: bytes):
CommonSegGroup.scan(self, rom_bytes)
if self.rom_start is not None and self.rom_end is not None:
self.disassemble_data(rom_bytes)
def split(self, rom_bytes: bytes):
super().split(rom_bytes)
if self.type.startswith(".") and not options.opts.disassemble_all:
return
if self.spim_section is None or not self.should_self_split():
return
path = self.asm_out_path()
path.parent.mkdir(parents=True, exist_ok=True)
self.print_file_boundaries()
with path.open("w", newline="\n") as f:
f.write('.include "macro.inc"\n\n')
preamble = options.opts.generated_s_preamble
if preamble:
f.write(preamble + "\n")
f.write(f".section {self.get_linker_section()}")
section_flags = self.get_section_flags()
if section_flags:
f.write(f', "{section_flags}"')
f.write("\n\n")
f.write(self.spim_section.disassemble())
def should_self_split(self) -> bool:
return options.opts.is_mode_active("data")
def should_scan(self) -> bool:
return True
def should_split(self) -> bool:
return True
def cache(self):
return [CommonSegCodeSubsegment.cache(self), CommonSegGroup.cache(self)]
def get_linker_section(self) -> str:
return ".data"
def get_linker_entries(self):
return CommonSegCodeSubsegment.get_linker_entries(self)
def disassemble_data(self, rom_bytes):
if not isinstance(self.rom_start, int):
log.error(
f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'"
)
# Supposedly logic error, not user error
assert isinstance(self.rom_end, int), self.rom_end
# Supposedly logic error, not user error
segment_rom_start = self.get_most_parent().rom_start
assert isinstance(segment_rom_start, int), segment_rom_start
if not isinstance(self.vram_start, int):
log.error(
f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'"
)
self.spim_section = make_data_section(
self.rom_start,
self.rom_end,
self.vram_start,
self.name,
rom_bytes,
segment_rom_start,
self.get_exclusive_ram_id(),
)
assert self.spim_section is not None
# Set rodata string encoding
# First check the global configuration
if options.opts.data_string_encoding is not None:
self.spim_section.get_section().stringEncoding = (
options.opts.data_string_encoding
)
# Then check the per-segment configuration in case we want to override the global one
if self.str_encoding is not None:
self.spim_section.get_section().stringEncoding = self.str_encoding
self.spim_section.analyze()
self.spim_section.set_comment_offset(self.rom_start)
rodata_encountered = False
for symbol in self.spim_section.get_section().symbolList:
symbols.create_symbol_from_spim_symbol(
self.get_most_parent(), symbol.contextSym
)
# Hint to the user that we are now in the .rodata section and no longer in the .data section (assuming rodata follows data)
if not rodata_encountered and self.get_most_parent().rodata_follows_data:
if symbol.contextSym.isJumpTable():
rodata_encountered = True
print(
f"Data segment {self.name}, symbol at vram {symbol.contextSym.vram:X} is a jumptable, indicating the start of the rodata section _may_ be near here."
)
print(
"Please note the real start of the rodata section may be way before this point."
)
if symbol.contextSym.vromAddress is not None:
print(f" - [0x{symbol.contextSym.vromAddress:X}, rodata]")

45
tools/splat/segtypes/common/databin.py
View File

@ -1,45 +0,0 @@
from pathlib import Path
from typing import Optional
from util import log, options
from segtypes.common.textbin import CommonSegTextbin
class CommonSegDatabin(CommonSegTextbin):
@staticmethod
def is_text() -> bool:
return False
@staticmethod
def is_data() -> bool:
return True
def get_linker_section(self) -> str:
return ".data"
def get_section_flags(self) -> Optional[str]:
return "wa"
def split(self, rom_bytes):
if self.rom_end is None:
log.error(
f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it"
)
self.write_bin(rom_bytes)
if self.sibling is None:
# textbin will write the incbin instead
s_path = self.out_path()
assert s_path is not None
s_path.parent.mkdir(parents=True, exist_ok=True)
with s_path.open("w") as f:
f.write('.include "macro.inc"\n\n')
preamble = options.opts.generated_s_preamble
if preamble:
f.write(preamble + "\n")
self.write_asm_contents(rom_bytes, f)

50
tools/splat/segtypes/common/decompressor.py
View File

@ -1,50 +0,0 @@
from typing import Optional, Any
from util import log, options
from util.n64.decompressor import Decompressor
from segtypes.n64.segment import N64Segment
class CommonSegDecompressor(N64Segment):
decompressor: Decompressor
compression_type = "" # "Mio0" -> filename.Mio0.o
def split(self, rom_bytes):
if self.decompressor is None:
log.error("Decompressor is not a standalone segment type")
out_dir = options.opts.asset_path / self.dir
out_dir.mkdir(parents=True, exist_ok=True)
if self.rom_end is None:
log.error(
f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it"
)
out_path = out_dir / f"{self.name}.bin"
with open(out_path, "wb") as f:
assert isinstance(self.rom_start, int)
assert isinstance(self.rom_end, int)
self.log(f"Decompressing {self.name}")
compressed_bytes = rom_bytes[self.rom_start : self.rom_end]
decompressed_bytes = self.decompressor.decompress(compressed_bytes)
f.write(decompressed_bytes)
self.log(f"Wrote {self.name} to {out_path}")
def get_linker_entries(self):
from segtypes.linker_entry import LinkerEntry
return [
LinkerEntry(
self,
[options.opts.asset_path / self.dir / f"{self.name}.bin"],
options.opts.asset_path
/ self.dir
/ f"{self.name}.{self.compression_type}",
self.get_linker_section_order(),
self.get_linker_section_linksection(),
self.is_noload(),
)
]

160
tools/splat/segtypes/common/group.py
View File

@ -1,160 +0,0 @@
from typing import List, Optional
from util import log
from segtypes.common.segment import CommonSegment
from segtypes.segment import Segment
class CommonSegGroup(CommonSegment):
def __init__(
self,
rom_start: Optional[int],
rom_end: Optional[int],
type: str,
name: str,
vram_start: Optional[int],
args: list,
yaml,
):
super().__init__(
rom_start,
rom_end,
type,
name,
vram_start,
args=args,
yaml=yaml,
)
self.subsegments: List[Segment] = self.parse_subsegments(yaml)
def get_next_seg_start(self, i, subsegment_yamls):
return (
self.rom_end
if i == len(subsegment_yamls) - 1
else Segment.parse_segment_start(subsegment_yamls[i + 1])
)
def parse_subsegments(self, yaml) -> List[Segment]:
ret: List[Segment] = []
if not yaml or "subsegments" not in yaml:
return ret
prev_start: Optional[int] = -1
last_rom_end = 0
for i, subsegment_yaml in enumerate(yaml["subsegments"]):
# endpos marker
if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1:
continue
typ = Segment.parse_segment_type(subsegment_yaml)
start = Segment.parse_segment_start(subsegment_yaml)
segment_class = Segment.get_class_for_type(typ)
end = self.get_next_seg_start(i, yaml["subsegments"])
if start is None:
# Attempt to infer the start address
if i == 0:
# The start address of this segment is the start address of the group
start = self.rom_start
else:
# The start address is the end address of the previous segment
start = last_rom_end
if start is not None and end is None:
est_size = segment_class.estimate_size(subsegment_yaml)
if est_size is not None:
end = start + est_size
if start is not None and prev_start is not None and start < prev_start:
log.error(
f"Error: Group segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})"
)
vram = None
if start is not None:
most_parent = self.get_most_parent()
if (
most_parent.vram_start is not None
and most_parent.rom_start is not None
):
vram = most_parent.vram_start + start - most_parent.rom_start
if segment_class.is_noload():
# Pretend bss's rom address is after the last actual rom segment
start = last_rom_end
# and it has a rom size of zero
end = last_rom_end
segment: Segment = Segment.from_yaml(
segment_class, subsegment_yaml, start, end, vram
)
segment.parent = self
if segment.special_vram_segment:
self.special_vram_segment = True
ret.append(segment)
prev_start = start
if end is not None:
last_rom_end = end
return ret
@property
def needs_symbols(self) -> bool:
for seg in self.subsegments:
if seg.needs_symbols:
return True
return False
def get_linker_entries(self):
return [entry for sub in self.subsegments for entry in sub.get_linker_entries()]
def scan(self, rom_bytes):
for sub in self.subsegments:
if sub.should_scan():
sub.scan(rom_bytes)
def split(self, rom_bytes):
for sub in self.subsegments:
if sub.should_split():
sub.split(rom_bytes)
def should_split(self) -> bool:
return self.extract
def should_scan(self) -> bool:
return self.extract
def cache(self):
c = []
for sub in self.subsegments:
c.append(sub.cache())
return c
def get_subsegment_for_ram(self, addr: int) -> Optional[Segment]:
for sub in self.subsegments:
if sub.contains_vram(addr):
return sub
return None
def get_next_subsegment_for_ram(self, addr: int) -> Optional[Segment]:
"""
Returns the first subsegment which comes after the specified address,
or None in case this address belongs to the last subsegment of this group
"""
for sub in self.subsegments:
if sub.vram_start is None:
continue
assert isinstance(sub.vram_start, int)
if sub.vram_start > addr:
return sub
return None

24
tools/splat/segtypes/common/hasm.py
View File

@ -1,24 +0,0 @@
from segtypes.common.asm import CommonSegAsm
class CommonSegHasm(CommonSegAsm):
def scan(self, rom_bytes: bytes):
if (
self.rom_start is not None
and self.rom_end is not None
and self.rom_start != self.rom_end
):
self.scan_code(rom_bytes, is_hasm=True)
def split(self, rom_bytes: bytes):
if not self.rom_start == self.rom_end and self.spim_section is not None:
out_path = self.out_path()
if out_path and not out_path.exists():
out_path.parent.mkdir(parents=True, exist_ok=True)
self.print_file_boundaries()
with open(out_path, "w", newline="\n") as f:
for line in self.get_file_header():
f.write(line + "\n")
f.write(self.spim_section.disassemble())

46
tools/splat/segtypes/common/header.py
View File

@ -1,46 +0,0 @@
from pathlib import Path
from util import options
from segtypes.common.segment import CommonSegment
class CommonSegHeader(CommonSegment):
@staticmethod
def is_data() -> bool:
return True
def should_split(self):
return self.extract and options.opts.is_mode_active("code")
@staticmethod
def get_line(typ, data, comment):
if typ == "ascii":
text = data.decode("ASCII").strip()
text = text.replace("\x00", "\\0") # escape NUL chars
dstr = '"' + text + '"'
else: # .word, .byte
dstr = "0x" + data.hex().upper()
dstr = dstr.ljust(20 - len(typ))
return f".{typ} {dstr} /* {comment} */"
def out_path(self) -> Path:
return options.opts.asm_path / self.dir / f"{self.name}.s"
def parse_header(self, rom_bytes):
return []
def split(self, rom_bytes):
header_lines = self.parse_header(rom_bytes)
src_path = self.out_path()
src_path.parent.mkdir(parents=True, exist_ok=True)
with open(src_path, "w", newline="\n") as f:
f.write("\n".join(header_lines))
self.log(f"Wrote {self.name} to {src_path}")
@staticmethod
def get_default_name(addr):
return "header"

62
tools/splat/segtypes/common/lib.py
View File

@ -1,62 +0,0 @@
from pathlib import Path
from typing import Optional
from util import log, options
from segtypes.linker_entry import LinkerEntry
from segtypes.n64.segment import N64Segment
class CommonSegLib(N64Segment):
def __init__(
self,
rom_start: Optional[int],
rom_end: Optional[int],
type: str,
name: str,
vram_start: Optional[int],
args: list,
yaml,
):
super().__init__(
rom_start,
rom_end,
type,
name,
vram_start,
args=args,
yaml=yaml,
)
if isinstance(yaml, dict):
log.error("Error: 'dict' not currently supported for 'lib' segment")
return
if len(args) < 1:
log.error(f"Error: {self.name} is missing object file")
return
self.extract = False
if len(args) > 1:
self.object, self.section = args[0], args[1]
else:
self.object, self.section = args[0], ".text"
def get_linker_section(self) -> str:
return self.section
def get_linker_entries(self):
path = options.opts.lib_path / self.name
object_path = Path(f"{path}.a:{self.object}.o")
return [
LinkerEntry(
self,
[path],
object_path,
self.get_linker_section_order(),
self.get_linker_section_linksection(),
self.is_noload(),
)
]

6
tools/splat/segtypes/common/rdata.py
View File

@ -1,6 +0,0 @@
from segtypes.common.rodata import CommonSegRodata
class CommonSegRdata(CommonSegRodata):
def get_linker_section(self) -> str:
return ".rdata"

106
tools/splat/segtypes/common/rodata.py
View File

@ -1,106 +0,0 @@
from typing import Optional, Set, Tuple
import spimdisasm
from segtypes.segment import Segment
from util import log, options, symbols
from segtypes.common.data import CommonSegData
from disassembler_section import make_rodata_section
class CommonSegRodata(CommonSegData):
def get_linker_section(self) -> str:
return ".rodata"
@staticmethod
def is_data() -> bool:
return False
@staticmethod
def is_rodata() -> bool:
return True
def get_possible_text_subsegment_for_symbol(
self, rodata_sym: spimdisasm.mips.symbols.SymbolBase
) -> Optional[Tuple[Segment, spimdisasm.common.ContextSymbol]]:
# Check if this rodata segment does not have a corresponding code file, try to look for one
if self.sibling is not None or not options.opts.pair_rodata_to_text:
return None
if not rodata_sym.shouldMigrate():
return None
if len(rodata_sym.contextSym.referenceFunctions) != 1:
return None
func = list(rodata_sym.contextSym.referenceFunctions)[0]
text_segment = self.parent.get_subsegment_for_ram(func.vram)
if text_segment is None or not text_segment.is_text():
return None
return text_segment, func
def disassemble_data(self, rom_bytes):
if not isinstance(self.rom_start, int):
log.error(
f"Segment '{self.name}' (type '{self.type}') requires a rom_start. Got '{self.rom_start}'"
)
# Supposedly logic error, not user error
assert isinstance(self.rom_end, int), self.rom_end
# Supposedly logic error, not user error
segment_rom_start = self.get_most_parent().rom_start
assert isinstance(segment_rom_start, int), segment_rom_start
if not isinstance(self.vram_start, int):
log.error(
f"Segment '{self.name}' (type '{self.type}') requires a vram address. Got '{self.vram_start}'"
)
self.spim_section = make_rodata_section(
self.rom_start,
self.rom_end,
self.vram_start,
self.name,
rom_bytes,
segment_rom_start,
self.get_exclusive_ram_id(),
)
assert self.spim_section is not None
# Set rodata string encoding
# First check the global configuration
if options.opts.string_encoding is not None:
self.spim_section.get_section().stringEncoding = (
options.opts.string_encoding
)
# Then check the per-segment configuration in case we want to override the global one
if self.str_encoding is not None:
self.spim_section.get_section().stringEncoding = self.str_encoding
self.spim_section.analyze()
self.spim_section.set_comment_offset(self.rom_start)
possible_text_segments: Set[Segment] = set()
for symbol in self.spim_section.get_section().symbolList:
generated_symbol = symbols.create_symbol_from_spim_symbol(
self.get_most_parent(), symbol.contextSym
)
generated_symbol.linker_section = self.get_linker_section()
possible_text = self.get_possible_text_subsegment_for_symbol(symbol)
if possible_text is not None:
text_segment, refenceeFunction = possible_text
if text_segment not in possible_text_segments:
print(
f"\nRodata segment '{self.name}' may belong to the text segment '{text_segment.name}'"
)
print(
f" Based on the usage from the function {refenceeFunction.getName()} to the symbol {symbol.getName()}"
)
possible_text_segments.add(text_segment)

45
tools/splat/segtypes/common/rodatabin.py
View File

@ -1,45 +0,0 @@
from pathlib import Path
from typing import Optional
from util import log, options
from segtypes.common.textbin import CommonSegTextbin
class CommonSegRodatabin(CommonSegTextbin):
@staticmethod
def is_text() -> bool:
return False
@staticmethod
def is_rodata() -> bool:
return True
def get_linker_section(self) -> str:
return ".rodata"
def get_section_flags(self) -> Optional[str]:
return "a"
def split(self, rom_bytes):
if self.rom_end is None:
log.error(
f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it"
)
self.write_bin(rom_bytes)
if self.sibling is None:
# textbin will write the incbin instead
s_path = self.out_path()
assert s_path is not None
s_path.parent.mkdir(parents=True, exist_ok=True)
with s_path.open("w") as f:
f.write('.include "macro.inc"\n\n')
preamble = options.opts.generated_s_preamble
if preamble:
f.write(preamble + "\n")
self.write_asm_contents(rom_bytes, f)

6
tools/splat/segtypes/common/sbss.py
View File

@ -1,6 +0,0 @@
from segtypes.common.data import CommonSegData
class CommonSegSbss(CommonSegData):
def get_linker_section(self) -> str:
return ".sbss"

6
tools/splat/segtypes/common/sdata.py
View File

@ -1,6 +0,0 @@
from segtypes.common.data import CommonSegData
class CommonSegSdata(CommonSegData):
def get_linker_section(self) -> str:
return ".sdata"

5
tools/splat/segtypes/common/segment.py
View File

@ -1,5 +0,0 @@
from segtypes.segment import Segment
class CommonSegment(Segment):
pass

114
tools/splat/segtypes/common/textbin.py
View File

@ -1,114 +0,0 @@
from pathlib import Path
from typing import Optional, TextIO
from util import log, options
from segtypes.common.segment import CommonSegment
class CommonSegTextbin(CommonSegment):
@staticmethod
def is_text() -> bool:
return True
def get_linker_section(self) -> str:
return ".text"
def get_section_flags(self) -> Optional[str]:
return "ax"
def out_path(self) -> Optional[Path]:
return options.opts.data_path / self.dir / f"{self.name}.s"
def bin_path(self) -> Path:
typ = self.type
if typ.startswith("."):
typ = typ[1:]
return options.opts.asset_path / self.dir / f"{self.name}.{typ}.bin"
def write_bin(self, rom_bytes):
binpath = self.bin_path()
binpath.parent.mkdir(parents=True, exist_ok=True)
assert isinstance(self.rom_start, int)
assert isinstance(self.rom_end, int)
binpath.write_bytes(rom_bytes[self.rom_start : self.rom_end])
self.log(f"Wrote {self.name} to {binpath}")
def write_asm_contents(self, rom_bytes, f: TextIO):
binpath = self.bin_path()
asm_label = options.opts.asm_function_macro
if not self.is_text():
asm_label = options.opts.asm_data_macro
assert isinstance(self.rom_start, int)
assert isinstance(self.rom_end, int)
f.write(f".section {self.get_linker_section()}")
section_flags = self.get_section_flags()
if section_flags:
f.write(f', "{section_flags}"')
f.write("\n\n")
# Check if there's a symbol at this address
sym = None
vram = self.rom_to_ram(self.rom_start)
if vram is not None:
sym = self.get_symbol(vram, in_segment=True)
if sym is not None:
f.write(f"{asm_label} {sym.name}\n")
sym.defined = True
f.write(f'.incbin "{binpath}"\n')
if sym is not None:
if self.is_text() and options.opts.asm_end_label != "":
f.write(f"{options.opts.asm_end_label} {sym.name}\n")
if sym.given_name_end is not None:
if (
sym.given_size is None
or sym.given_size == self.rom_end - self.rom_start
):
f.write(f"{asm_label} {sym.given_name_end}\n")
def split(self, rom_bytes):
if self.rom_end is None:
log.error(
f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it"
)
self.write_bin(rom_bytes)
s_path = self.out_path()
assert s_path is not None
s_path.parent.mkdir(parents=True, exist_ok=True)
with s_path.open("w") as f:
f.write('.include "macro.inc"\n\n')
preamble = options.opts.generated_s_preamble
if preamble:
f.write(preamble + "\n")
self.write_asm_contents(rom_bytes, f)
# We check against CommonSegTextbin instead of the specific type because the other incbins inherit from this class
if isinstance(self.data_sibling, CommonSegTextbin):
f.write("\n")
self.data_sibling.write_asm_contents(rom_bytes, f)
if isinstance(self.rodata_sibling, CommonSegTextbin):
f.write("\n")
self.rodata_sibling.write_asm_contents(rom_bytes, f)
def should_scan(self) -> bool:
return self.rom_start is not None and self.rom_end is not None
def should_split(self) -> bool:
return (
self.extract and self.should_scan()
) # only split if the segment was scanned first

8
tools/splat/segtypes/gc/apploader.py
View File

@ -1,8 +0,0 @@
import struct
from pathlib import Path
from segtypes.gc.segment import GCSegment
class GcSegApploader(GCSegment):
pass

67
tools/splat/segtypes/gc/bi2.py
View File

@ -1,67 +0,0 @@
import struct
from pathlib import Path
from util import options
from segtypes.gc.segment import GCSegment
class GcSegBi2(GCSegment):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def split(self, bi2_bytes):
lines = []
# Gathering variables
debug_monitor_size = struct.unpack_from(">I", bi2_bytes, 0x00)[0]
simulated_memory_size = struct.unpack_from(">I", bi2_bytes, 0x04)[0]
argument_offset = struct.unpack_from(">I", bi2_bytes, 0x08)[0]
debug_flag = struct.unpack_from(">I", bi2_bytes, 0x0C)[0]
track_offset = struct.unpack_from(">I", bi2_bytes, 0x10)[0]
track_size = struct.unpack_from(">I", bi2_bytes, 0x14)[0]
country_code_bi2 = struct.unpack_from(">I", bi2_bytes, 0x18)[0]
unk_int = struct.unpack_from(">I", bi2_bytes, 0x1C)[0]
unk_int_2 = struct.unpack_from(">I", bi2_bytes, 0x20)[0]
# Outputting .s file
lines.append(f"# GameCube disc image bi2 data, located at 0x440 in the disc.\n")
lines.append(f"# Generated by splat.\n\n")
lines.append(f".section .data\n\n")
lines.append(f"debug_monitor_size: .long 0x{debug_monitor_size:08X}\n")
lines.append(f"simulated_memory_size: .long 0x{simulated_memory_size:08X}\n\n")
lines.append(f"argument_offset: .long 0x{argument_offset:08X}\n\n")
lines.append(f"debug_flag: .long 0x{debug_flag:08X}\n\n")
lines.append(f"track_offset: .long 0x{track_offset:08X}\n")
lines.append(f"track_size: .long 0x{track_size:08X}\n\n")
lines.append(f"country_code_bi2: .long 0x{country_code_bi2:08X}\n\n")
lines.append(f"ukn_int_bi2: .long 0x{unk_int:08X}\n")
lines.append(f"ukn_int_bi2_2: .long 0x{unk_int_2:08X}\n\n")
lines.append(f".fill 0x00001FDC\n\n")
out_path = self.out_path()
out_path.parent.mkdir(parents=True, exist_ok=True)
with open(out_path, "w", encoding="utf-8") as f:
f.writelines(lines)
return
def should_split(self) -> bool:
return True
def out_path(self) -> Path:
return options.opts.asm_path / "sys" / "bi2.s"

117
tools/splat/segtypes/gc/bootinfo.py
View File

@ -1,117 +0,0 @@
import struct
from pathlib import Path
from util import options
from segtypes.gc.segment import GCSegment
class GcSegBootinfo(GCSegment):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def split(self, iso_bytes):
lines = []
gc_dvd_magic = struct.unpack_from(">I", iso_bytes, 0x1C)[0]
assert gc_dvd_magic == 0xC2339F3D
# Gathering variables
system_code = chr(iso_bytes[0x00])
game_code = iso_bytes[0x01:0x03].decode("utf-8")
region_code = chr(iso_bytes[0x03])
publisher_code = iso_bytes[0x04:0x06].decode("utf-8")
disc_id = iso_bytes[0x06]
game_version = iso_bytes[0x07]
audio_streaming = iso_bytes[0x08]
stream_buffer_size = iso_bytes[0x09]
name = iso_bytes[0x20:0x400].decode("utf-8").strip("\x00")
name_padding_len = 0x3E0 - len(name)
# The following is from YAGCD, don't know what they were for:
# https://web.archive.org/web/20220528011846/http://hitmen.c02.at/files/yagcd/yagcd/chap13.html#sec13.1
apploader_size = struct.unpack_from(">I", iso_bytes, 0x400)[0]
debug_monitor_address = struct.unpack_from(">I", iso_bytes, 0x404)[0]
# These on the other hand are easy to understand
dol_offset = struct.unpack_from(">I", iso_bytes, 0x420)[0]
fst_offset = struct.unpack_from(">I", iso_bytes, 0x424)[0]
fst_size = struct.unpack_from(">I", iso_bytes, 0x428)[0]
fst_max_size = struct.unpack_from(">I", iso_bytes, 0x42C)[0]
user_position = struct.unpack_from(">I", iso_bytes, 0x430)[0]
user_length = struct.unpack_from(">I", iso_bytes, 0x434)[0]
unk_int = struct.unpack_from(">I", iso_bytes, 0x438)[0]
# Outputting .s file
lines.append(f"# GameCube disc image boot data, located at 0x00 in the disc.\n")
lines.append(f"# Generated by splat.\n\n")
lines.append(f".section .data\n\n")
# Game ID stuff
lines.append(f'system_code: .ascii "{system_code}"\n')
lines.append(f'game_code: .ascii "{game_code}"\n')
lines.append(f'region_code: .ascii "{region_code}"\n')
lines.append(f'publisher_code: .ascii "{publisher_code}"\n\n')
lines.append(f"disc_id: .byte {disc_id:X}\n")
lines.append(f"game_version: .byte {game_version:X}\n")
lines.append(f"audio_streaming: .byte {audio_streaming:X}\n")
lines.append(f"stream_buffer_size: .byte {stream_buffer_size:X}\n\n")
# padding
lines.append(f".fill 0x12\n\n")
# GC magic number
lines.append(f"gc_magic: .long 0xC2339F3D\n\n")
# Long game name
lines.append(f'game_name: .ascii "{name}"\n')
lines.append(f".org 0x400\n\n")
lines.append(f"apploader_size: .long 0x{apploader_size:08X}\n\n")
# Unknown stuff gleaned from YAGCD
lines.append(f"debug_monitor_address: .long 0x{debug_monitor_address:08X}\n\n")
# More padding
lines.append(f".fill 0x18\n\n")
# DOL and FST data
lines.append(f"dol_offset: .long 0x{dol_offset:08X}\n")
lines.append(f"fst_offset: .long 0x{fst_offset:08X}\n\n")
lines.append(
f"# The FST is only allocated once per game boot, even in games with multiple disks. fst_max_size is used to ensure that\n"
)
lines.append(
f"# there is enough space allocated for the FST in the event that a game spans multiple disks, and one disk has a larger FST than another.\n"
)
lines.append(f"fst_size: .long 0x{fst_size:08X}\n")
lines.append(f"fst_max_size: .long 0x{fst_max_size:08X}\n\n")
# Honestly not sure what this data is for
lines.append(f"# Not even YAGCD knows what these are for.\n")
lines.append(f"user_position: .long 0x{user_position:08X}\n")
lines.append(f"user_length: .long 0x{user_length:08X}\n")
lines.append(f"unk_int: .long 0x{unk_int:08X}\n\n")
# Final padding
lines.append(f".word 0\n")
out_path = self.out_path()
out_path.parent.mkdir(parents=True, exist_ok=True)
with open(out_path, "w", encoding="utf-8") as f:
f.writelines(lines)
return
def should_split(self) -> bool:
return True
def out_path(self) -> Path:
return options.opts.asm_path / "sys" / "boot.s"

8
tools/splat/segtypes/gc/dol.py
View File

@ -1,8 +0,0 @@
import struct
from pathlib import Path
from segtypes.gc.segment import GCSegment
class GcSegDol(GCSegment):
pass

68
tools/splat/segtypes/gc/dolheader.py
View File

@ -1,68 +0,0 @@
from util import options
from segtypes.common.header import CommonSegHeader
class DolSegHeader(CommonSegHeader):
def parse_header(self, dol_bytes):
header_lines = []
header_lines.append(".section .data\n")
# Text file offsets
for i in range(0x00, 0x1C, 4):
header_lines.append(
self.get_line("word", dol_bytes[i : i + 4], f"Text {i / 4} Offset")
)
# Data file offsets
for i in range(0x1C, 0x48, 4):
header_lines.append(
self.get_line("word", dol_bytes[i : i + 4], f"Data {i / 4} Offset")
)
# Text RAM addresses
for i in range(0x48, 0x64, 4):
header_lines.append(
self.get_line(
"word",
dol_bytes[i : i + 4],
f"Text {(i - 0x48) / 4} Address",
)
)
# Data RAM addresses
for i in range(0x64, 0x90, 4):
header_lines.append(
self.get_line(
"word",
dol_bytes[i : i + 4],
f"Data {(i - 0x64) / 4} Address",
)
)
# Text file sizes
for i in range(0x90, 0xAC, 4):
header_lines.append(
self.get_line(
"word",
dol_bytes[i : i + 4],
f"Text {(i - 0x90) / 4} Size",
)
)
# Data file sizes
for i in range(0xAC, 0xD8, 4):
header_lines.append(
self.get_line(
"word",
dol_bytes[i : i + 4],
f"Data {(i - 0xAC) / 4} Size",
)
)
# BSS RAM address
header_lines.append(self.get_line("word", dol_bytes[0xD8:0xDC], "BSS Address"))
# BSS size
header_lines.append(self.get_line("word", dol_bytes[0xDC:0xE0], "BSS Size"))
# Entry point
header_lines.append(self.get_line("word", dol_bytes[0xE0:0xE4], "Entry Point"))
return header_lines

8
tools/splat/segtypes/gc/fst.py
View File

@ -1,8 +0,0 @@
import struct
from pathlib import Path
from segtypes.gc.segment import GCSegment
class GcSegFst(GCSegment):
pass

321
tools/splat/segtypes/gc/rarc.py
View File

@ -1,321 +0,0 @@
import struct
from enum import IntEnum
from pathlib import Path
from typing import List, Optional
from util import options
from util.gc.gcutil import read_string_from_bytes
from util.n64.Yay0decompress import Yay0Decompressor
from segtypes.gc.segment import GCSegment
# Represents the RARC archive format used by first-party Nintendo games.
class GcSegRarc(GCSegment):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
def split(self, file_bytes):
assert self.file_path is not None
archive = GCRARCArchive(self.file_path, file_bytes)
archive.build_hierarchy(file_bytes)
archive.emit(file_bytes)
def should_split(self) -> bool:
return True
class GCRARCArchive:
def __init__(self, file_path: Path, file_bytes):
self.file_path = file_path
self.compression = "none"
file_bytes = self.try_decompress_archive(file_bytes)
self.magic = struct.unpack_from(">I", file_bytes, 0x0000)[0]
self.file_size = struct.unpack_from(">I", file_bytes, 0x0004)[0]
self.data_header_offset = struct.unpack_from(">I", file_bytes, 0x0008)[0]
self.file_data_offset = struct.unpack_from(">I", file_bytes, 0x000C)[0] + 0x0020
self.total_file_data_size = struct.unpack_from(">I", file_bytes, 0x0010)[0]
self.mram_preload_size = struct.unpack_from(">I", file_bytes, 0x0014)[0]
self.aram_preload_size = struct.unpack_from(">I", file_bytes, 0x0018)[0]
self.data_header = GCRARCDataHeader(self.data_header_offset, file_bytes)
self.nodes: List[GCRARCNode] = []
def try_decompress_archive(self, file_bytes):
compression_scheme = struct.unpack_from(">I", file_bytes, 0x0000)[0]
# Yaz0
if compression_scheme == 0x59617A30:
self.compression = "yaz0"
return file_bytes
# Yay0
elif compression_scheme == 0x59617930:
self.compression = "yay0"
return Yay0Decompressor.decompress(file_bytes)
# Not compressed!
else:
return file_bytes
def build_hierarchy(self, file_bytes):
string_table_offset = self.data_header.string_table_offset
string_table_size = self.data_header.string_table_size
string_table_bytes = file_bytes[
string_table_offset : string_table_offset + string_table_size
]
# Load the file entries into their corresponding nodes.
for i in range(self.data_header.node_count):
offset = self.data_header.node_offset + i * 0x10
new_node = GCRARCNode(offset, file_bytes, string_table_bytes)
new_node.get_entries(
self.data_header.file_entry_offset, file_bytes, string_table_bytes
)
self.nodes.append(new_node)
# Now, organize the nodes into a hierarchy.
for n in self.nodes:
for e in n.entries:
# We're only looking for directory nodes, so ignore files.
if e.flags & int(GCRARCFlags.IS_FILE) != 0x00:
continue
if e.name == "." or e.name == "..":
continue
# This is the node that the current entry corresponds to.
dir_node = self.nodes[e.data_offset]
# Set up hierarchy relationship.
dir_node.parent = n
n.children.append(dir_node)
def emit(self, file_bytes):
assert options.opts.filesystem_path is not None
rel_path = self.file_path.relative_to(options.opts.filesystem_path / "files")
arc_root_path = options.opts.asset_path / rel_path.with_suffix("")
self.nodes[0].emit_to_filesystem_recursive(
arc_root_path, self.file_data_offset, file_bytes
)
self.emit_config(arc_root_path)
def emit_config(self, config_path: Path):
lines = []
lines.append(f'name: "{self.file_path.name}"\n')
if self.compression != "none":
lines.append(f"compression: {self.compression}\n")
lines.append(f"next_file_id: 0x{self.data_header.next_free_file_id:04X}\n")
lines.append(
f"sync_file_ids_to_indices: {self.data_header.sync_file_ids_to_indices}\n"
)
root_node = self.nodes[0]
lines.append("root_dir:\n")
lines.append(f' res_type: "{root_node.resource_type}"\n')
lines.append(f' name: "{root_node.name}"\n')
if len(root_node.entries) != 0:
lines.append(" entries:\n")
for e in root_node.entries:
entry_config = e.emit_config(2)
if entry_config != None:
lines.extend(entry_config)
if len(root_node.children) != 0:
lines.append(" subdirs:\n")
for n in root_node.children:
node_config = n.emit_config(2)
if node_config != None:
lines.extend(node_config)
with open(config_path / "arcinfo.yaml", "w", newline="\n") as f:
f.writelines(lines)
class GCRARCDataHeader:
def __init__(self, offset, file_bytes):
self.node_count = struct.unpack_from(">I", file_bytes, offset + 0x0000)[0]
self.node_offset = (
struct.unpack_from(">I", file_bytes, offset + 0x0004)[0] + 0x0020
)
self.file_entry_count = struct.unpack_from(">I", file_bytes, offset + 0x0008)[0]
self.file_entry_offset = (
struct.unpack_from(">I", file_bytes, offset + 0x000C)[0] + 0x0020
)
self.string_table_size = struct.unpack_from(">I", file_bytes, offset + 0x0010)[
0
]
self.string_table_offset = (
struct.unpack_from(">I", file_bytes, offset + 0x0014)[0] + 0x0020
)
self.next_free_file_id = struct.unpack_from(">H", file_bytes, offset + 0x0018)[
0
]
self.sync_file_ids_to_indices = bool(file_bytes[offset + 0x001A])
class GCRARCNode:
def __init__(self, offset, file_bytes, string_table_bytes):
self.resource_type = file_bytes[offset + 0x0000 : offset + 0x0004].decode(
"utf-8"
)
self.name_offset = struct.unpack_from(">I", file_bytes, offset + 0x0004)[0]
self.name_hash = struct.unpack_from(">H", file_bytes, offset + 0x0008)[0]
self.file_entry_count = struct.unpack_from(">H", file_bytes, offset + 0x000A)[0]
self.first_file_entry_index = struct.unpack_from(
">I", file_bytes, offset + 0x000C
)[0]
self.name = read_string_from_bytes(self.name_offset, string_table_bytes)
self.entries = []
self.parent: Optional[GCRARCNode] = None
self.children = []
def get_entries(self, file_entry_offset, file_bytes, string_table_bytes):
for i in range(self.file_entry_count):
entry_offset = file_entry_offset + (self.first_file_entry_index + i) * 0x14
new_entry = GCRARCFileEntry(entry_offset, file_bytes, string_table_bytes)
new_entry.parent_node = self
self.entries.append(new_entry)
def emit_to_filesystem_recursive(
self, root_path: Path, file_data_offset, file_bytes
):
dir_path = root_path / self.get_full_directory_path()
dir_path.mkdir(parents=True, exist_ok=True)
for n in self.children:
n.emit_to_filesystem_recursive(root_path, file_data_offset, file_bytes)
for e in self.entries:
e.emit_to_filesystem(root_path, file_data_offset, file_bytes)
def emit_config(self, level):
lines = []
lines.append(" " * level + f'- res_type: "{self.resource_type}"\n')
lines.append(" " * level + f' name: "{self.name}"\n')
if len(self.entries) != 0:
lines.append(" " * level + " entries:\n")
for e in self.entries:
entry_config = e.emit_config(level + 1)
if entry_config != None:
lines.extend(entry_config)
if len(self.children) != 0:
lines.append(" " * level + " subdirs:\n")
for n in self.children:
node_config = n.emit_config(level + 1)
if node_config != None:
lines.extend(node_config)
return lines
def print_recursive(self, level):
print((" " * level) + self.name)
for n in self.children:
n.print_recursive(level + 1)
def get_full_directory_path(self):
path_components: List[str] = []
node: Optional[GCRARCNode] = self
while node is not None:
path_components.insert(0, node.name)
node = node.parent
return Path(*path_components)
class GCRARCFileEntry:
def __init__(self, offset, file_bytes, string_table_bytes):
self.file_id = struct.unpack_from(">H", file_bytes, offset + 0x0000)[0]
self.name_hash = struct.unpack_from(">H", file_bytes, offset + 0x0002)[0]
self.flags = file_bytes[offset + 0x0004]
self.name_offset = (
struct.unpack_from(">I", file_bytes, offset + 0x0004)[0] & 0x00FFFFFF
)
self.data_offset = struct.unpack_from(">I", file_bytes, offset + 0x0008)[0]
self.data_size = struct.unpack_from(">I", file_bytes, offset + 0x000C)[0]
self.name = read_string_from_bytes(self.name_offset, string_table_bytes)
self.parent_node: Optional[GCRARCNode] = None
def emit_to_filesystem(self, dir_path: Path, file_data_offset, file_bytes):
if self.flags & int(GCRARCFlags.IS_DIR) != 0x00:
return
file_path = dir_path / self.get_full_file_path()
file_data = file_bytes[
file_data_offset
+ self.data_offset : file_data_offset
+ self.data_offset
+ self.data_size
]
with open(file_path, "wb") as f:
f.write(file_data)
def emit_config(self, level):
if self.flags & int(GCRARCFlags.IS_DIR) != 0x00:
return
lines = []
lines.append(" " * level + f' - name: "{self.name}"\n')
lines.append(" " * level + f" file_id: 0x{self.file_id:04X}\n")
if self.flags & int(GCRARCFlags.IS_COMPRESSED) != 0x00:
if self.flags & int(GCRARCFlags.IS_YAZ0_COMPRESSED) != 0x00:
lines.append(" " * level + f" compression: yaz0\n")
else:
lines.append(" " * level + f" compression: yay0\n")
if self.flags & int(GCRARCFlags.PRELOAD_TO_MRAM) == 0x00:
if self.flags & int(GCRARCFlags.PRELOAD_TO_ARAM) != 0x00:
lines.append(" " * level + f" preload_type: aram\n")
else:
lines.append(" " * level + f" preload_type: dvd\n")
return lines
def get_full_file_path(self):
path_components = [self.name]
node = self.parent_node
while node is not None:
path_components.insert(0, node.name)
node = node.parent
return Path("/".join(path_components))
class GCRARCFlags(IntEnum):
IS_FILE = 0x01
IS_DIR = 0x02
IS_COMPRESSED = 0x04
PRELOAD_TO_MRAM = 0x10
PRELOAD_TO_ARAM = 0x20
LOAD_FROM_DVD = 0x40
IS_YAZ0_COMPRESSED = 0x80

114
tools/splat/segtypes/gc/relheader.py
View File

@ -1,114 +0,0 @@
from util import options
from segtypes.common.header import CommonSegHeader
class RelSegHeader(CommonSegHeader):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
if isinstance(self.yaml, dict):
self.version: int = self.yaml.get("version", 0)
def parse_header(self, rel_bytes):
header_lines = []
header_lines.append(".section .data\n")
# Module ID
header_lines.append(self.get_line("word", rel_bytes[0x00:0x04], "Module ID"))
# Next module (filled at runtime)
header_lines.append(self.get_line("word", rel_bytes[0x04:0x08], "Next Module"))
# Last module (filled at runtime)
header_lines.append(self.get_line("word", rel_bytes[0x08:0x0C], "Last Module"))
# Section count
header_lines.append(
self.get_line("word", rel_bytes[0x0C:0x10], "Section Count")
)
# Section table offset
header_lines.append(
self.get_line("word", rel_bytes[0x10:0x14], "Section Table Offset")
)
# Module name offset (might be null)
header_lines.append(
self.get_line("word", rel_bytes[0x14:0x18], "Module Name Offset")
)
# Module name length
header_lines.append(
self.get_line("word", rel_bytes[0x18:0x1C], "Module Name Length")
)
# REL format version
header_lines.append(
self.get_line("word", rel_bytes[0x1C:0x20], "REL Format Version")
)
# BSS size
header_lines.append(self.get_line("word", rel_bytes[0x20:0x24], "BSS Size"))
# Relocation table offset
header_lines.append(
self.get_line("word", rel_bytes[0x24:0x28], "Relocation Table Offset")
)
# Import table offset
header_lines.append(
self.get_line("word", rel_bytes[0x28:0x2C], "Import Table Offset")
)
# Import table size
header_lines.append(
self.get_line("word", rel_bytes[0x2C:0x30], "Import Table Size")
)
# Prolog section index
header_lines.append(
self.get_line("byte", rel_bytes[0x30:0x31], "Prolog Section Index")
)
# Epilog section index
header_lines.append(
self.get_line("byte", rel_bytes[0x31:0x32], "Epilog Section Index")
)
# Unresolved section index
header_lines.append(
self.get_line("byte", rel_bytes[0x32:0x33], "Unresolved Section Index")
)
# BSS section index (filled at runtime)
header_lines.append(
self.get_line("byte", rel_bytes[0x33:0x34], "BSS Section Index")
)
# Prolog function offset
header_lines.append(
self.get_line("word", rel_bytes[0x34:0x38], "Prolog Function Offset")
)
# Epilog function offset
header_lines.append(
self.get_line("word", rel_bytes[0x38:0x3C], "Epilog Function Offset")
)
# Unresolved function offset
header_lines.append(
self.get_line("word", rel_bytes[0x3C:0x40], "Unresolved Function Offset")
)
# Version 1 is only 0x40 bytes long
if self.version <= 1:
return header_lines
# Alignment constraint
header_lines.append(
self.get_line("word", rel_bytes[0x40:0x44], "Alignment Constraint")
)
# BSS alignment constraint
header_lines.append(
self.get_line("word", rel_bytes[0x44:0x48], "BSS Alignment Constraint")
)
# Version 2 is only 0x48 bytes long
if self.version <= 2:
return header_lines
# Fix size
header_lines.append(self.get_line("word", rel_bytes[0x48:0x4C], "Fix Size"))
return header_lines

5
tools/splat/segtypes/gc/segment.py
View File

@ -1,5 +0,0 @@
from segtypes.segment import Segment
class GCSegment(Segment):
pass

641
tools/splat/segtypes/linker_entry.py
View File

@ -1,641 +0,0 @@
import os
import re
from functools import lru_cache
from pathlib import Path
from typing import Dict, List, OrderedDict, Set, Tuple, Union
from util import options
from segtypes.segment import Segment
from util.symbols import to_cname
# clean 'foo/../bar' to 'bar'
@lru_cache(maxsize=None)
def clean_up_path(path: Path) -> Path:
path_resolved = path.resolve()
base_resolved = options.opts.base_path.resolve()
try:
return path_resolved.relative_to(base_resolved)
except ValueError:
pass
# If the path wasn't relative to the splat file, use the working directory instead
cwd = Path(os.getcwd())
try:
return path_resolved.relative_to(cwd)
except ValueError:
pass
# If it wasn't relative to that too, then just return the path as-is
return path
def path_to_object_path(path: Path) -> Path:
path = clean_up_path(path)
if options.opts.use_o_as_suffix:
full_suffix = ".o"
else:
full_suffix = path.suffix + ".o"
if not str(path).startswith(str(options.opts.build_path)):
path = options.opts.build_path / path
return clean_up_path(path.with_suffix(full_suffix))
def write_file_if_different(path: Path, new_content: str):
if path.exists():
old_content = path.read_text()
else:
old_content = ""
if old_content != new_content:
path.parent.mkdir(parents=True, exist_ok=True)
with path.open("w") as f:
f.write(new_content)
def get_segment_rom_start(cname: str) -> str:
if options.opts.segment_symbols_style == "makerom":
return f"_{cname}SegmentRomStart"
return f"{cname}_ROM_START"
def get_segment_rom_end(cname: str) -> str:
if options.opts.segment_symbols_style == "makerom":
return f"_{cname}SegmentRomEnd"
return f"{cname}_ROM_END"
def get_segment_vram_start(cname: str) -> str:
if options.opts.segment_symbols_style == "makerom":
return f"_{cname}SegmentStart"
return f"{cname}_VRAM"
def get_segment_vram_end(cname: str) -> str:
if options.opts.segment_symbols_style == "makerom":
return f"_{cname}SegmentEnd"
return f"{cname}_VRAM_END"
def convert_section_name_to_linker_format(section_type: str) -> str:
assert section_type.startswith(".")
if options.opts.segment_symbols_style == "makerom":
if section_type == ".rodata":
return "RoData"
return section_type[1:].capitalize()
return to_cname(section_type.upper())
def get_segment_section_start(segment_name: str, section_type: str) -> str:
sec = convert_section_name_to_linker_format(section_type)
if options.opts.segment_symbols_style == "makerom":
return f"_{segment_name}Segment{sec}Start"
return f"{segment_name}{sec}_START"
def get_segment_section_end(segment_name: str, section_type: str) -> str:
sec = convert_section_name_to_linker_format(section_type)
if options.opts.segment_symbols_style == "makerom":
return f"_{segment_name}Segment{sec}End"
return f"{segment_name}{sec}_END"
def get_segment_section_size(segment_name: str, section_type: str) -> str:
sec = convert_section_name_to_linker_format(section_type)
if options.opts.segment_symbols_style == "makerom":
return f"_{segment_name}Segment{sec}Size"
return f"{segment_name}{sec}_SIZE"
def get_segment_vram_end_symbol_name(segment: Segment) -> str:
return get_segment_vram_end(segment.get_cname())
class LinkerEntry:
def __init__(
self,
segment: Segment,
src_paths: List[Path],
object_path: Path,
section_order: str,
section_link: str,
noload: bool = False,
):
self.segment = segment
self.src_paths = [clean_up_path(p) for p in src_paths]
self.section_order = section_order
self.section_link = section_link
self.noload = noload
self.bss_contains_common = segment.bss_contains_common
if self.section_link == "linker" or self.section_link == "linker_offset":
self.object_path = None
elif self.segment.type == "lib":
self.object_path = object_path
else:
self.object_path = path_to_object_path(object_path)
@property
def section_order_type(self) -> str:
if self.section_order == ".rdata":
return ".rodata"
else:
return self.section_order
@property
def section_link_type(self) -> str:
if self.section_link == ".rdata":
return ".rodata"
else:
return self.section_link
class LinkerWriter:
def __init__(self, is_partial: bool = False):
self.linker_discard_section: bool = options.opts.ld_discard_section
self.sections_allowlist: List[str] = options.opts.ld_sections_allowlist
self.sections_denylist: List[str] = options.opts.ld_sections_denylist
# Used to store all the linker entries - build tools may want this information
self.entries: List[LinkerEntry] = []
self.dependencies_entries: List[LinkerEntry] = []
self.buffer: List[str] = []
self.header_symbols: Set[str] = set()
self.is_partial: bool = is_partial
self._indent_level = 0
self._writeln("SECTIONS")
self._begin_block()
if not self.is_partial:
self._writeln(f"__romPos = {options.opts.ld_rom_start};")
if options.opts.gp is not None:
self._writeln("_gp = " + f"0x{options.opts.gp:X};")
# Write a series of statements which compute a symbol that represents the highest address among a list of segments' end addresses
def write_max_vram_end_sym(self, symbol: str, overlays: List[Segment]):
for segment in overlays:
if segment == overlays[0]:
self._writeln(
f"{symbol} = {get_segment_vram_end_symbol_name(segment)};"
)
else:
self._writeln(
f"{symbol} = MAX({symbol}, {get_segment_vram_end_symbol_name(segment)});"
)
# Adds all the entries of a segment to the linker script buffer
def add(self, segment: Segment, max_vram_syms: List[Tuple[str, List[Segment]]]):
entries = segment.get_linker_entries()
self.entries.extend(entries)
self.dependencies_entries.extend(entries)
seg_name = segment.get_cname()
for sym, segs in max_vram_syms:
self.write_max_vram_end_sym(sym, segs)
if options.opts.ld_legacy_generation:
self.add_legacy(segment, entries)
return
section_entries: OrderedDict[str, List[LinkerEntry]] = OrderedDict()
for l in segment.section_order:
if l in options.opts.ld_section_labels:
section_entries[l] = []
# Add all entries to section_entries
prev_entry = None
for entry in entries:
if entry.section_order_type in section_entries:
# Search for the very first section type
# This is required in case the very first entry is a type that's not listed on ld_section_labels (like linker_offset) because it would be dropped
prev_entry = entry.section_order_type
break
any_load = False
any_noload = False
for entry in entries:
if entry.section_order_type in section_entries:
section_entries[entry.section_order_type].append(entry)
elif prev_entry is not None:
# If this section is not present in section_order or ld_section_labels then pretend it is part of the last seen section, mainly for handling linker_offset
section_entries[prev_entry].append(entry)
any_load = any_load or not entry.noload
any_noload = any_noload or entry.noload
prev_entry = entry.section_order_type
seg_rom_start = get_segment_rom_start(seg_name)
self._write_symbol(seg_rom_start, "__romPos")
is_first = True
if any_load:
# Only emit normal segment if there's at least one normal entry
self._write_segment_sections(
segment, seg_name, section_entries, noload=False, is_first=is_first
)
is_first = False
if any_noload:
# Only emit NOLOAD segment if there is at least one noload entry
self._write_segment_sections(
segment, seg_name, section_entries, noload=True, is_first=is_first
)
is_first = False
self._end_segment(segment, all_bss=not any_load)
def add_legacy(self, segment: Segment, entries: List[LinkerEntry]):
seg_name = segment.get_cname()
# To keep track which sections has been started
started_sections: Dict[str, bool] = {
l: False for l in options.opts.ld_section_labels
}
# Find where sections are last seen
last_seen_sections: Dict[LinkerEntry, str] = {}
for entry in reversed(entries):
if (
entry.section_order_type in options.opts.ld_section_labels
and entry.section_order_type not in last_seen_sections.values()
):
last_seen_sections[entry] = entry.section_order_type
seg_rom_start = get_segment_rom_start(seg_name)
self._write_symbol(seg_rom_start, "__romPos")
self._begin_segment(segment, seg_name, noload=False, is_first=True)
i = 0
for entry in entries:
if entry.noload:
break
started = started_sections.get(entry.section_order_type, True)
if not started:
self._begin_section(seg_name, entry.section_order_type)
started_sections[entry.section_order_type] = True
self._write_linker_entry(entry)
if entry in last_seen_sections:
self._end_section(seg_name, entry.section_order_type)
i += 1
if any(entry.noload for entry in entries):
self._end_block()
self._begin_segment(segment, seg_name, noload=True, is_first=False)
for entry in entries[i:]:
started = started_sections.get(entry.section_order_type, True)
if not started:
self._begin_section(seg_name, entry.section_order_type)
started_sections[entry.section_order_type] = True
self._write_linker_entry(entry)
if entry in last_seen_sections:
self._end_section(seg_name, entry.section_order_type)
self._end_segment(segment, all_bss=False)
def add_referenced_partial_segment(
self, segment: Segment, max_vram_syms: List[Tuple[str, List[Segment]]]
):
entries = segment.get_linker_entries()
self.entries.extend(entries)
segments_path = options.opts.ld_partial_build_segments_path
assert segments_path is not None
seg_name = segment.get_cname()
for sym, segs in max_vram_syms:
self.write_max_vram_end_sym(sym, segs)
seg_rom_start = get_segment_rom_start(seg_name)
self._write_symbol(seg_rom_start, "__romPos")
any_load = any(not e.noload for e in entries)
is_first = True
if any_load:
# Only emit normal segment if there's at least one normal entry
self._begin_segment(segment, seg_name, noload=False, is_first=is_first)
for l in segment.section_order:
if l not in options.opts.ld_section_labels:
continue
if l == ".bss":
continue
entry = LinkerEntry(
segment, [], segments_path / f"{seg_name}.o", l, l, noload=False
)
self.dependencies_entries.append(entry)
self._write_linker_entry(entry)
is_first = False
if any(e.noload for e in entries):
# Only emit NOLOAD segment if there is at least one noload entry
if not is_first:
self._end_block()
self._begin_segment(segment, seg_name, noload=True, is_first=is_first)
# Check if any section has the bss_contains_common option
bss_contains_common = False
for entry in entries:
if entry.segment.bss_contains_common:
bss_contains_common = True
break
entry = LinkerEntry(
segment,
[],
segments_path / f"{seg_name}.o",
".bss",
".bss",
noload=True,
)
entry.bss_contains_common = bss_contains_common
self.dependencies_entries.append(entry)
self._write_linker_entry(entry)
self._end_segment(segment, all_bss=not any_load)
def add_partial_segment(self, segment: Segment):
entries = segment.get_linker_entries()
self.entries.extend(entries)
self.dependencies_entries.extend(entries)
seg_name = segment.get_cname()
section_entries: OrderedDict[str, List[LinkerEntry]] = OrderedDict()
for l in segment.section_order:
if l in options.opts.ld_section_labels:
section_entries[l] = []
# Add all entries to section_entries
prev_entry = None
for entry in entries:
if entry.section_order_type in section_entries:
section_entries[entry.section_order_type].append(entry)
elif prev_entry is not None:
# If this section is not present in section_order or ld_section_labels then pretend it is part of the last seen section, mainly for handling linker_offset
section_entries[prev_entry].append(entry)
prev_entry = entry.section_order_type
for section_name, entries in section_entries.items():
if len(entries) == 0:
continue
first_entry = entries[0]
self._begin_partial_segment(section_name, segment, first_entry.noload)
self._begin_section(seg_name, section_name)
for entry in entries:
self._write_linker_entry(entry)
self._end_section(seg_name, section_name)
self._end_partial_segment(section_name)
def save_linker_script(self, output_path: Path):
if len(self.sections_allowlist) > 0:
address = " 0"
if self.is_partial:
address = ""
for sect in self.sections_allowlist:
self._writeln(f"{sect}{address} :")
self._begin_block()
self._writeln(f"*({sect});")
self._end_block()
self._writeln("")
if self.linker_discard_section or len(self.sections_denylist) > 0:
self._writeln("/DISCARD/ :")
self._begin_block()
for sect in self.sections_denylist:
self._writeln(f"*({sect});")
if self.linker_discard_section:
self._writeln("*(*);")
self._end_block()
self._end_block() # SECTIONS
assert self._indent_level == 0
write_file_if_different(output_path, "\n".join(self.buffer) + "\n")
def save_symbol_header(self):
path = options.opts.ld_symbol_header_path
if path:
write_file_if_different(
path,
"#ifndef _HEADER_SYMBOLS_H_\n"
"#define _HEADER_SYMBOLS_H_\n"
"\n"
'#include "common.h"\n'
"\n"
+ "".join(
f"extern Addr {symbol};\n" for symbol in sorted(self.header_symbols)
)
+ "\n"
"#endif\n",
)
def save_dependencies_file(self, output_path: Path, target_elf_path: Path):
output = f"{target_elf_path}:"
for entry in self.dependencies_entries:
if entry.object_path is None:
continue
output += f" \\\n {entry.object_path}"
output += "\n"
for entry in self.dependencies_entries:
if entry.object_path is None:
continue
output += f"{entry.object_path}:\n"
write_file_if_different(output_path, output)
def _writeln(self, line: str):
if len(line) == 0:
self.buffer.append(line)
else:
self.buffer.append(" " * self._indent_level + line)
def _begin_block(self):
self._writeln("{")
self._indent_level += 1
def _end_block(self):
self._indent_level -= 1
self._writeln("}")
def _write_symbol(self, symbol: str, value: Union[str, int]):
symbol = to_cname(symbol)
if isinstance(value, int):
value = f"0x{value:X}"
self._writeln(f"{symbol} = {value};")
self.header_symbols.add(symbol)
def _begin_segment(
self, segment: Segment, seg_name: str, noload: bool, is_first: bool
):
if (
options.opts.ld_use_symbolic_vram_addresses
and segment.vram_symbol is not None
):
vram_str = segment.vram_symbol + " "
else:
vram_str = (
f"0x{segment.vram_start:X} "
if isinstance(segment.vram_start, int)
else ""
)
addr_str = " "
if is_first:
addr_str += f"{vram_str}"
if noload:
seg_name += "_bss"
addr_str += "(NOLOAD) "
seg_vram_start = get_segment_vram_start(seg_name)
self._write_symbol(seg_vram_start, f"ADDR(.{seg_name})")
line = f".{seg_name}{addr_str}:"
if not noload:
seg_rom_start = get_segment_rom_start(seg_name)
line += f" AT({seg_rom_start})"
if segment.subalign != None:
line += f" SUBALIGN({segment.subalign})"
self._writeln(line)
self._begin_block()
if segment.ld_fill_value is not None:
self._writeln(f"FILL(0x{segment.ld_fill_value:08X});")
def _end_segment(self, segment: Segment, all_bss=False):
self._end_block()
name = segment.get_cname()
if not all_bss:
self._writeln(f"__romPos += SIZEOF(.{name});")
# Align directive
if not options.opts.segment_end_before_align:
if segment.align:
self._writeln(f"__romPos = ALIGN(__romPos, {segment.align});")
seg_rom_end = get_segment_rom_end(name)
self._write_symbol(seg_rom_end, "__romPos")
self._write_symbol(get_segment_vram_end_symbol_name(segment), ".")
# Align directive
if options.opts.segment_end_before_align:
if segment.align:
self._writeln(f"__romPos = ALIGN(__romPos, {segment.align});")
self._writeln("")
def _begin_partial_segment(self, section_name: str, segment: Segment, noload: bool):
line = f"{section_name}"
if noload:
line += " (NOLOAD)"
line += " :"
if segment.subalign != None:
line += f" SUBALIGN({segment.subalign})"
self._writeln(line)
self._begin_block()
def _end_partial_segment(self, section_name: str, all_bss=False):
self._end_block()
self._writeln("")
def _begin_section(self, seg_name: str, cur_section: str) -> None:
section_start = get_segment_section_start(seg_name, cur_section)
self._write_symbol(section_start, ".")
def _end_section(self, seg_name: str, cur_section: str) -> None:
section_start = get_segment_section_start(seg_name, cur_section)
section_end = get_segment_section_end(seg_name, cur_section)
section_size = get_segment_section_size(seg_name, cur_section)
self._write_symbol(section_end, ".")
self._write_symbol(
section_size,
f"ABSOLUTE({section_end} - {section_start})",
)
def _write_linker_entry(self, entry: LinkerEntry):
if entry.section_link_type == "linker_offset":
self._write_symbol(f"{entry.segment.get_cname()}_OFFSET", ".")
return
# TODO: option to turn this off?
if (
entry.object_path
and entry.section_link_type == ".data"
and entry.segment.type != "lib"
):
path_cname = re.sub(
r"[^0-9a-zA-Z_]",
"_",
str(entry.segment.dir / entry.segment.name)
+ ".".join(entry.object_path.suffixes[:-1]),
)
self._write_symbol(path_cname, ".")
if entry.noload and entry.bss_contains_common:
self._writeln(f"{entry.object_path}(.bss COMMON .scommon);")
else:
wildcard = "*" if options.opts.ld_wildcard_sections else ""
self._writeln(f"{entry.object_path}({entry.section_link}{wildcard});")
def _write_segment_sections(
self,
segment: Segment,
seg_name: str,
section_entries: OrderedDict[str, List[LinkerEntry]],
noload: bool,
is_first: bool,
):
if not is_first:
self._end_block()
self._begin_segment(segment, seg_name, noload=noload, is_first=is_first)
for section_name, entries in section_entries.items():
if len(entries) == 0:
continue
first_entry = entries[0]
if first_entry.noload != noload:
continue
self._begin_section(seg_name, section_name)
for entry in entries:
self._write_linker_entry(entry)
self._end_section(seg_name, section_name)

0
tools/splat/segtypes/n64/__init__.py
View File

28
tools/splat/segtypes/n64/asm.py
View File

@ -1,28 +0,0 @@
from util import options
from segtypes.common.asm import CommonSegAsm
class N64SegAsm(CommonSegAsm):
@staticmethod
def get_file_header():
ret = []
ret.append('.include "macro.inc"')
ret.append("")
ret.append("/* assembler directives */")
ret.append(".set noat /* allow manual use of $at */")
ret.append(".set noreorder /* don't insert nops after branches */")
if options.opts.add_set_gp_64:
ret.append(
".set gp=64 /* allow use of 64-bit general purpose registers */"
)
ret.append("")
preamble = options.opts.generated_s_preamble
if preamble:
ret.append(preamble)
ret.append("")
ret.append('.section .text, "ax"')
ret.append("")
return ret

42
tools/splat/segtypes/n64/ci.py
View File

@ -1,42 +0,0 @@
from typing import Optional, TYPE_CHECKING
from util import log
from segtypes.n64.img import N64SegImg
if TYPE_CHECKING:
from segtypes.n64.palette import N64SegPalette
# Base class for CI4/CI8
class N64SegCi(N64SegImg):
def parse_palette_name(self, yaml, args) -> str:
ret = self.name
if isinstance(yaml, dict):
if "palette" in yaml:
ret = yaml["palette"]
elif len(args) > 2:
ret = args[2]
return ret
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.palette: "Optional[N64SegPalette]" = None
self.palette_name = self.parse_palette_name(self.yaml, self.args)
def scan(self, rom_bytes: bytes) -> None:
self.n64img.data = rom_bytes[self.rom_start : self.rom_end]
def split(self, rom_bytes):
if self.palette is None:
# TODO: output with blank palette
log.error(
f"no palette sibling segment exists\n(hint: add a segment with type 'palette' and name '{self.name}')"
)
assert self.palette is not None
self.palette.extract = False
self.n64img.palette = self.palette.parse_palette(rom_bytes)
super().split(rom_bytes)

8
tools/splat/segtypes/n64/ci4.py
View File

@ -1,8 +0,0 @@
import n64img.image
from segtypes.n64.ci import N64SegCi
class N64SegCi4(N64SegCi):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs, img_cls=n64img.image.CI4)

8
tools/splat/segtypes/n64/ci8.py
View File

@ -1,8 +0,0 @@
import n64img.image
from segtypes.n64.ci import N64SegCi
class N64SegCi8(N64SegCi):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs, img_cls=n64img.image.CI8)

277
tools/splat/segtypes/n64/gfx.py
View File

@ -1,277 +0,0 @@
"""
N64 f3dex display list splitter
Dumps out Gfx[] as a .inc.c file.
"""
import re
from typing import Optional
from pathlib import Path
from pygfxd import (
gfxd_buffer_to_string,
gfxd_cimg_callback,
gfxd_dl_callback,
gfxd_endian,
gfxd_execute,
gfxd_input_buffer,
gfxd_light_callback,
gfxd_lookat_callback,
gfxd_macro_dflt,
gfxd_macro_fn,
gfxd_mtx_callback,
gfxd_output_buffer,
gfxd_printf,
gfxd_puts,
gfxd_target,
gfxd_timg_callback,
gfxd_tlut_callback,
gfxd_vp_callback,
gfxd_vtx_callback,
gfxd_zimg_callback,
GfxdEndian,
gfxd_f3d,
gfxd_f3db,
gfxd_f3dex,
gfxd_f3dexb,
gfxd_f3dex2,
)
from segtypes.segment import Segment
from util import log, options
from util.log import error
from segtypes.common.codesubsegment import CommonSegCodeSubsegment
from util import symbols
LIGHTS_RE = re.compile(r"\*\(Lightsn \*\)0x[0-9A-F]{8}")
class N64SegGfx(CommonSegCodeSubsegment):
def __init__(
self,
rom_start: Optional[int],
rom_end: Optional[int],
type: str,
name: str,
vram_start: Optional[int],
args: list,
yaml,
):
super().__init__(
rom_start,
rom_end,
type,
name,
vram_start,
args=args,
yaml=yaml,
)
self.file_text = None
self.data_only = isinstance(yaml, dict) and yaml.get("data_only", False)
self.in_segment = not isinstance(yaml, dict) or yaml.get("in_segment", True)
def format_sym_name(self, sym) -> str:
return sym.name
def get_linker_section(self) -> str:
return ".data"
def out_path(self) -> Path:
return options.opts.asset_path / self.dir / f"{self.name}.gfx.inc.c"
def scan(self, rom_bytes: bytes):
self.file_text = self.disassemble_data(rom_bytes)
def get_gfxd_target(self):
opt = options.opts.gfx_ucode
if opt == "f3d":
return gfxd_f3d
elif opt == "f3db":
return gfxd_f3db
elif opt == "f3dex":
return gfxd_f3dex
elif opt == "f3dexb":
return gfxd_f3dexb
elif opt == "f3dex2":
return gfxd_f3dex2
else:
log.error(f"Unknown target {opt}")
def tlut_handler(self, addr, idx, count):
sym = self.create_symbol(
addr=addr, in_segment=self.in_segment, type="data", reference=True
)
gfxd_printf(self.format_sym_name(sym))
return 1
def timg_handler(self, addr, fmt, size, width, height, pal):
sym = self.create_symbol(
addr=addr, in_segment=self.in_segment, type="data", reference=True
)
gfxd_printf(self.format_sym_name(sym))
return 1
def cimg_handler(self, addr, fmt, size, width):
sym = self.create_symbol(
addr=addr, in_segment=self.in_segment, type="data", reference=True
)
gfxd_printf(self.format_sym_name(sym))
return 1
def zimg_handler(self, addr):
sym = self.create_symbol(
addr=addr, in_segment=self.in_segment, type="data", reference=True
)
gfxd_printf(self.format_sym_name(sym))
return 1
def dl_handler(self, addr):
# Look for 'Gfx'-typed symbols first
sym = self.retrieve_sym_type(symbols.all_symbols_dict, addr, "Gfx")
if not sym:
sym = self.create_symbol(
addr=addr, in_segment=self.in_segment, type="data", reference=True
)
gfxd_printf(self.format_sym_name(sym))
return 1
def mtx_handler(self, addr):
sym = self.create_symbol(
addr=addr, in_segment=self.in_segment, type="data", reference=True
)
gfxd_printf(f"&{self.format_sym_name(sym)}")
return 1
def lookat_handler(self, addr, count):
sym = self.create_symbol(
addr=addr, in_segment=self.in_segment, type="data", reference=True
)
gfxd_printf(self.format_sym_name(sym))
return 1
def light_handler(self, addr, count):
sym = self.create_symbol(
addr=addr, in_segment=self.in_segment, type="data", reference=True
)
gfxd_printf(self.format_sym_name(sym))
return 1
def vtx_handler(self, addr, count):
# Look for 'Vtx'-typed symbols first
sym = self.retrieve_sym_type(symbols.all_symbols_dict, addr, "Vtx")
if not sym:
sym = self.create_symbol(
addr=addr,
in_segment=self.in_segment,
type="Vtx",
reference=True,
search_ranges=True,
)
index = int((addr - sym.vram_start) / 0x10)
gfxd_printf(f"&{self.format_sym_name(sym)}[{index}]")
return 1
def vp_handler(self, addr):
sym = self.create_symbol(
addr=addr, in_segment=self.in_segment, type="data", reference=True
)
gfxd_printf(f"&{self.format_sym_name(sym)}")
return 1
def macro_fn(self):
gfxd_puts(" ")
gfxd_macro_dflt()
gfxd_puts(",\n")
return 0
def disassemble_data(self, rom_bytes):
assert isinstance(self.rom_start, int)
assert isinstance(self.rom_end, int)
assert isinstance(self.vram_start, int)
gfx_data = rom_bytes[self.rom_start : self.rom_end]
segment_length = len(gfx_data)
if (segment_length) % 8 != 0:
error(
f"Error: gfx segment {self.name} length ({segment_length}) is not a multiple of 8!"
)
out_str = "" if self.data_only else options.opts.generated_c_preamble + "\n\n"
sym = self.create_symbol(
addr=self.vram_start, in_segment=True, type="data", define=True
)
gfxd_input_buffer(gfx_data)
# TODO terrible guess at the size we'll need - improve this
outb = bytes([0] * segment_length * 100)
outbuf = gfxd_output_buffer(outb, len(outb))
gfxd_target(self.get_gfxd_target())
gfxd_endian(
GfxdEndian.big if options.opts.endianness == "big" else GfxdEndian.little, 4
)
# Callbacks
gfxd_macro_fn(self.macro_fn)
gfxd_tlut_callback(self.tlut_handler)
gfxd_timg_callback(self.timg_handler)
gfxd_cimg_callback(self.cimg_handler)
gfxd_zimg_callback(self.zimg_handler)
gfxd_dl_callback(self.dl_handler)
gfxd_mtx_callback(self.mtx_handler)
gfxd_lookat_callback(self.lookat_handler)
gfxd_light_callback(self.light_handler)
# gfxd_seg_callback ?
gfxd_vtx_callback(self.vtx_handler)
gfxd_vp_callback(self.vp_handler)
# gfxd_uctext_callback ?
# gfxd_ucdata_callback ?
# gfxd_dram_callback ?
gfxd_execute()
if self.data_only:
out_str += gfxd_buffer_to_string(outbuf)
else:
out_str += "Gfx " + self.format_sym_name(sym) + "[] = {\n"
out_str += gfxd_buffer_to_string(outbuf)
out_str += "};\n"
# Poor man's light fix until we get my libgfxd PR merged
def light_sub_func(match):
light = match.group(0)
addr = int(light[12:], 0)
sym = self.create_symbol(
addr=addr, in_segment=self.in_segment, type="data", reference=True
)
return self.format_sym_name(sym)
out_str = re.sub(LIGHTS_RE, light_sub_func, out_str)
return out_str
def split(self, rom_bytes: bytes):
if self.file_text and self.out_path():
self.out_path().parent.mkdir(parents=True, exist_ok=True)
with open(self.out_path(), "w", newline="\n") as f:
f.write(self.file_text)
def should_scan(self) -> bool:
return (
options.opts.is_mode_active("gfx")
and self.rom_start is not None
and self.rom_end is not None
)
def should_split(self) -> bool:
return self.extract and options.opts.is_mode_active("gfx")

28
tools/splat/segtypes/n64/hasm.py
View File

@ -1,28 +0,0 @@
from util import options
from segtypes.common.hasm import CommonSegHasm
class N64SegHasm(CommonSegHasm):
@staticmethod
def get_file_header():
ret = []
ret.append('.include "macro.inc"')
ret.append("")
ret.append("/* assembler directives */")
ret.append(".set noat /* allow manual use of $at */")
ret.append(".set noreorder /* don't insert nops after branches */")
if options.opts.add_set_gp_64:
ret.append(
".set gp=64 /* allow use of 64-bit general purpose registers */"
)
ret.append("")
preamble = options.opts.generated_s_preamble
if preamble:
ret.append(preamble)
ret.append("")
ret.append('.section .text, "ax"')
ret.append("")
return ret

50
tools/splat/segtypes/n64/header.py
View File

@ -1,50 +0,0 @@
from util import options
from segtypes.common.header import CommonSegHeader
class N64SegHeader(CommonSegHeader):
def parse_header(self, rom_bytes):
encoding = options.opts.header_encoding
header_lines = []
header_lines.append(".section .data\n")
header_lines.append(
self.get_line("word", rom_bytes[0x00:0x04], "PI BSB Domain 1 register")
)
header_lines.append(
self.get_line("word", rom_bytes[0x04:0x08], "Clockrate setting")
)
header_lines.append(
self.get_line("word", rom_bytes[0x08:0x0C], "Entrypoint address")
)
header_lines.append(self.get_line("word", rom_bytes[0x0C:0x10], "Revision"))
header_lines.append(self.get_line("word", rom_bytes[0x10:0x14], "Checksum 1"))
header_lines.append(self.get_line("word", rom_bytes[0x14:0x18], "Checksum 2"))
header_lines.append(self.get_line("word", rom_bytes[0x18:0x1C], "Unknown 1"))
header_lines.append(self.get_line("word", rom_bytes[0x1C:0x20], "Unknown 2"))
if encoding != "word":
header_lines.append(
'.ascii "'
+ rom_bytes[0x20:0x34].decode(encoding).strip().ljust(20)
+ '" /* Internal name */'
)
else:
for i in range(0x20, 0x34, 4):
header_lines.append(
self.get_line("word", rom_bytes[i : i + 4], "Internal name")
)
header_lines.append(self.get_line("word", rom_bytes[0x34:0x38], "Unknown 3"))
header_lines.append(self.get_line("word", rom_bytes[0x38:0x3C], "Cartridge"))
header_lines.append(
self.get_line("ascii", rom_bytes[0x3C:0x3E], "Cartridge ID")
)
header_lines.append(
self.get_line("ascii", rom_bytes[0x3E:0x3F], "Country code")
)
header_lines.append(self.get_line("byte", rom_bytes[0x3F:0x40], "Version"))
header_lines.append("")
return header_lines

9
tools/splat/segtypes/n64/i1.py
View File

@ -1,9 +0,0 @@
import n64img.image
from segtypes.n64.img import N64SegImg
class N64SegI1(N64SegImg):
def __init__(self, *args, **kwargs):
kwargs["img_cls"] = n64img.image.I1
super().__init__(*args, **kwargs)

9
tools/splat/segtypes/n64/i4.py
View File

@ -1,9 +0,0 @@
import n64img.image
from segtypes.n64.img import N64SegImg
class N64SegI4(N64SegImg):
def __init__(self, *args, **kwargs):
kwargs["img_cls"] = n64img.image.I4
super().__init__(*args, **kwargs)

9
tools/splat/segtypes/n64/i8.py
View File

@ -1,9 +0,0 @@
import n64img.image
from segtypes.n64.img import N64SegImg
class N64SegI8(N64SegImg):
def __init__(self, *args, **kwargs):
kwargs["img_cls"] = n64img.image.I8
super().__init__(*args, **kwargs)

9
tools/splat/segtypes/n64/ia16.py
View File

@ -1,9 +0,0 @@
import n64img.image
from segtypes.n64.img import N64SegImg
class N64SegIa16(N64SegImg):
def __init__(self, *args, **kwargs):
kwargs["img_cls"] = n64img.image.IA16
super().__init__(*args, **kwargs)

9
tools/splat/segtypes/n64/ia4.py
View File

@ -1,9 +0,0 @@
import n64img.image
from segtypes.n64.img import N64SegImg
class N64SegIa4(N64SegImg):
def __init__(self, *args, **kwargs):
kwargs["img_cls"] = n64img.image.IA4
super().__init__(*args, **kwargs)

9
tools/splat/segtypes/n64/ia8.py
View File

@ -1,9 +0,0 @@
import n64img.image
from segtypes.n64.img import N64SegImg
class N64SegIa8(N64SegImg):
def __init__(self, *args, **kwargs):
kwargs["img_cls"] = n64img.image.IA8
super().__init__(*args, **kwargs)

99
tools/splat/segtypes/n64/img.py
View File

@ -1,99 +0,0 @@
from pathlib import Path
from typing import Dict, List, Tuple, Type, Optional, Union
from n64img.image import Image
from util import log, options
from segtypes.n64.segment import N64Segment
class N64SegImg(N64Segment):
@staticmethod
def parse_dimensions(yaml: Union[Dict, List]) -> Tuple[int, int]:
if isinstance(yaml, dict):
return yaml["width"], yaml["height"]
else:
if len(yaml) < 5:
log.error(f"Error: {yaml} is missing width and height parameters")
return yaml[3], yaml[4]
def __init__(
self,
rom_start: Optional[int],
rom_end: Optional[int],
type: str,
name: str,
vram_start: Optional[int],
args: list,
yaml,
img_cls: Type[Image],
):
super().__init__(
rom_start,
rom_end,
type,
name,
vram_start,
args=args,
yaml=yaml,
)
if rom_start is None:
log.error(f"Error: {type} segment {name} rom start could not be determined")
self.n64img: Image = img_cls(b"", 0, 0)
if isinstance(yaml, dict):
self.n64img.flip_h = bool(yaml.get("flip_x", False))
self.n64img.flip_v = bool(yaml.get("flip_y", False))
self.width, self.height = self.parse_dimensions(yaml)
self.n64img.width = self.width
self.n64img.height = self.height
self.check_len()
def check_len(self) -> None:
expected_len = int(self.n64img.size())
assert isinstance(self.rom_start, int)
assert isinstance(self.rom_end, int)
assert isinstance(self.subalign, int)
actual_len = self.rom_end - self.rom_start
if actual_len > expected_len and actual_len - expected_len > self.subalign:
log.error(
f"Error: {self.name} should end at 0x{self.rom_start + expected_len:X}, but it ends at 0x{self.rom_end:X}\n(hint: add a 'bin' segment after it)"
)
def out_path(self) -> Path:
return options.opts.asset_path / self.dir / f"{self.name}.png"
def should_split(self) -> bool:
return options.opts.is_mode_active("img")
def split(self, rom_bytes):
path = self.out_path()
path.parent.mkdir(parents=True, exist_ok=True)
assert isinstance(self.rom_start, int)
assert isinstance(self.rom_end, int)
if self.n64img.data == b"":
self.n64img.data = rom_bytes[self.rom_start : self.rom_end]
self.n64img.write(path)
self.log(f"Wrote {self.name} to {path}")
@staticmethod
def estimate_size(yaml: Union[Dict, List]) -> int:
width, height = N64SegImg.parse_dimensions(yaml)
typ = N64Segment.parse_segment_type(yaml)
if typ == "ci4" or typ == "i4" or typ == "ia4":
return width * height // 2
elif typ in ("ia16", "rgba16"):
return width * height * 2
elif typ == "rgba32":
return width * height * 4
else:
return width * height

9
tools/splat/segtypes/n64/ipl3.py
View File

@ -1,9 +0,0 @@
from segtypes.common.code import CommonSegCode
from segtypes.common.hasm import CommonSegHasm
class N64SegIpl3(CommonSegHasm):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.special_vram_segment = True

14
tools/splat/segtypes/n64/linker_offset.py
View File

@ -1,14 +0,0 @@
from pathlib import Path
from segtypes.n64.segment import N64Segment
class N64SegLinker_offset(N64Segment):
def get_linker_entries(self):
from segtypes.linker_entry import LinkerEntry
return [
LinkerEntry(
self, [], Path(self.name), "linker_offset", "linker_offset", False
)
]

10
tools/splat/segtypes/n64/mio0.py
View File

@ -1,10 +0,0 @@
from util.n64.Mio0decompress import Mio0Decompressor
from segtypes.common.decompressor import CommonSegDecompressor
class N64SegMio0(CommonSegDecompressor):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.decompressor = Mio0Decompressor()
self.compression_type = "Mio0"

112
tools/splat/segtypes/n64/palette.py
View File

@ -1,112 +0,0 @@
from itertools import zip_longest
from pathlib import Path
from typing import Dict, List, Optional, Tuple, TYPE_CHECKING, Union
from util import log, options
from util.color import unpack_color
from segtypes.n64.segment import N64Segment
from util.symbols import to_cname
if TYPE_CHECKING:
from segtypes.n64.ci import N64SegCi as Raster
def iter_in_groups(iterable, n, fillvalue=None):
args = [iter(iterable)] * n
return zip_longest(*args, fillvalue=fillvalue)
VALID_SIZES = [0x20, 0x40, 0x80, 0x100, 0x200]
class N64SegPalette(N64Segment):
require_unique_name = False
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.raster: "Optional[Raster]" = None
# palette segments must be named as one of the following:
# 1) same as the relevant raster segment name (max. 1 palette)
# 2) relevant raster segment name + "." + unique palette name
# 3) unique, referencing the relevant raster segment using `raster_name`
self.raster_name = (
self.yaml.get("raster_name", self.name.split(".")[0])
if isinstance(self.yaml, dict)
else self.name.split(".")[0]
)
if self.extract:
if self.rom_end is None:
log.error(
f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it"
)
if not isinstance(self.yaml, dict) or "size" not in self.yaml:
assert self.rom_end is not None
assert self.rom_start is not None
actual_len = self.rom_end - self.rom_start
hint_msg = "(hint: add a 'bin' segment after it or specify the size in the segment)"
if actual_len > VALID_SIZES[-1]:
log.error(
f"Error: {self.name} (0x{actual_len:X} bytes) is too long, max 0x{VALID_SIZES[-1]:X})\n{hint_msg}"
)
if actual_len not in VALID_SIZES:
log.error(
f"Error: {self.name} (0x{actual_len:X} bytes) is not a valid palette size ({', '.join(hex(s) for s in VALID_SIZES)})\n{hint_msg}"
)
def get_cname(self) -> str:
return super().get_cname() + "_pal"
def split(self, rom_bytes):
if self.raster is None:
# TODO: output with no raster
log.error(f"orphaned palette segment: {self.name} lacks ci4/ci8 sibling")
assert self.raster is not None
self.raster.n64img.palette = self.parse_palette(rom_bytes) # type: ignore
self.raster.n64img.write(self.out_path())
self.raster.extract = False
def parse_palette(self, rom_bytes) -> List[Tuple[int, int, int, int]]:
data = rom_bytes[self.rom_start : self.rom_end]
palette = []
for a, b in iter_in_groups(data, 2):
palette.append(unpack_color([a, b]))
return palette
def out_path(self) -> Path:
return options.opts.asset_path / self.dir / f"{self.name}.png"
def should_split(self) -> bool:
return self.extract and options.opts.is_mode_active("img")
def get_linker_entries(self):
from segtypes.linker_entry import LinkerEntry
return [
LinkerEntry(
self,
[options.opts.asset_path / self.dir / f"{self.name}.png"],
options.opts.asset_path / self.dir / f"{self.name}.pal",
self.get_linker_section_order(),
self.get_linker_section_linksection(),
self.is_noload(),
)
]
@staticmethod
def estimate_size(yaml: Union[Dict, List]) -> int:
if isinstance(yaml, dict):
if "size" in yaml:
return int(yaml["size"])
return 0x20

9
tools/splat/segtypes/n64/rgba16.py
View File

@ -1,9 +0,0 @@
import n64img.image
from segtypes.n64.img import N64SegImg
class N64SegRgba16(N64SegImg):
def __init__(self, *args, **kwargs):
kwargs["img_cls"] = n64img.image.RGBA16
super().__init__(*args, **kwargs)

9
tools/splat/segtypes/n64/rgba32.py
View File

@ -1,9 +0,0 @@
import n64img.image
from segtypes.n64.img import N64SegImg
class N64SegRgba32(N64SegImg):
def __init__(self, *args, **kwargs):
kwargs["img_cls"] = n64img.image.RGBA32
super().__init__(*args, **kwargs)

10
tools/splat/segtypes/n64/rsp.py
View File

@ -1,10 +0,0 @@
import rabbitizer
from segtypes.common.hasm import CommonSegHasm
class N64SegRsp(CommonSegHasm):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.instr_category = rabbitizer.InstrCategory.RSP

5
tools/splat/segtypes/n64/segment.py
View File

@ -1,5 +0,0 @@
from segtypes.segment import Segment
class N64Segment(Segment):
pass

102
tools/splat/segtypes/n64/vtx.py
View File

@ -1,102 +0,0 @@
"""
N64 Vtx struct splitter
Dumps out Vtx as a .inc.c file.
Originally written by Mark Street (https://github.com/mkst)
"""
import struct
from pathlib import Path
from typing import Optional
from util import options, log
from segtypes.common.codesubsegment import CommonSegCodeSubsegment
class N64SegVtx(CommonSegCodeSubsegment):
def __init__(
self,
rom_start: Optional[int],
rom_end: Optional[int],
type: str,
name: str,
vram_start: Optional[int],
args: list,
yaml,
):
super().__init__(
rom_start,
rom_end,
type,
name,
vram_start,
args=args,
yaml=yaml,
)
self.file_text: Optional[str] = None
self.data_only = isinstance(yaml, dict) and yaml.get("data_only", False)
def format_sym_name(self, sym) -> str:
return sym.name
def get_linker_section(self) -> str:
return ".data"
def out_path(self) -> Path:
return options.opts.asset_path / self.dir / f"{self.name}.vtx.inc.c"
def scan(self, rom_bytes: bytes):
self.file_text = self.disassemble_data(rom_bytes)
def disassemble_data(self, rom_bytes) -> str:
assert isinstance(self.rom_start, int)
assert isinstance(self.rom_end, int)
assert isinstance(self.vram_start, int)
vertex_data = rom_bytes[self.rom_start : self.rom_end]
segment_length = len(vertex_data)
if (segment_length) % 16 != 0:
log.error(
f"Error: Vtx segment {self.name} length ({segment_length}) is not a multiple of 16!"
)
lines = []
if not self.data_only:
lines.append(options.opts.generated_c_preamble)
lines.append("")
vertex_count = segment_length // 16
sym = self.create_symbol(
addr=self.vram_start, in_segment=True, type="data", define=True
)
if not self.data_only:
lines.append(f"Vtx {self.format_sym_name(sym)}[{vertex_count}] = {{")
for vtx in struct.iter_unpack(">hhhHhhBBBB", vertex_data):
x, y, z, flg, t, c, r, g, b, a = vtx
vtx_string = f" {{{{{{ {x:5}, {y:5}, {z:5} }}, {flg}, {{ {t:5}, {c:5} }}, {{ {r:3}, {g:3}, {b:3}, {a:3} }}}}}},"
if flg != 0:
self.warn(f"Non-zero flag found in vertex data {self.name}!")
lines.append(vtx_string)
if not self.data_only:
lines.append("};")
# enforce newline at end of file
lines.append("")
return "\n".join(lines)
def split(self, rom_bytes: bytes):
if self.file_text and self.out_path():
self.out_path().parent.mkdir(parents=True, exist_ok=True)
with open(self.out_path(), "w", newline="\n") as f:
f.write(self.file_text)
def should_scan(self) -> bool:
return options.opts.is_mode_active("vtx")
def should_split(self) -> bool:
return self.extract and options.opts.is_mode_active("vtx")

10
tools/splat/segtypes/n64/yay0.py
View File

@ -1,10 +0,0 @@
from util.n64.Yay0decompress import Yay0Decompressor
from segtypes.common.decompressor import CommonSegDecompressor
class N64SegYay0(CommonSegDecompressor):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
self.decompressor = Yay0Decompressor()
self.compression_type = "Yay0"

0
tools/splat/segtypes/ps2/__init__.py
View File

23
tools/splat/segtypes/ps2/asm.py
View File

@ -1,23 +0,0 @@
from util import options
from segtypes.common.asm import CommonSegAsm
class Ps2SegAsm(CommonSegAsm):
@staticmethod
def get_file_header():
ret = []
ret.append('.include "macro.inc"')
ret.append("")
ret.append(".set noat")
ret.append(".set noreorder")
ret.append("")
preamble = options.opts.generated_s_preamble
if preamble:
ret.append(preamble)
ret.append("")
ret.append('.section .text, "ax"')
ret.append("")
return ret

0
tools/splat/segtypes/psx/__init__.py
View File

23
tools/splat/segtypes/psx/asm.py
View File

@ -1,23 +0,0 @@
from util import options
from segtypes.common.asm import CommonSegAsm
class PsxSegAsm(CommonSegAsm):
@staticmethod
def get_file_header():
ret = []
ret.append('.include "macro.inc"')
ret.append("")
ret.append(".set noat")
ret.append(".set noreorder")
ret.append("")
preamble = options.opts.generated_s_preamble
if preamble:
ret.append(preamble)
ret.append("")
ret.append('.section .text, "ax"')
ret.append("")
return ret

Some files were not shown because too many files have changed in this diff Show More