Merge pull request #163 from pmret/splat-update

Splat update

configure.py

@@ -10,7 +10,7 @@ from subprocess import PIPE
 import subprocess
 import hashlib
 
-sys.path.append(os.path.dirname(__file__) + "/tools/n64splat")
+sys.path.append(os.path.dirname(__file__) + "/tools/splat")
 import split
 
 INCLUDE_ASM_RE = re.compile(r"_INCLUDE_ASM\([^,]+, ([^,]+), ([^,)]+)") # note _ prefix
@@ -159,8 +159,8 @@ async def main():
     cpp = args.cpp or "cpp"
     ccache = "ccache" if cmd_exists("ccache") else ""
 
-    # compile n64splat dependencies
-    await shell("make -C tools/n64splat")
+    # compile splat dependencies
+    await shell("make -C tools/splat")
 
     # split assets
     print("Splitting segments from baserom", end="")

tools/compile_npc_sprite.py

@@ -3,7 +3,7 @@
 from sys import argv, path
 from pathlib import Path
 import os
-path.append(os.path.join(os.path.dirname(__file__), "n64splat"))
+path.append(os.path.join(os.path.dirname(__file__), "splat"))
 from splat_ext.PaperMarioNpcSprites import Sprite
 from convert_image import pack_color, iter_in_groups
 

tools/gen_sprite_animations_h.py

@@ -3,7 +3,7 @@
 from sys import argv, path
 from pathlib import Path
 import os
-path.append(os.path.join(os.path.dirname(__file__), "n64splat"))
+path.append(os.path.join(os.path.dirname(__file__), "splat"))
 from splat_ext.PaperMarioNpcSprites import Sprite
 
 if __name__ == "__main__":

tools/n64splat/Makefile

@@ -1,11 +0,0 @@
-UTIL_DIR := util
-
-default: all
-
-all: Yay0decompress
-
-Yay0decompress:
-	gcc $(UTIL_DIR)/Yay0decompress.c -fPIC -shared -O3 -o $(UTIL_DIR)/Yay0decompress
-
-clean:
-	rm -f $(UTIL_DIR)/Yay0decompress

tools/n64splat/README.md

@@ -1,8 +0,0 @@
-# n64splat
-A n64 rom splitting tool to assist with decompilation and modding projects
-
-For example usage, see https://github.com/ethteck/papermario
-The Makefile `setup` target calls n64splat with a config file that you can use for reference. More documentation coming soon.
-
-### Requirements
-Python package requirements can be installed via `pip3 install -r requirements.txt`

tools/splat.yaml

@@ -7,7 +7,8 @@ options:
   ld_o_replace_extension: False
   ld_addrs_header: include/ld_addrs.h
   extensions: splat_ext
-  symbol_addrs: tools/symbol_addrs.txt
+  symbol_addrs_path: tools/symbol_addrs.txt
+  platform: n64
 segments:
   - name: header
     type: header

tools/splat/.gitignore

@@ -2,4 +2,4 @@
 venv/
 .vscode/
 __pycache__/
-util/Yay0decompress
+util/n64/Yay0decompress

tools/splat/.gitrepo

@@ -4,9 +4,9 @@
 ; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme
 ;
 [subrepo]
-	remote = https://github.com/ethteck/n64splat.git
+	remote = https://github.com/ethteck/splat.git
 	branch = master
-	commit = 7574db712ef19ca420904c82d3559e9ac4b8c5f5
-	parent = 86760369a5ab977c037c21aebf6f10484570642f
+	commit = e2b731ab198c1d8400412ffd09252deca65253d2
+	parent = 4201a08a28b6d6f070b35f97e3ba726b5448893b
 	method = merge
 	cmdver = 0.4.3

tools/splat/CHANGELOG.md

@@ -0,0 +1,12 @@
+# splat Release Notes
+
+## 0.5
+* n64splat name changed to splat
+  * Some refactoring was done to support other platforms besides n64 in the future 
+    * New `platform` option, which defaults to `n64`
+  * This will cause breaking changes in custom segments, so please refer to one of the changes in one of the n64 base segments for details
+* Support for custom artifact paths 
+  * New `undefined_syms_auto_path` option
+  * New `undefined_funcs_auto_path` option
+  * New `cache_path` option
+  * (All path-like options' names now end with `_path`)

tools/splat/Makefile

@@ -0,0 +1,11 @@
+UTIL_DIR := util
+
+default: all
+
+all: Yay0decompress
+
+Yay0decompress:
+	gcc $(UTIL_DIR)/n64/Yay0decompress.c -fPIC -shared -O3 -o $(UTIL_DIR)/n64/Yay0decompress
+
+clean:
+	rm -f $(UTIL_DIR)/n64/Yay0decompress

tools/splat/README.md

@@ -0,0 +1,10 @@
+# splat
+A binary splitting tool to assist with decompilation and modding projects
+
+Currently, only N64 roms in the .z64 format are supported.
+
+For example usage, see https://github.com/pmret/papermario
+The Makefile `setup` target calls splat with a config file that you can use for reference. More documentation coming soon.
+
+### Requirements
+Python package requirements can be installed via `pip3 install -r requirements.txt`

tools/splat/create_config.py

@@ -1,13 +1,11 @@
 #! /usr/bin/env python3
 
-from capstone import *
-from capstone.mips import *
-
 import argparse
-from util import rominfo
+from util.n64 import rominfo
+from util.n64 import find_code_length
 from segtypes.code import N64SegCode
 
-parser = argparse.ArgumentParser(description="Create a splat config from a rom")
+parser = argparse.ArgumentParser(description="Create a splat config from a rom (currently only n64 .z64 roms supported)")
 parser.add_argument("rom", help="path to a .z64 rom")
 
 
@@ -26,11 +24,7 @@ options:
     with open(rom_path, "rb") as f:
       fbytes = f.read()
     
-    rom_addr = 0x1000
-
-    md = Cs(CS_ARCH_MIPS, CS_MODE_MIPS64 + CS_MODE_BIG_ENDIAN)
-    for insn in md.disasm(fbytes[rom_addr:], rom.entry_point):
-        rom_addr += 4
+    first_section_end = find_code_length.run(fbytes, 0x1000, rom.entry_point)
 
     segments = \
 """segments:
@@ -52,7 +46,7 @@ options:
   - type: bin
     start: 0x{:X}
   - [0x{:X}]
-""".format(rom.entry_point, rom_addr, rom.size)
+""".format(rom.entry_point, first_section_end, rom.size)
 
     outstr = header + segments
 

tools/splat/segtypes/n64/Yay0.py

@@ -1,7 +1,6 @@
 import os
-from segtypes.segment import N64Segment
-from pathlib import Path
-from util import Yay0decompress
+from segtypes.n64.segment import N64Segment
+from util.n64 import Yay0decompress
 
 class N64SegYay0(N64Segment):
     def split(self, rom_bytes, base_path):

tools/splat/segtypes/n64/bin.py

@@ -1,5 +1,5 @@
 import os
-from segtypes.segment import N64Segment
+from segtypes.n64.segment import N64Segment
 from pathlib import Path
 
 

tools/splat/segtypes/n64/ci4.py

@@ -1,4 +1,4 @@
-from segtypes.ci8 import N64SegCi8
+from segtypes.n64.ci8 import N64SegCi8
 
 class N64SegCi4(N64SegCi8):
     def parse_image(self, data):

tools/splat/segtypes/n64/ci8.py

@@ -1,8 +1,7 @@
-from segtypes.segment import N64Segment
-from segtypes.rgba16 import N64SegRgba16
+from segtypes.n64.rgba16 import N64SegRgba16
 import png
 import os
-from util import Yay0decompress
+from util.n64 import Yay0decompress
 
 
 class N64SegCi8(N64SegRgba16):

tools/splat/segtypes/n64/code.py

@@ -3,7 +3,7 @@ from capstone import *
 from capstone.mips import *
 
 from collections import OrderedDict
-from segtypes.segment import N64Segment, parse_segment_name
+from segtypes.n64.segment import N64Segment
 import os
 from pathlib import Path, PurePath
 from ranges import Range, RangeDict
@@ -508,7 +508,7 @@ class N64SegCode(N64Segment):
             if sym_type in ["float", "double"]:
                 if sym_type == "float":
                     float_str = floats.format_f32_imm(bits)
-                elif sym_type == "double":
+                else:
                     float_str = floats.format_f64_imm(bits)
 
                 # Fall back to .word if we see weird float values

tools/splat/segtypes/n64/header.py

@@ -1,7 +1,6 @@
 import os
-from segtypes.segment import N64Segment
+from segtypes.n64.segment import N64Segment
 from pathlib import Path
-from util import rominfo
 
 class N64SegHeader(N64Segment):
     def should_run(self):

tools/splat/segtypes/n64/i4.py

@@ -1,4 +1,4 @@
-from segtypes.rgba16 import N64SegRgba16
+from segtypes.n64.rgba16 import N64SegRgba16
 import png
 from math import ceil
 

tools/splat/segtypes/n64/i8.py

@@ -1,4 +1,4 @@
-from segtypes.i4 import N64SegI4
+from segtypes.n64.i4 import N64SegI4
 from math import ceil
 
 class N64SegI8(N64SegI4):

tools/splat/segtypes/n64/ia16.py

@@ -1,4 +1,4 @@
-from segtypes.ia4 import N64SegIa4
+from segtypes.n64.ia4 import N64SegIa4
 
 class N64SegIa8(N64SegIa4):
     def parse_image(self, data):

tools/splat/segtypes/n64/ia4.py

@@ -1,5 +1,4 @@
-import os
-from segtypes.rgba16 import N64SegRgba16
+from segtypes.n64.rgba16 import N64SegRgba16
 import png
 from math import ceil
 

tools/splat/segtypes/n64/ia8.py

@@ -1,5 +1,4 @@
-from segtypes.ia4 import N64SegIa4
-import png
+from segtypes.n64.ia4 import N64SegIa4
 from math import ceil
 
 class N64SegIa8(N64SegIa4):

tools/splat/segtypes/n64/palette.py

@@ -1,9 +1,9 @@
 import os
-from segtypes.segment import N64Segment
+from segtypes.n64.segment import N64Segment
 from util.color import unpack_color
+from util.n64 import Yay0decompress
 from util.iter import iter_in_groups
 
-
 class N64SegPalette(N64Segment):
     require_unique_name = False
 

tools/splat/segtypes/n64/rgba16.py

@@ -1,7 +1,6 @@
 import os
-from segtypes.segment import N64Segment
-from pathlib import Path
-from util import Yay0decompress
+from segtypes.n64.segment import N64Segment
+from util.n64 import Yay0decompress
 import png
 from math import ceil
 from util.color import unpack_color

tools/splat/segtypes/n64/rgba32.py

@@ -1,4 +1,4 @@
-from segtypes.rgba16 import N64SegRgba16
+from segtypes.n64.rgba16 import N64SegRgba16
 
 class N64SegRgba32(N64SegRgba16):
     def parse_image(self, data):

tools/splat/segtypes/n64/segment.py

@@ -0,0 +1,4 @@
+from segtypes.segment import Segment
+
+class N64Segment(Segment):
+    pass

tools/splat/segtypes/segment.py

@@ -1,7 +1,4 @@
-import os
 from pathlib import Path, PurePath
-import re
-import json
 from util import log
 
 default_subalign = 16
@@ -43,7 +40,7 @@ def parse_segment_subalign(segment):
     return default_subalign
 
 
-class N64Segment:
+class Segment:
     require_unique_name = True
 
     def __init__(self, segment, next_segment, options):
@@ -52,8 +49,7 @@ class N64Segment:
         self.type = parse_segment_type(segment)
         self.name = parse_segment_name(segment, self.__class__)
         self.vram_addr = parse_segment_vram(segment)
-        self.ld_name_override = segment.get(
-            "ld_name", None) if type(segment) is dict else None
+        self.ld_name_override = segment.get("ld_name", None) if type(segment) is dict else None
         self.options = options
         self.config = segment
         self.subalign = parse_segment_subalign(segment)

tools/splat/split.py

@@ -5,16 +5,12 @@ import importlib
 import importlib.util
 import os
 from ranges import Range, RangeDict
-import re
 from pathlib import Path
-import segtypes
-import sys
 import yaml
 import pickle
 from colorama import Style, Fore
-from collections import OrderedDict
-from segtypes.segment import N64Segment, parse_segment_type
-from segtypes.code import N64SegCode
+from segtypes.segment import parse_segment_type
+from segtypes.n64.code import N64SegCode
 from util import log
 
 parser = argparse.ArgumentParser(
@@ -81,11 +77,23 @@ def parse_file_start(split_file):
 
 
 def get_symbol_addrs_path(repo_path, options):
-    return os.path.join(repo_path, options.get("symbol_addrs", "symbol_addrs.txt"))
+    return os.path.join(repo_path, options.get("symbol_addrs_path", "symbol_addrs.txt"))
 
 
 def get_undefined_syms_path(repo_path, options):
-    return os.path.join(repo_path, options.get("undefined_syms", "undefined_syms.txt"))
+    return os.path.join(repo_path, options.get("undefined_syms_path", "undefined_syms.txt"))
+
+
+def get_undefined_syms_auto_path(repo_path, options):
+    return os.path.join(repo_path, options.get("undefined_syms_auto_path", "undefined_syms_auto.txt"))
+
+
+def get_undefined_funcs_auto_path(repo_path, options):
+    return os.path.join(repo_path, options.get("undefined_funcs_auto_path", "undefined_funcs_auto.txt"))
+
+
+def get_cache_path(repo_path, options):
+    return os.path.join(repo_path, options.get("cache_path", ".splat_cache"))
 
 
 def gather_symbols(symbol_addrs_path, undefined_syms_path):
@@ -156,13 +164,13 @@ def gather_c_variables(undefined_syms_path):
     return vars
 
 
-def get_base_segment_class(seg_type):
+def get_base_segment_class(seg_type, platform):
     try:
-        segmodule = importlib.import_module("segtypes." + seg_type)
+        segmodule = importlib.import_module(f"segtypes.{platform}.{seg_type}")
     except ModuleNotFoundError:
         return None
 
-    return getattr(segmodule, "N64Seg" + seg_type[0].upper() + seg_type[1:])
+    return getattr(segmodule, f"{platform.upper()}Seg{seg_type[0].upper()}{seg_type[1:]}")
 
 
 def get_extension_dir(options, config_path):
@@ -171,21 +179,23 @@ def get_extension_dir(options, config_path):
     return os.path.join(Path(config_path).parent, options["extensions"])
 
 
-def get_extension_class(options, config_path, seg_type):
+def get_extension_class(options, config_path, seg_type, platform):
     ext_dir = get_extension_dir(options, config_path)
     if ext_dir == None:
         return None
 
     try:
-        ext_spec = importlib.util.spec_from_file_location(f"segtypes.{seg_type}", os.path.join(ext_dir, f"{seg_type}.py"))
+        ext_spec = importlib.util.spec_from_file_location(f"{platform}.segtypes.{seg_type}", os.path.join(ext_dir, f"{seg_type}.py"))
         ext_mod = importlib.util.module_from_spec(ext_spec)
         ext_spec.loader.exec_module(ext_mod)
     except Exception as err:
         log.write(err, status="error")
         return None
 
-    return getattr(ext_mod, "N64Seg" + seg_type[0].upper() + seg_type[1:])
+    return getattr(ext_mod, f"{platform.upper()}Seg{seg_type[0].upper()}{seg_type[1:]}")
 
+def get_platform(options):
+    return options.get("platform", "n64")
 
 def fmt_size(size):
     if size > 1000000:
@@ -207,24 +217,18 @@ def initialize_segments(options, config_path, config_segments):
 
         seg_type = parse_segment_type(segment)
 
-        segment_class = get_base_segment_class(seg_type)
+        platform = get_platform(options)
+
+        segment_class = get_base_segment_class(seg_type, platform)
         if segment_class == None:
             # Look in extensions
-            segment_class = get_extension_class(options, config_path, seg_type)
+            segment_class = get_extension_class(options, config_path, seg_type, platform)
 
         if segment_class == None:
             log.write(f"fatal error: could not load segment type '{seg_type}'\n(hint: confirm your extension directory is configured correctly)", status="error")
             return 2
 
-        try:
         segment = segment_class(segment, config_segments[i + 1], options)
-        except (IndexError, KeyError) as e:
-            try:
-                segment = N64Segment(segment, config_segments[i + 1], options)
-                segment.error(e)
-            except Exception as e:
-                log.write(f"fatal error (segment type = {seg_type}): " + str(e), status="error")
-                return 2
 
         if segment_class.require_unique_name:
             if segment.name in seen_segment_names:
@@ -254,6 +258,7 @@ def main(rom_path, config_path, repo_path, modes, verbose, ignore_cache=False):
     symbol_addrs_path = get_symbol_addrs_path(repo_path, options)
     undefined_syms_path = get_undefined_syms_path(repo_path, options)
     provided_symbols, c_func_labels_to_add, special_labels, ranges = gather_symbols(symbol_addrs_path, undefined_syms_path)
+    platform = get_platform(options)
 
     processed_segments = []
     ld_sections = []
@@ -267,7 +272,7 @@ def main(rom_path, config_path, repo_path, modes, verbose, ignore_cache=False):
     seg_cached = {}
 
     # Load cache
-    cache_path = Path(repo_path) / ".splat_cache"
+    cache_path = get_cache_path(repo_path, options)
     try:
         with open(cache_path, "rb") as f:
             cache = pickle.load(f)
@@ -278,7 +283,7 @@ def main(rom_path, config_path, repo_path, modes, verbose, ignore_cache=False):
     all_segments = initialize_segments(options, config_path, config["segments"])
 
     for segment in all_segments:
-        if type(segment) == N64SegCode:
+        if platform == "n64" and type(segment) == N64SegCode: # remove special-case sometime
             segment.all_functions = defined_funcs
             segment.provided_symbols = provided_symbols
             segment.special_labels = special_labels
@@ -314,7 +319,7 @@ def main(rom_path, config_path, repo_path, modes, verbose, ignore_cache=False):
                     if len(segment.errors) == 0:
                         processed_segments.append(segment)
 
-                        if type(segment) == N64SegCode:
+                        if platform == "n64" and type(segment) == N64SegCode: # edge case
                             undefined_funcs |= segment.glabels_to_add
                             defined_funcs = {**defined_funcs, **segment.glabels_added}
                             undefined_syms |= segment.undefined_syms_to_add
@@ -335,21 +340,23 @@ def main(rom_path, config_path, repo_path, modes, verbose, ignore_cache=False):
         write_ldscript(config['basename'], repo_path, ld_sections, options)
 
     # Write undefined_funcs_auto.txt
+    undefined_funcs_auto_path = get_undefined_funcs_auto_path(repo_path, options)
     if verbose:
-        log.write(f"saving undefined_funcs_auto.txt")
+        log.write(f"saving {undefined_funcs_auto_path}")
     c_predefined_funcs = set(provided_symbols.keys())
     to_write = sorted(undefined_funcs - set(defined_funcs.values()) - c_predefined_funcs)
     if len(to_write) > 0:
-        with open(os.path.join(repo_path, "undefined_funcs_auto.txt"), "w", newline="\n") as f:
+        with open(undefined_funcs_auto_path, "w", newline="\n") as f:
             for line in to_write:
                 f.write(line + " = 0x" + line.split("_")[1][:8].upper() + ";\n")
 
     # write undefined_syms_auto.txt
+    undefined_syms_auto_path = get_undefined_syms_auto_path(repo_path, options)
     if verbose:
-        log.write(f"saving undefined_syms_auto.txt")
+        log.write(f"saving {undefined_syms_auto_path}")
     to_write = sorted(undefined_syms, key=lambda x:x[0])
     if len(to_write) > 0:
-        with open(os.path.join(repo_path, "undefined_syms_auto.txt"), "w", newline="\n") as f:
+        with open(undefined_syms_auto_path, "w", newline="\n") as f:
             for sym in to_write:
                 f.write(f"{sym[0]} = 0x{sym[1]:X};\n")
 

tools/splat/util/n64/find_code_length.py

@@ -4,9 +4,6 @@ from capstone import *
 from capstone.mips import *
 
 import argparse
-import hashlib
-import rominfo
-import zlib
 
 md = Cs(CS_ARCH_MIPS, CS_MODE_MIPS64 + CS_MODE_BIG_ENDIAN)
 
@@ -33,7 +30,7 @@ def run(rom_bytes, start_offset, vram, end_offset=None):
 def main():
     args = parser.parse_args()
 
-    rom_bytes = rominfo.read_rom(args.rom)
+    rom_bytes = open(args.rom, "rb").read()
     start = int(args.start, 0)
     end = None
     vram = int(args.vram, 0)

tools/splat/util/n64/rominfo.py

@@ -6,7 +6,7 @@ import zlib
 
 parser = argparse.ArgumentParser(description='Gives information on n64 roms')
 parser.add_argument('rom', help='path to a .z64 rom')
-parser.add_argument('--encoding', help='Text encoding the game header is using, defaults to ASCII, see docs.python.org/2.4/lib/standard-encodings.html for valid encodings', default='ASCII')
+parser.add_argument('--encoding', help='Text encoding the game header is using; see docs.python.org/3/library/codecs.html#standard-encodings for valid encodings', default='ASCII')
 
 country_codes = {
     0x37: "Beta",
@@ -71,7 +71,7 @@ def get_info_bytes(rom_bytes, encoding):
     try:
         name = rom_bytes[0x20:0x34].decode(encoding).strip()
     except:
-        print("n64splat could not decode the game name, try using a different encoding by passing the --encoding argument (see docs.python.org/2.4/lib/standard-encodings.html for valid encodings)")
+        print("splat could not decode the game name; try using a different encoding by passing the --encoding argument (see docs.python.org/3/library/codecs.html#standard-encodings for valid encodings)")
         exit(1)
 
     country_code = rom_bytes[0x3E]

tools/splat/util/n64/symbol.py

@@ -0,0 +1,12 @@
+class N64Symbol:
+
+    @staticmethod
+    def get_default_name(vram):
+        return f"D_{vram:X}"
+
+    def __init__(self, vram, rom=None, name=None, segment=None, length=4):
+        self.vram = vram
+        self.rom = rom
+        self.name = name if name else self.get_default_name(vram)
+        self.segment = segment
+        self.length = length

tools/splat_ext/PaperMarioMapFS.py

@@ -1,7 +1,7 @@
 import os
-from segtypes.segment import N64Segment
+from segtypes.n64.segment import N64Segment
 from pathlib import Path
-from util import Yay0decompress
+from util.n64 import Yay0decompress
 
 
 def decode_null_terminated_ascii(data):

tools/splat_ext/PaperMarioMessages.py

@@ -1,4 +1,4 @@
-from segtypes.segment import N64Segment
+from segtypes.n64.segment import N64Segment
 from pathlib import Path
 
 CHARSET = {

tools/splat_ext/PaperMarioNpcSprites.py

@@ -1,6 +1,6 @@
-from segtypes.segment import N64Segment
+from segtypes.n64.segment import N64Segment
 from pathlib import Path
-from util import Yay0decompress
+from util.n64 import Yay0decompress
 from util.iter import iter_in_groups
 from util.color import unpack_color
 import png