Configure updates for modding (#262)

* root-level configure script

* fix configure on macos?

* fix macos again maybe

* fix messages, add asset stack

* generate map headers from xml maps

* fix sprite_dir asset

* git subrepo pull (merge) --force tools/splat

subrepo:
  subdir:   "tools/splat"
  merged:   "5e36c45558"
upstream:
  origin:   "https://github.com/ethteck/splat.git"
  branch:   "master"
  commit:   "5e36c45558"
git-subrepo:
  version:  "0.4.3"
  origin:   "???"
  commit:   "???"

.gitignore

@@ -29,14 +29,15 @@ build.ninja
 baserom.*
 build/
 !tools/build
-/assets/jp
-/assets/us
 /docs/doxygen/
 /include/ld_addrs.h
 /include/message_ids.h
 /include/sprite/
 /include/map
 
+# Assets
+/assets/*
+
 # Star Rod
 /sprite/SpriteTable.xml
 /mod.cfg

tools/build/configure.py

@@ -6,7 +6,11 @@ import sys
 import ninja_syntax
 from glob import glob
 
+# Configuration:
 VERSIONS = ["us", "jp"]
+DO_SHA1_CHECK = True
+
+# Paths:
 ROOT = Path(__file__).parent.parent.parent
 BUILD_TOOLS = ROOT / "tools" / "build" # directory where this file is (TODO: use relative_to)
 YAY0_COMPRESS_TOOL = f"{BUILD_TOOLS}/yay0/Yay0compress"
@@ -59,10 +63,16 @@ def write_ninja_rules(ninja: ninja_syntax.Writer, cpp: str):
         command=f"{cross}objcopy $in $out -O binary && {BUILD_TOOLS}/rom/n64crc $out",
     )
 
-    ninja.rule("sha1sum",
+    if DO_SHA1_CHECK:
-        description="check $in",
+        ninja.rule("sha1sum",
-        command=f"sha1sum -c $in && touch $out",
+            description="check $in",
-    )
+            command=f"sha1sum -c $in && touch $out",
+        )
+    else:
+        ninja.rule("sha1sum",
+            description="check $in",
+            command=f"touch $out",
+        )
 
     ninja.rule("cc",
         description="cc($version) $in",
@@ -133,6 +143,8 @@ def write_ninja_rules(ninja: ninja_syntax.Writer, cpp: str):
         command=f"$python {BUILD_TOOLS}/mapfs/combine.py $out $in",
     )
 
+    ninja.rule("map_header", command=f"$python {BUILD_TOOLS}/mapfs/map_header.py $in > $out")
+
 def write_ninja_for_tools(ninja: ninja_syntax.Writer):
     ninja.rule("cc_tool",
         description="cc_tool $in",
@@ -165,6 +177,7 @@ class Configure:
             verbose=False,
         )
         self.linker_entries = split.linker_writer.entries[:]
+        self.asset_stack = split.config["asset_stack"]
 
     def build_path(self) -> Path:
         return Path(f"ver/{self.version}/build")
@@ -190,9 +203,7 @@ class Configure:
         out = []
 
         for path in src_paths:
-            if path.parents[0] == "assets":
+            path = self.resolve_asset_path(path)
-                # TODO resolve asset
-                pass
 
             if path.is_dir():
                 out.extend(glob(str(path) + "/**/*", recursive=True))
@@ -201,6 +212,19 @@ class Configure:
 
         return out
 
+    def resolve_asset_path(self, path: Path) -> Path:
+        parts = list(path.parts)
+
+        if parts[0] == "assets":
+            for d in self.asset_stack:
+                parts[1] = d
+                new_path = Path("/".join(parts))
+                if new_path.exists():
+                    return new_path
+
+        # ¯\_(ツ)_/¯
+        return path
+
     def write_ninja(self, ninja: ninja_syntax.Writer, skip_outputs: Set[str]):
         import segtypes
         import segtypes.n64.data # Doesn't get imported on jp for some odd reason (should maybe be a * import?)
@@ -302,7 +326,7 @@ class Configure:
                     variables = {
                         "sprite_id": sprite_id,
                         "sprite_name": sprite_name,
-                        "sprite_dir": str(sprite_dir),
+                        "sprite_dir": str(self.resolve_asset_path(sprite_dir)),
                     }
 
                     build(bin_path, [sprite_dir], "sprite", variables=variables)
@@ -351,6 +375,25 @@ class Configure:
                     elif name.endswith("_tex"):
                         compress = False
                         bin_path = path
+                    elif name.endswith("_shape"):
+                        map_name = "_".join(name.split("_")[:-1])
+
+                        # Handle map XML files, if they exist (TODO: have splat output these)
+                        map_xml = self.resolve_asset_path(Path(f"assets/{self.version}") / seg.dir / seg.name / (map_name + ".xml"))
+                        if map_xml.exists():
+                            # Build a header file for this map
+                            build(
+                                self.build_path() / "include" / seg.dir / seg.name / (map_name + ".h"),
+                                [map_xml],
+                                "map_header",
+                            )
+
+                            # NOTE: we don't build the map xml into a _shape or _hit file (yet); the Star Rod Map Editor
+                            # is able to build the xml nonmatchingly into assets/star_rod_build/mapfs/*.bin for people
+                            # who want that (i.e. modders). 'star_rod_build' should be added to asset_stack also.
+
+                        compress = True
+                        bin_path = path
                     else:
                         compress = True
                         bin_path = path

tools/build/mapfs/map_header.py

@@ -0,0 +1,97 @@
+#! /usr/bin/python3
+
+from sys import argv, stderr
+from os import path
+from xml.dom.minidom import parse
+
+def eprint(*args, **kwargs):
+    print(*args, file=stderr, **kwargs)
+
+if __name__ == "__main__":
+    _, xml_path = argv
+    xml = parse(xml_path)
+
+    map_name = path.basename(xml_path)[:-4]
+
+    print("#include \"common.h\"")
+    print("#include \"map.h\"")
+    print("")
+    print("#ifndef NAMESPACE")
+    print(f"#define NAMESPACE {map_name}")
+    print("")
+
+    seen_names = set()
+    entrys = []
+    npc_id = 0
+
+    for marker in xml.getElementsByTagName("Marker"):
+        marker_type = marker.getAttribute("type")
+        if marker_type == "Root" or marker_type == "Group":
+            continue
+
+        map_object = marker.getElementsByTagName("MapObject")[0]
+        name = marker_type + "_" + map_object.getAttribute("name")
+
+        if name in seen_names:
+            continue
+        seen_names.add(name)
+
+        x, y, z = [p for p in marker.getAttribute("pos").split(",")]
+        yaw = marker.getAttribute("yaw") + "f"
+
+        if marker_type == "NPC":
+            npc_id += 1
+            print(f"#define {name} {npc_id}")
+
+        if marker_type == "Entry":
+            entrys.append(name)
+
+        print(f"#define {name}_x {x}")
+        print(f"#define {name}_y {y}")
+        print(f"#define {name}_z {z}")
+        print(f"#define {name}_vec3d {x}, {y}, {z}")
+        print(f"#define {name}_vec4d {x}, {y}, {z}, (((s32){yaw}))")
+        print(f"#define {name}_vec3f {x}.0f, {y}.0f, {z}.0f")
+        print(f"#define {name}_vec4f {x}.0f, {y}.0f, {z}.0f, {yaw}")
+        print(f"#define {name}_yaw {yaw}")
+        print("")
+
+    for i, entry in enumerate(entrys):
+        print(f"#define {entry} {i}")
+    print(f"EntryList {map_name}_entryList = {{")
+    for entry in entrys:
+        print("    " + entry + "_vec4f,")
+    print("};")
+
+    print("")
+
+    for collider in xml.getElementsByTagName("Collider"):
+        map_object = collider.getElementsByTagName("MapObject")[0]
+        name = map_object.getAttribute("name")
+        idx = "0x" + map_object.getAttribute("id")
+
+        if name in seen_names or " " in name:
+            continue
+        seen_names.add(name)
+
+        print(f"#define Collider_{name} {idx}")
+
+    print("")
+
+    for model in xml.getElementsByTagName("Model"):
+        map_object = model.getElementsByTagName("MapObject")[0]
+        name = map_object.getAttribute("name")
+        idx = "0x" + map_object.getAttribute("id")
+
+        if name in seen_names or " " in name:
+            continue
+        seen_names.add(name)
+
+        print(f"#define Model_{name} {idx}")
+
+    print("")
+    print("#endif")
+    print("")
+
+    for i, entry in enumerate(entrys):
+        print(f"#define {map_name}_{entry[len('Entry_'):]} {i}")

tools/build/msg/parse_compile.py

@@ -753,9 +753,15 @@ if __name__ == "__main__":
 
                         message.bytes += [0xFF, 0x1A, 0, index, delay]
                     elif command == "animloop":
-                        message.bytes += [0xFF, 0x1B, 0, 0]
+                        if len(args) != 2:
+                            print(f"{filename}:{lineno}: {command} command requires 2 parameters")
+                            exit(1)
+                        message.bytes += [0xFF, 0x1B, args[0], args[1]]
                     elif command == "animdone":
-                        message.bytes += [0xFF, 0x1C, 0]
+                        if len(args) != 1:
+                            print(f"{filename}:{lineno}: {command} command requires 1 parameter")
+                            exit(1)
+                        message.bytes += [0xFF, 0x1C, args[0]]
                     elif command == "setcursorpos":
                         index = named_args.get("index")
                         pos = named_args.get("pos")

tools/splat/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/ethteck/splat.git
 	branch = master
-	commit = a44631e1941e8451897bc93faad8c8d6a45ea696
+	commit = 5e36c4555865e8192653217ca75afe0da661651a
-	parent = ba8528bb245492473d6290c9c16f7fa55d0593f1
+	parent = e7de26b831ba89b66285a4847c8c28e562f19ca2
 	method = merge
 	cmdver = 0.4.3

tools/splat/segtypes/linker_entry.py

@@ -9,8 +9,8 @@ def clean_up_path(path: Path) -> Path:
     return path.resolve().relative_to(options.get_base_path().resolve())
 
 def path_to_object_path(path: Path) -> Path:
-    path = options.get_build_path() / path.with_suffix(path.suffix + ".o").relative_to(options.get_base_path())
+    path = clean_up_path(path)
-    return clean_up_path(path)
+    return options.get_build_path() / path.with_suffix(path.suffix + ".o")
 
 def write_file_if_different(path: Path, new_content: str):
     if path.exists():
@@ -35,8 +35,11 @@ class LinkerEntry:
     def __init__(self, segment: Segment, src_paths: List[Path], object_path: Path, section: str):
         self.segment = segment
         self.src_paths = [clean_up_path(p) for p in src_paths]
-        self.object_path = path_to_object_path(object_path)
         self.section = section
+        if self.section == "linker":
+            self.object_path = None
+        else:
+            self.object_path = path_to_object_path(object_path)
 
 class LinkerWriter():
     def __init__(self):
@@ -62,6 +65,7 @@ class LinkerWriter():
             if entry.section == "linker": # TODO: isinstance is preferable
                 self._end_block()
                 self._begin_segment(entry.segment)
+                continue
 
             start = entry.segment.rom_start
             if isinstance(start, int):
@@ -137,10 +141,12 @@ class LinkerWriter():
     def _begin_segment(self, segment: Segment):
         # force location if not shiftable/auto
         if not self.shiftable and isinstance(segment.rom_start, int):
-            self._writeln(f". = 0x{segment.rom_start:X};")
+            self._writeln(f"__romPos = 0x{segment.rom_start:X};")
         else:
-            # align
+            # TODO: align 0x10, preferably
-            self._writeln(f". = ALIGN(0x10);")
+            pass
+
+        self._writeln(f". = __romPos;")
 
         vram = segment.vram_start
         vram_str = f"0x{vram:X}" if isinstance(vram, int) else ""
@@ -150,7 +156,7 @@ class LinkerWriter():
         else:
             name = to_cname(segment.name)
 
-        self._write_symbol(f"{name}_ROM_START", ".")
+        self._write_symbol(f"{name}_ROM_START", "__romPos")
         self._write_symbol(f"{name}_VRAM", f"ADDR(.{name})")
         self._writeln(f".{name} {vram_str} : AT({name}_ROM_START) SUBALIGN({segment.subalign})")
         self._begin_block()
@@ -166,7 +172,9 @@ class LinkerWriter():
         # force end if not shiftable/auto
         if not self.shiftable and isinstance(segment.rom_start, int) and isinstance(segment.rom_end, int):
             self._write_symbol(f"{to_cname(name)}_ROM_END", segment.rom_end)
+            self._writeln(f"__romPos = 0x{segment.rom_end:X};")
         else:
-            self._write_symbol(f"{to_cname(name)}_ROM_END", ".")
+            self._writeln(f"__romPos += SIZEOF(.{name});")
+            self._write_symbol(f"{to_cname(name)}_ROM_END", "__romPos")
 
         self._writeln("")

tools/splat/segtypes/n64/asm.py

@@ -27,7 +27,7 @@ class N64SegAsm(N64SegCodeSubsegment):
 
     def scan(self, rom_bytes: bytes):
         if self.rom_start != "auto" and self.rom_end != "auto" and self.rom_start != self.rom_end:
-            self.funcs_text = self.disassemble_code(rom_bytes)
+            self.funcs_text = self.disassemble_code(rom_bytes, options.get("asm_endlabels", False))
 
     def split(self, rom_bytes: bytes):
         if not self.rom_start == self.rom_end:

tools/splat/segtypes/n64/c.py

@@ -10,7 +10,7 @@ from util import options
 
 class N64SegC(N64SegCodeSubsegment):
     defined_funcs: Set[str] = set()
-    
+
     STRIP_C_COMMENTS_RE = re.compile(
         r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"',
         re.DOTALL | re.MULTILINE
@@ -135,7 +135,7 @@ class N64SegC(N64SegCodeSubsegment):
         c_lines = self.get_c_preamble()
 
         for func in funcs_text:
-            func_name = self.get_symbol(func, type="func", local_only=True).name
+            func_name = self.parent.get_symbol(func, type="func", local_only=True).name
             if options.get_compiler() == "GCC":
                 c_lines.append("INCLUDE_ASM(s32, \"{}\", {});".format(self.name, func_name))
             else:

tools/splat/segtypes/n64/codesubsegment.py

@@ -39,7 +39,7 @@ class N64SegCodeSubsegment(Segment):
     def is_branch_insn(mnemonic):
         return (mnemonic.startswith("b") and not mnemonic.startswith("binsl") and not mnemonic == "break") or mnemonic == "j"
 
-    def disassemble_code(self, rom_bytes):
+    def disassemble_code(self, rom_bytes, addsuffix=False):
         insns = [insn for insn in N64SegCodeSubsegment.md.disasm(rom_bytes[self.rom_start : self.rom_end], self.vram_start)]
 
         funcs = self.process_insns(insns, self.rom_start)
@@ -50,7 +50,7 @@ class N64SegCodeSubsegment(Segment):
 
         funcs = self.determine_symbols(funcs)
         self.gather_jumptable_labels(rom_bytes)
-        return self.add_labels(funcs)
+        return self.add_labels(funcs, addsuffix)
 
     def process_insns(self, insns, rom_addr):
         assert(isinstance(self.parent, N64SegCode))
@@ -256,7 +256,7 @@ class N64SegCodeSubsegment(Segment):
             ret[func_addr] = func
         return ret
 
-    def add_labels(self, funcs):
+    def add_labels(self, funcs, addsuffix):
         ret = {}
 
         for func in funcs:
@@ -310,6 +310,9 @@ class N64SegCodeSubsegment(Segment):
 
                 if insn[0].mnemonic != "branch" and insn[0].mnemonic.startswith("b") or insn[0].mnemonic.startswith("j"):
                     indent_next = True
+            
+            if addsuffix:            
+                func_text.append(f"endlabel {sym.name}")
 
             ret[func] = (func_text, rom_addr)
 

tools/splat/split.py

@@ -1,6 +1,6 @@
 #! /usr/bin/python3
 
-from typing import Dict, List, Union, Set
+from typing import Dict, List, Union, Set, Any
 import argparse
 import pylibyaml
 import yaml
@@ -22,6 +22,7 @@ parser.add_argument("--verbose", action="store_true", help="Enable debug logging
 parser.add_argument("--use-cache", action="store_true", help="Only split changed segments in config")
 
 linker_writer: LinkerWriter
+config: Dict[str, Any]
 
 def fmt_size(size):
     if size > 1000000:
@@ -106,6 +107,8 @@ def do_statistics(seg_sizes, rom_bytes, seg_split, seg_cached):
     log.write(f"{'unknown':>20}: {fmt_size(unk_size):>8} ({unk_ratio:.2%}) from unknown bin files")
 
 def main(config_path, base_dir, target_path, modes, verbose, use_cache=True):
+    global config
+
     # Load config
     with open(config_path) as f:
         config = yaml.load(f.read(), Loader=yaml.SafeLoader)

tools/splat/util/options.py

@@ -22,12 +22,12 @@ def initialize(config: Dict, config_path: str, base_path=None, target_path=None)
 
 def set(opt, val):
     opts[opt] = val
-    
+
 def get(opt, default=None):
     return opts.get(opt, default)
 
 def get_platform() -> str:
-    return opts.get("platform", "N64")
+    return opts.get("platform", "n64")
 
 def get_compiler() -> str:
     return opts.get("compiler", "IDO")

tools/splat_ext/PaperMarioMessages.py

@@ -257,8 +257,8 @@ CHARSET = {
         0x18: {None: lambda d: (f"[Image index={d[0]} pos={(d[1] << 8) + d[2]},{d[3]} hasBorder={d[4]} alpha={d[5]} fadeAmount={d[6]}]\n", 7)},
         0x19: {None: lambda d: (f"[HideImage fadeAmount={d[0]}]\n", 1)},
         0x1A: {None: lambda d: (f"[AnimDelay index={d[1]} delay={d[2]}]", 3)},
-        0x1B: {None: lambda d: (f"[AnimLoop]", 2)},
+        0x1B: {None: lambda d: (f"[AnimLoop {d[0]} {d[1]}]", 2)},
-        0x1C: {None: lambda d: (f"[AnimDone]", 1)},
+        0x1C: {None: lambda d: (f"[AnimDone {d[0]}]", 1)},
         0x1E: {None: lambda d: (f"[Cursor {d[0]}]", 1)},
         0x1F: {None: lambda d: (f"[EndChoice {d[0]}]", 1)},
         0x20: {None: lambda d: (f"[SetCancel {d[0]}]", 1)},