From 060486c85ea5e957ce290a714f41744c4a1d61d1 Mon Sep 17 00:00:00 2001 From: alex Date: Sun, 18 Apr 2021 14:26:00 +0100 Subject: [PATCH] Configure updates for modding (#262) * root-level configure script * fix configure on macos? * fix macos again maybe * fix messages, add asset stack * generate map headers from xml maps * fix sprite_dir asset * git subrepo pull (merge) --force tools/splat subrepo: subdir: "tools/splat" merged: "5e36c45558" upstream: origin: "https://github.com/ethteck/splat.git" branch: "master" commit: "5e36c45558" git-subrepo: version: "0.4.3" origin: "???" commit: "???" --- .gitignore | 5 +- tools/build/configure.py | 59 +++++++++++-- tools/build/mapfs/map_header.py | 97 ++++++++++++++++++++++ tools/build/msg/parse_compile.py | 10 ++- tools/splat/.gitrepo | 4 +- tools/splat/segtypes/linker_entry.py | 24 ++++-- tools/splat/segtypes/n64/asm.py | 2 +- tools/splat/segtypes/n64/c.py | 4 +- tools/splat/segtypes/n64/codesubsegment.py | 9 +- tools/splat/split.py | 5 +- tools/splat/util/options.py | 4 +- tools/splat_ext/PaperMarioMessages.py | 4 +- 12 files changed, 194 insertions(+), 33 deletions(-) create mode 100644 tools/build/mapfs/map_header.py diff --git a/.gitignore b/.gitignore index cefa97b006..43f199ee57 100644 --- a/.gitignore +++ b/.gitignore @@ -29,14 +29,15 @@ build.ninja baserom.* build/ !tools/build -/assets/jp -/assets/us /docs/doxygen/ /include/ld_addrs.h /include/message_ids.h /include/sprite/ /include/map +# Assets +/assets/* + # Star Rod /sprite/SpriteTable.xml /mod.cfg diff --git a/tools/build/configure.py b/tools/build/configure.py index 9c68f3acf8..3f70c28124 100755 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -6,7 +6,11 @@ import sys import ninja_syntax from glob import glob +# Configuration: VERSIONS = ["us", "jp"] +DO_SHA1_CHECK = True + +# Paths: ROOT = Path(__file__).parent.parent.parent BUILD_TOOLS = ROOT / "tools" / "build" # directory where this file is (TODO: use relative_to) YAY0_COMPRESS_TOOL = f"{BUILD_TOOLS}/yay0/Yay0compress" @@ -59,10 +63,16 @@ def write_ninja_rules(ninja: ninja_syntax.Writer, cpp: str): command=f"{cross}objcopy $in $out -O binary && {BUILD_TOOLS}/rom/n64crc $out", ) - ninja.rule("sha1sum", - description="check $in", - command=f"sha1sum -c $in && touch $out", - ) + if DO_SHA1_CHECK: + ninja.rule("sha1sum", + description="check $in", + command=f"sha1sum -c $in && touch $out", + ) + else: + ninja.rule("sha1sum", + description="check $in", + command=f"touch $out", + ) ninja.rule("cc", description="cc($version) $in", @@ -133,6 +143,8 @@ def write_ninja_rules(ninja: ninja_syntax.Writer, cpp: str): command=f"$python {BUILD_TOOLS}/mapfs/combine.py $out $in", ) + ninja.rule("map_header", command=f"$python {BUILD_TOOLS}/mapfs/map_header.py $in > $out") + def write_ninja_for_tools(ninja: ninja_syntax.Writer): ninja.rule("cc_tool", description="cc_tool $in", @@ -165,6 +177,7 @@ class Configure: verbose=False, ) self.linker_entries = split.linker_writer.entries[:] + self.asset_stack = split.config["asset_stack"] def build_path(self) -> Path: return Path(f"ver/{self.version}/build") @@ -190,9 +203,7 @@ class Configure: out = [] for path in src_paths: - if path.parents[0] == "assets": - # TODO resolve asset - pass + path = self.resolve_asset_path(path) if path.is_dir(): out.extend(glob(str(path) + "/**/*", recursive=True)) @@ -201,6 +212,19 @@ class Configure: return out + def resolve_asset_path(self, path: Path) -> Path: + parts = list(path.parts) + + if parts[0] == "assets": + for d in self.asset_stack: + parts[1] = d + new_path = Path("/".join(parts)) + if new_path.exists(): + return new_path + + # ¯\_(ツ)_/¯ + return path + def write_ninja(self, ninja: ninja_syntax.Writer, skip_outputs: Set[str]): import segtypes import segtypes.n64.data # Doesn't get imported on jp for some odd reason (should maybe be a * import?) @@ -302,7 +326,7 @@ class Configure: variables = { "sprite_id": sprite_id, "sprite_name": sprite_name, - "sprite_dir": str(sprite_dir), + "sprite_dir": str(self.resolve_asset_path(sprite_dir)), } build(bin_path, [sprite_dir], "sprite", variables=variables) @@ -351,6 +375,25 @@ class Configure: elif name.endswith("_tex"): compress = False bin_path = path + elif name.endswith("_shape"): + map_name = "_".join(name.split("_")[:-1]) + + # Handle map XML files, if they exist (TODO: have splat output these) + map_xml = self.resolve_asset_path(Path(f"assets/{self.version}") / seg.dir / seg.name / (map_name + ".xml")) + if map_xml.exists(): + # Build a header file for this map + build( + self.build_path() / "include" / seg.dir / seg.name / (map_name + ".h"), + [map_xml], + "map_header", + ) + + # NOTE: we don't build the map xml into a _shape or _hit file (yet); the Star Rod Map Editor + # is able to build the xml nonmatchingly into assets/star_rod_build/mapfs/*.bin for people + # who want that (i.e. modders). 'star_rod_build' should be added to asset_stack also. + + compress = True + bin_path = path else: compress = True bin_path = path diff --git a/tools/build/mapfs/map_header.py b/tools/build/mapfs/map_header.py new file mode 100644 index 0000000000..59841f9b6d --- /dev/null +++ b/tools/build/mapfs/map_header.py @@ -0,0 +1,97 @@ +#! /usr/bin/python3 + +from sys import argv, stderr +from os import path +from xml.dom.minidom import parse + +def eprint(*args, **kwargs): + print(*args, file=stderr, **kwargs) + +if __name__ == "__main__": + _, xml_path = argv + xml = parse(xml_path) + + map_name = path.basename(xml_path)[:-4] + + print("#include \"common.h\"") + print("#include \"map.h\"") + print("") + print("#ifndef NAMESPACE") + print(f"#define NAMESPACE {map_name}") + print("") + + seen_names = set() + entrys = [] + npc_id = 0 + + for marker in xml.getElementsByTagName("Marker"): + marker_type = marker.getAttribute("type") + if marker_type == "Root" or marker_type == "Group": + continue + + map_object = marker.getElementsByTagName("MapObject")[0] + name = marker_type + "_" + map_object.getAttribute("name") + + if name in seen_names: + continue + seen_names.add(name) + + x, y, z = [p for p in marker.getAttribute("pos").split(",")] + yaw = marker.getAttribute("yaw") + "f" + + if marker_type == "NPC": + npc_id += 1 + print(f"#define {name} {npc_id}") + + if marker_type == "Entry": + entrys.append(name) + + print(f"#define {name}_x {x}") + print(f"#define {name}_y {y}") + print(f"#define {name}_z {z}") + print(f"#define {name}_vec3d {x}, {y}, {z}") + print(f"#define {name}_vec4d {x}, {y}, {z}, (((s32){yaw}))") + print(f"#define {name}_vec3f {x}.0f, {y}.0f, {z}.0f") + print(f"#define {name}_vec4f {x}.0f, {y}.0f, {z}.0f, {yaw}") + print(f"#define {name}_yaw {yaw}") + print("") + + for i, entry in enumerate(entrys): + print(f"#define {entry} {i}") + print(f"EntryList {map_name}_entryList = {{") + for entry in entrys: + print(" " + entry + "_vec4f,") + print("};") + + print("") + + for collider in xml.getElementsByTagName("Collider"): + map_object = collider.getElementsByTagName("MapObject")[0] + name = map_object.getAttribute("name") + idx = "0x" + map_object.getAttribute("id") + + if name in seen_names or " " in name: + continue + seen_names.add(name) + + print(f"#define Collider_{name} {idx}") + + print("") + + for model in xml.getElementsByTagName("Model"): + map_object = model.getElementsByTagName("MapObject")[0] + name = map_object.getAttribute("name") + idx = "0x" + map_object.getAttribute("id") + + if name in seen_names or " " in name: + continue + seen_names.add(name) + + print(f"#define Model_{name} {idx}") + + print("") + print("#endif") + print("") + + for i, entry in enumerate(entrys): + print(f"#define {map_name}_{entry[len('Entry_'):]} {i}") diff --git a/tools/build/msg/parse_compile.py b/tools/build/msg/parse_compile.py index 92ae3b854e..98b58f2970 100755 --- a/tools/build/msg/parse_compile.py +++ b/tools/build/msg/parse_compile.py @@ -753,9 +753,15 @@ if __name__ == "__main__": message.bytes += [0xFF, 0x1A, 0, index, delay] elif command == "animloop": - message.bytes += [0xFF, 0x1B, 0, 0] + if len(args) != 2: + print(f"{filename}:{lineno}: {command} command requires 2 parameters") + exit(1) + message.bytes += [0xFF, 0x1B, args[0], args[1]] elif command == "animdone": - message.bytes += [0xFF, 0x1C, 0] + if len(args) != 1: + print(f"{filename}:{lineno}: {command} command requires 1 parameter") + exit(1) + message.bytes += [0xFF, 0x1C, args[0]] elif command == "setcursorpos": index = named_args.get("index") pos = named_args.get("pos") diff --git a/tools/splat/.gitrepo b/tools/splat/.gitrepo index 8132b59447..22839fe0b6 100644 --- a/tools/splat/.gitrepo +++ b/tools/splat/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/ethteck/splat.git branch = master - commit = a44631e1941e8451897bc93faad8c8d6a45ea696 - parent = ba8528bb245492473d6290c9c16f7fa55d0593f1 + commit = 5e36c4555865e8192653217ca75afe0da661651a + parent = e7de26b831ba89b66285a4847c8c28e562f19ca2 method = merge cmdver = 0.4.3 diff --git a/tools/splat/segtypes/linker_entry.py b/tools/splat/segtypes/linker_entry.py index 9b28ef83d6..d5fa68e347 100644 --- a/tools/splat/segtypes/linker_entry.py +++ b/tools/splat/segtypes/linker_entry.py @@ -9,8 +9,8 @@ def clean_up_path(path: Path) -> Path: return path.resolve().relative_to(options.get_base_path().resolve()) def path_to_object_path(path: Path) -> Path: - path = options.get_build_path() / path.with_suffix(path.suffix + ".o").relative_to(options.get_base_path()) - return clean_up_path(path) + path = clean_up_path(path) + return options.get_build_path() / path.with_suffix(path.suffix + ".o") def write_file_if_different(path: Path, new_content: str): if path.exists(): @@ -35,8 +35,11 @@ class LinkerEntry: def __init__(self, segment: Segment, src_paths: List[Path], object_path: Path, section: str): self.segment = segment self.src_paths = [clean_up_path(p) for p in src_paths] - self.object_path = path_to_object_path(object_path) self.section = section + if self.section == "linker": + self.object_path = None + else: + self.object_path = path_to_object_path(object_path) class LinkerWriter(): def __init__(self): @@ -62,6 +65,7 @@ class LinkerWriter(): if entry.section == "linker": # TODO: isinstance is preferable self._end_block() self._begin_segment(entry.segment) + continue start = entry.segment.rom_start if isinstance(start, int): @@ -137,10 +141,12 @@ class LinkerWriter(): def _begin_segment(self, segment: Segment): # force location if not shiftable/auto if not self.shiftable and isinstance(segment.rom_start, int): - self._writeln(f". = 0x{segment.rom_start:X};") + self._writeln(f"__romPos = 0x{segment.rom_start:X};") else: - # align - self._writeln(f". = ALIGN(0x10);") + # TODO: align 0x10, preferably + pass + + self._writeln(f". = __romPos;") vram = segment.vram_start vram_str = f"0x{vram:X}" if isinstance(vram, int) else "" @@ -150,7 +156,7 @@ class LinkerWriter(): else: name = to_cname(segment.name) - self._write_symbol(f"{name}_ROM_START", ".") + self._write_symbol(f"{name}_ROM_START", "__romPos") self._write_symbol(f"{name}_VRAM", f"ADDR(.{name})") self._writeln(f".{name} {vram_str} : AT({name}_ROM_START) SUBALIGN({segment.subalign})") self._begin_block() @@ -166,7 +172,9 @@ class LinkerWriter(): # force end if not shiftable/auto if not self.shiftable and isinstance(segment.rom_start, int) and isinstance(segment.rom_end, int): self._write_symbol(f"{to_cname(name)}_ROM_END", segment.rom_end) + self._writeln(f"__romPos = 0x{segment.rom_end:X};") else: - self._write_symbol(f"{to_cname(name)}_ROM_END", ".") + self._writeln(f"__romPos += SIZEOF(.{name});") + self._write_symbol(f"{to_cname(name)}_ROM_END", "__romPos") self._writeln("") diff --git a/tools/splat/segtypes/n64/asm.py b/tools/splat/segtypes/n64/asm.py index 3030d50741..2f78e616de 100644 --- a/tools/splat/segtypes/n64/asm.py +++ b/tools/splat/segtypes/n64/asm.py @@ -27,7 +27,7 @@ class N64SegAsm(N64SegCodeSubsegment): def scan(self, rom_bytes: bytes): if self.rom_start != "auto" and self.rom_end != "auto" and self.rom_start != self.rom_end: - self.funcs_text = self.disassemble_code(rom_bytes) + self.funcs_text = self.disassemble_code(rom_bytes, options.get("asm_endlabels", False)) def split(self, rom_bytes: bytes): if not self.rom_start == self.rom_end: diff --git a/tools/splat/segtypes/n64/c.py b/tools/splat/segtypes/n64/c.py index a177b5294c..f613d6497d 100644 --- a/tools/splat/segtypes/n64/c.py +++ b/tools/splat/segtypes/n64/c.py @@ -10,7 +10,7 @@ from util import options class N64SegC(N64SegCodeSubsegment): defined_funcs: Set[str] = set() - + STRIP_C_COMMENTS_RE = re.compile( r'//.*?$|/\*.*?\*/|\'(?:\\.|[^\\\'])*\'|"(?:\\.|[^\\"])*"', re.DOTALL | re.MULTILINE @@ -135,7 +135,7 @@ class N64SegC(N64SegCodeSubsegment): c_lines = self.get_c_preamble() for func in funcs_text: - func_name = self.get_symbol(func, type="func", local_only=True).name + func_name = self.parent.get_symbol(func, type="func", local_only=True).name if options.get_compiler() == "GCC": c_lines.append("INCLUDE_ASM(s32, \"{}\", {});".format(self.name, func_name)) else: diff --git a/tools/splat/segtypes/n64/codesubsegment.py b/tools/splat/segtypes/n64/codesubsegment.py index 9bd4855fa3..daf8f4a669 100644 --- a/tools/splat/segtypes/n64/codesubsegment.py +++ b/tools/splat/segtypes/n64/codesubsegment.py @@ -39,7 +39,7 @@ class N64SegCodeSubsegment(Segment): def is_branch_insn(mnemonic): return (mnemonic.startswith("b") and not mnemonic.startswith("binsl") and not mnemonic == "break") or mnemonic == "j" - def disassemble_code(self, rom_bytes): + def disassemble_code(self, rom_bytes, addsuffix=False): insns = [insn for insn in N64SegCodeSubsegment.md.disasm(rom_bytes[self.rom_start : self.rom_end], self.vram_start)] funcs = self.process_insns(insns, self.rom_start) @@ -50,7 +50,7 @@ class N64SegCodeSubsegment(Segment): funcs = self.determine_symbols(funcs) self.gather_jumptable_labels(rom_bytes) - return self.add_labels(funcs) + return self.add_labels(funcs, addsuffix) def process_insns(self, insns, rom_addr): assert(isinstance(self.parent, N64SegCode)) @@ -256,7 +256,7 @@ class N64SegCodeSubsegment(Segment): ret[func_addr] = func return ret - def add_labels(self, funcs): + def add_labels(self, funcs, addsuffix): ret = {} for func in funcs: @@ -310,6 +310,9 @@ class N64SegCodeSubsegment(Segment): if insn[0].mnemonic != "branch" and insn[0].mnemonic.startswith("b") or insn[0].mnemonic.startswith("j"): indent_next = True + + if addsuffix: + func_text.append(f"endlabel {sym.name}") ret[func] = (func_text, rom_addr) diff --git a/tools/splat/split.py b/tools/splat/split.py index 83f8eb8001..bc028cac99 100755 --- a/tools/splat/split.py +++ b/tools/splat/split.py @@ -1,6 +1,6 @@ #! /usr/bin/python3 -from typing import Dict, List, Union, Set +from typing import Dict, List, Union, Set, Any import argparse import pylibyaml import yaml @@ -22,6 +22,7 @@ parser.add_argument("--verbose", action="store_true", help="Enable debug logging parser.add_argument("--use-cache", action="store_true", help="Only split changed segments in config") linker_writer: LinkerWriter +config: Dict[str, Any] def fmt_size(size): if size > 1000000: @@ -106,6 +107,8 @@ def do_statistics(seg_sizes, rom_bytes, seg_split, seg_cached): log.write(f"{'unknown':>20}: {fmt_size(unk_size):>8} ({unk_ratio:.2%}) from unknown bin files") def main(config_path, base_dir, target_path, modes, verbose, use_cache=True): + global config + # Load config with open(config_path) as f: config = yaml.load(f.read(), Loader=yaml.SafeLoader) diff --git a/tools/splat/util/options.py b/tools/splat/util/options.py index ed18a0564c..4a2aee48d8 100644 --- a/tools/splat/util/options.py +++ b/tools/splat/util/options.py @@ -22,12 +22,12 @@ def initialize(config: Dict, config_path: str, base_path=None, target_path=None) def set(opt, val): opts[opt] = val - + def get(opt, default=None): return opts.get(opt, default) def get_platform() -> str: - return opts.get("platform", "N64") + return opts.get("platform", "n64") def get_compiler() -> str: return opts.get("compiler", "IDO") diff --git a/tools/splat_ext/PaperMarioMessages.py b/tools/splat_ext/PaperMarioMessages.py index 7fa62fd703..856172194b 100644 --- a/tools/splat_ext/PaperMarioMessages.py +++ b/tools/splat_ext/PaperMarioMessages.py @@ -257,8 +257,8 @@ CHARSET = { 0x18: {None: lambda d: (f"[Image index={d[0]} pos={(d[1] << 8) + d[2]},{d[3]} hasBorder={d[4]} alpha={d[5]} fadeAmount={d[6]}]\n", 7)}, 0x19: {None: lambda d: (f"[HideImage fadeAmount={d[0]}]\n", 1)}, 0x1A: {None: lambda d: (f"[AnimDelay index={d[1]} delay={d[2]}]", 3)}, - 0x1B: {None: lambda d: (f"[AnimLoop]", 2)}, - 0x1C: {None: lambda d: (f"[AnimDone]", 1)}, + 0x1B: {None: lambda d: (f"[AnimLoop {d[0]} {d[1]}]", 2)}, + 0x1C: {None: lambda d: (f"[AnimDone {d[0]}]", 1)}, 0x1E: {None: lambda d: (f"[Cursor {d[0]}]", 1)}, 0x1F: {None: lambda d: (f"[EndChoice {d[0]}]", 1)}, 0x20: {None: lambda d: (f"[SetCancel {d[0]}]", 1)},