diff --git a/tools/build/configure.py b/tools/build/configure.py index 508b32e9fb..6e505553b4 100755 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -32,7 +32,7 @@ def exec_shell(command: List[str]) -> str: return ret.stdout def write_ninja_rules(ninja: ninja_syntax.Writer, cpp: str, cppflags: str, extra_cflags: str, use_ccache: bool, - non_matching: bool): + non_matching: bool, debug: bool): # platform-specific if sys.platform == "darwin": iconv = "tools/iconv.py UTF-8 SHIFT-JIS" @@ -78,9 +78,17 @@ def write_ninja_rules(ninja: ninja_syntax.Writer, cpp: str, cppflags: str, extra command=f"{cross}ld -T ver/$version/build/undefined_syms.txt -T ver/$version/undefined_syms_auto.txt -T ver/$version/undefined_funcs_auto.txt -Map $mapfile --no-check-sections -T $in -o $out", ) + objcopy_sections = "" + if debug: + ninja.rule("genobjcopy", + description="generate $out", + command=f"$python {BUILD_TOOLS}/genobjcopy.py $in $out", + ) + objcopy_sections = "@ver/$version/build/objcopy_sections.txt " + ninja.rule("z64", description="rom $out", - command=f"{cross}objcopy $in $out -O binary && {BUILD_TOOLS}/rom/n64crc $out", + command=f"{cross}objcopy {objcopy_sections} $in $out -O binary && {BUILD_TOOLS}/rom/n64crc $out", ) ninja.rule("sha1sum", @@ -227,7 +235,7 @@ class Configure: self.version_path = ROOT / f"ver/{version}" self.linker_entries = None - def split(self, assets: bool, code: bool): + def split(self, assets: bool, code: bool, debug: bool): import split modes = ["ld"] @@ -237,12 +245,16 @@ class Configure: if code: modes.extend(["code", "c", "data", "rodata"]) + splat_file = [str(self.version_path / "splat.yaml")] + if debug: + splat_file += [str(self.version_path / "splat-debug.yaml")] + split.main( - str(self.version_path / "splat.yaml"), + splat_file, None, str(self.version_path / "baserom.z64"), modes, - verbose=False, + verbose=True, ) self.linker_entries = split.linker_writer.entries[:] self.asset_stack = split.config["asset_stack"] @@ -250,6 +262,9 @@ class Configure: def build_path(self) -> Path: return Path(f"ver/{self.version}/build") + def objcopy_sections_path(self) -> Path: + return self.build_path() / "objcopy_sections.txt" + def undefined_syms_path(self) -> Path: return self.build_path() / "undefined_syms.txt" @@ -296,7 +311,7 @@ class Configure: # ¯\_(ツ)_/¯ return path - def write_ninja(self, ninja: ninja_syntax.Writer, skip_outputs: Set[str], non_matching: bool): + def write_ninja(self, ninja: ninja_syntax.Writer, skip_outputs: Set[str], non_matching: bool, debug: bool): import segtypes import segtypes.common.data import segtypes.n64.Yay0 @@ -615,6 +630,14 @@ class Configure: else: raise Exception(f"don't know how to build {seg.__class__.__name__} '{seg.name}'") + # Create objcopy section list + if debug: + ninja.build( + str(self.objcopy_sections_path()), + "genobjcopy", + str(self.build_path() / "elf_sections.txt"), + ) + # Run undefined_syms through cpp ninja.build( str(self.undefined_syms_path()), @@ -623,11 +646,15 @@ class Configure: ) # Build elf, z64, ok + additional_objects = [str(self.undefined_syms_path())] + if debug: + additional_objects += [str(self.objcopy_sections_path())] + ninja.build( str(self.elf_path()), "ld", str(self.linker_script_path()), - implicit=[str(obj) for obj in built_objects] + [str(self.undefined_syms_path())], + implicit=[str(obj) for obj in built_objects] + additional_objects, variables={ "version": self.version, "mapfile": str(self.map_path()) }, ) ninja.build( @@ -635,6 +662,7 @@ class Configure: "z64", str(self.elf_path()), implicit=[CRC_TOOL], + variables={ "version": self.version }, ) ninja.build( str(self.rom_ok_path()), @@ -731,14 +759,14 @@ if __name__ == "__main__": cflags += " -g1" if not args.no_warn: - cflags += "-Wuninitialized -Wmissing-braces -Wimplicit -Wredundant-decls -Wstrict-prototypes" + cflags += " -Wuninitialized -Wmissing-braces -Wimplicit -Wredundant-decls -Wstrict-prototypes" # add splat to python import path sys.path.append(str((ROOT / args.splat).resolve())) ninja = ninja_syntax.Writer(open(str(ROOT / "build.ninja"), "w"), width=9999) - write_ninja_rules(ninja, args.cpp or "cpp", cppflags, cflags, args.ccache, args.non_matching) + write_ninja_rules(ninja, args.cpp or "cpp", cppflags, cflags, args.ccache, args.non_matching, args.debug) write_ninja_for_tools(ninja) skip_files = set() @@ -753,8 +781,8 @@ if __name__ == "__main__": if not first_configure: first_configure = configure - configure.split(not args.no_split_assets, args.split_code) - configure.write_ninja(ninja, skip_files, args.non_matching) + configure.split(not args.no_split_assets, args.split_code, args.debug) + configure.write_ninja(ninja, skip_files, args.non_matching, args.debug) all_rom_oks.append(str(configure.rom_ok_path())) diff --git a/tools/build/genobjcopy.py b/tools/build/genobjcopy.py new file mode 100644 index 0000000000..4d24023772 --- /dev/null +++ b/tools/build/genobjcopy.py @@ -0,0 +1,23 @@ +#! /usr/bin/python3 +import sys, os + +#Under normal compilation we rely on splat to use a discard option in the ldscript +#to not include sections in the elf then just output all sections, however under debug we want +#to have debug sections. +#In debugging mode splat is told to output a list of sections it is custom creating, which are +#all of the sections we export to the z64 file with an objcopy. The below chunk of code is +#responsible for adding -j to each of the names and outputting a file for objcopy to use +#so we can still generate a elf file with all the extra debugging sections and still output +# the required sections to the .z64 without outputting everything. + +if __name__ == "__main__": + infile, outfile = sys.argv[1:] + + #generate output based on input + file_data = open(infile,"r").read().split("\n") + if len(file_data[-1]) == 0: + file_data.pop() + + outdata = "-j " + " -j ".join(file_data) + with open(outfile, "w") as f: + f.write(outdata) diff --git a/tools/splat/.gitrepo b/tools/splat/.gitrepo index 87bc0e0cd5..85186c61c7 100644 --- a/tools/splat/.gitrepo +++ b/tools/splat/.gitrepo @@ -6,7 +6,7 @@ [subrepo] remote = https://github.com/ethteck/splat.git branch = master - commit = 0efa552c5d3cf866b6325486868714787261673d - parent = 35fa67cd8de20bf6dacdb92f1ac8411d3e5f09cf + commit = e5838f0b063425e843d44091b8654962995829bd + parent = a1965af227c11f4dde5a7114352db3d24f0bc6a9 method = merge cmdver = 0.4.3 diff --git a/tools/splat/segtypes/linker_entry.py b/tools/splat/segtypes/linker_entry.py index 0daa120110..4eabc17b30 100644 --- a/tools/splat/segtypes/linker_entry.py +++ b/tools/splat/segtypes/linker_entry.py @@ -66,6 +66,7 @@ class LinkerEntry: class LinkerWriter(): def __init__(self): self.shiftable: bool = options.get("shiftable", False) + self.linker_discard_section: bool = options.get("linker_discard_section", True) self.entries: List[LinkerEntry] = [] self.buffer: List[str] = [] @@ -154,10 +155,11 @@ class LinkerWriter(): self._end_segment(segment) def save_linker_script(self): - self._writeln("/DISCARD/ :") - self._begin_block() - self._writeln("*(*);") - self._end_block() + if self.linker_discard_section: + self._writeln("/DISCARD/ :") + self._begin_block() + self._writeln("*(*);") + self._end_block() self._end_block() # SECTIONS diff --git a/tools/splat/split.py b/tools/splat/split.py index daad202655..a140e79c87 100755 --- a/tools/splat/split.py +++ b/tools/splat/split.py @@ -8,7 +8,7 @@ import yaml import pickle from colorama import Style, Fore from segtypes.segment import Segment -from segtypes.linker_entry import LinkerWriter +from segtypes.linker_entry import LinkerWriter, to_cname from util import log from util import options from util import symbols @@ -17,7 +17,7 @@ from util import palettes VERSION = "0.7.10" parser = argparse.ArgumentParser(description="Split a rom given a rom, a config, and output directory") -parser.add_argument("config", help="path to a compatible config .yaml file") +parser.add_argument("config", help="path to a compatible config .yaml file", nargs='+') parser.add_argument("--target", help="path to a file to split (.z64 rom)") parser.add_argument("--basedir", help="a directory in which to extract the rom") parser.add_argument("--modes", nargs="+", default="all") @@ -109,14 +109,42 @@ def do_statistics(seg_sizes, rom_bytes, seg_split, seg_cached): log.write(f"{typ:>20}: {fmt_size(tmp_size):>8} ({tmp_ratio:.2%}) {Fore.GREEN}{seg_split[typ]} split{Style.RESET_ALL}, {Style.DIM}{seg_cached[typ]} cached") log.write(f"{'unknown':>20}: {fmt_size(unk_size):>8} ({unk_ratio:.2%}) from unknown bin files") +def merge_configs(main_config, additional_config): + # Merge rules are simple + # For each key in the dictionary + # - If list then append to list + # - If a dictionary then repeat merge on sub dictionary entries + # - Else assume string or number and replace entry + + for curkey in additional_config: + if curkey not in main_config: + main_config[curkey] = additional_config[curkey] + elif type(main_config[curkey]) != type(additional_config[curkey]): + log.error(f"Type for key {curkey} in configs does not match") + else: + # keys exist and match, see if a list to append + if type(main_config[curkey]) == list: + main_config[curkey] += additional_config[curkey] + elif type(main_config[curkey]) == dict: + #need to merge sub areas + main_config[curkey] = merge_configs(main_config[curkey], additional_config[curkey]) + else: + #not a list or dictionary, must be a number or string, overwrite + main_config[curkey] = additional_config[curkey] + + return main_config + def main(config_path, base_dir, target_path, modes, verbose, use_cache=True): global config log.write(f"splat {VERSION}") # Load config - with open(config_path) as f: - config = yaml.load(f.read(), Loader=yaml.SafeLoader) + config = {} + for entry in config_path: + with open(entry) as f: + additional_config = yaml.load(f.read(), Loader=yaml.SafeLoader) + config = merge_configs(config, additional_config) options.initialize(config, config_path, base_dir, target_path) options.set("modes", modes) @@ -236,6 +264,15 @@ def main(config_path, base_dir, target_path, modes, verbose, use_cache=True): linker_writer.save_linker_script() linker_writer.save_symbol_header() + # write elf_sections.txt - this only lists the generated sections in the elf, not sub sections + # that the elf combines into one section + if options.get_create_elf_section_list_auto(): + section_list = "" + for segment in all_segments: + section_list += "." + to_cname(segment.name) + "\n" + with open(options.get_elf_section_list_path(), "w", newline="\n") as f: + f.write(section_list) + # Write undefined_funcs_auto.txt if options.get_create_undefined_funcs_auto(): to_write = [s for s in symbols.all_symbols if s.referenced and not s.defined and not s.dead and s.type == "func"] diff --git a/tools/splat/util/options.py b/tools/splat/util/options.py index efa5f40160..42b378a5eb 100644 --- a/tools/splat/util/options.py +++ b/tools/splat/util/options.py @@ -4,7 +4,7 @@ from util import log opts = {} -def initialize(config: Dict, config_path: str, base_path=None, target_path=None): +def initialize(config: Dict, config_path, base_path=None, target_path=None): global opts opts = dict(config.get("options", {})) @@ -14,7 +14,7 @@ def initialize(config: Dict, config_path: str, base_path=None, target_path=None) if not "base_path" in opts: log.error("Error: Base output dir not specified as a command line arg or via the config yaml (base_path)") - opts["base_path"] = Path(config_path).parent / opts["base_path"] + opts["base_path"] = Path(config_path[0]).parent / opts["base_path"] if not target_path: if "target_path" not in opts: @@ -77,6 +77,12 @@ def get_create_undefined_syms_auto() -> bool: def get_undefined_syms_auto_path(): return get_base_path() / opts.get("undefined_syms_auto_path", "undefined_syms_auto.txt") +def get_create_elf_section_list_auto(): + return opts.get("create_elf_section_list_auto", False) + +def get_elf_section_list_path(): + return get_base_path() / opts.get("elf_section_list_path", "elf_sections.txt") + def get_symbol_addrs_path(): return get_base_path() / opts.get("symbol_addrs_path", "symbol_addrs.txt") diff --git a/ver/jp/splat-debug.yaml b/ver/jp/splat-debug.yaml new file mode 100644 index 0000000000..ea52050ebd --- /dev/null +++ b/ver/jp/splat-debug.yaml @@ -0,0 +1,4 @@ +options: + linker_discard_section: False + create_elf_section_list_auto: True + elf_section_list_path: ver/jp/build/elf_sections.txt diff --git a/ver/us/splat-debug.yaml b/ver/us/splat-debug.yaml new file mode 100644 index 0000000000..1c53be5d40 --- /dev/null +++ b/ver/us/splat-debug.yaml @@ -0,0 +1,4 @@ +options: + linker_discard_section: False + create_elf_section_list_auto: True + elf_section_list_path: ver/us/build/elf_sections.txt