More autos (#1097)

* prep

* git subrepo pull --force tools/splat

subrepo:
  subdir:   "tools/splat"
  merged:   "aab26aab63"
upstream:
  origin:   "https://github.com/ethteck/splat.git"
  branch:   "master"
  commit:   "aab26aab63"
git-subrepo:
  version:  "0.4.5"
  origin:   "https://github.com/ingydotnet/git-subrepo"
  commit:   "aa416e4"

* fix

* blah

tools/build/configure.py

@@ -624,7 +624,11 @@ class Configure:
                     )
                 # Not dead cod
                 else:
-                    if non_matching or seg.get_most_parent().name not in ["main", "engine1", "engine2"]:
+                    if non_matching or seg.get_most_parent().name not in [
+                        "main",
+                        "engine1",
+                        "engine2",
+                    ]:
                         cflags += " -fno-common"
                     build(
                         entry.object_path,
@@ -1171,6 +1175,7 @@ if __name__ == "__main__":
         "version",
         nargs="*",
         default=[],
+        choices=[*VERSIONS, []],
         help="Version(s) to configure for. Most tools will operate on the first-provided only. Supported versions: "
         + ",".join(VERSIONS),
     )

tools/splat/.gitrepo

@@ -6,7 +6,7 @@
 [subrepo]
 	remote = https://github.com/ethteck/splat.git
 	branch = master
-	commit = 7869ef2e51a5974fb8e583e45e8b47793d3fecd2
+	commit = aab26aab63c40fe0bffeb21bf27677881bdb6910
-	parent = 03b911bbea92dc1d78522d8de3cc41c2fd45ecbc
+	parent = 943a5d67594b76f0d7f030ae9fc613e7c5fe046c
 	method = merge
 	cmdver = 0.4.5

tools/splat/CHANGELOG.md

@@ -1,9 +1,33 @@
 # splat Release Notes
 
+### 0.16.1
+* Various changes so that series of image and palette subsegments can have `auto` rom addresses (as long as the first can find its rom address from the parent segment or its own definition)
+
+### 0.16.0
+
+* Add option `detect_redundant_function_end`. It tries to detect redundant and unreferenced functions ends and merge them together.
+  * This option is ignored if the compiler is not set to IDO.
+  * This type of codegen is only affected by flags `-g`, `-g1` and `-g2`.
+  * This option can also be overriden per file.
+* Disable `include_macro_inc` by default for IDO projects.
+* Disable `asm_emit_size_directive` by default for SN64 projects.
+* `spimdisasm` 1.16.0 or above is now required.
+
+### 0.15.4
+
+* Try to assign a segment to an user-declared symbol if the user declared the rom address.
+  * Helps to disambiguate symbols for same-address overlays.
+
+### 0.15.3
+
+* Disabled `asm_emit_size_directive` by default for IDO projects.
+
 ### 0.15.2
+
 * Various cleanup and fixes to support more liberal use of `auto` for rom addresses
 
 ### 0.15.1
+
 * Made some modifications such that linker object paths should be simpler in some circumstances
 
 ### 0.15.0
@@ -349,7 +373,7 @@ The `auto_all_sections` option, when set to true, will automatically add `all_`
     * Code and ASM modes have been combined into the `code` mode
 * BREAKING: The `name` attribute of a segment now should no longer be a subdirectory but rather a meaningful name for the segment which will be used as the name of the linker section. If your `name` was previously a directory, please change it into a `dir`.
 * BREAKING: `subsections` has been renamed to `subsegments`
-* New `dir` segment attribute specifies a subdirectory into which files will be saved. You can combine `dir` ("foo") with a subsection file name containing a subdirectory ("bar/out"), and the paths will be joined (foo/bar/out.c)
+* New `dir` segment attribute specifies a subdirectory into which files will be saved. You can combine `dir` ("foo") with a subsegment name containing a subdirectory ("bar/out"), and the paths will be joined (foo/bar/out.c)
   * If the `dir` attribute is specified but the `name` isn't, the `name` becomes `dir` with directory separation slashes replaced with underscores (foo/bar/baz -> foo_bar_baz)
 * BREAKING: Many configuration options have been renamed. `_dir` options have been changed to the suffix `_path`.
 * BREAKING: Assets (non-code, like `bin` and images) are now placed in the directory `asset_path` (defaults to `assets`).

tools/splat/disassembler/spimdisasm_disassembler.py

@@ -8,7 +8,7 @@ from typing import Set
 
 class SpimdisasmDisassembler(disassembler.Disassembler):
     # This value should be kept in sync with the version listed on requirements.txt
-    SPIMDISASM_MIN = (1, 15, 0)
+    SPIMDISASM_MIN = (1, 16, 0)
 
     def configure(self, opts: SplatOpts):
         # Configure spimdisasm
@@ -73,6 +73,10 @@ class SpimdisasmDisassembler(disassembler.Disassembler):
         elif selected_compiler == compiler.IDO:
             spimdisasm.common.GlobalConfig.COMPILER = spimdisasm.common.Compiler.IDO
 
+        spimdisasm.common.GlobalConfig.DETECT_REDUNDANT_FUNCTION_END = (
+            opts.detect_redundant_function_end
+        )
+
         spimdisasm.common.GlobalConfig.GP_VALUE = opts.gp
 
         spimdisasm.common.GlobalConfig.ASM_TEXT_LABEL = opts.asm_function_macro

tools/splat/requirements.txt

@@ -4,7 +4,7 @@ tqdm
 intervaltree
 colorama
 # This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py
-spimdisasm>=1.15.0
+spimdisasm>=1.16.0
 rabbitizer>=1.7.0
 pygfxd
 n64img>=0.1.4

tools/splat/segtypes/common/code.py

@@ -178,12 +178,12 @@ class CommonSegCode(CommonSegGroup):
         # Mark any manually added dot types
         cur_section = None
 
-        for i, subsection_yaml in enumerate(segment_yaml["subsegments"]):
+        for i, subsegment_yaml in enumerate(segment_yaml["subsegments"]):
             # endpos marker
-            if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1:
+            if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1:
                 continue
 
-            typ = Segment.parse_segment_type(subsection_yaml)
+            typ = Segment.parse_segment_type(subsegment_yaml)
             if typ.startswith("all_"):
                 typ = typ[4:]
             if not typ.startswith("."):
@@ -218,15 +218,15 @@ class CommonSegCode(CommonSegGroup):
 
         inserts = self.find_inserts(found_sections)
 
-        last_rom_end = 0
+        last_rom_end = None
 
-        for i, subsection_yaml in enumerate(segment_yaml["subsegments"]):
+        for i, subsegment_yaml in enumerate(segment_yaml["subsegments"]):
             # endpos marker
-            if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1:
+            if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1:
                 continue
 
-            typ = Segment.parse_segment_type(subsection_yaml)
+            typ = Segment.parse_segment_type(subsegment_yaml)
-            start = Segment.parse_segment_start(subsection_yaml)
+            start = Segment.parse_segment_start(subsegment_yaml)
 
             # Add dummy segments to be expanded later
             if typ.startswith("all_"):
@@ -253,11 +253,21 @@ class CommonSegCode(CommonSegGroup):
 
             end = self.get_next_seg_start(i, segment_yaml["subsegments"])
 
-            if (
+            if start is None:
-                isinstance(start, int)
+                # Attempt to infer the start address
-                and isinstance(prev_start, int)
+                if i == 0:
-                and start < prev_start
+                    # The start address of this segment is the start address of the group
-            ):
+                    start = self.rom_start
+                else:
+                    # The start address is the end address of the previous segment
+                    start = last_rom_end
+
+            if start is not None and end is None:
+                est_size = segment_class.estimate_size(subsegment_yaml)
+                if est_size is not None:
+                    end = start + est_size
+
+            if start is not None and prev_start is not None and start < prev_start:
                 log.error(
                     f"Error: Group segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})"
                 )
@@ -274,7 +284,7 @@ class CommonSegCode(CommonSegGroup):
                 end = last_rom_end
 
             segment: Segment = Segment.from_yaml(
-                segment_class, subsection_yaml, start, end, vram
+                segment_class, subsegment_yaml, start, end, vram
             )
 
             segment.sibling = base_segments.get(segment.name, None)

tools/splat/segtypes/common/codesubsegment.py

@@ -33,6 +33,12 @@ class CommonSegCodeSubsegment(Segment):
         elif options.opts.platform == "psx":
             self.instr_category = rabbitizer.InstrCategory.R3000GTE
 
+        self.detect_redundant_function_end: Optional[bool] = (
+            self.yaml.get("detect_redundant_function_end", None)
+            if isinstance(self.yaml, dict)
+            else None
+        )
+
     @property
     def needs_symbols(self) -> bool:
         return True
@@ -72,6 +78,9 @@ class CommonSegCodeSubsegment(Segment):
 
         self.spim_section.get_section().isHandwritten = is_hasm
         self.spim_section.get_section().instrCat = self.instr_category
+        self.spim_section.get_section().detectRedundantFunctionEnd = (
+            self.detect_redundant_function_end
+        )
 
         self.spim_section.analyze()
         self.spim_section.set_comment_offset(self.rom_start)

tools/splat/segtypes/common/group.py

@@ -45,23 +45,33 @@ class CommonSegGroup(CommonSegment):
         prev_start: Optional[int] = -1
         last_rom_end = 0
 
-        for i, subsection_yaml in enumerate(yaml["subsegments"]):
+        for i, subsegment_yaml in enumerate(yaml["subsegments"]):
             # endpos marker
-            if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1:
+            if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1:
                 continue
 
-            typ = Segment.parse_segment_type(subsection_yaml)
+            typ = Segment.parse_segment_type(subsegment_yaml)
-            start = Segment.parse_segment_start(subsection_yaml)
+            start = Segment.parse_segment_start(subsegment_yaml)
 
             segment_class = Segment.get_class_for_type(typ)
 
             end = self.get_next_seg_start(i, yaml["subsegments"])
 
-            if (
+            if start is None:
-                isinstance(start, int)
+                # Attempt to infer the start address
-                and isinstance(prev_start, int)
+                if i == 0:
-                and start < prev_start
+                    # The start address of this segment is the start address of the group
-            ):
+                    start = self.rom_start
+                else:
+                    # The start address is the end address of the previous segment
+                    start = last_rom_end
+
+            if start is not None and end is None:
+                est_size = segment_class.estimate_size(subsegment_yaml)
+                if est_size is not None:
+                    end = start + est_size
+
+            if start is not None and prev_start is not None and start < prev_start:
                 log.error(
                     f"Error: Group segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})"
                 )
@@ -82,7 +92,7 @@ class CommonSegGroup(CommonSegment):
                 end = last_rom_end
 
             segment: Segment = Segment.from_yaml(
-                segment_class, subsection_yaml, start, end, vram
+                segment_class, subsegment_yaml, start, end, vram
             )
             segment.parent = self
             if segment.special_vram_segment:

tools/splat/segtypes/n64/img.py

@@ -1,5 +1,5 @@
 from pathlib import Path
-from typing import Type, Optional
+from typing import Dict, List, Tuple, Type, Optional, Union
 
 from n64img.image import Image
 from util import log, options
@@ -8,6 +8,15 @@ from segtypes.n64.segment import N64Segment
 
 
 class N64SegImg(N64Segment):
+    @staticmethod
+    def parse_dimensions(yaml: Union[Dict, List]) -> Tuple[int, int]:
+        if isinstance(yaml, dict):
+            return yaml["width"], yaml["height"]
+        else:
+            if len(yaml) < 5:
+                log.error(f"Error: {yaml} is missing width and height parameters")
+            return yaml[3], yaml[4]
+
     def __init__(
         self,
         rom_start: Optional[int],
@@ -29,23 +38,16 @@ class N64SegImg(N64Segment):
             yaml=yaml,
         )
 
-        self.n64img: Image = img_cls(None, 0, 0)
+        if rom_start is None:
+            log.error(f"Error: {type} segment {name} rom start could not be determined")
+
+        self.n64img: Image = img_cls(b"", 0, 0)
 
         if isinstance(yaml, dict):
-            if self.extract:
-                self.width = yaml["width"]
-                self.height = yaml["height"]
-
             self.n64img.flip_h = bool(yaml.get("flip_x", False))
             self.n64img.flip_v = bool(yaml.get("flip_y", False))
-        else:
+
-            if self.extract:
+        self.width, self.height = self.parse_dimensions(yaml)
-                if len(yaml) < 5:
-                    log.error(
-                        f"Error: {self.name} is missing width and height parameters"
-                    )
-                self.width = yaml[3]
-                self.height = yaml[4]
 
         self.n64img.width = self.width
         self.n64img.height = self.height
@@ -53,22 +55,21 @@ class N64SegImg(N64Segment):
         self.check_len()
 
     def check_len(self) -> None:
-        if self.extract:
+        expected_len = int(self.n64img.size())
-            expected_len = int(self.n64img.size())
+        assert isinstance(self.rom_start, int)
-            assert isinstance(self.rom_start, int)
+        assert isinstance(self.rom_end, int)
-            assert isinstance(self.rom_end, int)
+        assert isinstance(self.subalign, int)
-            assert isinstance(self.subalign, int)
+        actual_len = self.rom_end - self.rom_start
-            actual_len = self.rom_end - self.rom_start
+        if actual_len > expected_len and actual_len - expected_len > self.subalign:
-            if actual_len > expected_len and actual_len - expected_len > self.subalign:
+            log.error(
-                log.error(
+                f"Error: {self.name} should end at 0x{self.rom_start + expected_len:X}, but it ends at 0x{self.rom_end:X}\n(hint: add a 'bin' segment after it)"
-                    f"Error: {self.name} should end at 0x{self.rom_start + expected_len:X}, but it ends at 0x{self.rom_end:X}\n(hint: add a 'bin' segment after it)"
+            )
-                )
 
     def out_path(self) -> Path:
         return options.opts.asset_path / self.dir / f"{self.name}.png"
 
     def should_split(self) -> bool:
-        return self.extract and options.opts.is_mode_active("img")
+        return options.opts.is_mode_active("img")
 
     def split(self, rom_bytes):
         path = self.out_path()
@@ -77,8 +78,22 @@ class N64SegImg(N64Segment):
         assert isinstance(self.rom_start, int)
         assert isinstance(self.rom_end, int)
 
-        if self.n64img.data is None:
+        if self.n64img.data == b"":
             self.n64img.data = rom_bytes[self.rom_start : self.rom_end]
         self.n64img.write(path)
 
         self.log(f"Wrote {self.name} to {path}")
+
+    @staticmethod
+    def estimate_size(yaml: Union[Dict, List]) -> int:
+        width, height = N64SegImg.parse_dimensions(yaml)
+        typ = N64Segment.parse_segment_type(yaml)
+
+        if typ == "ci4" or typ == "i4" or typ == "ia4":
+            return width * height // 2
+        elif typ in ("ia16", "rgba16"):
+            return width * height * 2
+        elif typ == "rgba32":
+            return width * height * 4
+        else:
+            return width * height

tools/splat/segtypes/n64/palette.py

@@ -1,6 +1,6 @@
 from itertools import zip_longest
 from pathlib import Path
-from typing import List, Optional, Tuple, TYPE_CHECKING
+from typing import Dict, List, Optional, Tuple, TYPE_CHECKING, Union
 
 from util import log, options
 from util.color import unpack_color
@@ -16,6 +16,9 @@ def iter_in_groups(iterable, n, fillvalue=None):
     return zip_longest(*args, fillvalue=fillvalue)
 
 
+VALID_SIZES = [0x20, 0x40, 0x80, 0x100, 0x200]
+
+
 class N64SegPalette(N64Segment):
     require_unique_name = False
 
@@ -40,18 +43,21 @@ class N64SegPalette(N64Segment):
                     f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it"
                 )
 
-            if self.max_length() and isinstance(self.rom_end, int):
+            if not isinstance(self.yaml, dict) or "size" not in self.yaml:
-                expected_len = int(self.max_length())
+                assert self.rom_end is not None
-                assert isinstance(self.rom_end, int)
+                assert self.rom_start is not None
-                assert isinstance(self.rom_start, int)
-                assert isinstance(self.subalign, int)
                 actual_len = self.rom_end - self.rom_start
-                if (
+
-                    actual_len > expected_len
+                hint_msg = "(hint: add a 'bin' segment after it or specify the size in the segment)"
-                    and actual_len - expected_len > self.subalign
+
-                ):
+                if actual_len > VALID_SIZES[-1]:
                     log.error(
-                        f"Error: {self.name} should end at 0x{self.rom_start + expected_len:X}, but it ends at 0x{self.rom_end:X}\n(hint: add a 'bin' segment after it)"
+                        f"Error: {self.name} (0x{actual_len:X} bytes) is too long, max 0x{VALID_SIZES[-1]:X})\n{hint_msg}"
+                    )
+
+                if actual_len not in VALID_SIZES:
+                    log.error(
+                        f"Error: {self.name} (0x{actual_len:X} bytes) is not a valid palette size ({', '.join(hex(s) for s in VALID_SIZES)})\n{hint_msg}"
                     )
 
     def split(self, rom_bytes):
@@ -66,9 +72,6 @@ class N64SegPalette(N64Segment):
         self.raster.extract = False
 
     def parse_palette(self, rom_bytes) -> List[Tuple[int, int, int, int]]:
-        assert isinstance(self.rom_start, int)
-        assert isinstance(self.rom_end, int)
-
         data = rom_bytes[self.rom_start : self.rom_end]
         palette = []
 
@@ -77,9 +80,6 @@ class N64SegPalette(N64Segment):
 
         return palette
 
-    def max_length(self):
-        return 256 * 2
-
     def out_path(self) -> Path:
         return options.opts.asset_path / self.dir / f"{self.name}.png"
 
@@ -97,3 +97,10 @@ class N64SegPalette(N64Segment):
                 self.get_linker_section(),
             )
         ]
+
+    @staticmethod
+    def estimate_size(yaml: Union[Dict, List]) -> int:
+        if isinstance(yaml, dict):
+            if "size" in yaml:
+                return int(yaml["size"])
+        return 0x20

tools/splat/segtypes/segment.py

@@ -304,6 +304,10 @@ class Segment:
     def is_noload() -> bool:
         return False
 
+    @staticmethod
+    def estimate_size(yaml: Union[Dict, List]) -> Optional[int]:
+        return None
+
     @property
     def needs_symbols(self) -> bool:
         return False
@@ -455,9 +459,6 @@ class Segment:
     def warn(self, msg: str):
         self.warnings.append(msg)
 
-    def max_length(self):
-        return None
-
     @staticmethod
     def get_default_name(addr) -> str:
         return f"{addr:X}"

tools/splat/split.py

@@ -19,7 +19,7 @@ from segtypes.linker_entry import (
 from segtypes.segment import Segment
 from util import log, options, palettes, symbols, relocs
 
-VERSION = "0.15.2"
+VERSION = "0.16.1"
 
 parser = argparse.ArgumentParser(
     description="Split a rom given a rom, a config, and output directory"

tools/splat/util/compiler.py

@@ -1,4 +1,5 @@
 from dataclasses import dataclass
+from typing import Optional
 
 
 @dataclass
@@ -11,6 +12,7 @@ class Compiler:
     c_newline: str = "\n"
     asm_inc_header: str = ""
     include_macro_inc: bool = True
+    asm_emit_size_directive: Optional[bool] = None
 
 
 GCC = Compiler(
@@ -26,9 +28,10 @@ SN64 = Compiler(
     asm_end_label=".end",
     c_newline="\r\n",
     include_macro_inc=False,
+    asm_emit_size_directive=False,
 )
 
-IDO = Compiler("IDO")
+IDO = Compiler("IDO", include_macro_inc=False, asm_emit_size_directive=False)
 
 compiler_for_name = {"GCC": GCC, "SN64": SN64, "IDO": IDO}
 

tools/splat/util/options.py

@@ -175,6 +175,8 @@ class SplatOpts:
     asm_generated_by: bool
     # Tells the disassembler to try disassembling functions with unknown instructions instead of falling back to disassembling as raw data
     disasm_unknown: bool
+    # Tries to detect redundant and unreferenced functions ends and merge them together. This option is ignored if the compiler is not set to IDO.
+    detect_redundant_function_end: bool
 
     ################################################################################
     # N64-specific options
@@ -289,6 +291,11 @@ def _parse_yaml(
     )
     asm_path: Path = p.parse_path(base_path, "asm_path", "asm")
 
+    asm_emit_size_directive = p.parse_optional_opt("asm_emit_size_directive", bool)
+    # If option not provided then use the compiler default
+    if asm_emit_size_directive is None:
+        asm_emit_size_directive = comp.asm_emit_size_directive
+
     def parse_endianness() -> Literal["big", "little"]:
         endianness = p.parse_opt_within(
             "endianness",
@@ -389,7 +396,7 @@ def _parse_yaml(
         ),
         asm_data_macro=p.parse_opt("asm_data_macro", str, comp.asm_data_macro),
         asm_end_label=p.parse_opt("asm_end_label", str, comp.asm_end_label),
-        asm_emit_size_directive=p.parse_optional_opt("asm_emit_size_directive", bool),
+        asm_emit_size_directive=asm_emit_size_directive,
         include_macro_inc=p.parse_opt(
             "include_macro_inc", bool, comp.include_macro_inc
         ),
@@ -431,6 +438,9 @@ def _parse_yaml(
         filesystem_path=p.parse_optional_path(base_path, "filesystem_path"),
         asm_generated_by=p.parse_opt("asm_generated_by", bool, True),
         disasm_unknown=p.parse_opt("disasm_unknown", bool, False),
+        detect_redundant_function_end=p.parse_opt(
+            "detect_redundant_function_end", bool, True
+        ),
     )
     p.check_no_unread_opts()
     return ret

tools/splat/util/symbols.py

@@ -72,13 +72,21 @@ def to_cname(symbol_name: str) -> str:
     return symbol_name
 
 
-def handle_sym_addrs(path: Path, sym_addrs_lines: List[str], all_segments):
+def handle_sym_addrs(
+    path: Path, sym_addrs_lines: List[str], all_segments: "List[Segment]"
+):
     def get_seg_for_name(name: str) -> Optional["Segment"]:
         for segment in all_segments:
             if segment.name == name:
                 return segment
         return None
 
+    def get_seg_for_rom(rom: int) -> Optional["Segment"]:
+        for segment in all_segments:
+            if segment.contains_rom(rom):
+                return segment
+        return None
+
     for line_num, line in enumerate(
         tqdm.tqdm(sym_addrs_lines, desc=f"Loading symbols ({path.stem})")
     ):
@@ -234,6 +242,9 @@ def handle_sym_addrs(path: Path, sym_addrs_lines: List[str], all_segments):
 
                 continue
 
+            if sym.segment is None and sym.rom is not None:
+                sym.segment = get_seg_for_rom(sym.rom)
+
             if sym.segment:
                 sym.segment.add_symbol(sym)
 

ver/pal/splat.yaml

@@ -2963,11 +2963,11 @@ segments:
       - [0x3D8E30, ci4, D_09000420_3AB450, 32, 64]
       - [0x3D9230, palette, D_09000420_3AB450]
       - [0x3D9250, ci4, D_09000840_3AB870, 32, 64]
-      - [0x3D9650, palette, D_09000840_3AB870]
+      - {start: 0x3D9650, type: palette, name: D_09000840_3AB870, size: 0x10}
       - [0x3D9660, ci4, D_09000C50_3ABC80, 32, 64]
       - [0x3D9A60, palette, D_09000C50_3ABC80]
       - [0x3D9A80, ci4, D_09001070_3AC0A0, 32, 64]
-      - [0x3D9E80, palette, D_09001070_3AC0A0]
+      - {start: 0x3D9E80, type: palette, name: D_09001070_3AC0A0, size: 0x10}
       - [0x3D9E90, ci4, D_09001480_3AC4B0, 32, 64]
       - [0x3DA290, palette, D_09001480_3AC4B0]
       - [0x3DA2B0, ci4, D_090018A0_3AC8D0, 32, 64]