pmret/papermario
1
0
Fork 0
mirror of https://github.com/pmret/papermario.git synced 2024-11-08 12:02:30 +01:00
Code Issues Packages Projects Releases Wiki Activity

More autos (#1097)

Browse Source 
* prep

* git subrepo pull --force tools/splat

subrepo:
  subdir:   "tools/splat"
  merged:   "aab26aab63"
upstream:
  origin:   "https://github.com/ethteck/splat.git"
  branch:   "master"
  commit:   "aab26aab63"
git-subrepo:
  version:  "0.4.5"
  origin:   "https://github.com/ingydotnet/git-subrepo"
  commit:   "aa416e4"

* fix

* blah
This commit is contained in:
Ethan Roseman 2023-07-25 00:37:23 +09:00 committed by GitHub
parent 76386ce361
commit ccc8e8e46b
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
17 changed files with 809 additions and 702 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
tools/build/configure.py
View File

@ -624,7 +624,11 @@ class Configure:
)
# Not dead cod
else:
if non_matching or seg.get_most_parent().name not in ["main", "engine1", "engine2"]:
if non_matching or seg.get_most_parent().name not in [
"main",
"engine1",
"engine2",
]:
cflags += " -fno-common"
build(
entry.object_path,
@ -1171,6 +1175,7 @@ if __name__ == "__main__":
"version",
nargs="*",
default=[],
choices=[*VERSIONS, []],
help="Version(s) to configure for. Most tools will operate on the first-provided only. Supported versions: "
+ ",".join(VERSIONS),
)

4
tools/splat/.gitrepo
View File

@ -6,7 +6,7 @@
[subrepo]
remote = https://github.com/ethteck/splat.git
branch = master
commit = 7869ef2e51a5974fb8e583e45e8b47793d3fecd2
parent = 03b911bbea92dc1d78522d8de3cc41c2fd45ecbc
commit = aab26aab63c40fe0bffeb21bf27677881bdb6910
parent = 943a5d67594b76f0d7f030ae9fc613e7c5fe046c
method = merge
cmdver = 0.4.5

26
tools/splat/CHANGELOG.md
View File

@ -1,9 +1,33 @@
# splat Release Notes
### 0.16.1
* Various changes so that series of image and palette subsegments can have `auto` rom addresses (as long as the first can find its rom address from the parent segment or its own definition)
### 0.16.0
* Add option `detect_redundant_function_end`. It tries to detect redundant and unreferenced functions ends and merge them together.
* This option is ignored if the compiler is not set to IDO.
* This type of codegen is only affected by flags `-g`, `-g1` and `-g2`.
* This option can also be overriden per file.
* Disable `include_macro_inc` by default for IDO projects.
* Disable `asm_emit_size_directive` by default for SN64 projects.
* `spimdisasm` 1.16.0 or above is now required.
### 0.15.4
* Try to assign a segment to an user-declared symbol if the user declared the rom address.
* Helps to disambiguate symbols for same-address overlays.
### 0.15.3
* Disabled `asm_emit_size_directive` by default for IDO projects.
### 0.15.2
* Various cleanup and fixes to support more liberal use of `auto` for rom addresses
### 0.15.1
* Made some modifications such that linker object paths should be simpler in some circumstances
### 0.15.0
@ -349,7 +373,7 @@ The `auto_all_sections` option, when set to true, will automatically add `all_`
* Code and ASM modes have been combined into the `code` mode
* BREAKING: The `name` attribute of a segment now should no longer be a subdirectory but rather a meaningful name for the segment which will be used as the name of the linker section. If your `name` was previously a directory, please change it into a `dir`.
* BREAKING: `subsections` has been renamed to `subsegments`
* New `dir` segment attribute specifies a subdirectory into which files will be saved. You can combine `dir` ("foo") with a subsection file name containing a subdirectory ("bar/out"), and the paths will be joined (foo/bar/out.c)
* New `dir` segment attribute specifies a subdirectory into which files will be saved. You can combine `dir` ("foo") with a subsegment name containing a subdirectory ("bar/out"), and the paths will be joined (foo/bar/out.c)
* If the `dir` attribute is specified but the `name` isn't, the `name` becomes `dir` with directory separation slashes replaced with underscores (foo/bar/baz -> foo_bar_baz)
* BREAKING: Many configuration options have been renamed. `_dir` options have been changed to the suffix `_path`.
* BREAKING: Assets (non-code, like `bin` and images) are now placed in the directory `asset_path` (defaults to `assets`).

6
tools/splat/disassembler/spimdisasm_disassembler.py
View File

@ -8,7 +8,7 @@ from typing import Set
class SpimdisasmDisassembler(disassembler.Disassembler):
# This value should be kept in sync with the version listed on requirements.txt
SPIMDISASM_MIN = (1, 15, 0)
SPIMDISASM_MIN = (1, 16, 0)
def configure(self, opts: SplatOpts):
# Configure spimdisasm
@ -73,6 +73,10 @@ class SpimdisasmDisassembler(disassembler.Disassembler):
elif selected_compiler == compiler.IDO:
spimdisasm.common.GlobalConfig.COMPILER = spimdisasm.common.Compiler.IDO
spimdisasm.common.GlobalConfig.DETECT_REDUNDANT_FUNCTION_END = (
opts.detect_redundant_function_end
)
spimdisasm.common.GlobalConfig.GP_VALUE = opts.gp
spimdisasm.common.GlobalConfig.ASM_TEXT_LABEL = opts.asm_function_macro

2
tools/splat/requirements.txt
View File

@ -4,7 +4,7 @@ tqdm
intervaltree
colorama
# This value should be keep in sync with the version listed on disassembler/spimdisasm_disassembler.py
spimdisasm>=1.15.0
spimdisasm>=1.16.0
rabbitizer>=1.7.0
pygfxd
n64img>=0.1.4

38
tools/splat/segtypes/common/code.py
View File

@ -178,12 +178,12 @@ class CommonSegCode(CommonSegGroup):
# Mark any manually added dot types
cur_section = None
for i, subsection_yaml in enumerate(segment_yaml["subsegments"]):
for i, subsegment_yaml in enumerate(segment_yaml["subsegments"]):
# endpos marker
if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1:
if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1:
continue
typ = Segment.parse_segment_type(subsection_yaml)
typ = Segment.parse_segment_type(subsegment_yaml)
if typ.startswith("all_"):
typ = typ[4:]
if not typ.startswith("."):
@ -218,15 +218,15 @@ class CommonSegCode(CommonSegGroup):
inserts = self.find_inserts(found_sections)
last_rom_end = 0
last_rom_end = None
for i, subsection_yaml in enumerate(segment_yaml["subsegments"]):
for i, subsegment_yaml in enumerate(segment_yaml["subsegments"]):
# endpos marker
if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1:
if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1:
continue
typ = Segment.parse_segment_type(subsection_yaml)
start = Segment.parse_segment_start(subsection_yaml)
typ = Segment.parse_segment_type(subsegment_yaml)
start = Segment.parse_segment_start(subsegment_yaml)
# Add dummy segments to be expanded later
if typ.startswith("all_"):
@ -253,11 +253,21 @@ class CommonSegCode(CommonSegGroup):
end = self.get_next_seg_start(i, segment_yaml["subsegments"])
if (
isinstance(start, int)
and isinstance(prev_start, int)
and start < prev_start
):
if start is None:
# Attempt to infer the start address
if i == 0:
# The start address of this segment is the start address of the group
start = self.rom_start
else:
# The start address is the end address of the previous segment
start = last_rom_end
if start is not None and end is None:
est_size = segment_class.estimate_size(subsegment_yaml)
if est_size is not None:
end = start + est_size
if start is not None and prev_start is not None and start < prev_start:
log.error(
f"Error: Group segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})"
)
@ -274,7 +284,7 @@ class CommonSegCode(CommonSegGroup):
end = last_rom_end
segment: Segment = Segment.from_yaml(
segment_class, subsection_yaml, start, end, vram
segment_class, subsegment_yaml, start, end, vram
)
segment.sibling = base_segments.get(segment.name, None)

9
tools/splat/segtypes/common/codesubsegment.py
View File

@ -33,6 +33,12 @@ class CommonSegCodeSubsegment(Segment):
elif options.opts.platform == "psx":
self.instr_category = rabbitizer.InstrCategory.R3000GTE
self.detect_redundant_function_end: Optional[bool] = (
self.yaml.get("detect_redundant_function_end", None)
if isinstance(self.yaml, dict)
else None
)
@property
def needs_symbols(self) -> bool:
return True
@ -72,6 +78,9 @@ class CommonSegCodeSubsegment(Segment):
self.spim_section.get_section().isHandwritten = is_hasm
self.spim_section.get_section().instrCat = self.instr_category
self.spim_section.get_section().detectRedundantFunctionEnd = (
self.detect_redundant_function_end
)
self.spim_section.analyze()
self.spim_section.set_comment_offset(self.rom_start)

30
tools/splat/segtypes/common/group.py
View File

@ -45,23 +45,33 @@ class CommonSegGroup(CommonSegment):
prev_start: Optional[int] = -1
last_rom_end = 0
for i, subsection_yaml in enumerate(yaml["subsegments"]):
for i, subsegment_yaml in enumerate(yaml["subsegments"]):
# endpos marker
if isinstance(subsection_yaml, list) and len(subsection_yaml) == 1:
if isinstance(subsegment_yaml, list) and len(subsegment_yaml) == 1:
continue
typ = Segment.parse_segment_type(subsection_yaml)
start = Segment.parse_segment_start(subsection_yaml)
typ = Segment.parse_segment_type(subsegment_yaml)
start = Segment.parse_segment_start(subsegment_yaml)
segment_class = Segment.get_class_for_type(typ)
end = self.get_next_seg_start(i, yaml["subsegments"])
if (
isinstance(start, int)
and isinstance(prev_start, int)
and start < prev_start
):
if start is None:
# Attempt to infer the start address
if i == 0:
# The start address of this segment is the start address of the group
start = self.rom_start
else:
# The start address is the end address of the previous segment
start = last_rom_end
if start is not None and end is None:
est_size = segment_class.estimate_size(subsegment_yaml)
if est_size is not None:
end = start + est_size
if start is not None and prev_start is not None and start < prev_start:
log.error(
f"Error: Group segment {self.name} contains subsegments which are out of ascending rom order (0x{prev_start:X} followed by 0x{start:X})"
)
@ -82,7 +92,7 @@ class CommonSegGroup(CommonSegment):
end = last_rom_end
segment: Segment = Segment.from_yaml(
segment_class, subsection_yaml, start, end, vram
segment_class, subsegment_yaml, start, end, vram
)
segment.parent = self
if segment.special_vram_segment:

49
tools/splat/segtypes/n64/img.py
View File

@ -1,5 +1,5 @@
from pathlib import Path
from typing import Type, Optional
from typing import Dict, List, Tuple, Type, Optional, Union
from n64img.image import Image
from util import log, options
@ -8,6 +8,15 @@ from segtypes.n64.segment import N64Segment
class N64SegImg(N64Segment):
@staticmethod
def parse_dimensions(yaml: Union[Dict, List]) -> Tuple[int, int]:
if isinstance(yaml, dict):
return yaml["width"], yaml["height"]
else:
if len(yaml) < 5:
log.error(f"Error: {yaml} is missing width and height parameters")
return yaml[3], yaml[4]
def __init__(
self,
rom_start: Optional[int],
@ -29,23 +38,16 @@ class N64SegImg(N64Segment):
yaml=yaml,
)
self.n64img: Image = img_cls(None, 0, 0)
if rom_start is None:
log.error(f"Error: {type} segment {name} rom start could not be determined")
self.n64img: Image = img_cls(b"", 0, 0)
if isinstance(yaml, dict):
if self.extract:
self.width = yaml["width"]
self.height = yaml["height"]
self.n64img.flip_h = bool(yaml.get("flip_x", False))
self.n64img.flip_v = bool(yaml.get("flip_y", False))
else:
if self.extract:
if len(yaml) < 5:
log.error(
f"Error: {self.name} is missing width and height parameters"
)
self.width = yaml[3]
self.height = yaml[4]
self.width, self.height = self.parse_dimensions(yaml)
self.n64img.width = self.width
self.n64img.height = self.height
@ -53,7 +55,6 @@ class N64SegImg(N64Segment):
self.check_len()
def check_len(self) -> None:
if self.extract:
expected_len = int(self.n64img.size())
assert isinstance(self.rom_start, int)
assert isinstance(self.rom_end, int)
@ -68,7 +69,7 @@ class N64SegImg(N64Segment):
return options.opts.asset_path / self.dir / f"{self.name}.png"
def should_split(self) -> bool:
return self.extract and options.opts.is_mode_active("img")
return options.opts.is_mode_active("img")
def split(self, rom_bytes):
path = self.out_path()
@ -77,8 +78,22 @@ class N64SegImg(N64Segment):
assert isinstance(self.rom_start, int)
assert isinstance(self.rom_end, int)
if self.n64img.data is None:
if self.n64img.data == b"":
self.n64img.data = rom_bytes[self.rom_start : self.rom_end]
self.n64img.write(path)
self.log(f"Wrote {self.name} to {path}")
@staticmethod
def estimate_size(yaml: Union[Dict, List]) -> int:
width, height = N64SegImg.parse_dimensions(yaml)
typ = N64Segment.parse_segment_type(yaml)
if typ == "ci4" or typ == "i4" or typ == "ia4":
return width * height // 2
elif typ in ("ia16", "rgba16"):
return width * height * 2
elif typ == "rgba32":
return width * height * 4
else:
return width * height

41
tools/splat/segtypes/n64/palette.py
View File

@ -1,6 +1,6 @@
from itertools import zip_longest
from pathlib import Path
from typing import List, Optional, Tuple, TYPE_CHECKING
from typing import Dict, List, Optional, Tuple, TYPE_CHECKING, Union
from util import log, options
from util.color import unpack_color
@ -16,6 +16,9 @@ def iter_in_groups(iterable, n, fillvalue=None):
return zip_longest(*args, fillvalue=fillvalue)
VALID_SIZES = [0x20, 0x40, 0x80, 0x100, 0x200]
class N64SegPalette(N64Segment):
require_unique_name = False
@ -40,18 +43,21 @@ class N64SegPalette(N64Segment):
f"segment {self.name} needs to know where it ends; add a position marker [0xDEADBEEF] after it"
)
if self.max_length() and isinstance(self.rom_end, int):
expected_len = int(self.max_length())
assert isinstance(self.rom_end, int)
assert isinstance(self.rom_start, int)
assert isinstance(self.subalign, int)
if not isinstance(self.yaml, dict) or "size" not in self.yaml:
assert self.rom_end is not None
assert self.rom_start is not None
actual_len = self.rom_end - self.rom_start
if (
actual_len > expected_len
and actual_len - expected_len > self.subalign
):
hint_msg = "(hint: add a 'bin' segment after it or specify the size in the segment)"
if actual_len > VALID_SIZES[-1]:
log.error(
f"Error: {self.name} should end at 0x{self.rom_start + expected_len:X}, but it ends at 0x{self.rom_end:X}\n(hint: add a 'bin' segment after it)"
f"Error: {self.name} (0x{actual_len:X} bytes) is too long, max 0x{VALID_SIZES[-1]:X})\n{hint_msg}"
)
if actual_len not in VALID_SIZES:
log.error(
f"Error: {self.name} (0x{actual_len:X} bytes) is not a valid palette size ({', '.join(hex(s) for s in VALID_SIZES)})\n{hint_msg}"
)
def split(self, rom_bytes):
@ -66,9 +72,6 @@ class N64SegPalette(N64Segment):
self.raster.extract = False
def parse_palette(self, rom_bytes) -> List[Tuple[int, int, int, int]]:
assert isinstance(self.rom_start, int)
assert isinstance(self.rom_end, int)
data = rom_bytes[self.rom_start : self.rom_end]
palette = []
@ -77,9 +80,6 @@ class N64SegPalette(N64Segment):
return palette
def max_length(self):
return 256 * 2
def out_path(self) -> Path:
return options.opts.asset_path / self.dir / f"{self.name}.png"
@ -97,3 +97,10 @@ class N64SegPalette(N64Segment):
self.get_linker_section(),
)
]
@staticmethod
def estimate_size(yaml: Union[Dict, List]) -> int:
if isinstance(yaml, dict):
if "size" in yaml:
return int(yaml["size"])
return 0x20

7
tools/splat/segtypes/segment.py
View File

@ -304,6 +304,10 @@ class Segment:
def is_noload() -> bool:
return False
@staticmethod
def estimate_size(yaml: Union[Dict, List]) -> Optional[int]:
return None
@property
def needs_symbols(self) -> bool:
return False
@ -455,9 +459,6 @@ class Segment:
def warn(self, msg: str):
self.warnings.append(msg)
def max_length(self):
return None
@staticmethod
def get_default_name(addr) -> str:
return f"{addr:X}"

2
tools/splat/split.py
View File

@ -19,7 +19,7 @@ from segtypes.linker_entry import (
from segtypes.segment import Segment
from util import log, options, palettes, symbols, relocs
VERSION = "0.15.2"
VERSION = "0.16.1"
parser = argparse.ArgumentParser(
description="Split a rom given a rom, a config, and output directory"

5
tools/splat/util/compiler.py
View File

@ -1,4 +1,5 @@
from dataclasses import dataclass
from typing import Optional
@dataclass
@ -11,6 +12,7 @@ class Compiler:
c_newline: str = "\n"
asm_inc_header: str = ""
include_macro_inc: bool = True
asm_emit_size_directive: Optional[bool] = None
GCC = Compiler(
@ -26,9 +28,10 @@ SN64 = Compiler(
asm_end_label=".end",
c_newline="\r\n",
include_macro_inc=False,
asm_emit_size_directive=False,
)
IDO = Compiler("IDO")
IDO = Compiler("IDO", include_macro_inc=False, asm_emit_size_directive=False)
compiler_for_name = {"GCC": GCC, "SN64": SN64, "IDO": IDO}

12
tools/splat/util/options.py
View File

@ -175,6 +175,8 @@ class SplatOpts:
asm_generated_by: bool
# Tells the disassembler to try disassembling functions with unknown instructions instead of falling back to disassembling as raw data
disasm_unknown: bool
# Tries to detect redundant and unreferenced functions ends and merge them together. This option is ignored if the compiler is not set to IDO.
detect_redundant_function_end: bool
################################################################################
# N64-specific options
@ -289,6 +291,11 @@ def _parse_yaml(
)
asm_path: Path = p.parse_path(base_path, "asm_path", "asm")
asm_emit_size_directive = p.parse_optional_opt("asm_emit_size_directive", bool)
# If option not provided then use the compiler default
if asm_emit_size_directive is None:
asm_emit_size_directive = comp.asm_emit_size_directive
def parse_endianness() -> Literal["big", "little"]:
endianness = p.parse_opt_within(
"endianness",
@ -389,7 +396,7 @@ def _parse_yaml(
),
asm_data_macro=p.parse_opt("asm_data_macro", str, comp.asm_data_macro),
asm_end_label=p.parse_opt("asm_end_label", str, comp.asm_end_label),
asm_emit_size_directive=p.parse_optional_opt("asm_emit_size_directive", bool),
asm_emit_size_directive=asm_emit_size_directive,
include_macro_inc=p.parse_opt(
"include_macro_inc", bool, comp.include_macro_inc
),
@ -431,6 +438,9 @@ def _parse_yaml(
filesystem_path=p.parse_optional_path(base_path, "filesystem_path"),
asm_generated_by=p.parse_opt("asm_generated_by", bool, True),
disasm_unknown=p.parse_opt("disasm_unknown", bool, False),
detect_redundant_function_end=p.parse_opt(
"detect_redundant_function_end", bool, True
),
)
p.check_no_unread_opts()
return ret

13
tools/splat/util/symbols.py
View File

@ -72,13 +72,21 @@ def to_cname(symbol_name: str) -> str:
return symbol_name
def handle_sym_addrs(path: Path, sym_addrs_lines: List[str], all_segments):
def handle_sym_addrs(
path: Path, sym_addrs_lines: List[str], all_segments: "List[Segment]"
):
def get_seg_for_name(name: str) -> Optional["Segment"]:
for segment in all_segments:
if segment.name == name:
return segment
return None
def get_seg_for_rom(rom: int) -> Optional["Segment"]:
for segment in all_segments:
if segment.contains_rom(rom):
return segment
return None
for line_num, line in enumerate(
tqdm.tqdm(sym_addrs_lines, desc=f"Loading symbols ({path.stem})")
):
@ -234,6 +242,9 @@ def handle_sym_addrs(path: Path, sym_addrs_lines: List[str], all_segments):
continue
if sym.segment is None and sym.rom is not None:
sym.segment = get_seg_for_rom(sym.rom)
if sym.segment:
sym.segment.add_symbol(sym)

4
ver/pal/splat.yaml
View File

@ -2963,11 +2963,11 @@ segments:
- [0x3D8E30, ci4, D_09000420_3AB450, 32, 64]
- [0x3D9230, palette, D_09000420_3AB450]
- [0x3D9250, ci4, D_09000840_3AB870, 32, 64]
- [0x3D9650, palette, D_09000840_3AB870]
- {start: 0x3D9650, type: palette, name: D_09000840_3AB870, size: 0x10}
- [0x3D9660, ci4, D_09000C50_3ABC80, 32, 64]
- [0x3D9A60, palette, D_09000C50_3ABC80]
- [0x3D9A80, ci4, D_09001070_3AC0A0, 32, 64]
- [0x3D9E80, palette, D_09001070_3AC0A0]
- {start: 0x3D9E80, type: palette, name: D_09001070_3AC0A0, size: 0x10}
- [0x3D9E90, ci4, D_09001480_3AC4B0, 32, 64]
- [0x3DA290, palette, D_09001480_3AC4B0]
- [0x3DA2B0, ci4, D_090018A0_3AC8D0, 32, 64]

1236
ver/us/splat.yaml
View File

File diff suppressed because it is too large Load Diff