papermario/tools/build/sprite/sprites.py
Ethan Roseman 8837fbdf65
Player sprites & more (#1055)
* WIP work on sprites (sprite_stuff.py)

* cleanup of various stuff

* separate compiler installation into separate script

* wipz

* more

* renames, bugfixes

* more

* very grood

* cleanin

* goods and services

* oopth

* oopth2

* Parse palette data from xml

* more work

* more wipperz

* more

* it working

* git subrepo pull --force tools/splat

subrepo:
  subdir:   "tools/splat"
  merged:   "e72a868f9f"
upstream:
  origin:   "https://github.com/ethteck/splat.git"
  branch:   "master"
  commit:   "e72a868f9f"
git-subrepo:
  version:  "0.4.5"
  origin:   "https://github.com/ingydotnet/git-subrepo"
  commit:   "aa416e4"

* fix symbol_addrs for new splat

* upd8s

* Use generated header, other versions, fixes

* fixes & formatting

* wip fusing npc + player extraction & cleanup

* remove npc_files

* buildin

* fix some bugs

* Cleanup, yay0s separately

* cleen

* cleanup

* Respect stack during build

* jp spritz

* dun

* fix c files

---------

Co-authored-by: pixel-stuck <mathmcclintic@gmail.com>
2023-06-26 19:27:37 +09:00

685 lines
21 KiB
Python
Executable File

#! /usr/bin/env
import argparse
from dataclasses import dataclass
from functools import cache
from pathlib import Path
import sys
from typing import List
sys.path.append(str(Path(__file__).parent.parent.parent))
sys.path.append(str(Path(__file__).parent.parent.parent / "splat"))
from splat_ext.pm_sprites import (
BACK_PALETTE_XML,
LIST_END_BYTES,
MAX_COMPONENTS_XML,
NPC_SPRITE_MEDADATA_XML_FILENAME,
PALETTE_GROUPS_XML,
PALETTE_XML,
PLAYER_SPRITE_MEDADATA_XML_FILENAME,
SPECIAL_RASTER,
PlayerRaster,
RasterTableEntry,
)
from splat_ext.sprite_common import AnimComponent, iter_in_groups
import os
import png # type: ignore
import struct
import subprocess
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from typing import Dict, List, Tuple
TOOLS_DIR = Path(
os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
)
sys.path.append(str(TOOLS_DIR))
ASSET_DIR = TOOLS_DIR.parent / "assets"
def pack_color(r, g, b, a) -> int:
r = r >> 3
g = g >> 3
b = b >> 3
a = a >> 7
return (r << 11) | (g << 6) | (b << 1) | a
def get_player_sprite_metadata() -> Tuple[str, List[str], List[str]]:
orderings_tree = ET.parse(get_asset_path(PLAYER_SPRITE_MEDADATA_XML_FILENAME))
build_info = str(orderings_tree.getroot()[0].text)
sprite_order: List[str] = []
for sprite_tag in orderings_tree.getroot()[1]:
sprite_order.append(sprite_tag.attrib["name"])
raster_order: List[str] = []
for raster_tag in orderings_tree.getroot()[2]:
raster_order.append(raster_tag.attrib["name"])
return build_info, sprite_order, raster_order
def get_npc_sprite_metadata() -> List[str]:
orderings_tree = ET.parse(get_asset_path(NPC_SPRITE_MEDADATA_XML_FILENAME))
sprite_order: List[str] = []
for sprite_tag in orderings_tree.getroot()[0]:
sprite_order.append(sprite_tag.attrib["name"])
return sprite_order
@dataclass
class CI4Info:
offset: int
width: int
height: int
data: bytes
@property
def size(self):
return self.width * self.height // 2
RASTER_CACHE: Dict[str, CI4Info] = {}
PALETTE_CACHE: Dict[str, bytes] = {}
PLAYER_XML_CACHE: Dict[str, ET.Element] = {}
# TODO perhaps encode this better
SPECIAL_RASTER_BYTES = (
b"\x80\x30\x02\x10\x00\x00\x02\x00\x00\x00\x00\x01\x00\x10\x00\x00"
)
@cache
def get_asset_path(asset: str) -> Path:
for sdir in ASSET_STACK:
potential_path = ASSET_DIR / sdir / "sprite" / asset
if potential_path.exists():
return potential_path
raise FileNotFoundError(f"Could not find asset {asset}")
def cache_player_rasters(raster_order: List[str]):
# Read all player rasters and cache them
cur_offset = 0
for raster_name in raster_order:
png_path = get_asset_path(Path("player/rasters") / f"{raster_name}.png")
# "Weird" raster
if os.path.getsize(png_path) == 0x10:
RASTER_CACHE[raster_name] = CI4Info(0, 0, cur_offset, SPECIAL_RASTER_BYTES)
cur_offset += 0x10
continue
with open(png_path, "rb") as f:
reader = png.Reader(f)
width, height, rows, _ = reader.read()
img_bytes = b""
for row in rows:
for a, b in iter_in_groups(row, 2):
byte: int = (a << 4) | b
byte = byte & 0xFF
img_bytes += byte.to_bytes(1, byteorder="big")
RASTER_CACHE[raster_name] = CI4Info(cur_offset, width, height, img_bytes)
cur_offset += RASTER_CACHE[raster_name].size
def player_raster_from_xml(xml: ET.Element, back: bool = False) -> PlayerRaster:
palette = int(xml.attrib[PALETTE_XML], 16)
is_special = "special" in xml.attrib
if back in xml.attrib:
img_name = xml.attrib["back"]
else:
img_name = xml.attrib["src"]
raster_info = RASTER_CACHE[img_name[:-4]]
return PlayerRaster(
0,
raster_info.width,
raster_info.height,
palette,
is_special,
)
def player_xml_to_bytes(xml: ET.Element) -> List[bytes]:
has_back = False
out_bytes = b""
back_out_bytes = b""
max_components = int(xml.attrib[MAX_COMPONENTS_XML])
num_variations = int(xml.attrib[PALETTE_GROUPS_XML])
# Animations
animations: List[List[AnimComponent]] = []
for anim_xml in xml[2]:
comps: List[AnimComponent] = []
for comp_xml in anim_xml:
comp: AnimComponent = AnimComponent.from_xml(comp_xml)
comps.append(comp)
animations.append(comps)
cur_offset = 0x10 # header
cur_offset += (len(animations) + 1) * 4 # animation offsets and list terminator
animation_offsets: List[int] = []
total_anim_bytes = b""
for anim in animations:
animation_offsets.append(cur_offset)
animation_bytes: bytes = b""
cur_offset += (len(anim) + 1) * 4
comp_offsets = []
for comp in anim:
cmd_start_offset = cur_offset
for cmd in comp.commands:
animation_bytes += int.to_bytes(cmd, 2, "big")
cur_offset += 2
if comp.size % 2 != 0:
animation_bytes += b"\x00\x00"
cur_offset += 2
comp_offsets.append(cur_offset)
animation_bytes += int.to_bytes(cmd_start_offset, 4, "big")
animation_bytes += int.to_bytes(comp.size * 2, 2, "big")
animation_bytes += struct.pack(">hhh", comp.x, comp.y, comp.z)
cur_offset += 12
offset_bytes = b""
for offset in comp_offsets:
offset_bytes += int.to_bytes(offset, 4, "big")
offset_bytes += LIST_END_BYTES
blah_bytes = offset_bytes + animation_bytes
total_anim_bytes += blah_bytes
animation_offset_bytes: bytes = b""
for offset in animation_offsets:
animation_offset_bytes += int.to_bytes(offset, 4, "big")
animation_offset_bytes += LIST_END_BYTES
total_anim_bytes = animation_offset_bytes + total_anim_bytes
out_bytes += total_anim_bytes
# Pad out_bytes to 0x8
while len(out_bytes) % 8 != 0:
out_bytes += b"\x00"
# Back sprite sheets don't have animations, so just -1 and 0 padding
# TODO is there always padding needed, or is this just a result of the vanilla data alignment?
back_out_bytes += LIST_END_BYTES + b"\x00\x00\x00\x00"
# Palettes
palette_list_start = len(out_bytes) + 0x10
palette_list_start_back = len(back_out_bytes) + 0x10
palette_bytes: bytes = b""
palette_bytes_back: bytes = b""
for palette_xml in xml[0]:
source = palette_xml.attrib["src"]
front_only = bool(palette_xml.get("front_only", False))
if source not in PALETTE_CACHE:
palette_path = get_asset_path(Path("player/palettes") / source)
with open(palette_path, "rb") as f:
img = png.Reader(f)
img.preamble(True)
palette = img.palette(alpha="force")
pal: bytes = b""
for rgba in palette:
if rgba[3] not in (0, 0xFF):
print("alpha mask mode but translucent pixels used")
color = pack_color(*rgba)
pal += int.to_bytes(color, 2, "big")
PALETTE_CACHE[source] = pal
palette_bytes += PALETTE_CACHE[source]
if not front_only:
palette_bytes_back += PALETTE_CACHE[source]
# Pad out_bytes to 0x8
while len(out_bytes) % 8 != 0:
out_bytes += b"\x00"
out_bytes += palette_bytes
back_out_bytes += palette_bytes_back
# Rasters
raster_list_start = len(out_bytes) + 0x10
raster_list_start_back = len(back_out_bytes) + 0x10
raster_bytes: bytes = b""
raster_bytes_back: bytes = b""
raster_offset = 0
for raster_xml in xml[1]:
if "back" in raster_xml.attrib:
has_back = True
r = player_raster_from_xml(raster_xml, back=False)
raster_bytes += struct.pack(
">IBBBB", raster_offset, r.width, r.height, r.palette_idx, 0xFF
)
raster_offset += r.width * r.height // 2
if has_back:
raster_offset = 0
for raster_xml in xml[1]:
is_back = False
r = player_raster_from_xml(raster_xml, back=is_back)
if "back" in raster_xml.attrib:
is_back = True
width = r.width
height = r.height
else:
special = raster_xml.attrib["special"].split(",")
width = int(special[0], 16)
height = int(special[1], 16)
palette = int(raster_xml.attrib.get(BACK_PALETTE_XML, r.palette_idx))
raster_bytes_back += struct.pack(
">IBBBB", raster_offset, width, height, palette, 0xFF
)
if is_back:
raster_offset += width * height // 2
else:
raster_offset += 0x10
out_bytes += raster_bytes
back_out_bytes += raster_bytes_back
raster_list_offset = len(out_bytes) + 0x10
raster_list_offset_back = len(back_out_bytes) + 0x10
# Raster file offsets
raster_offsets_bytes = b""
raster_offsets_bytes_back = b""
for i in range(len(xml[1])):
raster_offsets_bytes += int.to_bytes(raster_list_start + i * 8, 4, "big")
raster_offsets_bytes_back += int.to_bytes(
raster_list_start_back + i * 8, 4, "big"
)
raster_offsets_bytes += LIST_END_BYTES
raster_offsets_bytes_back += LIST_END_BYTES
out_bytes += raster_offsets_bytes
back_out_bytes += raster_offsets_bytes_back
# Palette file offsets
palette_list_offset = len(out_bytes) + 0x10
palette_list_offset_back = len(back_out_bytes) + 0x10
palette_offsets_bytes = b""
palette_offsets_bytes_back = b""
for i, palette_xml in enumerate(xml[0]):
palette_offsets_bytes += int.to_bytes(palette_list_start + i * 0x20, 4, "big")
front_only = bool(palette_xml.attrib.get("front_only", False))
if not front_only:
palette_offsets_bytes_back += int.to_bytes(
palette_list_start_back + i * 0x20, 4, "big"
)
palette_offsets_bytes += LIST_END_BYTES
palette_offsets_bytes_back += LIST_END_BYTES
out_bytes += palette_offsets_bytes
back_out_bytes += palette_offsets_bytes_back
header = struct.pack(
">IIII",
raster_list_offset,
palette_list_offset,
max_components,
num_variations,
)
out_bytes = header + out_bytes
ret = [out_bytes]
if has_back:
back_header = struct.pack(
">IIII",
raster_list_offset_back,
palette_list_offset_back,
0,
num_variations,
)
back_out_bytes = back_header + back_out_bytes
ret.append(back_out_bytes)
return ret
def xml_has_back(xml: ET.Element) -> bool:
for raster_xml in xml[1]:
if "back" in raster_xml.attrib:
return True
return False
def write_player_sprite_header(
sprite_order: List[str],
out_file: Path,
) -> None:
ifdef_name = "_PLAYER_SPRITE_H_"
sprite_id = 1
player_sprites: Dict[str, int] = {}
player_rasters: Dict[str, Dict[str, int]] = {}
player_palettes: Dict[str, Dict[str, int]] = {}
player_anims: Dict[str, Dict[str, int]] = {}
max_sprite_sizes: Dict[str, int] = {}
for sprite_name in sprite_order:
sprite_xml = PLAYER_XML_CACHE[sprite_name]
has_back = xml_has_back(sprite_xml)
player_sprites[f"SPR_{sprite_name}"] = sprite_id
player_rasters[sprite_name] = {}
player_palettes[sprite_name] = {}
player_anims[sprite_name] = {}
for palette_xml in sprite_xml[0]:
palette_id = int(palette_xml.attrib["id"], 0x10)
palette_name = palette_xml.attrib["name"]
player_palettes[sprite_name][
f"SPR_PAL_{sprite_name}_{palette_name}"
] = palette_id
for anim_id, anim_xml in enumerate(sprite_xml[2]):
anim_name = anim_xml.attrib["name"]
if palette_id > 0:
anim_name = f"{palette_name}_{anim_name}"
player_anims[sprite_name][f"ANIM_{sprite_name}_{anim_name}"] = (
(sprite_id << 16) | (palette_id << 8) | anim_id
)
max_size = 0
for raster_xml in sprite_xml[1]:
raster_id = int(raster_xml.attrib["id"], 0x10)
raster_name = raster_xml.attrib["name"]
player_rasters[sprite_name][
f"SPR_IMG_{sprite_name}_{raster_name}"
] = raster_id
raster = RASTER_CACHE[raster_xml.attrib["src"][:-4]]
if max_size < raster.size:
max_size = raster.size
max_sprite_sizes[sprite_name] = max_size
sprite_id += 1
if has_back:
player_sprites[f"SPR_{sprite_name}_Back"] = sprite_id
max_size = 0
for raster_xml in sprite_xml[1]:
if "back" in raster_xml.attrib:
raster = RASTER_CACHE[raster_xml.attrib["back"][:-4]]
if max_size < raster.size:
max_size = raster.size
max_sprite_sizes[f"{sprite_name}_Back"] = max_size
sprite_id += 1
out_file.parent.mkdir(exist_ok=True, parents=True)
with open(out_file, "w") as f:
f.write(f"#ifndef {ifdef_name}\n")
f.write(f"#define {ifdef_name}\n\n")
# PlayerSprites
f.write("enum PlayerSprites {\n")
for sprite_name, sprite_id in player_sprites.items():
f.write(f" {sprite_name} = 0x{sprite_id:X},\n")
f.write("};\n\n")
for sprite_name in max_sprite_sizes:
f.write(
f"#define MAX_IMG_{sprite_name} 0x{max_sprite_sizes[sprite_name]:04X}\n"
)
f.write("\n")
for sprite_name in sprite_order:
f.write(f"// {sprite_name}\n")
for raster_name, raster_id in player_rasters[sprite_name].items():
f.write(f"#define {raster_name} 0x{raster_id:02X}\n")
f.write("\n")
for palette_name, palette_id in player_palettes[sprite_name].items():
f.write(f"#define {palette_name} 0x{palette_id:02X}\n")
f.write("\n")
for anim_name, anim_id in player_anims[sprite_name].items():
f.write(f"#define {anim_name} 0x{anim_id:X}\n")
f.write("\n")
f.write(f"#endif // {ifdef_name}\n")
def build_player_sprites(sprite_order: List[str], build_dir: Path) -> bytes:
sprite_bytes: List[bytes] = []
for sprite_name in sprite_order:
sprite_bytes.extend(player_xml_to_bytes(PLAYER_XML_CACHE[sprite_name]))
# Compress sprite bytes
compressed_sprite_bytes: bytes = b""
yay0_cur_offset = 4 * (len(sprite_bytes) + 1)
list_bytes: bytes = struct.pack(">I", yay0_cur_offset)
# TODO figure out how to use tmp files if possible
build_dir.mkdir(exist_ok=True, parents=True)
yay0_in_path = build_dir / "yay0_bytes.bin"
yay0_out_path = build_dir / "yay0_bytes.Yay0"
for sprite_byte in sprite_bytes:
with open(yay0_in_path, "wb") as f:
f.write(sprite_byte)
subprocess.run(
[
str(TOOLS_DIR / "build/yay0/Yay0compress"),
yay0_in_path,
yay0_out_path,
]
)
with open(yay0_out_path, "rb") as f:
yay0_bytes = f.read()
# Pad to 0x8
yay0_bytes += b"\0" * (8 - (len(yay0_bytes) % 8))
compressed_sprite_bytes += yay0_bytes
yay0_cur_offset += len(yay0_bytes)
list_bytes += struct.pack(">I", yay0_cur_offset)
os.remove(yay0_in_path)
os.remove(yay0_out_path)
return list_bytes + compressed_sprite_bytes
def build_npc_sprites(sprite_order: List[str], build_dir: Path) -> bytes:
compressed_sprite_bytes: bytes = b""
yay0_cur_offset = 4 * (len(sprite_order) + 1)
list_bytes: bytes = struct.pack(">I", yay0_cur_offset)
for sprite_name in sprite_order:
with open(build_dir / "npc" / f"{sprite_name}.Yay0", "rb") as f:
yay0_bytes = f.read()
# Add 0s to pad to 0x8
yay0_bytes_len = (len(yay0_bytes) + 0x7) & ~0x7
yay0_bytes += b"\0" * (yay0_bytes_len - len(yay0_bytes))
compressed_sprite_bytes += yay0_bytes
yay0_cur_offset += len(yay0_bytes)
list_bytes += struct.pack(">I", yay0_cur_offset)
return list_bytes + compressed_sprite_bytes
def build_player_rasters(sprite_order: List[str], raster_order: List[str]) -> bytes:
packed_raster_data = b""
raster_info_offsets: list[int] = []
rtes: List[RasterTableEntry] = []
num_sheets = 0
# Get raster data
for sprite_name in sprite_order:
sprite_xml = PLAYER_XML_CACHE[sprite_name]
sheet_rtes: List[RasterTableEntry] = []
sheet_rtes_back: List[RasterTableEntry] = []
has_back = False
for raster_xml in sprite_xml[1]:
if "back" in raster_xml.attrib:
has_back = True
if has_back:
if "back" in raster_xml.attrib:
png_info = RASTER_CACHE[raster_xml.attrib["back"][:-4]]
sheet_rtes_back.append(
RasterTableEntry(png_info.offset, png_info.size)
)
else:
sheet_rtes_back.append(RasterTableEntry(SPECIAL_RASTER, 0x10))
png_info = RASTER_CACHE[raster_xml.attrib["src"][:-4]]
sheet_rtes.append(RasterTableEntry(png_info.offset, png_info.size))
raster_info_offsets.append(len(rtes))
num_sheets += 1
rtes.extend(sheet_rtes)
if has_back:
raster_info_offsets.append(len(rtes))
num_sheets += 1
rtes.extend(sheet_rtes_back)
raster_info_offsets.append(len(rtes)) # Final 'offset' (size of list)
info_list_bytes = b""
for offset in raster_info_offsets:
info_list_bytes += struct.pack(">I", offset)
separators_offset = 0x10
infos_offset = separators_offset + (num_sheets + 1) * 4
rasters_offset = infos_offset + (len(rtes) + 1) * 4
# Align raster_offset_start to 0x10 offset
rasters_offset = (rasters_offset + 0xF) & ~0xF
for rte in rtes:
if rte.offset == SPECIAL_RASTER:
packed_raster_data += struct.pack(">I", 0x0011F880)
continue
packed_info = (rte.size >> 4) << 20
packed_info |= (rte.offset + rasters_offset) & 0xFFFFF
packed_raster_data += struct.pack(">I", packed_info)
# This is the missing raster from before
packed_raster_data += struct.pack(">I", 0x06C9CD50)
header = struct.pack(">IIII", separators_offset, infos_offset, rasters_offset, 0)
ret = header + info_list_bytes + packed_raster_data
# Align cumulative data to 0x10 offset
ret += b"\0" * ((0x10 - len(ret)) & 0xF)
raster_bytes = b""
for raster_name in raster_order[:-1]: # Skip last raster
png_info = RASTER_CACHE[raster_name]
raster_bytes += png_info.data
ret += raster_bytes
return ret
ASSET_STACK: List[Path]
def build(
out_file: Path,
player_header_path: Path,
build_dir: Path,
asset_stack: List[Path],
) -> None:
global ASSET_STACK
ASSET_STACK = asset_stack
build_info, player_sprite_order, player_raster_order = get_player_sprite_metadata()
npc_sprite_order = get_npc_sprite_metadata()
cache_player_rasters(player_raster_order)
# Read and cache player XMLs
for sprite_name in player_sprite_order:
sprite_xml = ET.parse(
get_asset_path(Path(f"player/{sprite_name}.xml"))
).getroot()
PLAYER_XML_CACHE[sprite_name] = sprite_xml
# Encode build_info to bytes and pad to 0x10
build_info_bytes = build_info.encode("ascii")
build_info_bytes += b"\0" * (0x10 - len(build_info_bytes))
player_sprite_bytes = build_player_sprites(
player_sprite_order, build_dir / "player"
)
player_raster_bytes = build_player_rasters(player_sprite_order, player_raster_order)
npc_sprite_bytes = build_npc_sprites(npc_sprite_order, build_dir)
built_raster_info_offset = 0x10 + len(player_raster_bytes)
compressed_sprite_bytes_offset = built_raster_info_offset + len(player_sprite_bytes)
npc_sprites_offset = compressed_sprite_bytes_offset + len(npc_sprite_bytes)
major_file_divisons = struct.pack(
">IIII",
0x10,
built_raster_info_offset,
compressed_sprite_bytes_offset,
npc_sprites_offset,
)
final_data = (
build_info_bytes
+ major_file_divisons
+ player_raster_bytes
+ player_sprite_bytes
+ npc_sprite_bytes
)
with open(out_file, "wb") as f:
f.write(final_data)
write_player_sprite_header(player_sprite_order, player_header_path)
if __name__ == "__main__":
parser = argparse.ArgumentParser(description="Builds sprite blobs")
parser.add_argument("out")
parser.add_argument("player_header_out")
parser.add_argument("build_dir")
parser.add_argument("asset_stack")
args = parser.parse_args()
build(
Path(args.out),
Path(args.player_header_out),
Path(args.build_dir),
[Path(d) for d in args.asset_stack.split(",")],
)