papermario/tools/build/mapfs/shape.py

import argparse
from pathlib import Path
import struct
from abc import ABC
from collections import deque
from io import TextIOWrapper
from typing import List, Dict, Optional

BASE_ADDR = 0x80210000

NODE_TYPE_ROOT = 7
NODE_TYPE_MODEL = 2
NODE_TYPE_GROUP = 5
NODE_TYPE_SPECIAL_GROUP = 10

GFX_LOAD_VTX = 0x01
GFX_DRAW_TRI = 0x05
GFX_DRAW_TRIS = 0x06
GFX_RDP_PIPE_SYNC = 0xE7
GFX_POP_MATRIX = 0xD8
GFX_GEOMETRYMODE = 0xD9
GFX_LOAD_MATRIX = 0xDA
GFX_START_DL = 0xDE
GFX_END_DL = 0xDF

ALIGN_16 = "__attribute__ ((aligned (16))) "


def read_ascii_string(bytes: bytearray, addr: int) -> str:
    start = addr - BASE_ADDR
    length = 0

    for char in bytes[start:]:
        if char == 0:
            break
        length += 1

    return bytes[start : start + length].decode("ascii")


def get_shape_type_name(id: int) -> str:
    if id == NODE_TYPE_ROOT:
        return "SHAPE_TYPE_ROOT"
    elif id == NODE_TYPE_MODEL:
        return "SHAPE_TYPE_MODEL"
    elif id == NODE_TYPE_GROUP:
        return "SHAPE_TYPE_GROUP"
    elif id == NODE_TYPE_SPECIAL_GROUP:
        return "SHAPE_TYPE_SPECIAL_GROUP"
    raise Exception(f"Unknown shape type {id}")


class Segment(ABC):
    def __init__(self, addr: int, name: str):
        self.addr = addr
        self.name = name
        self.count = 0
        self.model_name = ""

    def get_sym(self) -> str:
        if self.model_name != "":
            return f"N({self.name}_{self.model_name})"
        else:
            return f"N({self.name})"

    def __str__(self):
        return str(self.__class__.__name__[:-7]) + " " + self.get_sym()

    def scan(self, shape):
        pass

    def print(self, shape):
        shape.print(hex(self.addr) + " : " + str(self))


class HeaderSegment(Segment):
    def __init__(self, addr: int, name: str):
        super().__init__(addr, name)

    def scan(self, shape):
        start = self.addr - BASE_ADDR
        (
            self.ptr_root_node,
            self.ptr_vtx_table,
            self.ptr_model_names,
            self.ptr_collider_names,
            self.ptr_zone_names,
        ) = struct.unpack(">IIIII", shape.file_bytes[start : start + 20])

        # note: do not push model root yet
        shape.root_node = NodeSegment(self.ptr_root_node, "Node")

        shape.vtx_table = shape.push(VertexTableSegment(self.ptr_vtx_table, "VertexTable"))
        shape.model_names = shape.push(StringListSegment(self.ptr_model_names, "ModelNames"))
        shape.collider_names = shape.push(StringListSegment(self.ptr_collider_names, "ColliderNames"))
        shape.zone_names = shape.push(StringListSegment(self.ptr_zone_names, "ZoneNames"))

    def print(self, shape):
        shape.print(f"ShapeFileHeader {self.get_sym()} = {{")
        shape.print(f"    .root = &{shape.get_symbol(self.ptr_root_node)},")
        shape.print(f"    .vertexTable = {shape.get_symbol(self.ptr_vtx_table)},")
        shape.print(f"    .modelNames = {shape.get_symbol(self.ptr_model_names)},")
        shape.print(f"    .colliderNames = {shape.get_symbol(self.ptr_collider_names)},")
        if self.ptr_zone_names != 0:
            shape.print(f"    .zoneNames = {shape.get_symbol(self.ptr_zone_names)},")
        shape.print("};")


class VertexTableSegment(Segment):
    def __init__(self, addr: int, name: str):
        super().__init__(addr, name)

    def print(self, shape):
        pos = self.addr - BASE_ADDR
        shape.print(f"Vtx_t {self.get_sym()}[] = {{")

        for idx in range(self.count):
            (
                x,
                y,
                z,
                flag,
                u,
                v,
                r,
                g,
                b,
                a,
            ) = struct.unpack(">hhhhhhBBBB", shape.file_bytes[pos : pos + 16])
            pos += 16

            shape.print(
                f"    {{{{ {x:4}, {y:4}, {z:4} }}, {flag}, {{ {u:6}, {v:6} }}, {{ {r:3}, {g:3}, {b:3}, {a:3} }}}},"
            )

        shape.print("};")


class VectorListSegment(Segment):
    def __init__(self, addr: int, name: str):
        super().__init__(addr, name)

    def print(self, shape):
        next = shape.get_segment_after(self)

        pos = self.addr - BASE_ADDR
        end = next.addr - BASE_ADDR
        count = (end - pos) // 12

        shape.print(f"Vec3f {self.get_sym()}[] = {{")

        for _ in range(count):
            (x, y, z) = struct.unpack(">fff", shape.file_bytes[pos : pos + 12])
            pos += 12

            shape.print(f"    {{ {x}, {y}, {z} }},")

        shape.print("};")


class StringListSegment(Segment):
    def __init__(self, addr: int, name: str):
        super().__init__(addr, name)
        self.list: deque

    def scan(self, shape):
        self.list = deque()
        pos = self.addr - BASE_ADDR
        while True:
            (ptr_str,) = struct.unpack(">I", shape.file_bytes[pos : pos + 4])
            pos += 4

            string = read_ascii_string(shape.file_bytes, ptr_str)

            if string == "db":
                break

            self.list.append(string)

    def print(self, shape):
        align_attribute = ""
        if self.name == "ModelNames":
            align_attribute = ALIGN_16

        shape.print(f"{align_attribute}char* {self.get_sym()}[] = {{")

        for name in self.list:
            shape.print(f'    "{name}",')
        shape.print('    "db",')
        shape.print("};")


class NodeSegment(Segment):
    def __init__(self, addr: int, name: str):
        super().__init__(addr, name)

    def scan(self, shape):
        pos = self.addr - BASE_ADDR
        (
            self.node_type,
            self.ptr_display_data,
            self.num_properties,
            self.ptr_property_list,
            self.ptr_group_data,
        ) = struct.unpack(">IIIII", shape.file_bytes[pos : pos + 20])

        self.model_name = shape.model_name_map[self.addr]
        shape.push(GroupDataSegment(self.ptr_group_data, "GroupData", self.model_name))
        shape.push(DisplayDataSegment(self.ptr_display_data, "DisplayData", self.model_name))
        shape.push(
            PropertyListSegment(
                self.ptr_property_list,
                "Properties",
                self.model_name,
                self.num_properties,
            )
        )

    def print(self, shape):
        shape.print(f"ModelNode {self.get_sym()} = {{")
        shape.print(f"    .type = {get_shape_type_name(self.node_type)},")
        if self.ptr_group_data != 0:
            shape.print(f"    .groupData = &{shape.get_symbol(self.ptr_group_data)},")
        shape.print(f"    .displayData = &{shape.get_symbol(self.ptr_display_data)},")
        shape.print(f"    .propertyList = {shape.get_symbol(self.ptr_property_list)},")
        shape.print(f"    .numProperties = {self.num_properties},")
        shape.print("};")


class NodeListSegment(Segment):
    def __init__(self, addr: int, name: str, model_name: str, num_children: int):
        super().__init__(addr, name)
        self.model_name = model_name
        self.count = num_children
        self.children: List[NodeSegment] = []

    def scan(self, shape):
        pos = self.addr - BASE_ADDR

        for _ in range(self.count):
            (ptr_child,) = struct.unpack(">I", shape.file_bytes[pos : pos + 4])
            pos += 4

            self.children.append(ptr_child)
            shape.push(NodeSegment(ptr_child, "Node"))

    def print(self, shape):
        shape.print(f"ModelNode* {self.get_sym()}[] = {{")

        for addr in self.children:
            shape.print(f"    &{shape.get_symbol(addr)},")
        shape.print("};")


class PropertyListSegment(Segment):
    def __init__(self, addr: int, name: str, model_name: str, count: int):
        super().__init__(addr, name)
        self.model_name = model_name
        self.count = count

    def print(self, shape):
        pos = self.addr - BASE_ADDR
        shape.print(f"ModelNodeProperty {self.get_sym()}[] = {{")

        for _ in range(self.count):
            (
                key,
                fmt,
                value,
            ) = struct.unpack(">III", shape.file_bytes[pos : pos + 12])
            pos += 12

            if key == 0x5E:
                if value == 0:
                    shape.print(f"    {{ .key = {hex(key)}, .dataType = {fmt}, .data = {{ .p = NULL }}}},")
                else:
                    tex_name = read_ascii_string(shape.file_bytes, value)
                    shape.print(f'    {{ .key = {hex(key)}, .dataType = {fmt}, .data = {{ .p = "{tex_name}" }}}},')
            elif key == 0x5F:
                shape.print(f"    {{ .key = {hex(key)}, .dataType = {fmt}, .data = {{ .s = {hex(value)} }}}},")
            else:
                if fmt == 0:  # int
                    shape.print(f"    {{ .key = {hex(key)}, .dataType = {fmt}, .data = {{ .s = {hex(value)} }}}},")
                elif fmt == 1:  # float
                    temp = struct.pack(">I", value)
                    (f,) = struct.unpack(">f", temp)
                    shape.print(f"    {{ .key = {hex(key)}, .dataType = {fmt}, .data = {{ .f = {f} }}}},")
                elif fmt == 2:  # pointer
                    shape.print(
                        f'    {{ .key = {hex(key)}, .dataType = {fmt}, .data = {{ .p = "{shape.get_symbol(value)}" }}}},'
                    )
                else:
                    raise Exception(f"Invalid property: 0x{key:08X} 0x{fmt:08X} 0x{value:08X}")

        shape.print("};")


class GroupDataSegment(Segment):
    def __init__(self, addr: int, name: str, model_name: str):
        super().__init__(addr, name)
        self.model_name = model_name

    def scan(self, shape):
        start = self.addr - BASE_ADDR
        (
            self.ptr_transform_mtx,
            self.ptr_lights,
            self.num_lights,
            self.num_children,
            self.ptr_children,
        ) = struct.unpack(">IIIII", shape.file_bytes[start : start + 20])

        shape.push(NodeListSegment(self.ptr_children, "Children", self.model_name, self.num_children))
        shape.push(LightSetSegment(self.ptr_lights, "Lights", self.model_name, self.num_lights))
        shape.push(MatrixSegment(self.ptr_transform_mtx, "Mtx", self.model_name))

    def print(self, shape):
        shape.print(f"ModelGroupData {self.get_sym()} = {{")
        if self.ptr_transform_mtx != 0:
            shape.print(f"    .transformMatrix = (Mtx*) &{shape.get_symbol(self.ptr_transform_mtx)},")
        shape.print(f"    .lightingGroup = (Lightsn*) &{shape.get_symbol(self.ptr_lights)},")
        shape.print(f"    .numLights = {self.num_lights},")
        shape.print(f"    .childList = {shape.get_symbol(self.ptr_children)},")
        shape.print(f"    .numChildren = {self.num_children},")
        shape.print("};")


class LightSetSegment(Segment):
    def __init__(self, addr: int, name: str, model_name: str, count: int):
        super().__init__(addr, name)
        self.model_name = model_name
        self.count = count

    def print(self, shape):
        next = shape.get_segment_after(self)
        pos = self.addr - BASE_ADDR
        end = next.addr - BASE_ADDR

        shape.print(f"// num: {self.count}")
        shape.print(f"s32 {self.get_sym()}[] = {{")
        while pos < end:
            (v,) = struct.unpack(">I", shape.file_bytes[pos : pos + 4])
            pos += 4
            shape.print(f"    0x{v:08X},")
        shape.print("};")


class MatrixSegment(Segment):
    def __init__(self, addr: int, name: str, model_name: str):
        super().__init__(addr, name)
        self.model_name = model_name

    def print(self, shape):
        pos = self.addr - BASE_ADDR
        shape.print(f"Matrix4s {self.get_sym()} = {{")

        shape.print("    .whole = {")
        for i in range(4):
            (a, b, c, d) = struct.unpack(">hhhh", shape.file_bytes[pos : pos + 8])
            pos += 8
            shape.print(f"        {{ {hex(a):4}, {hex(b):4}, {hex(c):4}, {hex(d):4} }},")
        shape.print("    },")

        shape.print("    .frac = {")
        for i in range(4):
            (a, b, c, d) = struct.unpack(">hhhh", shape.file_bytes[pos : pos + 8])
            pos += 8
            shape.print(f"        {{ {hex(a):4}, {hex(b):4}, {hex(c):4}, {hex(d):4} }},")
        shape.print("    },")

        shape.print("};")


class DisplayDataSegment(Segment):
    def __init__(self, addr: int, name: str, model_name: str):
        super().__init__(addr, name)
        self.model_name = model_name

    def scan(self, shape):
        start = self.addr - BASE_ADDR
        (self.ptr_display_list,) = struct.unpack(">I", shape.file_bytes[start : start + 4])

        gfx_segment = shape.push(DisplayListSegment(self.ptr_display_list, "Gfx", self.model_name))
        # Gfx segments may have been already visited during root Gfx traversal
        # so we will now force the associated model name to be the current model
        gfx_segment.model_name = self.model_name

    def print(self, shape):
        start = self.addr - BASE_ADDR
        shape.print(f"ModelDisplayData {self.get_sym()} = {{")
        shape.print(f"    .displayList = {shape.get_symbol(self.ptr_display_list)},")
        shape.print("};")


class DisplayListSegment(Segment):
    def __init__(self, addr: int, name: str, model_name: str):
        super().__init__(addr, name)
        self.model_name = model_name

    def scan(self, shape):
        pos = self.addr - BASE_ADDR
        while True:
            (w1, w2) = struct.unpack(">II", shape.file_bytes[pos : pos + 8])
            pos += 8

            op = w1 >> 24
            if op == GFX_END_DL:
                break
            elif op == GFX_START_DL:
                shape.push(DisplayListSegment(w2, f"Gfx_{hex(w2)[2:].upper()}", self.model_name))
            elif op == GFX_LOAD_MATRIX:
                shape.push(MatrixSegment(w2, f"Mtx_{hex(w2)[2:]}", model_name=self.model_name))
            elif op == GFX_LOAD_VTX:
                num = (w1 >> 12) & 0xFFF
                idx = (w2 - shape.vtx_table.addr) // 0x10
                if shape.vtx_table.count < idx + num:
                    shape.vtx_table.count = idx + num

    def get_geometry_flags(self, bits: int) -> str:
        flags = []

        if (bits & 0x00000400) != 0:
            flags.append("G_CULL_BACK")

        if (bits & 0x00020000) != 0:
            flags.append("G_LIGHTING")

        if (bits & 0x00200000) != 0:
            flags.append("G_SHADING_SMOOTH")

        return " | ".join(flags)

    def print(self, shape):
        pos = self.addr - BASE_ADDR
        shape.print(f"Gfx {self.get_sym()}[] = {{")
        while True:
            (w1, w2) = struct.unpack(">II", shape.file_bytes[pos : pos + 8])
            pos += 8

            op = w1 >> 24

            if op == GFX_LOAD_VTX:
                num = (w1 >> 12) & 0x00000FFF
                end = (w1 & 0x00000FFF) // 2
                buf_pos = end - num
                index = (w2 - shape.vtx_table.addr) // 0x10
                shape.print(f"    gsSPVertex(&{shape.vtx_table.get_sym()}[{index}], {num}, {buf_pos}),")
            elif op == GFX_DRAW_TRI:
                i = (w1 & 0x00FF0000) >> 16
                j = (w1 & 0x0000FF00) >> 8
                k = w1 & 0x000000FF
                shape.print(f"    gsSP1Triangle({i // 2}, {j // 2}, {k // 2}, 0),")
            elif op == GFX_DRAW_TRIS:
                a = (w1 & 0x00FF0000) >> 16
                b = (w1 & 0x0000FF00) >> 8
                c = w1 & 0x000000FF
                d = (w2 & 0x00FF0000) >> 16
                e = (w2 & 0x0000FF00) >> 8
                f = w2 & 0x000000FF
                shape.print(f"    gsSP2Triangles({a // 2}, {b // 2}, {c // 2}, 0, {d // 2}, {e // 2}, {f // 2}, 0),")
            elif op == GFX_RDP_PIPE_SYNC:
                shape.print("    gsDPPipeSync(),")
            elif op == GFX_POP_MATRIX:
                shape.print("    gsSPPopMatrix(G_MTX_MODELVIEW),")
            elif op == GFX_GEOMETRYMODE:
                if w1 == 0xD9FFFFFF:
                    flags = self.get_geometry_flags(w2)
                    shape.print(f"    gsSPSetGeometryMode({flags}),")
                else:
                    flags = self.get_geometry_flags(~(w1 | 0xFF000000))
                    shape.print(f"    gsSPClearGeometryMode({flags}),")
            elif op == GFX_LOAD_MATRIX:
                shape.print(f"    gsSPMatrix(&{shape.get_symbol(w2)}, G_MTX_PUSH | G_MTX_MUL | G_MTX_MODELVIEW),")
            elif op == GFX_START_DL:
                shape.print(f"    gsSPDisplayList({shape.get_symbol(w2)}),")
            elif op == GFX_END_DL:
                shape.print("    gsSPEndDisplayList(),")
                break

        shape.print("};")

        if self.model_name == "root":
            next = shape.get_segment_after(self)
            end = next.addr - BASE_ADDR

            shape.print(f"\ns32 N(PostGfxPad)[] = {{")
            while pos < end:
                (v,) = struct.unpack(">I", shape.file_bytes[pos : pos + 4])
                pos += 4
                shape.print(f"    0x{v:08X},")
            shape.print("};")


class ShapeFile:
    def __init__(self, map_name: str, file_bytes: bytes):
        self.map_name = map_name
        self.file_bytes = file_bytes
        self.out_file: TextIOWrapper
        self.pending: List[Segment] = []
        self.visited: Dict[int, Segment] = {}
        self.model_name_map: Dict[int, str] = {}
        self.root_node: Optional[Segment] = None
        self.vtx_table: Optional[Segment] = None
        self.model_names: Optional[StringListSegment] = None
        self.collider_names: Optional[Segment] = None
        self.zone_names: Optional[Segment] = None

    def push(self, segment: Segment):
        if segment.addr == 0:
            return None

        if segment.addr in self.visited:
            return self.visited[segment.addr]

        self.pending.append(segment)
        self.visited[segment.addr] = segment
        return segment

    def get_symbol(self, addr) -> str:
        if not addr in self.visited:
            raise Exception(f"Encountered unknown pointer: {hex(addr)}")

        return self.visited[addr].get_sym()

    def print(self, string: str):
        if self.out_file != None:
            self.out_file.write(string + "\n")
        else:
            print(string)

    # traverse the model tree and create a mapping from addr -> name which we will use during the second scan pass
    def build_model_name_map(self, node_addr: int, names: deque):
        node_start = node_addr - BASE_ADDR
        (
            node_type,
            ptr_display_list,
            num_properties,
            ptr_property_list,
            ptr_group_data,
        ) = struct.unpack(">IIIII", self.file_bytes[node_start : node_start + 20])

        if node_type == NODE_TYPE_MODEL:
            # set name for this model node
            self.model_name_map[node_addr] = names.pop()
            return

        group_start = ptr_group_data - BASE_ADDR
        (
            ptr_transform_mtx,
            ptr_lights,
            num_lights,
            num_children,
            ptr_children,
        ) = struct.unpack(">IIIII", self.file_bytes[group_start : group_start + 20])

        child_start = ptr_children - BASE_ADDR

        for i in range(num_children):
            (ptr_child,) = struct.unpack(">I", self.file_bytes[child_start : child_start + 4])
            self.build_model_name_map(ptr_child, names)
            child_start += 4

        # set name for this group node
        if node_type == NODE_TYPE_ROOT:
            self.model_name_map[node_addr] = "root"
        else:
            self.model_name_map[node_addr] = names.pop()

    def print_prologue(self, segments):
        assert self.root_node is not None
        assert self.vtx_table is not None
        assert self.model_names is not None
        assert self.collider_names is not None

        self.print('#include "common.h"')
        self.print('#include "model.h"')
        self.print("")
        self.print(f"#define NAMESPACE {self.map_name}_shape")
        self.print("")
        self.print(f"extern ModelNode {self.root_node.get_sym()};")
        self.print(f"extern Vtx_t {self.vtx_table.get_sym()}[];")
        self.print(f"extern char* {self.model_names.get_sym()}[];")
        self.print(f"extern char* {self.collider_names.get_sym()}[];")

        if self.zone_names is not None:
            self.print(f"extern char* {self.zone_names.get_sym()}[];")
        for segment in segments:
            if isinstance(segment, MatrixSegment):
                self.print(f"extern Matrix4s {segment.get_sym()};")
        self.print("")

    def digest(self):
        # first pass just scans the header and string lists
        self.push(HeaderSegment(BASE_ADDR, "Header"))

        while len(self.pending) > 0:
            segment = self.pending.pop()
            segment.scan(self)

        assert self.model_names is not None
        assert self.root_node is not None

        # traverse the model tree to create initial name map
        model_names = deque(self.model_names.list)
        model_names.reverse()
        self.build_model_name_map(self.root_node.addr, model_names)

        # second pass scans the model tree and subordinate data structures
        self.push(self.root_node)
        self.push(VectorListSegment(self.root_node.addr + 0x14, "UnknownVectors"))

        while len(self.pending) > 0:
            segment = self.pending.pop()
            segment.scan(self)

        # create a sorted segment map
        segment_addrs = list(self.visited.keys())
        segment_addrs.sort()
        self.sorted_segments = {i: self.visited[i] for i in segment_addrs}

    def get_segment_after(self, seg: Segment) -> Optional[Segment]:
        keys = list(self.sorted_segments.keys())
        idx = keys.index(seg.addr)
        if idx + 1 < len(keys):
            next_addr = keys[idx + 1]
            ret = self.sorted_segments[next_addr]
        else:
            ret = None
        return ret

    def write_to_c(self, out_file):
        self.out_file = out_file
        self.print_prologue(self.sorted_segments.values())

        for addr, seg in self.sorted_segments.items():
            self.print(f"// {hex(seg.addr - BASE_ADDR)}")
            seg.print(self)
            self.print("")


def run(in_bin: Path, out: Path) -> None:
    map_name = "_".join(in_bin.stem.split("_")[:-1])

    with open(in_bin, "rb") as f:
        file_bytes = f.read()
        shape = ShapeFile(map_name, file_bytes)
        shape.digest()

        with open(out, "w") as out_file:
            shape.write_to_c(out_file)


if __name__ == "__main__":
    parser = argparse.ArgumentParser()
    parser.add_argument("in_bin", type=Path, help="input binary file")
    parser.add_argument("out_c", type=Path, help="output text file")
    args = parser.parse_args()

    run(args.in_bin, args.out_c)