From 236aa9b7183c9433d137977be250bb74cbd3d249 Mon Sep 17 00:00:00 2001 From: Alex Bates Date: Sun, 18 Oct 2020 04:54:32 +0100 Subject: [PATCH] decode assets fs encoding is wip --- .editorconfig | 3 ++ .gitignore | 3 ++ Makefile | 20 ++++++++-- tools/build_assets_fs.py | 82 ++++++++++++++++++++++++++++++++++++++++ tools/splat.yaml | 4 +- 5 files changed, 107 insertions(+), 5 deletions(-) create mode 100755 tools/build_assets_fs.py diff --git a/.editorconfig b/.editorconfig index 6e394e06ab..537ddfac48 100644 --- a/.editorconfig +++ b/.editorconfig @@ -8,3 +8,6 @@ insert_final_newline = true [*.yaml] indent_size = 2 + +[Makefile] +indent_type = tab diff --git a/.gitignore b/.gitignore index 05030ee51f..bd79c043b6 100644 --- a/.gitignore +++ b/.gitignore @@ -8,6 +8,9 @@ venv/ ctx.c expected/ +# Assets +assets + # Build artifacts *.ld *.z64 diff --git a/Makefile b/Makefile index b6f7492006..6a68cf422a 100644 --- a/Makefile +++ b/Makefile @@ -12,6 +12,7 @@ ASM_DIRS := asm asm/os INCLUDE_DIRS := include include/PR src DATA_DIRS := bin YAY0_DIRS := bin/Yay0 +ASSETS_FS_DIRS := assets/fs # Source code files C_FILES := $(foreach dir,$(SRC_DIRS),$(wildcard $(dir)/*.c)) @@ -21,11 +22,13 @@ ifdef PM_HEADER_REBUILD endif DATA_FILES := $(foreach dir,$(DATA_DIRS),$(wildcard $(dir)/*.bin)) YAY0_FILES := $(foreach dir,$(YAY0_DIRS),$(wildcard $(dir)/*.bin)) +ASSETS_FS_FILES := $(foreach dir,$(ASSETS_FS_DIRS),$(wildcard $(dir)/*.*)) # Object files O_FILES := $(foreach file,$(C_FILES),$(BUILD_DIR)/$(file:.c=.o)) \ - $(foreach file,$(S_FILES),$(BUILD_DIR)/$(file:.s=.o)) \ - $(foreach file,$(DATA_FILES),$(BUILD_DIR)/$(file:.bin=.o)) \ + $(foreach file,$(S_FILES),$(BUILD_DIR)/$(file:.s=.o)) \ + $(foreach file,$(DATA_FILES),$(BUILD_DIR)/$(file:.bin=.o)) \ + $(foreach dir,$(ASSETS_FS_DIRS),$(BUILD_DIR)/$(dir).o) \ YAY0_FILES := $(foreach file,$(YAY0_FILES),$(BUILD_DIR)/$(file:.bin=.bin.Yay0)) @@ -56,7 +59,7 @@ LDFLAGS = -T undefined_syms.txt -T undefined_funcs.txt -T $(LD_SCRIPT) -Map $ ######################## Targets ############################# -$(foreach dir,$(SRC_DIRS) $(ASM_DIRS) $(DATA_DIRS) ,$(shell mkdir -p build/$(dir))) +$(foreach dir,$(SRC_DIRS) $(ASM_DIRS) $(DATA_DIRS) $(ASSETS_FS_DIRS) ,$(shell mkdir -p build/$(dir))) default: all @@ -71,7 +74,7 @@ submodules: git submodule update --init --recursive split: - rm -rf $(DATA_DIRS) && ./tools/n64splat/split.py baserom.z64 tools/splat.yaml . --modes ld bin Yay0 + rm -rf $(DATA_DIRS) && ./tools/n64splat/split.py baserom.z64 tools/splat.yaml . --modes ld bin Yay0 PaperMarioMapFS split-all: rm -rf $(DATA_DIRS) && ./tools/n64splat/split.py baserom.z64 tools/splat.yaml . --modes all @@ -104,6 +107,15 @@ $(BUILD_DIR)/%.bin.Yay0: %.bin tools/Yay0compress $< $<.Yay0 $(LD) -r -b binary -o $@ $<.Yay0 +$(BUILD_DIR)/assets/fs/%: $(ASSETS_FS_FILES) + tools/build_assets_fs.py $* + +$(BUILD_DIR)/assets/fs.bin: assets/fs.json tools/build_assets_fs.py $(foreach file,$(ASSETS_FS_FILES),build/$(file)) + tools/build_assets_fs.py + +$(BUILD_DIR)/assets/fs.o: $(BUILD_DIR)/assets/fs.bin + $(LD) -r -b binary -o $@ $< + $(BUILD_DIR)/$(TARGET).bin: $(BUILD_DIR)/$(TARGET).elf $(OBJCOPY) $< $@ -O binary diff --git a/tools/build_assets_fs.py b/tools/build_assets_fs.py new file mode 100755 index 0000000000..607866bee9 --- /dev/null +++ b/tools/build_assets_fs.py @@ -0,0 +1,82 @@ +#! /usr/bin/python3 + +import os +import json +import sys +from subprocess import call +from pathlib import Path +import shutil + +tools_dir = Path(__file__).parent.absolute() + +with open("assets/fs.json", "r") as f: + config = json.loads(f.read()) + +def build_mapfs(src_dir, build_dir, out_bin): + with open(out_bin, "wb") as f: + f.write(config["title"].encode("ascii")) + + next_data_pos = 0x20 + (len(config["assets"]) + 1) * 0x1C + + asset_idx = 0 + for asset in config["assets"]: + offset = 0x20 + asset_idx * 0x1C + + + src_path = Path(src_dir, asset["path"]) + build_path = Path(build_dir, asset["path"]) + + name = asset["name"] + src_size = src_path.stat().st_size + compressed_size = build_path.stat().st_size + 1 + + print({ + "name": name, + "offset": (next_data_pos + 0x20), + "size": compressed_size, + "decompressed_size": src_size, + }) + + # write TOC row + f.seek(offset) + f.write(name.encode('ascii')) + f.seek(offset + 0x10) + f.write((next_data_pos - 0x20).to_bytes(4, byteorder="big")) + f.write(compressed_size.to_bytes(4, byteorder="big")) + f.write(src_size.to_bytes(4, byteorder="big")) + + # write data + f.seek(next_data_pos) + f.write(build_path.read_bytes()) + next_data_pos += compressed_size + + asset_idx += 1 + + f.seek(0x20 + asset_idx * 0x1C) + f.write(b"end_data") + +def build_file(src_dir, out_dir, filename): + asset = None + for a in config["assets"]: + if (a["path"] == filename): + asset = a + + if not asset: + print("asset not configured in json file") + exit(1) + + src_path = Path(src_dir, filename) + out_path = Path(out_dir, filename) + + if asset["compress"]: + call([f"{tools_dir}/Yay0compress", src_path, out_path]) + else: + shutil.copy(src_path, out_path) + +if __name__ == "__main__": + if len(sys.argv) > 1: + # copy (and compress if required) the given file + build_file("assets/fs", "build/assets/fs", sys.argv[1]) + else: + # build the aggregate file + build_mapfs("assets/fs", "build/assets/fs", "build/assets/fs.bin") diff --git a/tools/splat.yaml b/tools/splat.yaml index 296b4bb128..10ddf7925e 100644 --- a/tools/splat.yaml +++ b/tools/splat.yaml @@ -764,6 +764,8 @@ segments: - [0x1B81E88, "Yay0"] - [0x1B82058, "Yay0"] - [0x1B82202, "bin"] - - [0x1E40000, "bin", "map_assets.fs"] # todo add fs support + - name: assets/fs + type: PaperMarioMapFS + start: 0x1E40000 - [0x27FEE22, "bin"] - [0x2800000]