From ad275eb02c359444a93f24b95656cd0ed6aae3ea Mon Sep 17 00:00:00 2001 From: Ethan Roseman Date: Mon, 19 Oct 2020 17:29:40 -0400 Subject: [PATCH] Makefile cleanup, quieter splata, assist.py --- Makefile | 51 +++++++------- tools/assist.py | 177 ++++++++++++++++++++++++++++++++++++++++++++++++ tools/n64splat | 2 +- 3 files changed, 203 insertions(+), 27 deletions(-) create mode 100755 tools/assist.py diff --git a/Makefile b/Makefile index 81e2e16563..7ff3e45c8f 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ SHELL=/bin/bash -o pipefail # BUILD_DIR is location where all build artifacts are placed BUILD_DIR = build -WORLD_AREAS := area_dgb area_arn area_dro area_end area_flo area_gv area_hos area_isk area_iwa area_jan area_kgr area_kkj area_kmr area_kpa area_kzn area_mac area_mgm area_mim area_nok area_obk area_omo area_osr area_pra area_sam area_sbk area_tik area_trd area_tst +WORLD_AREAS := $(foreach dir, $(wildcard src/world/*), $(dir:src/world/%=%)) SRC_DIRS := src src/os src/os/nusys $(foreach area,$(WORLD_AREAS),src/world/$(area)) ASM_DIRS := asm asm/os @@ -29,20 +29,19 @@ O_FILES := $(foreach file,$(C_FILES),$(BUILD_DIR)/$(file:.c=.o)) \ $(foreach file,$(S_FILES),$(BUILD_DIR)/$(file:.s=.o)) \ $(foreach file,$(DATA_FILES),$(BUILD_DIR)/$(file:.bin=.o)) \ $(foreach dir,$(ASSETS_FS_DIRS),$(BUILD_DIR)/$(dir).o) \ + $(foreach file,$(YAY0_FILES),$(BUILD_DIR)/$(file:.bin=.Yay0.o)) -YAY0_FILES := $(foreach file,$(YAY0_FILES),$(BUILD_DIR)/$(file:.bin=.Yay0.o)) +####################### Tools ######################### -####################### Other Tools ######################### - -# N64 tools -TOOLS_DIR = tools -N64CKSUM = $(TOOLS_DIR)/n64crc +TOOLS = tools +N64CKSUM = $(TOOLS)/n64crc +SPLAT = ./$(TOOLS)/n64splat/split.py baserom.z64 $(TOOLS)/splat.yaml . ##################### Compiler Options ####################### CROSS = mips-linux-gnu- AS = $(CROSS)as -OLD_AS = $(TOOLS_DIR)/mips-nintendo-nu64-as -CC = $(TOOLS_DIR)/cc1 +OLD_AS = $(TOOLS)/mips-nintendo-nu64-as +CC = $(TOOLS)/cc1 LD = $(CROSS)ld OBJDUMP = $(CROSS)objdump OBJCOPY = $(CROSS)objcopy @@ -72,25 +71,19 @@ submodules: git submodule update --init --recursive split: - rm -rf $(DATA_DIRS) && ./tools/n64splat/split.py baserom.z64 tools/splat.yaml . --modes ld bin Yay0 PaperMarioMapFS + rm -rf $(DATA_DIRS) && $(SPLAT) --modes ld bin Yay0 PaperMarioMapFS split-all: - rm -rf $(DATA_DIRS) && ./tools/n64splat/split.py baserom.z64 tools/splat.yaml . --modes all - -$(TARGET).ld: tools/splat.yaml - ./tools/n64splat/split.py baserom.z64 tools/splat.yaml . --modes ld + rm -rf $(DATA_DIRS) && $(SPLAT) --modes all setup: clean submodules split - make -C tools + make -C $(TOOLS) print-% : ; $(info $* is a $(flavor $*) variable set to [$($*)]) @true $(BUILD_DIR): mkdir -p $(BUILD_DIR) -$(BUILD_DIR)/$(TARGET).elf: $(O_FILES) $(YAY0_FILES) $(LD_SCRIPT) - @$(LD) $(LDFLAGS) -o $@ $(O_FILES) - $(BUILD_DIR)/%.o: %.s $(AS) $(ASFLAGS) -o $@ $< @@ -100,20 +93,26 @@ $(BUILD_DIR)/%.o: %.c $(H_FILES) $(BUILD_DIR)/%.o: %.bin $(LD) -r -b binary -o $@ $< -$(BUILD_DIR)/%.Yay0.o: %.bin - mkdir -p build/bin/Yay0 - tools/Yay0compress $< $<.Yay0 - $(LD) -r -b binary -o $@ $<.Yay0 - $(BUILD_DIR)/assets/fs/%: $(ASSETS_FS_FILES) - tools/build_assets_fs.py $* + $(TOOLS)/build_assets_fs.py $* -$(BUILD_DIR)/assets/fs.bin: assets/fs.json tools/build_assets_fs.py $(foreach file,$(ASSETS_FS_FILES),build/$(file)) - tools/build_assets_fs.py +$(BUILD_DIR)/assets/fs.bin: assets/fs.json $(TOOLS)/build_assets_fs.py $(foreach file,$(ASSETS_FS_FILES),build/$(file)) + $(TOOLS)/build_assets_fs.py $(BUILD_DIR)/assets/fs.o: $(BUILD_DIR)/assets/fs.bin $(LD) -r -b binary -o $@ $< +$(BUILD_DIR)/%.Yay0.o: %.bin + mkdir -p build/bin/Yay0 + $(TOOLS)/Yay0compress $< $<.Yay0 + $(LD) -r -b binary -o $@ $<.Yay0 + +$(LD_SCRIPT): $(TOOLS)/splat.yaml + $(SPLAT) --modes ld + +$(BUILD_DIR)/$(TARGET).elf: $(O_FILES) $(LD_SCRIPT) + @$(LD) $(LDFLAGS) -o $@ $(O_FILES) + $(BUILD_DIR)/$(TARGET).bin: $(BUILD_DIR)/$(TARGET).elf $(OBJCOPY) $< $@ -O binary diff --git a/tools/assist.py b/tools/assist.py new file mode 100755 index 0000000000..67ee32784a --- /dev/null +++ b/tools/assist.py @@ -0,0 +1,177 @@ +#!/usr/bin/python3 + +import argparse +from collections import OrderedDict +import os +import re +import pickle +import sys + +script_dir = os.path.dirname(os.path.realpath(__file__)) +root_dir = script_dir + "/../" +asm_dir = root_dir + "asm/nonmatchings/" +build_dir = root_dir + "build/" + +def read_rom(): + with open("baserom.z64", "rb") as f: + return f.read() + + +def find_dir(query): + for root, dirs, files in os.walk(asm_dir): + for d in dirs: + if d == query: + return os.path.join(root, d) + return None + + +def get_all_s_files(): + ret = set() + for root, dirs, files in os.walk(asm_dir): + for f in files: + if f.endswith(".s"): + ret.add(f[:-2]) + return ret + + +def get_symbol_bytes(offsets, func): + if func not in offsets or "start" not in offsets[func] or "end" not in offsets[func]: + return None + start = offsets[func]["start"] + end = offsets[func]["end"] + return list(rom_bytes[start:end]) + + +def parse_map(fname): + ram_offset = None + cur_file = "" + syms = {} + prev_sym = None + prev_line = "" + with open(fname) as f: + for line in f: + if "load address" in line: + if "noload" in line or "noload" in prev_line: + ram_offset = None + continue + ram = int(line[16 : 16 + 18], 0) + rom = int(line[59 : 59 + 18], 0) + ram_offset = ram - rom + continue + prev_line = line + + if ( + ram_offset is None + or "=" in line + or "*fill*" in line + or " 0x" not in line + ): + continue + ram = int(line[16 : 16 + 18], 0) + rom = ram - ram_offset + fn = line.split()[-1] + if "0x" in fn: + ram_offset = None + elif "/" in fn: + cur_file = fn + else: + syms[fn] = (rom, cur_file, prev_sym, ram) + prev_sym = fn + return syms + + +def get_map_offsets(syms): + offsets = {} + for sym in syms: + prev_sym = syms[sym][2] + if sym not in offsets: + offsets[sym] = {} + if prev_sym not in offsets: + offsets[prev_sym] = {} + offsets[sym]["start"] = syms[sym][0] + offsets[prev_sym]["end"] = syms[sym][0] + return offsets + + +def diff_syms(qb, tb): + if len(tb) < 8: + return 0 + + if len(qb) > len(tb): + larger = qb + smaller = tb + else: + larger = tb + smaller = qb + + len_ratio = len(smaller) / len(larger) + + n_bytes = len(smaller) + matches = 0 + for i in range(0, n_bytes, 4): + if smaller[i] == larger[i]: + matches += 4 + return (matches / n_bytes) * len_ratio + + +def get_matches(query): + query_bytes = get_symbol_bytes(map_offsets, query) + if query_bytes is None: + sys.exit("Symbol '" + query + "' not found") + + ret = {} + for symbol in map_offsets: + if symbol is not None and query != symbol: + target_bytes = get_symbol_bytes(map_offsets, symbol) + if target_bytes is not None: + score = diff_syms(query_bytes, target_bytes) + if score >= args.threshold: + ret[symbol] = score + return OrderedDict(sorted(ret.items(), key=lambda kv: kv[1], reverse=True)) + + +def do_query(query): + matches = get_matches(query) + num_matches = len(matches) + + if num_matches == 0: + print(query + " - found no matches") + return + + i = 0 + more_str = ":" + if args.num_out < num_matches: + more_str = " (showing only " + str(args.num_out) + "):" + + print(query + " - found " + str(num_matches) + " matches total" + more_str) + for match in matches: + if i == args.num_out: + break + match_str = "{:.2f} - {}".format(matches[match], match) + if match not in s_files: + match_str += " (decompiled)" + print(match_str) + i += 1 + print() + +parser = argparse.ArgumentParser(description="Tools to assist with decomp") +parser.add_argument("query", help="function or file") +parser.add_argument("--threshold", help="score threshold between 0 and 1 (higher is more restrictive)", type=float, default=0.95, required=False) +parser.add_argument("--num-out", help="number of functions to display", type=int, default=10, required=False) + +args = parser.parse_args() + +rom_bytes = read_rom() +map_syms = parse_map(build_dir + "papermario.map") +map_offsets = get_map_offsets(map_syms) + +s_files = get_all_s_files() + +query_dir = find_dir(args.query) + +if query_dir is not None: + files = os.listdir(query_dir) + for f_name in files: + do_query(f_name[:-2]) +else: + do_query(args.query) diff --git a/tools/n64splat b/tools/n64splat index a81e396af9..19241c6a52 160000 --- a/tools/n64splat +++ b/tools/n64splat @@ -1 +1 @@ -Subproject commit a81e396af93929a863c7fc9ea988fb526037bb7d +Subproject commit 19241c6a5221e6737922ac5c4d40f3d0ce990c17