Makefile cleanup, quieter splata, assist.py

Makefile

@@ -5,7 +5,7 @@ SHELL=/bin/bash -o pipefail
 # BUILD_DIR is location where all build artifacts are placed
 BUILD_DIR = build
 
-WORLD_AREAS := area_dgb area_arn area_dro area_end area_flo area_gv area_hos area_isk area_iwa area_jan area_kgr area_kkj area_kmr area_kpa area_kzn area_mac area_mgm area_mim area_nok area_obk area_omo area_osr area_pra area_sam area_sbk area_tik area_trd area_tst
+WORLD_AREAS := $(foreach dir, $(wildcard src/world/*), $(dir:src/world/%=%))
 
 SRC_DIRS := src src/os src/os/nusys $(foreach area,$(WORLD_AREAS),src/world/$(area))
 ASM_DIRS := asm asm/os
@@ -29,20 +29,19 @@ O_FILES := $(foreach file,$(C_FILES),$(BUILD_DIR)/$(file:.c=.o)) \
 		   $(foreach file,$(S_FILES),$(BUILD_DIR)/$(file:.s=.o)) \
 		   $(foreach file,$(DATA_FILES),$(BUILD_DIR)/$(file:.bin=.o)) \
 		   $(foreach dir,$(ASSETS_FS_DIRS),$(BUILD_DIR)/$(dir).o) \
+		   $(foreach file,$(YAY0_FILES),$(BUILD_DIR)/$(file:.bin=.Yay0.o))
 
-YAY0_FILES := $(foreach file,$(YAY0_FILES),$(BUILD_DIR)/$(file:.bin=.Yay0.o))
+####################### Tools #########################
 
-####################### Other Tools #########################
+TOOLS = tools
-
+N64CKSUM = $(TOOLS)/n64crc
-# N64 tools
+SPLAT = ./$(TOOLS)/n64splat/split.py baserom.z64 $(TOOLS)/splat.yaml .
-TOOLS_DIR = tools
-N64CKSUM = $(TOOLS_DIR)/n64crc
 
 ##################### Compiler Options #######################
 CROSS = mips-linux-gnu-
 AS = $(CROSS)as
-OLD_AS = $(TOOLS_DIR)/mips-nintendo-nu64-as
+OLD_AS = $(TOOLS)/mips-nintendo-nu64-as
-CC = $(TOOLS_DIR)/cc1
+CC = $(TOOLS)/cc1
 LD = $(CROSS)ld
 OBJDUMP = $(CROSS)objdump
 OBJCOPY = $(CROSS)objcopy
@@ -72,25 +71,19 @@ submodules:
 	git submodule update --init --recursive
 
 split:
-	rm -rf $(DATA_DIRS) && ./tools/n64splat/split.py baserom.z64 tools/splat.yaml . --modes ld bin Yay0 PaperMarioMapFS
+	rm -rf $(DATA_DIRS) && $(SPLAT) --modes ld bin Yay0 PaperMarioMapFS
 
 split-all:
-	rm -rf $(DATA_DIRS) && ./tools/n64splat/split.py baserom.z64 tools/splat.yaml . --modes all
+	rm -rf $(DATA_DIRS) && $(SPLAT) --modes all
-
-$(TARGET).ld: tools/splat.yaml
-	./tools/n64splat/split.py baserom.z64 tools/splat.yaml . --modes ld
 
 setup: clean submodules split
-	make -C tools
+	make -C $(TOOLS)
 
 print-% : ; $(info $* is a $(flavor $*) variable set to [$($*)]) @true
 
 $(BUILD_DIR):
 	mkdir -p $(BUILD_DIR)
 
-$(BUILD_DIR)/$(TARGET).elf: $(O_FILES) $(YAY0_FILES) $(LD_SCRIPT)
-	@$(LD) $(LDFLAGS) -o $@ $(O_FILES)
-
 $(BUILD_DIR)/%.o: %.s
 	$(AS) $(ASFLAGS) -o $@ $<
 
@@ -100,20 +93,26 @@ $(BUILD_DIR)/%.o: %.c $(H_FILES)
 $(BUILD_DIR)/%.o: %.bin
 	$(LD) -r -b binary -o $@ $<
 
-$(BUILD_DIR)/%.Yay0.o: %.bin
-	mkdir -p build/bin/Yay0
-	tools/Yay0compress $< $<.Yay0
-	$(LD) -r -b binary -o $@ $<.Yay0
-
 $(BUILD_DIR)/assets/fs/%: $(ASSETS_FS_FILES)
-	tools/build_assets_fs.py $*
+	$(TOOLS)/build_assets_fs.py $*
 
-$(BUILD_DIR)/assets/fs.bin: assets/fs.json tools/build_assets_fs.py $(foreach file,$(ASSETS_FS_FILES),build/$(file))
+$(BUILD_DIR)/assets/fs.bin: assets/fs.json $(TOOLS)/build_assets_fs.py $(foreach file,$(ASSETS_FS_FILES),build/$(file))
-	tools/build_assets_fs.py
+	$(TOOLS)/build_assets_fs.py
 
 $(BUILD_DIR)/assets/fs.o: $(BUILD_DIR)/assets/fs.bin
 	$(LD) -r -b binary -o $@ $<
 
+$(BUILD_DIR)/%.Yay0.o: %.bin
+	mkdir -p build/bin/Yay0
+	$(TOOLS)/Yay0compress $< $<.Yay0
+	$(LD) -r -b binary -o $@ $<.Yay0
+
+$(LD_SCRIPT): $(TOOLS)/splat.yaml
+	$(SPLAT) --modes ld
+
+$(BUILD_DIR)/$(TARGET).elf: $(O_FILES) $(LD_SCRIPT)
+	@$(LD) $(LDFLAGS) -o $@ $(O_FILES)
+
 $(BUILD_DIR)/$(TARGET).bin: $(BUILD_DIR)/$(TARGET).elf
 	$(OBJCOPY) $< $@ -O binary
 

tools/assist.py

@@ -0,0 +1,177 @@
+#!/usr/bin/python3
+
+import argparse
+from collections import OrderedDict
+import os
+import re
+import pickle
+import sys
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = script_dir + "/../"
+asm_dir = root_dir + "asm/nonmatchings/"
+build_dir = root_dir + "build/"
+
+def read_rom():
+    with open("baserom.z64", "rb") as f:
+        return f.read()
+
+
+def find_dir(query):
+    for root, dirs, files in os.walk(asm_dir):
+        for d in dirs:
+            if d == query:
+                return os.path.join(root, d)
+    return None
+
+
+def get_all_s_files():
+    ret = set()
+    for root, dirs, files in os.walk(asm_dir):
+        for f in files:
+            if f.endswith(".s"):
+                ret.add(f[:-2])
+    return ret
+
+
+def get_symbol_bytes(offsets, func):
+    if func not in offsets or "start" not in offsets[func] or "end" not in offsets[func]:
+        return None
+    start = offsets[func]["start"]
+    end = offsets[func]["end"]
+    return list(rom_bytes[start:end])
+
+
+def parse_map(fname):
+    ram_offset = None
+    cur_file = "<no file>"
+    syms = {}
+    prev_sym = None
+    prev_line = ""
+    with open(fname) as f:
+        for line in f:
+            if "load address" in line:
+                if "noload" in line or "noload" in prev_line:
+                    ram_offset = None
+                    continue
+                ram = int(line[16 : 16 + 18], 0)
+                rom = int(line[59 : 59 + 18], 0)
+                ram_offset = ram - rom
+                continue
+            prev_line = line
+
+            if (
+                ram_offset is None
+                or "=" in line
+                or "*fill*" in line
+                or " 0x" not in line
+            ):
+                continue
+            ram = int(line[16 : 16 + 18], 0)
+            rom = ram - ram_offset
+            fn = line.split()[-1]
+            if "0x" in fn:
+                ram_offset = None
+            elif "/" in fn:
+                cur_file = fn
+            else:
+                syms[fn] = (rom, cur_file, prev_sym, ram)
+                prev_sym = fn
+    return syms
+
+
+def get_map_offsets(syms):
+    offsets = {}
+    for sym in syms:
+        prev_sym = syms[sym][2]
+        if sym not in offsets:
+            offsets[sym] = {}
+        if prev_sym not in offsets:
+            offsets[prev_sym] = {}
+        offsets[sym]["start"] = syms[sym][0]
+        offsets[prev_sym]["end"] = syms[sym][0]
+    return offsets
+
+
+def diff_syms(qb, tb):
+    if len(tb) < 8:
+        return 0
+
+    if len(qb) > len(tb):
+        larger = qb
+        smaller = tb
+    else:
+        larger = tb
+        smaller = qb
+    
+    len_ratio = len(smaller) / len(larger)
+
+    n_bytes = len(smaller)
+    matches = 0
+    for i in range(0, n_bytes, 4):
+        if smaller[i] == larger[i]:
+            matches += 4
+    return (matches / n_bytes) * len_ratio
+
+
+def get_matches(query):
+    query_bytes = get_symbol_bytes(map_offsets, query)
+    if query_bytes is None:
+        sys.exit("Symbol '" + query + "' not found")
+
+    ret = {}
+    for symbol in map_offsets:
+        if symbol is not None and query != symbol:
+            target_bytes = get_symbol_bytes(map_offsets, symbol)
+            if target_bytes is not None:
+                score = diff_syms(query_bytes, target_bytes)
+                if score >= args.threshold:
+                    ret[symbol] = score
+    return OrderedDict(sorted(ret.items(), key=lambda kv: kv[1], reverse=True))
+
+
+def do_query(query):
+    matches = get_matches(query)
+    num_matches = len(matches)
+
+    if num_matches == 0:
+        print(query + " - found no matches")
+        return
+
+    i = 0
+    more_str = ":"
+    if args.num_out < num_matches:
+        more_str = " (showing only " + str(args.num_out) + "):"
+
+    print(query + " - found " + str(num_matches) + " matches total" + more_str)
+    for match in matches:
+        if i == args.num_out:
+            break
+        match_str = "{:.2f} - {}".format(matches[match], match)
+        if match not in s_files:
+           match_str += " (decompiled)"
+        print(match_str)
+        i += 1
+    print()
+
+parser = argparse.ArgumentParser(description="Tools to assist with decomp")
+parser.add_argument("query", help="function or file")
+parser.add_argument("--threshold", help="score threshold between 0 and 1 (higher is more restrictive)", type=float, default=0.95, required=False)
+parser.add_argument("--num-out", help="number of functions to display", type=int, default=10, required=False)
+
+args = parser.parse_args()
+
+rom_bytes = read_rom()
+map_syms = parse_map(build_dir + "papermario.map")
+map_offsets = get_map_offsets(map_syms)
+
+s_files = get_all_s_files()
+
+query_dir = find_dir(args.query)
+
+if query_dir is not None:
+    files = os.listdir(query_dir)
+    for f_name in files:
+        do_query(f_name[:-2])
+else:
+    do_query(args.query)

tools/n64splat

@@ -1 +1 @@
-Subproject commit a81e396af93929a863c7fc9ea988fb526037bb7d
+Subproject commit 19241c6a5221e6737922ac5c4d40f3d0ce990c17