Progress on tools

tools/add_to_symbol_addrs.py

@@ -1,61 +1,150 @@
 #!/usr/bin/python3
 
-import argparse
 import os
 import re
+import subprocess
+import sys
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
 root_dir = script_dir + "/../"
 asm_dir = root_dir + "asm/nonmatchings/"
 
-symbol_addrs_txt = os.path.join(script_dir, "symbol_addrs.txt")
-papermario_map = os.path.join(root_dir, "build/papermario.map")
+symbol_addrs_path = os.path.join(script_dir, "symbol_addrs.txt")
+elf_path = os.path.join(root_dir, "build", "papermario.elf")
+map_path = os.path.join(root_dir, "build", "papermario.map")
 
-with open(symbol_addrs_txt, "r") as f:
-    symbols = set()
+map_symbols = {}
+symbol_addrs = []
+elf_symbols = []
 
-    for line in f.readlines():
-        line = line.split(";")[0]
-
-        s = [s.strip() for s in line.split("=", 1)]
-        name = s[0]
-
-        symbols.add(name)
-
-with open(symbol_addrs_txt, "a") as symbol_addrs:
-    for root, dirs, files in os.walk(asm_dir):
-        for f_name in files:
-            if f_name.endswith(".s"):
-                f_path = os.path.join(root, f_name)
-
-                with open(f_path, "r") as f:
-                    lines = f.readlines()
-                    line = lines[3]
-                    try:
-                        addr = lines[4].split(" ")[2]
-                    except:
-                        addr = ""
-
-                if not line.startswith("glabel ") or not addr.startswith("80"):
-                    #print(f"??? {f_path}")
-                    continue
-
-                func_name = line.split(" ")[1].rstrip()
-
-                if not func_name in symbols:
-                    symbol_addrs.write(f"{func_name} = 0x{addr}; // type:func\n")
-                    symbols.add(func_name)
-                    print(func_name)
-
-    with open(papermario_map, "r") as f:
-        for match in re.compile(f"^\s+0x([a-f0-9]+)\s+([_a-zA-Z0-9]+)$", re.MULTILINE).finditer(f.read()):
-            addr = int(match.group(1), 16)
-            func = match.group(2)
-
-            if func.startswith("0") or func.startswith("_binary_"):
+def scan_map():
+    ram_offset = None
+    cur_file = "<no file>"
+    prev_line = ""
+    with open(map_path) as f:
+        for line in f:
+            if "load address" in line:
+                ram = int(line[16 : 16 + 18], 0)
+                rom = int(line[59 : 59 + 18], 0)
+                ram_offset = ram - rom
                 continue
 
-            if func not in symbols:
-                symbol_addrs.write(f"{func} = 0x{addr:X};\n")
-                symbols.add(func)
-                print(func)
+            prev_line = line
+
+            if (
+                ram_offset is None
+                or "=" in line
+                or "*fill*" in line
+                or " 0x" not in line
+            ):
+                continue
+
+            ram = int(line[16 : 16 + 18], 0)
+            rom = ram - ram_offset
+            sym = line.split()[-1]
+
+            if "0x" in sym:
+                ram_offset = None
+                continue
+            elif "/" in sym:
+                cur_file = sym
+                continue
+
+            map_symbols[sym] = (rom, cur_file, ram)
+
+def read_symbol_addrs():
+    unique_lines = set()
+
+    with open(symbol_addrs_path, "r") as f:
+        for line in f.readlines():
+                unique_lines.add(line)
+
+        for line in unique_lines:
+            main, ext = line.rstrip().split(";")
+            args = ext.split("//")[-1].strip().split(" ")
+            name, addr = main.split(" = ")
+            symbol_addrs.append((name, int(addr, 0), args))
+
+def read_elf():
+    try:
+        result = subprocess.run(['objdump', '-x', elf_path], stdout=subprocess.PIPE)
+        objdump_lines = result.stdout.decode().split("\n")
+    except:
+        print(f"Error: Could not run objdump on {elf_path} - make sure that the project is built")
+        sys.exit(1)
+
+    for line in objdump_lines:
+        if " F " in line or " O " in line or " *ABS*" in line:
+            components = line.split()
+            name = components[-1]
+
+            if "/" in name or \
+               name.startswith("_") or \
+               name.startswith("jtbl_") or \
+               re.match(r"L[0-9A-F]{8}", name):
+                continue
+
+            addr = int(components[0], 16)
+            if " F " in line:
+                type = "func"
+            else:
+                type = "data"
+
+            opts = [f"type:{type}"]
+
+            if name in map_symbols:
+                opts.append(f"rom:0x{map_symbols[name][0]:X}")
+            elif re.match(".*_[0-9A-F]{8}_[0-9A-F]{6}", name):
+                rom = name.split("_")[-1]
+                opts.append(f"rom:0x{rom}")
+
+            elf_symbols.append((name, addr, opts))
+
+def reconcile_symbols():
+    print(f"Processing {str(len(elf_symbols))} elf symbols...")
+
+    for i, sym in enumerate(elf_symbols):
+        if i % 1000 == 0:
+            print(i)
+        name_match = None
+        rom_match = None
+
+        for sym2 in symbol_addrs:
+
+            # Name
+            if not name_match:
+                if sym[0] == sym2[0]:
+                    name_match = sym2
+
+                    if sym[1] != sym2[1]:
+                        print(f"Address mismatch! {sym[0]} is 0x{sym[1]:X} in the elf and 0x{sym2[1]} in symbol_addrs")
+
+                    if not rom_match:
+                        if sym[2] == sym2[2]:
+                            rom_match = sym2
+
+            # Rom
+            if not rom_match:
+                # Todo account for either or both syms not containing a rom addr
+                if sym[2] == sym2[2]:
+                    rom_match = sym2
+
+                    if not name_match:
+                        if sym[0] == sym2[1]:
+                            name_match = sym2
+
+        if not name_match and not rom_match:
+            # Todo add new symbol to symbol_addrs
+            pass
+        elif not name_match:
+            #todo rename symbol in symbol_addrs
+            pass
+        elif not rom_match:
+            # todo add rom addr in symbol_addrs
+            pass
+
+
+scan_map()
+read_symbol_addrs()
+read_elf()
+reconcile_symbols()

tools/sym_info.py

@@ -0,0 +1,142 @@
+#!/usr/bin/env python3
+
+import os.path
+import argparse
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+root_dir = os.path.abspath(os.path.join(script_dir, ".."))
+
+parser = argparse.ArgumentParser(
+    description="Display various information about a symbol or address."
+)
+parser.add_argument(
+    "name",
+    type=str,
+    default="",
+    help="symbol name or ROM/RAM address to lookup"
+)
+parser.add_argument(
+    "-e",
+    "--expected",
+    dest="use_expected",
+    action="store_true",
+    help="use the map file in expected/build/ instead of build/"
+)
+args = parser.parse_args()
+
+mymap = os.path.join(root_dir, "build", "papermario.map")
+if args.use_expected:
+    mymap = os.path.join(root_dir, "expected", "build", "papermario.map")
+
+if not os.path.isfile(mymap):
+    print(f"{mymap} must exist.")
+    exit(1)
+
+
+def search_address(target_addr):
+    is_ram = target_addr & 0x80000000
+    ram_offset = None
+    prev_ram = 0
+    prev_rom = 0
+    prev_sym = "<start of rom>"
+    cur_file = "<no file>"
+    prev_file = cur_file
+    prev_line = ""
+    with open(mymap) as f:
+        for line in f:
+            if "load address" in line:
+                # Ignore .bss sections if we're looking for a ROM address
+                if not is_ram and (".bss" in line or ".bss" in prev_line):
+                    ram_offset = None
+                    continue
+                ram = int(line[16 : 16 + 18], 0)
+                rom = int(line[59 : 59 + 18], 0)
+                ram_offset = ram - rom
+                continue
+
+            prev_line = line
+
+            if (
+                ram_offset is None
+                or "=" in line
+                or "*fill*" in line
+                or " 0x" not in line
+            ):
+                continue
+
+            ram = int(line[16 : 16 + 18], 0)
+            rom = ram - ram_offset
+            sym = line.split()[-1]
+
+            if "0x" in sym:
+                ram_offset = None
+                continue
+            if "/" in sym:
+                cur_file = sym
+                continue
+
+            if rom == target_addr or (is_ram and ram == target_addr):
+                return f"{sym} (RAM 0x{ram:X}, ROM 0x{rom:X}, {cur_file})"
+            if rom > target_addr or (is_ram and ram > target_addr):
+                offset = target_addr - prev_ram if is_ram else target_addr - prev_rom
+                return f"at 0x{offset:X} bytes inside {prev_sym} (RAM 0x{prev_ram:X}, ROM 0x{prev_rom:X}, {prev_file})"
+
+            prev_ram = ram
+            prev_rom = rom
+            prev_sym = sym
+            prev_file = cur_file
+
+    return "at end of rom?"
+
+
+def search_symbol(target_sym):
+    ram_offset = None
+    cur_file = "<no file>"
+    prev_line = ""
+    with open(mymap) as f:
+        for line in f:
+            if "load address" in line:
+                ram = int(line[16 : 16 + 18], 0)
+                rom = int(line[59 : 59 + 18], 0)
+                ram_offset = ram - rom
+                continue
+
+            prev_line = line
+
+            if (
+                ram_offset is None
+                or "=" in line
+                or "*fill*" in line
+                or " 0x" not in line
+            ):
+                continue
+
+            ram = int(line[16 : 16 + 18], 0)
+            rom = ram - ram_offset
+            sym = line.split()[-1]
+
+            if "0x" in sym:
+                ram_offset = None
+                continue
+            elif "/" in sym:
+                cur_file = sym
+                continue
+
+            if sym == target_sym:
+                return (rom, cur_file, ram)
+
+    return None
+
+
+try:
+    target_addr = int(args.name, 0)
+    print(args.name, "is", search_address(target_addr))
+except ValueError:
+    sym_info = search_symbol(args.name)
+    if sym_info is not None:
+        sym_rom = sym_info[0]
+        sym_file = sym_info[1]
+        sym_ram = sym_info[2]
+        print(f"Symbol {args.name} (RAM: 0x{sym_ram:08X}, ROM: 0x{sym_rom:06X}, {sym_file})")
+    else:
+        print(f"Symbol {args.name} not found in map file {mymap}")