From 191ef16e9407eff277ef72506cffee27f3a617fd Mon Sep 17 00:00:00 2001 From: Ethan Roseman Date: Sat, 16 Jan 2021 01:20:52 +0900 Subject: [PATCH 1/2] update diff.py --- diff.py | 836 ++++++++++++++++++++++++++++++++++++-------------------- 1 file changed, 540 insertions(+), 296 deletions(-) diff --git a/diff.py b/diff.py index 1f7ff7e12e..8f5daeada2 100755 --- a/diff.py +++ b/diff.py @@ -1,10 +1,28 @@ #!/usr/bin/env python3 +# PYTHON_ARGCOMPLETE_OK +import argparse import sys +from typing import ( + Any, + Dict, + List, + Match, + NamedTuple, + NoReturn, + Optional, + Set, + Tuple, + Union, + Callable, + Pattern, +) -def fail(msg): + +def fail(msg: str) -> NoReturn: print(msg, file=sys.stderr) sys.exit(1) + # Prefer to use diff_settings.py from the current working directory sys.path.insert(0, ".") try: @@ -19,21 +37,25 @@ try: import argcomplete # type: ignore except ModuleNotFoundError: argcomplete = None -import argparse -parser = argparse.ArgumentParser(description="Diff MIPS assembly.") +parser = argparse.ArgumentParser(description="Diff MIPS or AArch64 assembly.") + +start_argument = parser.add_argument( + "start", + help="Function name or address to start diffing from.", +) -start_argument = parser.add_argument("start", help="Function name or address to start diffing from.") if argcomplete: - def complete_symbol(**kwargs): - prefix = kwargs["prefix"] - if prefix == "": + + def complete_symbol( + prefix: str, parsed_args: argparse.Namespace, **kwargs: object + ) -> List[str]: + if not prefix or prefix.startswith("-"): # skip reading the map file, which would # result in a lot of useless completions return [] - parsed_args = kwargs["parsed_args"] - config = {} - diff_settings.apply(config, parsed_args) + config: Dict[str, Any] = {} + diff_settings.apply(config, parsed_args) # type: ignore mapfile = config.get("mapfile") if not mapfile: return [] @@ -64,20 +86,28 @@ if argcomplete: pos = data.find(search, endPos) completes.append(match) return completes + setattr(start_argument, "completer", complete_symbol) -parser.add_argument("end", nargs="?", help="Address to end diff at.") +parser.add_argument( + "end", + nargs="?", + help="Address to end diff at.", +) parser.add_argument( "-o", dest="diff_obj", action="store_true", - help="Diff .o files rather than a whole binary. This makes it possible to see symbol names. (Recommended)", + help="Diff .o files rather than a whole binary. This makes it possible to " + "see symbol names. (Recommended)", ) parser.add_argument( "-e", "--elf", dest="diff_elf_symbol", - help="Diff a given function in two ELFs, one being stripped and the other one non-stripped. Requires objdump from binutils 2.33+.", + metavar="SYMBOL", + help="Diff a given function in two ELFs, one being stripped and the other " + "one non-stripped. Requires objdump from binutils 2.33+.", ) parser.add_argument( "--source", @@ -114,6 +144,7 @@ parser.add_argument( dest="skip_lines", type=int, default=0, + metavar="LINES", help="Skip the first N lines of output.", ) parser.add_argument( @@ -164,12 +195,22 @@ parser.add_argument( ) parser.add_argument( "-3", - "--threeway", + "--threeway=prev", dest="threeway", - action="store_true", + action="store_const", + const="prev", help="Show a three-way diff between target asm, current asm, and asm " "prior to -w rebuild. Requires -w.", ) +parser.add_argument( + "-b", + "--threeway=base", + dest="threeway", + action="store_const", + const="base", + help="Show a three-way diff between target asm, current asm, and asm " + "when diff.py was started. Requires -w.", +) parser.add_argument( "--width", dest="column_width", @@ -182,7 +223,8 @@ parser.add_argument( dest="algorithm", default="levenshtein", choices=["levenshtein", "difflib"], - help="Diff algorithm to use.", + help="Diff algorithm to use. Levenshtein gives the minimum diff, while difflib " + "aims for long sections of equal opcodes. Defaults to %(default)s.", ) parser.add_argument( "--max-size", @@ -194,14 +236,17 @@ parser.add_argument( ) # Project-specific flags, e.g. different versions/make arguments. -if hasattr(diff_settings, "add_custom_arguments"): - diff_settings.add_custom_arguments(parser) # type: ignore +add_custom_arguments_fn = getattr(diff_settings, "add_custom_arguments", None) +if add_custom_arguments_fn: + add_custom_arguments_fn(parser) if argcomplete: argcomplete.autocomplete(parser) # ==== IMPORTS ==== +# (We do imports late to optimize auto-complete performance.) + import re import os import ast @@ -212,7 +257,6 @@ import itertools import threading import queue import time -from typing import Any, Dict, List, NamedTuple, Optional, Set, Tuple, Union MISSING_PREREQUISITES = ( @@ -233,20 +277,22 @@ args = parser.parse_args() # Set imgs, map file and make flags in a project-specific manner. config: Dict[str, Any] = {} -diff_settings.apply(config, args) +diff_settings.apply(config, args) # type: ignore -arch = config.get("arch", "mips") -baseimg = config.get("baseimg", None) -myimg = config.get("myimg", None) -mapfile = config.get("mapfile", None) -makeflags = config.get("makeflags", []) -source_directories = config.get("source_directories", None) -objdump_executable = config.get("objdump_executable", None) +arch: str = config.get("arch", "mips") +baseimg: Optional[str] = config.get("baseimg") +myimg: Optional[str] = config.get("myimg") +mapfile: Optional[str] = config.get("mapfile") +makeflags: List[str] = config.get("makeflags", []) +source_directories: Optional[List[str]] = config.get("source_directories") +objdump_executable: Optional[str] = config.get("objdump_executable") +map_format: str = config.get("map_format", "gnu") +mw_build_dir: str = config.get("mw_build_dir", "build/") -MAX_FUNCTION_SIZE_LINES = args.max_lines -MAX_FUNCTION_SIZE_BYTES = MAX_FUNCTION_SIZE_LINES * 4 +MAX_FUNCTION_SIZE_LINES: int = args.max_lines +MAX_FUNCTION_SIZE_BYTES: int = MAX_FUNCTION_SIZE_LINES * 4 -COLOR_ROTATION = [ +COLOR_ROTATION: List[str] = [ Fore.MAGENTA, Fore.CYAN, Fore.GREEN, @@ -258,14 +304,16 @@ COLOR_ROTATION = [ Fore.LIGHTBLACK_EX, ] -BUFFER_CMD = ["tail", "-c", str(10 ** 9)] -LESS_CMD = ["less", "-SRic", "-#6"] +BUFFER_CMD: List[str] = ["tail", "-c", str(10 ** 9)] +LESS_CMD: List[str] = ["less", "-SRic", "-#6"] -DEBOUNCE_DELAY = 0.1 -FS_WATCH_EXTENSIONS = [".c", ".h"] +DEBOUNCE_DELAY: float = 0.1 +FS_WATCH_EXTENSIONS: List[str] = [".c", ".h"] # ==== LOGIC ==== +ObjdumpCommand = Tuple[List[str], str, Optional[str]] + if args.algorithm == "levenshtein": try: import Levenshtein # type: ignore @@ -278,6 +326,9 @@ if args.source: except ModuleNotFoundError as e: fail(MISSING_PREREQUISITES.format(e.name)) +if args.threeway and not args.watch: + fail("Threeway diffing requires -w.") + if objdump_executable is None: for objdump_cand in ["mips-linux-gnu-objdump", "mips64-elf-objdump"]: try: @@ -299,35 +350,41 @@ if not objdump_executable: ) -def eval_int(expr, emsg=None): +def maybe_eval_int(expr: str) -> Optional[int]: try: ret = ast.literal_eval(expr) if not isinstance(ret, int): raise Exception("not an integer") return ret except Exception: - if emsg is not None: - fail(emsg) return None -def eval_line_num(expr): +def eval_int(expr: str, emsg: str) -> int: + ret = maybe_eval_int(expr) + if ret is None: + fail(emsg) + return ret + + +def eval_line_num(expr: str) -> int: return int(expr.strip().replace(":", ""), 16) -def run_make(target, capture_output=False): - if capture_output: - return subprocess.run( - ["make"] + makeflags + [target], - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - ) - else: - subprocess.check_call(["make"] + makeflags + [target]) +def run_make(target: str) -> None: + subprocess.check_call(["make"] + makeflags + [target]) -def restrict_to_function(dump, fn_name): - out = [] +def run_make_capture_output(target: str) -> "subprocess.CompletedProcess[bytes]": + return subprocess.run( + ["make"] + makeflags + [target], + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + ) + + +def restrict_to_function(dump: str, fn_name: str) -> str: + out: List[str] = [] search = f"<{fn_name}>:" found = False for line in dump.split("\n"): @@ -340,13 +397,13 @@ def restrict_to_function(dump, fn_name): return "\n".join(out) -def maybe_get_objdump_source_flags(): +def maybe_get_objdump_source_flags() -> List[str]: if not args.source: return [] flags = [ "--source", - "--source-comment=| ", + "--source-comment=│ ", "-l", ] @@ -356,8 +413,9 @@ def maybe_get_objdump_source_flags(): return flags -def run_objdump(cmd): +def run_objdump(cmd: ObjdumpCommand) -> str: flags, target, restrict = cmd + assert objdump_executable, "checked previously" out = subprocess.check_output( [objdump_executable] + arch_flags + flags + [target], universal_newlines=True ) @@ -366,53 +424,76 @@ def run_objdump(cmd): return out -base_shift = eval_int( +base_shift: int = eval_int( args.base_shift, "Failed to parse --base-shift (-S) argument as an integer." ) -def search_map_file(fn_name): +def search_map_file(fn_name: str) -> Tuple[Optional[str], Optional[int]]: if not mapfile: fail(f"No map file configured; cannot find function {fn_name}.") try: with open(mapfile) as f: - lines = f.read().split("\n") + contents = f.read() except Exception: fail(f"Failed to open map file {mapfile} for reading.") - try: - cur_objfile = None - ram_to_rom = None - cands = [] - last_line = "" - for line in lines: - if line.startswith(" .text"): - cur_objfile = line.split()[3] - if "load address" in line: - tokens = last_line.split() + line.split() - ram = int(tokens[1], 0) - rom = int(tokens[5], 0) - ram_to_rom = rom - ram - if line.endswith(" " + fn_name): - ram = int(line.split()[0], 0) - if cur_objfile is not None and ram_to_rom is not None: - cands.append((cur_objfile, ram + ram_to_rom)) - last_line = line - except Exception as e: - import traceback + if map_format == 'gnu': + lines = contents.split("\n") - traceback.print_exc() - fail(f"Internal error while parsing map file") + try: + cur_objfile = None + ram_to_rom = None + cands = [] + last_line = "" + for line in lines: + if line.startswith(" .text"): + cur_objfile = line.split()[3] + if "load address" in line: + tokens = last_line.split() + line.split() + ram = int(tokens[1], 0) + rom = int(tokens[5], 0) + ram_to_rom = rom - ram + if line.endswith(" " + fn_name): + ram = int(line.split()[0], 0) + if cur_objfile is not None and ram_to_rom is not None: + cands.append((cur_objfile, ram + ram_to_rom)) + last_line = line + except Exception as e: + import traceback - if len(cands) > 1: - fail(f"Found multiple occurrences of function {fn_name} in map file.") - if len(cands) == 1: - return cands[0] + traceback.print_exc() + fail(f"Internal error while parsing map file") + + if len(cands) > 1: + fail(f"Found multiple occurrences of function {fn_name} in map file.") + if len(cands) == 1: + return cands[0] + elif map_format == 'mw': + # ram elf rom object name + find = re.findall(re.compile(r' \S+ \S+ (\S+) (\S+) . ' + fn_name + r'(?: \(entry of \.(?:init|text)\))? \t(\S+)'), contents) + if len(find) > 1: + fail(f"Found multiple occurrences of function {fn_name} in map file.") + if len(find) == 1: + rom = int(find[0][1],16) + objname = find[0][2] + # The metrowerks linker map format does not contain the full object path, so we must complete it manually. + objfiles = [os.path.join(dirpath, f) for dirpath, _, filenames in os.walk(mw_build_dir) for f in filenames if f == objname] + if len(objfiles) > 1: + all_objects = "\n".join(objfiles) + fail(f"Found multiple objects of the same name {objname} in {mw_build_dir}, cannot determine which to diff against: \n{all_objects}") + if len(objfiles) == 1: + objfile = objfiles[0] + # TODO Currently the ram-rom conversion only works for diffing ELF executables, but it would likely be more convenient to diff DOLs. + # At this time it is recommended to always use -o when running the diff script as this mode does not make use of the ram-rom conversion + return objfile, rom + else: + fail(f"Linker map format {map_format} unrecognised.") return None, None -def dump_elf(): +def dump_elf() -> Tuple[str, ObjdumpCommand, ObjdumpCommand]: if not baseimg or not myimg: fail("Missing myimg/baseimg in config.") if base_shift: @@ -442,7 +523,7 @@ def dump_elf(): ) -def dump_objfile(): +def dump_objfile() -> Tuple[str, ObjdumpCommand, ObjdumpCommand]: if base_shift: fail("--base-shift not compatible with -o") if args.end is not None: @@ -472,12 +553,12 @@ def dump_objfile(): ) -def dump_binary(): +def dump_binary() -> Tuple[str, ObjdumpCommand, ObjdumpCommand]: if not baseimg or not myimg: fail("Missing myimg/baseimg in config.") if args.make: run_make(myimg) - start_addr = eval_int(args.start) + start_addr = maybe_eval_int(args.start) if start_addr is None: _, start_addr = search_map_file(args.start) if start_addr is None: @@ -486,7 +567,7 @@ def dump_binary(): end_addr = eval_int(args.end, "End address must be an integer expression.") else: end_addr = start_addr + MAX_FUNCTION_SIZE_BYTES - objdump_flags = ["-Dz", "-bbinary", "-mmips", "-EB"] + objdump_flags = ["-Dz", "-bbinary", "-EB"] flags1 = [ f"--start-address={start_addr + base_shift}", f"--stop-address={end_addr + base_shift}", @@ -499,9 +580,9 @@ def dump_binary(): ) -# Alignment with ANSI colors is broken, let's fix it. -def ansi_ljust(s, width): - needed = width - ansiwrap.ansilen(s) +def ansi_ljust(s: str, width: int) -> str: + """Like s.ljust(width), but accounting for ANSI colors.""" + needed: int = width - ansiwrap.ansilen(s) if needed > 0: return s + " " * needed else: @@ -511,7 +592,9 @@ def ansi_ljust(s, width): if arch == "mips": re_int = re.compile(r"[0-9]+") re_comment = re.compile(r"<.*?>") - re_reg = re.compile(r"\$?\b(a[0-3]|t[0-9]|s[0-8]|at|v[01]|f[12]?[0-9]|f3[01]|k[01]|fp|ra)\b") + re_reg = re.compile( + r"\$?\b(a[0-3]|t[0-9]|s[0-8]|at|v[01]|f[12]?[0-9]|f3[01]|k[01]|fp|ra|zero)\b" + ) re_sprel = re.compile(r"(?<=,)([0-9]+|0x[0-9a-f]+)\(sp\)") re_large_imm = re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}") re_imm = re.compile(r"(\b|-)([0-9]+|0x[0-9a-fA-F]+)\b(?!\(sp)|%(lo|hi)\([^)]*\)") @@ -530,7 +613,19 @@ if arch == "mips": "bc1fl", } branch_instructions = branch_likely_instructions.union( - {"b", "beq", "bne", "beqz", "bnez", "bgez", "bgtz", "blez", "bltz", "bc1t", "bc1f"} + { + "b", + "beq", + "bne", + "beqz", + "bnez", + "bgez", + "bgtz", + "blez", + "bltz", + "bc1t", + "bc1f", + } ) instructions_with_address_immediates = branch_instructions.union({"jal", "j"}) elif arch == "aarch64": @@ -545,13 +640,71 @@ elif arch == "aarch64": arch_flags = [] forbidden = set(string.ascii_letters + "_") branch_likely_instructions = set() - branch_instructions = {"bl", "b", "b.eq", "b.ne", "b.cs", "b.hs", "b.cc", "b.lo", "b.mi", "b.pl", "b.vs", "b.vc", "b.hi", "b.ls", "b.ge", "b.lt", "b.gt", "b.le", "cbz", "cbnz", "tbz", "tbnz"} + branch_instructions = { + "bl", + "b", + "b.eq", + "b.ne", + "b.cs", + "b.hs", + "b.cc", + "b.lo", + "b.mi", + "b.pl", + "b.vs", + "b.vc", + "b.hi", + "b.ls", + "b.ge", + "b.lt", + "b.gt", + "b.le", + "cbz", + "cbnz", + "tbz", + "tbnz", + } instructions_with_address_immediates = branch_instructions.union({"adrp"}) +elif arch == "ppc": + re_int = re.compile(r"[0-9]+") + re_comment = re.compile(r"(<.*?>|//.*$)") + re_reg = re.compile(r"\$?\b([rf][0-9]+)\b") + re_sprel = re.compile(r"(?<=,)(-?[0-9]+|-?0x[0-9a-f]+)\(r1\)") + re_large_imm = re.compile(r"-?[1-9][0-9]{2,}|-?0x[0-9a-f]{3,}") + re_imm = re.compile(r"(\b|-)([0-9]+|0x[0-9a-fA-F]+)\b(?!\(r1)|[^@]*@(ha|h|lo)") + arch_flags = [] + forbidden = set(string.ascii_letters + "_") + branch_likely_instructions = set() + branch_instructions = { + "b", + "beq", + "beq+", + "beq-", + "bne", + "bne+", + "bne-", + "blt", + "blt+", + "blt-", + "ble", + "ble+", + "ble-", + "bdnz", + "bdnz+", + "bdnz-", + "bge", + "bge+", + "bge-", + "bgt", + "bgt+", + "bgt-", + } + instructions_with_address_immediates = branch_instructions.union({"bl"}) else: - fail("Unknown architecture.") + fail(f"Unknown architecture: {arch}") -def hexify_int(row, pat): +def hexify_int(row: str, pat: Match[str]) -> str: full = pat.group(0) if len(full) <= 1: # leave one-digit ints alone @@ -564,11 +717,14 @@ def hexify_int(row, pat): return hex(int(full)) -def parse_relocated_line(line): +def parse_relocated_line(line: str) -> Tuple[str, str, str]: try: ind2 = line.rindex(",") except ValueError: - ind2 = line.rindex("\t") + try: + ind2 = line.rindex("\t") + except ValueError: + ind2 = line.rindex(" ") before = line[: ind2 + 1] after = line[ind2 + 1 :] ind2 = after.find("(") @@ -581,7 +737,7 @@ def parse_relocated_line(line): return before, imm, after -def process_mips_reloc(row, prev): +def process_mips_reloc(row: str, prev: str) -> str: before, imm, after = parse_relocated_line(prev) repl = row.split()[-1] if imm != "0": @@ -602,12 +758,48 @@ def process_mips_reloc(row, prev): # correct addend for each, but objdump doesn't give us the order of # the relocations, so we can't find the right LO16. :( repl = f"%hi({repl})" + elif "R_MIPS_26" in row: + # Function calls + pass + elif "R_MIPS_PC16" in row: + # Branch to glabel. This gives confusing output, but there's not much + # we can do here. + pass else: - assert "R_MIPS_26" in row, f"unknown relocation type '{row}'" + assert False, f"unknown relocation type '{row}' for line '{prev}'" return before + repl + after -def pad_mnemonic(line): +def process_ppc_reloc(row: str, prev: str) -> str: + assert any(r in row for r in ["R_PPC_REL24", "R_PPC_ADDR16", "R_PPC_EMB_SDA21"]), f"unknown relocation type '{row}' for line '{prev}'" + before, imm, after = parse_relocated_line(prev) + repl = row.split()[-1] + if "R_PPC_REL24" in row: + # function calls + pass + elif "R_PPC_ADDR16_HI" in row: + # absolute hi of addr + repl = f"{repl}@h" + elif "R_PPC_ADDR16_HA" in row: + # adjusted hi of addr + repl = f"{repl}@ha" + elif "R_PPC_ADDR16_LO" in row: + # lo of addr + repl = f"{repl}@l" + elif "R_PPC_ADDR16" in row: + # 16-bit absolute addr + if "+0x7" in repl: + # remove the very large addends as they are an artifact of (label-_SDA(2)_BASE_) + # computations and are unimportant in a diff setting. + if int(repl.split("+")[1],16) > 0x70000000: + repl = repl.split("+")[0] + elif "R_PPC_EMB_SDA21" in row: + # small data area + pass + return before + repl + after + + +def pad_mnemonic(line: str) -> str: if "\t" not in line: return line mn, args = line.split("\t", 1) @@ -692,7 +884,7 @@ def make_difference_normalizer() -> DifferenceNormalizer: return DifferenceNormalizer() -def process(lines): +def process(lines: List[str]) -> List[Line]: normalizer = make_difference_normalizer() skip_next = False source_lines = [] @@ -701,7 +893,7 @@ def process(lines): if lines and not lines[-1]: lines.pop() - output = [] + output: List[Line] = [] stop_after_delay_slot = False for row in lines: if args.diff_obj and (">:" in row or not row): @@ -723,6 +915,11 @@ def process(lines): output[-1] = output[-1]._replace(original=new_original) continue + if "R_PPC_" in row: + new_original = process_ppc_reloc(row, output[-1].original) + output[-1] = output[-1]._replace(original=new_original) + continue + m_comment = re.search(re_comment, row) comment = m_comment[0] if m_comment else None row = re.sub(re_comment, "", row) @@ -730,10 +927,16 @@ def process(lines): tabs = row.split("\t") row = "\t".join(tabs[2:]) line_num = tabs[0].strip() - row_parts = row.split("\t", 1) + + if "\t" in row: + row_parts = row.split("\t", 1) + else: + # powerpc-eabi-objdump doesn't use tabs + row_parts = [part.lstrip() for part in row.split(" ", 1)] mnemonic = row_parts[0].strip() + if mnemonic not in instructions_with_address_immediates: - row = re.sub(re_int, lambda s: hexify_int(row, s), row) + row = re.sub(re_int, lambda m: hexify_int(row, m), row) original = row normalized_original = normalizer.normalize(mnemonic, original) if skip_next: @@ -781,16 +984,18 @@ def process(lines): return output -def format_single_line_diff(line1, line2, column_width): - return f"{ansi_ljust(line1,column_width)}{line2}" +def format_single_line_diff(line1: str, line2: str, column_width: int) -> str: + return ansi_ljust(line1, column_width) + line2 class SymbolColorer: - def __init__(self, base_index): + symbol_colors: Dict[str, str] + + def __init__(self, base_index: int) -> None: self.color_index = base_index self.symbol_colors = {} - def color_symbol(self, s, t=None): + def color_symbol(self, s: str, t: Optional[str] = None) -> str: try: color = self.symbol_colors[s] except: @@ -801,53 +1006,54 @@ class SymbolColorer: return f"{color}{t}{Fore.RESET}" -def normalize_imms(row): +def normalize_imms(row: str) -> str: return re.sub(re_imm, "", row) -def normalize_stack(row): +def normalize_stack(row: str) -> str: return re.sub(re_sprel, "addr(sp)", row) -def split_off_branch(line): +def split_off_branch(line: str) -> Tuple[str, str]: parts = line.split(",") if len(parts) < 2: parts = line.split(None, 1) off = len(line) - len(parts[-1]) return line[:off], line[off:] +ColorFunction = Callable[[str], str] -def color_imms(out1, out2): - g1 = [] - g2 = [] - re.sub(re_imm, lambda s: g1.append(s.group()), out1) - re.sub(re_imm, lambda s: g2.append(s.group()), out2) - if len(g1) == len(g2): - diffs = [x != y for (x, y) in zip(g1, g2)] - it = iter(diffs) +def color_fields(pat: Pattern[str], out1: str, out2: str, color1: ColorFunction, color2: Optional[ColorFunction]=None) -> Tuple[str, str]: + diffs = [of.group() != nf.group() for (of, nf) in zip(pat.finditer(out1), pat.finditer(out2))] - def maybe_color(s): - return f"{Fore.LIGHTBLUE_EX}{s}{Style.RESET_ALL}" if next(it) else s + it = iter(diffs) + def maybe_color(color: ColorFunction, s: str) -> str: + return color(s) if next(it, False) else f"{Style.RESET_ALL}{s}" + + out1 = pat.sub(lambda m: maybe_color(color1, m.group()), out1) + it = iter(diffs) + out2 = pat.sub(lambda m: maybe_color(color2 or color1, m.group()), out2) - out1 = re.sub(re_imm, lambda s: maybe_color(s.group()), out1) - it = iter(diffs) - out2 = re.sub(re_imm, lambda s: maybe_color(s.group()), out2) return out1, out2 -def color_branch_imms(br1, br2): +def color_branch_imms(br1: str, br2: str) -> Tuple[str, str]: if br1 != br2: br1 = f"{Fore.LIGHTBLUE_EX}{br1}{Style.RESET_ALL}" br2 = f"{Fore.LIGHTBLUE_EX}{br2}{Style.RESET_ALL}" return br1, br2 -def diff_sequences_difflib(seq1, seq2): +def diff_sequences_difflib( + seq1: List[str], seq2: List[str] +) -> List[Tuple[str, int, int, int, int]]: differ = difflib.SequenceMatcher(a=seq1, b=seq2, autojunk=False) return differ.get_opcodes() -def diff_sequences(seq1, seq2): +def diff_sequences( + seq1: List[str], seq2: List[str] +) -> List[Tuple[str, int, int, int, int]]: if ( args.algorithm != "levenshtein" or len(seq1) * len(seq2) > 4 * 10 ** 8 @@ -857,9 +1063,9 @@ def diff_sequences(seq1, seq2): # The Levenshtein library assumes that we compare strings, not lists. Convert. # (Per the check above we know we have fewer than 0x110000 unique elements, so chr() works.) - remapping = {} + remapping: Dict[str, str] = {} - def remap(seq): + def remap(seq: List[str]) -> str: seq = seq[:] for i in range(len(seq)): val = remapping.get(seq[i]) @@ -869,17 +1075,41 @@ def diff_sequences(seq1, seq2): seq[i] = val return "".join(seq) - seq1 = remap(seq1) - seq2 = remap(seq2) - return Levenshtein.opcodes(seq1, seq2) + rem1 = remap(seq1) + rem2 = remap(seq2) + return Levenshtein.opcodes(rem1, rem2) # type: ignore + + +def diff_lines( + lines1: List[Line], + lines2: List[Line], +) -> List[Tuple[Optional[Line], Optional[Line]]]: + ret = [] + for (tag, i1, i2, j1, j2) in diff_sequences( + [line.mnemonic for line in lines1], + [line.mnemonic for line in lines2], + ): + for line1, line2 in itertools.zip_longest(lines1[i1:i2], lines2[j1:j2]): + if tag == "replace": + if line1 is None: + tag = "insert" + elif line2 is None: + tag = "delete" + elif tag == "insert": + assert line1 is None + elif tag == "delete": + assert line2 is None + ret.append((line1, line2)) + + return ret class OutputLine: base: Optional[str] fmt2: str - key2: str + key2: Optional[str] - def __init__(self, base: Optional[str], fmt2: str, key2: str) -> None: + def __init__(self, base: Optional[str], fmt2: str, key2: Optional[str]) -> None: self.base = base self.fmt2 = fmt2 self.key2 = key2 @@ -919,139 +1149,128 @@ def do_diff(basedump: str, mydump: str) -> List[OutputLine]: btset.add(bt + ":") sc.color_symbol(bt + ":") - for (tag, i1, i2, j1, j2) in diff_sequences( - [line.mnemonic for line in lines1], [line.mnemonic for line in lines2] - ): - for line1, line2 in itertools.zip_longest(lines1[i1:i2], lines2[j1:j2]): - if tag == "replace": - if line1 is None: - tag = "insert" - elif line2 is None: - tag = "delete" - elif tag == "insert": - assert line1 is None - elif tag == "delete": - assert line2 is None + for (line1, line2) in diff_lines(lines1, lines2): + line_color1 = line_color2 = sym_color = Fore.RESET + line_prefix = " " + if line1 and line2 and line1.diff_row == line2.diff_row: + if line1.normalized_original == line2.normalized_original: + out1 = line1.original + out2 = line2.original + elif line1.diff_row == "": + out1 = f"{Style.BRIGHT}{Fore.LIGHTBLACK_EX}{line1.original}" + out2 = f"{Style.BRIGHT}{Fore.LIGHTBLACK_EX}{line2.original}" + else: + mnemonic = line1.original.split()[0] + out1, out2 = line1.original, line2.original + branch1 = branch2 = "" + if mnemonic in instructions_with_address_immediates: + out1, branch1 = split_off_branch(line1.original) + out2, branch2 = split_off_branch(line2.original) + branchless1 = out1 + branchless2 = out2 + out1, out2 = color_fields(re_imm, out1, out2, lambda s: f"{Fore.LIGHTBLUE_EX}{s}{Style.RESET_ALL}") - line_color1 = line_color2 = sym_color = Fore.RESET - line_prefix = " " - if line1 and line2 and line1.diff_row == line2.diff_row: - if line1.normalized_original == line2.normalized_original: - out1 = line1.original - out2 = line2.original - elif line1.diff_row == "": - out1 = f"{Style.BRIGHT}{Fore.LIGHTBLACK_EX}{line1.original}" - out2 = f"{Style.BRIGHT}{Fore.LIGHTBLACK_EX}{line2.original}" - else: - mnemonic = line1.original.split()[0] - out1, out2 = line1.original, line2.original - branch1 = branch2 = "" - if mnemonic in instructions_with_address_immediates: - out1, branch1 = split_off_branch(line1.original) - out2, branch2 = split_off_branch(line2.original) - branchless1 = out1 - branchless2 = out2 - out1, out2 = color_imms(out1, out2) + same_relative_target = False + if line1.branch_target is not None and line2.branch_target is not None: + relative_target1 = eval_line_num(line1.branch_target) - eval_line_num(line1.line_num) + relative_target2 = eval_line_num(line2.branch_target) - eval_line_num(line2.line_num) + same_relative_target = relative_target1 == relative_target2 - same_relative_target = False - if line1.branch_target is not None and line2.branch_target is not None: - relative_target1 = eval_line_num(line1.branch_target) - eval_line_num(line1.line_num) - relative_target2 = eval_line_num(line2.branch_target) - eval_line_num(line2.line_num) - same_relative_target = relative_target1 == relative_target2 + if not same_relative_target: + branch1, branch2 = color_branch_imms(branch1, branch2) + out1 += branch1 + out2 += branch2 + if normalize_imms(branchless1) == normalize_imms(branchless2): if not same_relative_target: - branch1, branch2 = color_branch_imms(branch1, branch2) - - out1 += branch1 - out2 += branch2 - if normalize_imms(branchless1) == normalize_imms(branchless2): - if not same_relative_target: - # only imms differences - sym_color = Fore.LIGHTBLUE_EX - line_prefix = "i" + # only imms differences + sym_color = Fore.LIGHTBLUE_EX + line_prefix = "i" + else: + out1, out2 = color_fields(re_sprel, out1, out2, sc3.color_symbol, sc4.color_symbol) + if normalize_stack(branchless1) == normalize_stack(branchless2): + # only stack differences (luckily stack and imm + # differences can't be combined in MIPS, so we + # don't have to think about that case) + sym_color = Fore.YELLOW + line_prefix = "s" else: - out1 = re.sub( - re_sprel, lambda s: sc3.color_symbol(s.group()), out1, - ) - out2 = re.sub( - re_sprel, lambda s: sc4.color_symbol(s.group()), out2, - ) - if normalize_stack(branchless1) == normalize_stack(branchless2): - # only stack differences (luckily stack and imm - # differences can't be combined in MIPS, so we - # don't have to think about that case) - sym_color = Fore.YELLOW - line_prefix = "s" - else: - # regs differences and maybe imms as well - out1 = re.sub( - re_reg, lambda s: sc1.color_symbol(s.group()), out1 + # regs differences and maybe imms as well + out1, out2 = color_fields(re_reg, out1, out2, sc1.color_symbol, sc2.color_symbol) + line_color1 = line_color2 = sym_color = Fore.YELLOW + line_prefix = "r" + elif line1 and line2: + line_prefix = "|" + line_color1 = Fore.LIGHTBLUE_EX + line_color2 = Fore.LIGHTBLUE_EX + sym_color = Fore.LIGHTBLUE_EX + out1 = line1.original + out2 = line2.original + elif line1: + line_prefix = "<" + line_color1 = sym_color = Fore.RED + out1 = line1.original + out2 = "" + elif line2: + line_prefix = ">" + line_color2 = sym_color = Fore.GREEN + out1 = "" + out2 = line2.original + + if args.source and line2 and line2.comment: + out2 += f" {line2.comment}" + + def format_part( + out: str, + line: Optional[Line], + line_color: str, + btset: Set[str], + sc: SymbolColorer, + ) -> Optional[str]: + if line is None: + return None + in_arrow = " " + out_arrow = "" + if args.show_branches: + if line.line_num in btset: + in_arrow = sc.color_symbol(line.line_num, "~>") + line_color + if line.branch_target is not None: + out_arrow = " " + sc.color_symbol(line.branch_target + ":", "~>") + out = pad_mnemonic(out) + return f"{line_color}{line.line_num} {in_arrow} {out}{Style.RESET_ALL}{out_arrow}" + + part1 = format_part(out1, line1, line_color1, bts1, sc5) + part2 = format_part(out2, line2, line_color2, bts2, sc6) + key2 = line2.original if line2 else None + + mid = f"{sym_color}{line_prefix}" + + if line2: + for source_line in line2.source_lines: + color = Style.DIM + # File names and function names + if source_line and source_line[0] != "│": + color += Style.BRIGHT + # Function names + if source_line.endswith("():"): + # Underline. Colorama does not provide this feature, unfortunately. + color += "\u001b[4m" + try: + source_line = cxxfilt.demangle( + source_line[:-3], external_only=False ) - out2 = re.sub( - re_reg, lambda s: sc2.color_symbol(s.group()), out2 - ) - line_color1 = line_color2 = sym_color = Fore.YELLOW - line_prefix = "r" - elif line1 and line2: - line_prefix = "|" - line_color1 = Fore.LIGHTBLUE_EX - line_color2 = Fore.LIGHTBLUE_EX - sym_color = Fore.LIGHTBLUE_EX - out1 = line1.original - out2 = line2.original - elif line1: - line_prefix = "<" - line_color1 = sym_color = Fore.RED - out1 = line1.original - out2 = "" - elif line2: - line_prefix = ">" - line_color2 = sym_color = Fore.GREEN - out1 = "" - out2 = line2.original + except: + pass + output.append( + OutputLine( + None, + f" {color}{source_line}{Style.RESET_ALL}", + source_line, + ) + ) - if args.source and line2 and line2.comment: - out2 += f" {line2.comment}" - - def format_part(out: str, line: Optional[Line], line_color: str, btset: Set[str], sc: SymbolColorer) -> Optional[str]: - if line is None: - return None - in_arrow = " " - out_arrow = "" - if args.show_branches: - if line.line_num in btset: - in_arrow = sc.color_symbol(line.line_num, "~>") + line_color - if line.branch_target is not None: - out_arrow = " " + sc.color_symbol(line.branch_target + ":", "~>") - out = pad_mnemonic(out) - return f"{line_color}{line.line_num} {in_arrow} {out}{Style.RESET_ALL}{out_arrow}" - - part1 = format_part(out1, line1, line_color1, bts1, sc5) - part2 = format_part(out2, line2, line_color2, bts2, sc6) - key2 = line2.original if line2 else "" - - mid = f"{sym_color}{line_prefix}" - - if line2: - for source_line in line2.source_lines: - color = Style.DIM - # File names and function names - if source_line and source_line[0] != "|": - color += Style.BRIGHT - # Function names - if source_line.endswith("():"): - # Underline. Colorama does not provide this feature, unfortunately. - color += "\u001b[4m" - try: - source_line = cxxfilt.demangle( - source_line[:-3], external_only=False - ) - except: - pass - output.append(OutputLine(None, f" {color}{source_line}{Style.RESET_ALL}", source_line)) - - fmt2 = mid + " " + (part2 or "") - output.append(OutputLine(part1, fmt2, key2)) + fmt2 = mid + " " + (part2 or "") + output.append(OutputLine(part1, fmt2, key2)) return output @@ -1070,12 +1289,14 @@ def chunk_diff(diff: List[OutputLine]) -> List[Union[List[OutputLine], OutputLin return chunks -def format_diff(old_diff: List[OutputLine], new_diff: List[OutputLine]) -> Tuple[str, List[str]]: +def format_diff( + old_diff: List[OutputLine], new_diff: List[OutputLine] +) -> Tuple[str, List[str]]: old_chunks = chunk_diff(old_diff) new_chunks = chunk_diff(new_diff) output: List[Tuple[str, OutputLine, OutputLine]] = [] assert len(old_chunks) == len(new_chunks), "same target" - empty = OutputLine("", "", "") + empty = OutputLine("", "", None) for old_chunk, new_chunk in zip(old_chunks, new_chunks): if isinstance(old_chunk, list): assert isinstance(new_chunk, list) @@ -1088,18 +1309,19 @@ def format_diff(old_diff: List[OutputLine], new_diff: List[OutputLine]) -> Tuple if tag in ["equal", "replace"]: for i, j in zip(range(i1, i2), range(j1, j2)): output.append(("", old_chunk[i], new_chunk[j])) - elif tag == "insert": - for j in range(j1, j2): + if tag in ["insert", "replace"]: + for j in range(j1 + i2 - i1, j2): output.append(("", empty, new_chunk[j])) - else: - for i in range(i1, i2): + if tag in ["delete", "replace"]: + for i in range(i1 + j2 - j1, i2): output.append(("", old_chunk[i], empty)) else: assert isinstance(new_chunk, OutputLine) + assert new_chunk.base # old_chunk.base and new_chunk.base have the same text since # both diffs are based on the same target, but they might # differ in color. Use the new version. - output.append((new_chunk.base or "", old_chunk, new_chunk)) + output.append((new_chunk.base, old_chunk, new_chunk)) # TODO: status line, with e.g. approximate permuter score? width = args.column_width @@ -1116,29 +1338,35 @@ def format_diff(old_diff: List[OutputLine], new_diff: List[OutputLine]) -> Tuple diff_lines = [ ansi_ljust(base, width) + new.fmt2 for (base, old, new) in output - if base or new.key2 + if base or new.key2 is not None ] return header_line, diff_lines -def debounced_fs_watch(targets, outq, debounce_delay): +def debounced_fs_watch( + targets: List[str], + outq: "queue.Queue[Optional[float]]", + debounce_delay: float, +) -> None: import watchdog.events # type: ignore import watchdog.observers # type: ignore - class WatchEventHandler(watchdog.events.FileSystemEventHandler): - def __init__(self, queue, file_targets): + class WatchEventHandler(watchdog.events.FileSystemEventHandler): # type: ignore + def __init__( + self, queue: "queue.Queue[float]", file_targets: List[str] + ) -> None: self.queue = queue self.file_targets = file_targets - def on_modified(self, ev): + def on_modified(self, ev: object) -> None: if isinstance(ev, watchdog.events.FileModifiedEvent): self.changed(ev.src_path) - def on_moved(self, ev): + def on_moved(self, ev: object) -> None: if isinstance(ev, watchdog.events.FileMovedEvent): self.changed(ev.dest_path) - def should_notify(self, path): + def should_notify(self, path: str) -> bool: for target in self.file_targets: if path == target: return True @@ -1148,13 +1376,13 @@ def debounced_fs_watch(targets, outq, debounce_delay): return True return False - def changed(self, path): + def changed(self, path: str) -> None: if self.should_notify(path): self.queue.put(time.time()) - def debounce_thread(): - listenq = queue.Queue() - file_targets = [] + def debounce_thread() -> NoReturn: + listenq: "queue.Queue[float]" = queue.Queue() + file_targets: List[str] = [] event_handler = WatchEventHandler(listenq, file_targets) observer = watchdog.observers.Observer() observed = set() @@ -1190,19 +1418,29 @@ def debounced_fs_watch(targets, outq, debounce_delay): class Display: - def __init__(self, basedump, mydump): + basedump: str + mydump: str + emsg: Optional[str] + last_diff_output: Optional[List[OutputLine]] + pending_update: Optional[Tuple[str, bool]] + ready_queue: "queue.Queue[None]" + watch_queue: "queue.Queue[Optional[float]]" + less_proc: "Optional[subprocess.Popen[bytes]]" + + def __init__(self, basedump: str, mydump: str) -> None: self.basedump = basedump self.mydump = mydump self.emsg = None self.last_diff_output = None - def run_less(self): + def run_less(self) -> "Tuple[subprocess.Popen[bytes], subprocess.Popen[bytes]]": if self.emsg is not None: output = self.emsg else: diff_output = do_diff(self.basedump, self.mydump) last_diff_output = self.last_diff_output or diff_output - self.last_diff_output = diff_output + if args.threeway != "base" or not self.last_diff_output: + self.last_diff_output = diff_output header, diff_lines = format_diff(last_diff_output, diff_output) header_lines = [header] if header else [] output = "\n".join(header_lines + diff_lines[args.skip_lines :]) @@ -1215,17 +1453,19 @@ class Display: BUFFER_CMD, stdin=subprocess.PIPE, stdout=subprocess.PIPE ) less_proc = subprocess.Popen(LESS_CMD, stdin=buffer_proc.stdout) + assert buffer_proc.stdin + assert buffer_proc.stdout buffer_proc.stdin.write(output.encode()) buffer_proc.stdin.close() buffer_proc.stdout.close() return (buffer_proc, less_proc) - def run_sync(self): + def run_sync(self) -> None: proca, procb = self.run_less() procb.wait() proca.wait() - def run_async(self, watch_queue): + def run_async(self, watch_queue: "queue.Queue[Optional[float]]") -> None: self.watch_queue = watch_queue self.ready_queue = queue.Queue() self.pending_update = None @@ -1233,10 +1473,10 @@ class Display: dthread.start() self.ready_queue.get() - def display_thread(self): + def display_thread(self) -> None: proca, procb = self.run_less() self.less_proc = procb - self.ready_queue.put(0) + self.ready_queue.put(None) while True: ret = procb.wait() proca.wait() @@ -1255,19 +1495,19 @@ class Display: self.emsg = msg proca, procb = self.run_less() self.less_proc = procb - self.ready_queue.put(0) + self.ready_queue.put(None) else: # terminated by user, or killed self.watch_queue.put(None) - self.ready_queue.put(0) + self.ready_queue.put(None) break - def progress(self, msg): + def progress(self, msg: str) -> None: # Write message to top-left corner sys.stdout.write("\x1b7\x1b[1;1f{}\x1b8".format(msg + " ")) sys.stdout.flush() - def update(self, text, error): + def update(self, text: str, error: bool) -> None: if not error and not self.emsg and text == self.mydump: self.progress("Unchanged. ") return @@ -1277,14 +1517,14 @@ class Display: self.less_proc.kill() self.ready_queue.get() - def terminate(self): + def terminate(self) -> None: if not self.less_proc: return self.less_proc.kill() self.ready_queue.get() -def main(): +def main() -> None: if args.diff_elf_symbol: make_target, basecmd, mycmd = dump_elf() elif args.diff_obj: @@ -1314,23 +1554,27 @@ def main(): else: if not args.make: yn = input( - "Warning: watch-mode (-w) enabled without auto-make (-m). You will have to run make manually. Ok? (Y/n) " + "Warning: watch-mode (-w) enabled without auto-make (-m). " + "You will have to run make manually. Ok? (Y/n) " ) if yn.lower() == "n": return if args.make: watch_sources = None - if hasattr(diff_settings, "watch_sources_for_target"): - watch_sources = diff_settings.watch_sources_for_target(make_target) + watch_sources_for_target_fn = getattr( + diff_settings, "watch_sources_for_target", None + ) + if watch_sources_for_target_fn: + watch_sources = watch_sources_for_target_fn(make_target) watch_sources = watch_sources or source_directories if not watch_sources: fail("Missing source_directories config, don't know what to watch.") else: watch_sources = [make_target] - q = queue.Queue() + q: "queue.Queue[Optional[float]]" = queue.Queue() debounced_fs_watch(watch_sources, q, DEBOUNCE_DELAY) display.run_async(q) - last_build = 0 + last_build = 0.0 try: while True: t = q.get() @@ -1341,7 +1585,7 @@ def main(): last_build = time.time() if args.make: display.progress("Building...") - ret = run_make(make_target, capture_output=True) + ret = run_make_capture_output(make_target) if ret.returncode != 0: display.update( ret.stderr.decode("utf-8-sig", "replace") From 1d3df1761bf871f05166b4950994df7f455ecfb7 Mon Sep 17 00:00:00 2001 From: Ethan Roseman Date: Sat, 16 Jan 2021 02:00:00 +0900 Subject: [PATCH 2/2] Diff.py update + removing make stuff --- Makefile | 313 ----------------------------------------------- diff.py | 44 +++---- diff_settings.py | 2 +- sources.mk | 64 ---------- 4 files changed, 23 insertions(+), 400 deletions(-) delete mode 100644 Makefile delete mode 100644 sources.mk diff --git a/Makefile b/Makefile deleted file mode 100644 index f712579516..0000000000 --- a/Makefile +++ /dev/null @@ -1,313 +0,0 @@ -### Build Options ### - -# Override these options in settings.mk or with `make SETTING=value'. - -BASEROM = baserom.z64 -TARGET = papermario -COMPARE = 1 -NON_MATCHING = 0 -WATCH_INCLUDES = 1 -WSL_ELEVATE_GUI = 1 - -# Fail early if baserom does not exist -ifeq ($(wildcard $(BASEROM)),) -$(error Baserom `$(BASEROM)' not found.) -endif - -# NON_MATCHING=1 implies COMPARE=0 -ifeq ($(NON_MATCHING),1) -override COMPARE=0 -endif - -# PERMUTER=1 implies WATCH_INCLUDES=0 -ifeq ($(PERMUTER),1) -override WATCH_INCLUDES=0 -endif - - -### Output ### - -BUILD_DIR := build -ROM := $(TARGET).z64 -ELF := $(BUILD_DIR)/$(TARGET).elf -LD_SCRIPT := $(TARGET).ld -LD_MAP := $(BUILD_DIR)/$(TARGET).map -ASSETS_BIN := $(BUILD_DIR)/bin/assets/assets.bin -MSG_BIN := $(BUILD_DIR)/msg.bin -NPC_BIN := $(BUILD_DIR)/sprite/npc.bin - - -### Tools ### - -PYTHON := python3 -N64CKSUM := tools/n64crc -SPLAT_YAML := tools/splat.yaml -SPLAT = $(PYTHON) tools/n64splat/split.py $(BASEROM) $(SPLAT_YAML) . -YAY0COMPRESS = tools/Yay0compress -EMULATOR = mupen64plus - - -CROSS := mips-linux-gnu- -AS := $(CROSS)as -OLD_AS := tools/mips-nintendo-nu64-as -CC := tools/cc1 -CPP := cpp -LD := $(CROSS)ld -OBJCOPY := $(CROSS)objcopy - -WSL := 0 -JAVA := java - -UNAME_S := $(shell uname -s) -ifeq ($(UNAME_S),Linux) - OS=linux - ICONV := iconv --from UTF-8 --to SHIFT-JIS - - ifeq ($(findstring microsoft,$(shell cat /proc/sys/kernel/osrelease)),microsoft) - WSL := 1 - ifeq ($(WSL_ELEVATE_GUI),1) - JAVA := powershell.exe -command java - endif -endif -endif -ifeq ($(UNAME_S),Darwin) - OS=mac - ICONV := tools/iconv.py UTF-8 SHIFT-JIS -endif - -OLD_AS=tools/$(OS)/mips-nintendo-nu64-as -CC=tools/$(OS)/cc1 - -### Compiler Options ### - -CPPFLAGS := -Iinclude -Isrc -D _LANGUAGE_C -D _FINALROM -ffreestanding -DF3DEX_GBI_2 -D_MIPS_SZLONG=32 -Wundef -Wcomment -ASFLAGS := -EB -Iinclude -march=vr4300 -mtune=vr4300 -OLDASFLAGS := -EB -Iinclude -G 0 -CFLAGS := -O2 -quiet -G 0 -mcpu=vr4300 -mfix4300 -mips3 -mgp32 -mfp32 -Wimplicit -Wuninitialized -Wshadow -LDFLAGS := -T undefined_syms.txt -T undefined_syms_auto.txt -T undefined_funcs.txt -T undefined_funcs_auto.txt -T $(BUILD_DIR)/$(LD_SCRIPT) -Map $(LD_MAP) --no-check-sections - -ifeq ($(WATCH_INCLUDES),1) -CPPMFLAGS = -MP -MD -MF $@.mk -MT $(BUILD_DIR)/$*.d -MDEPS = $(BUILD_DIR)/%.d -endif - -ifeq ($(NON_MATCHING),1) -CPPFLAGS += -DNON_MATCHING -endif - --include settings.mk - -### Sources ### - -include sources.mk - -ifeq ($(PERMUTER),1) -override OBJECTS:=$(filter %.c.o, $(OBJECTS)) -endif - -%.d: ; - -ifeq ($(WATCH_INCLUDES),1) --include $(foreach obj, $(OBJECTS), $(obj).mk) -endif - -NPC_DIRS := $(foreach npc, $(NPC_SPRITES), sprite/npc/$(npc)) - -GENERATED_HEADERS := include/ld_addrs.h $(foreach dir, $(NPC_DIRS), include/$(dir).h) - - -### Targets ### - -clean: - rm -rf $(BUILD_DIR) $(LD_SCRIPT) - -clean-all: - rm -rf $(BUILD_DIR) bin msg img sprite .splat_cache $(LD_SCRIPT) - -clean-code: - rm -rf $(BUILD_DIR)/src - -tools: - -setup: clean-all tools - @make split - -split: - make $(LD_SCRIPT) -W $(SPLAT_YAML) - -split-%: - $(SPLAT) --modes ld $* --verbose - -split-all: - $(SPLAT) --modes all - -test: $(ROM) - $(EMULATOR) $< - -%.bin: $(LD_SCRIPT) - -# Compressed files -%.Yay0: % - @mkdir -p $(shell dirname $@) - $(YAY0COMPRESS) $< $@ -# $(BUILD_DIR)/%.bin.Yay0: %.bin -# @mkdir -p $(shell dirname $@) -# $(YAY0COMPRESS) $< $@ - -# Data objects -# $(BUILD_DIR)/%.bin.o: %.bin -# @mkdir -p $(shell dirname $@) -# $(LD) -r -b binary -o $@ $< - -# Compressed data objects -# $(BUILD_DIR)/%.Yay0.o: $(BUILD_DIR)/%.bin.Yay0 -# @mkdir -p $(shell dirname $@) -# $(LD) -r -b binary -o $@ $< - -# Compile C files -# $(BUILD_DIR)/%.c.o: %.c $(MDEPS) | $(GENERATED_HEADERS) -# @mkdir -p $(shell dirname $@) -# $(CPP) $(CPPFLAGS) -o - $(CPPMFLAGS) $< | $(ICONV) | $(CC) $(CFLAGS) -o - | $(OLD_AS) $(OLDASFLAGS) -o $@ - - -# # Compile C files (with DSL macros) -# $(foreach cfile, $(DSL_C_FILES), $(BUILD_DIR)/$(cfile).o): $(BUILD_DIR)/%.c.o: %.c $(MDEPS) tools/compile_dsl_macros.py | $(GENERATED_HEADERS) -# @mkdir -p $(shell dirname $@) -# $(CPP) $(CPPFLAGS) -o - $< $(CPPMFLAGS) | $(PYTHON) tools/compile_dsl_macros.py | $(ICONV) | $(CC) $(CFLAGS) -o - | $(OLD_AS) $(OLDASFLAGS) -o $@ - - -# Assemble handwritten ASM -# $(BUILD_DIR)/%.s.o: %.s -# @mkdir -p $(shell dirname $@) -# $(AS) $(ASFLAGS) -o $@ $< - -# # Data -# $(BUILD_DIR)/data/%.data.o: asm/data/%.data.s -# @mkdir -p $(shell dirname $@) -# $(AS) $(ASFLAGS) -o $@ $< - -# # Rodata -# $(BUILD_DIR)/rodata/%.rodata.o: asm/data/%.rodata.s -# @mkdir -p $(shell dirname $@) -# $(AS) $(ASFLAGS) -o $@ $< - -# Images -# $(BUILD_DIR)/%.png.o: $(BUILD_DIR)/%.png -# $(LD) -r -b binary -o $@ $< -# $(BUILD_DIR)/%.rgba16.png: %.png -# @mkdir -p $(shell dirname $@) -# $(PYTHON) tools/convert_image.py rgba16 $< $@ $(IMG_FLAGS) -# $(BUILD_DIR)/%.rgba32.png: %.png -# @mkdir -p $(shell dirname $@) -# $(PYTHON) tools/convert_image.py rgba32 $< $@ $(IMG_FLAGS) -# $(BUILD_DIR)/%.ci8.png: %.png -# @mkdir -p $(shell dirname $@) -# $(PYTHON) tools/convert_image.py ci8 $< $@ $(IMG_FLAGS) -# $(BUILD_DIR)/%.ci4.png: %.png -# @mkdir -p $(shell dirname $@) -# $(PYTHON) tools/convert_image.py ci4 $< $@ $(IMG_FLAGS) -# $(BUILD_DIR)/%.palette.png: %.png -# @mkdir -p $(shell dirname $@) -# $(PYTHON) tools/convert_image.py palette $< $@ $(IMG_FLAGS) -# $(BUILD_DIR)/%.ia4.png: %.png -# @mkdir -p $(shell dirname $@) -# $(PYTHON) tools/convert_image.py ia4 $< $@ $(IMG_FLAGS) -# $(BUILD_DIR)/%.ia8.png: %.png -# @mkdir -p $(shell dirname $@) -# $(PYTHON) tools/convert_image.py ia8 $< $@ $(IMG_FLAGS) -# $(BUILD_DIR)/%.ia16.png: %.png -# @mkdir -p $(shell dirname $@) -# $(PYTHON) tools/convert_image.py ia16 $< $@ $(IMG_FLAGS) -# $(BUILD_DIR)/%.i4.png: %.png -# @mkdir -p $(shell dirname $@) -# $(PYTHON) tools/convert_image.py i4 $< $@ $(IMG_FLAGS) -# $(BUILD_DIR)/%.i8.png: %.png -# @mkdir -p $(shell dirname $@) -# $(PYTHON) tools/convert_image.py i8 $< $@ $(IMG_FLAGS) - -# Assets -# ASSET_FILES := $(foreach asset, $(ASSETS), $(BUILD_DIR)/bin/assets/$(asset)) -# YAY0_ASSET_FILES := $(foreach asset, $(filter-out %_tex, $(ASSET_FILES)), $(asset).Yay0) -# $(BUILD_DIR)/bin/assets/%: bin/assets/%.bin -# @mkdir -p $(shell dirname $@) -# @cp $< $@ -# $(ASSETS_BIN): $(ASSET_FILES) $(YAY0_ASSET_FILES) sources.mk -# @mkdir -p $(shell dirname $@) -# @echo "building $@" -# @$(PYTHON) tools/build_assets_bin.py $@ $(ASSET_FILES) -# $(ASSETS_BIN:.bin=.o): $(ASSETS_BIN) -# $(LD) -r -b binary -o $@ $< - -# Messages -# $(MSG_BIN): $(MESSAGES) -# @mkdir -p $(shell dirname $@) -# @echo "building $@" -# @$(PYTHON) tools/compile_messages.py $@ /dev/null $(MESSAGES) -# $(MSG_BIN:.bin=.o): $(MSG_BIN) -# $(LD) -r -b binary -o $@ $< - -# Sprites -# $(foreach npc, $(NPC_SPRITES), $(eval $(BUILD_DIR)/sprite/npc/$(npc):: $(shell find sprite/npc/$(npc) -type f 2> /dev/null))) # dependencies -# NPC_YAY0 := $(foreach npc, $(NPC_SPRITES), $(BUILD_DIR)/sprite/npc/$(npc).Yay0) -# $(BUILD_DIR)/sprite/npc/%:: sprite/npc/% tools/compile_npc_sprite.py -# @mkdir -p $(shell dirname $@) -# $(PYTHON) tools/compile_npc_sprite.py $@ $< -# $(NPC_BIN): $(NPC_YAY0) tools/compile_npc_sprites.py -# @mkdir -p $(shell dirname $@) -# @echo "building $@" -# @$(PYTHON) tools/compile_npc_sprites.py $@ $(NPC_YAY0) -# $(NPC_BIN:.bin=.o): $(NPC_BIN) -# $(LD) -r -b binary -o $@ $< -# include/sprite/npc/%.h: sprite/npc/%/SpriteSheet.xml tools/gen_sprite_animations_h.py -# @mkdir -p $(shell dirname $@) -# @echo "building $@" -# @$(PYTHON) tools/gen_sprite_animations_h.py $@ sprite/npc/$* $(NPC_DIRS) - - -### Linker ### - -# $(LD_SCRIPT): $(SPLAT_YAML) -# $(SPLAT) --modes ld bin Yay0 PaperMarioMapFS PaperMarioMessages img PaperMarioNpcSprites --new - -# $(BUILD_DIR)/$(LD_SCRIPT): $(LD_SCRIPT) -# @mkdir -p $(shell dirname $@) -# $(CPP) -P -DBUILD_DIR=$(BUILD_DIR) -o $@ $< - -# $(ROM): $(BUILD_DIR)/$(TARGET).bin -# @cp $< $@ -# ifeq ($(COMPARE),1) -# @sha1sum -c checksum.sha1 || (echo 'The build succeeded, but did not match the base ROM. This is expected if you are making changes to the game. To skip this check, use "make COMPARE=0".' && false) -# endif - -# $(BUILD_DIR)/$(TARGET).elf: $(BUILD_DIR)/$(LD_SCRIPT) $(OBJECTS) -# $(LD) $(LDFLAGS) -o $@ - -# $(BUILD_DIR)/$(TARGET).bin: $(BUILD_DIR)/$(TARGET).elf -# $(OBJCOPY) $< $@ -O binary - -# include/ld_addrs.h: $(BUILD_DIR)/$(LD_SCRIPT) -# grep -E "[^\. ]+ =" $< -o | sed 's/^/extern void* /; s/ =/;/' > $@ - - -### Star Rod (optional) ### - -STAR_ROD := cd tools/star-rod && $(JAVA) -jar StarRod.jar - -sprite/SpriteTable.xml: tools/star-rod sources.mk - $(PYTHON) tools/star-rod/spritetable.xml.py $(NPC_SPRITES) > $@ - -editor: tools/star-rod sprite/SpriteTable.xml - $(STAR_ROD) - - -### Make Settings ### - -.PHONY: clean tools test setup split editor $(ROM) -.DELETE_ON_ERROR: -.SECONDARY: -.PRECIOUS: $(ROM) %.Yay0 -.DEFAULT_GOAL := $(ROM) - -# Remove built-in implicit rules to improve performance -MAKEFLAGS += --no-builtin-rules - -# Fail targets if any command in the pipe exits with error -SHELL = /bin/bash -e -o pipefail diff --git a/diff.py b/diff.py index 8f5daeada2..25ab215507 100755 --- a/diff.py +++ b/diff.py @@ -283,7 +283,7 @@ arch: str = config.get("arch", "mips") baseimg: Optional[str] = config.get("baseimg") myimg: Optional[str] = config.get("myimg") mapfile: Optional[str] = config.get("mapfile") -makeflags: List[str] = config.get("makeflags", []) +build_command: List[str] = config.get("make_command", ["make", *config.get("makeflags", [])]) source_directories: Optional[List[str]] = config.get("source_directories") objdump_executable: Optional[str] = config.get("objdump_executable") map_format: str = config.get("map_format", "gnu") @@ -372,12 +372,12 @@ def eval_line_num(expr: str) -> int: def run_make(target: str) -> None: - subprocess.check_call(["make"] + makeflags + [target]) + subprocess.check_call(build_command + [target]) def run_make_capture_output(target: str) -> "subprocess.CompletedProcess[bytes]": return subprocess.run( - ["make"] + makeflags + [target], + build_command + [target], stderr=subprocess.PIPE, stdout=subprocess.PIPE, ) @@ -676,28 +676,28 @@ elif arch == "ppc": forbidden = set(string.ascii_letters + "_") branch_likely_instructions = set() branch_instructions = { - "b", - "beq", - "beq+", - "beq-", - "bne", - "bne+", - "bne-", - "blt", - "blt+", - "blt-", + "b", + "beq", + "beq+", + "beq-", + "bne", + "bne+", + "bne-", + "blt", + "blt+", + "blt-", "ble", "ble+", - "ble-", - "bdnz", - "bdnz+", + "ble-", + "bdnz", + "bdnz+", "bdnz-", - "bge", - "bge+", - "bge-", - "bgt", - "bgt+", - "bgt-", + "bge", + "bge+", + "bge-", + "bgt", + "bgt+", + "bgt-", } instructions_with_address_immediates = branch_instructions.union({"bl"}) else: diff --git a/diff_settings.py b/diff_settings.py index 52c1cfe713..965287954e 100644 --- a/diff_settings.py +++ b/diff_settings.py @@ -5,4 +5,4 @@ def apply(config, args): config['myimg'] = 'papermario.z64' config['mapfile'] = 'build/papermario.map' config['source_directories'] = ['.'] - config['makeflags'] = ['COMPARE=0', 'WATCH_INCLUDES=0'] + config['make_command'] = ['ninja'] diff --git a/sources.mk b/sources.mk deleted file mode 100644 index eb50f99b69..0000000000 --- a/sources.mk +++ /dev/null @@ -1,64 +0,0 @@ -OBJECTS := $(foreach OBJECT, $(shell $(PYTHON) tools/n64splat/list_objects.py $(SPLAT_YAML)), $(BUILD_DIR)/$(OBJECT)) - -DSL_C_FILES := $(shell grep -lrF "SCRIPT" src) - -MAPS := \ - dro_01 dro_02 \ - hos_00 hos_01 hos_02 hos_03 hos_04 hos_05 hos_06 hos_10 hos_20 \ - isk_01 isk_02 isk_03 isk_04 isk_05 isk_06 isk_07 isk_08 isk_09 isk_10 isk_11 isk_12 isk_13 isk_14 isk_16 isk_18 isk_19 \ - iwa_00 iwa_01 iwa_02 iwa_03 iwa_04 iwa_10 iwa_11 \ - osr_00 osr_01 osr_02 osr_03 kkj_00 kkj_01 kkj_02 kkj_03 kkj_10 kkj_11 kkj_12 kkj_13 kkj_14 kkj_15 kkj_16 kkj_17 kkj_18 kkj_19 kkj_20 kkj_21 kkj_22 kkj_23 kkj_24 kkj_25 kkj_26 kkj_27 kkj_28 kkj_29 \ - kmr_00 kmr_02 kmr_03 kmr_04 kmr_05 kmr_06 kmr_07 kmr_09 kmr_10 kmr_11 kmr_12 kmr_20 kmr_30 \ - kpa_01 kpa_03 kpa_04 kpa_08 kpa_09 kpa_10 kpa_11 kpa_12 kpa_13 kpa_14 kpa_15 kpa_16 kpa_17 kpa_32 kpa_33 kpa_40 kpa_41 kpa_50 kpa_52 kpa_60 kpa_61 kpa_62 kpa_63 kpa_70 kpa_80 kpa_90 kpa_91 kpa_94 kpa_95 kpa_96 kpa_102 kpa_111 kpa_112 kpa_113 kpa_115 kpa_116 kpa_117 kpa_118 kpa_119 kpa_121 kpa_130 kpa_133 kpa_134 \ - machi mac_00 mac_01 mac_02 mac_03 mac_04 mac_05 mac_06 \ - tik_01 tik_02 tik_03 tik_04 tik_05 tik_06 tik_07 tik_08 tik_09 tik_10 tik_12 tik_14 tik_15 tik_17 tik_18 tik_19 tik_20 tik_21 tik_22 tik_23 tik_25 \ - kgr_01 kgr_02 \ - nok_01 nok_02 nok_03 nok_04 nok_11 nok_12 nok_13 nok_14 nok_15 \ - sbk_00 sbk_01 sbk_02 sbk_03 sbk_04 sbk_05 sbk_06 sbk_10 sbk_11 sbk_12 sbk_13 sbk_14 sbk_15 sbk_16 sbk_20 sbk_21 sbk_22 sbk_23 sbk_24 sbk_25 sbk_26 sbk_30 sbk_31 sbk_32 sbk_33 sbk_34 sbk_35 sbk_36 sbk_40 sbk_41 sbk_42 sbk_43 sbk_44 sbk_45 sbk_46 sbk_50 sbk_51 sbk_52 sbk_53 sbk_54 sbk_55 sbk_56 sbk_60 sbk_61 sbk_62 sbk_63 sbk_64 sbk_65 sbk_66 sbk_99 \ - trd_00 trd_01 trd_02 trd_03 trd_04 trd_05 trd_06 trd_07 trd_08 trd_09 trd_10 \ - tst_01 tst_02 tst_03 tst_04 tst_10 tst_11 tst_12 tst_13 tst_20 \ - jan_00 jan_01 jan_02 jan_03 jan_04 jan_05 jan_06 jan_07 jan_08 jan_09 jan_10 jan_11 jan_12 jan_13 jan_14 jan_15 jan_16 jan_17 jan_18 jan_19 jan_22 jan_23 \ - mim_01 mim_02 mim_03 mim_04 mim_05 mim_06 mim_07 mim_08 mim_09 mim_10 mim_11 mim_12 \ - obk_01 obk_02 obk_03 obk_04 obk_05 obk_06 obk_07 obk_08 obk_09 \ - arn_02 arn_03 arn_04 arn_05 arn_07 arn_08 arn_09 arn_10 arn_11 arn_12 arn_13 arn_20 \ - dgb_01 dgb_02 dgb_03 dgb_04 dgb_05 dgb_06 dgb_07 dgb_08 dgb_09 dgb_10 dgb_11 dgb_12 dgb_13 dgb_14 dgb_15 dgb_16 dgb_17 dgb_18 \ - kzn_01 kzn_02 kzn_03 kzn_04 kzn_05 kzn_06 kzn_07 kzn_08 kzn_09 kzn_10 kzn_11 kzn_17 kzn_18 kzn_19 kzn_20 kzn_22 kzn_23 \ - flo_00 flo_03 flo_07 flo_08 flo_09 flo_10 flo_11 flo_12 flo_13 flo_14 flo_15 flo_16 flo_17 flo_18 flo_19 flo_21 flo_22 flo_23 flo_24 flo_25 \ - sam_01 sam_02 sam_03 sam_04 sam_05 sam_06 sam_07 sam_08 sam_09 sam_10 sam_11 sam_12 \ - pra_01 pra_02 pra_03 pra_04 pra_05 pra_09 pra_10 pra_11 pra_13 pra_14 pra_15 pra_16 pra_18 pra_19 pra_20 pra_21 pra_22 pra_29 pra_31 pra_32 pra_33 pra_34 pra_35 pra_40 \ - omo_01 omo_02 omo_03 omo_04 omo_05 omo_06 omo_07 omo_08 omo_09 omo_10 omo_11 omo_12 omo_13 omo_14 omo_15 omo_16 omo_17 \ - end_00 end_01 \ - mgm_00 mgm_01 mgm_02 mgm_03 \ - gv_01 \ - kmr_bt03 kmr_bt04 kmr_bt05 kmr_bt06 \ - nok_bt01 nok_bt02 nok_bt03 nok_bt04 \ - trd_bt00 trd_bt01 trd_bt02 trd_bt03 trd_bt04 trd_bt05 \ - iwa_bt01 iwa_bt02 \ - sbk_bt02 \ - isk_bt01 isk_bt02 isk_bt03 isk_bt04 isk_bt05 isk_bt06 isk_bt07 isk_bt08 \ - arn_bt01 arn_bt02 arn_bt03 arn_bt04 arn_bt05 arn_bt06 \ - dgb_bt01 dgb_bt02 dgb_bt03 dgb_bt04 dgb_bt05 \ - mim_bt01 \ - omo_bt01 omo_bt02 omo_bt03 omo_bt04 omo_bt05 omo_bt06 omo_bt07 \ - kgr_bt01 flo_bt01 flo_bt02 flo_bt03 flo_bt04 flo_bt05 flo_bt06 \ - jan_bt00 jan_bt01 jan_bt02 jan_bt03 jan_bt04 \ - kzn_bt01 kzn_bt02 kzn_bt04 kzn_bt05 sam_bt01 sam_bt02 sam_bt03 sam_bt04 \ - tik_bt01 tik_bt02 tik_bt03 tik_bt04 tik_bt05 \ - pra_bt01 pra_bt02 pra_bt03 pra_bt04 mac_bt01 mac_bt02 \ - kpa_bt01 kpa_bt02 kpa_bt03 kpa_bt04 kpa_bt05 kpa_bt07 kpa_bt08 kpa_bt09 kpa_bt11 kpa_bt13 kpa_bt14 \ - hos_bt01 hos_bt02 \ - kkj_bt01 kkj_bt02 - -ASSETS := \ - $(foreach map, $(MAPS), $(map)_shape $(map)_hit) \ - mac_tex tik_tex kgr_tex kmr_tex iwa_tex sbk_tex dro_tex isk_tex trd_tex nok_tex hos_tex kpa_tex osr_tex kkj_tex tst_tex jan_tex mim_tex obk_tex arn_tex dgb_tex kzn_tex flo_tex sam_tex pra_tex omo_tex end_tex mgm_tex gv__tex \ - kmr_bg nok_bg sbk_bg sbk3_bg iwa_bg hos_bg arn_bg obk_bg omo_bg yos_bg jan_bg fla_bg flb_bg sra_bg yki_bg sam_bg kpa_bg title_bg \ - title_data \ - party_kurio party_kameki party_pinki party_pareta party_resa party_akari party_opuku party_pokopi - -MESSAGES := $(shell find msg -type f -name "*.msg" 2> /dev/null) - -NPC_SPRITES := world_goombario world_kooper world_bombette world_parakarry world_bow world_watt world_sushie world_lakilester battle_goombario battle_kooper battle_bombette battle_parakarry battle_bow battle_watt battle_sushie battle_lakilester kooper_without_shell world_eldstar world_mamar world_skolar world_muskular world_misstar world_klevar world_kalmar battle_eldstar battle_mamar battle_skolar battle_muskular battle_misstar battle_klevar battle_kalmar twink jr_troopa spiked_jr_troopa spiked_para_jr_troopa mage_jr_troopa para_jr_troopa goomba spiked_goomba paragoomba koopa_troopa para_troopa fuzzy bob_omb bullet_bill bill_blaster monty_mole cleft pokey battle_bandit buzzy_beetle swooper stone_chomp putrid_piranha piranha_plant sentinel world_clubba battle_clubba shy_guy groove_guy sky_guy pyro_guy spy_guy medi_guy fuzzipede jungle_guy heart_plant hurt_plant m_bush bubble kent_c_koopa dayzee lakitu spiny bzzap ruff_puff spike_top duplighost albino_dino blooper baby_blooper gulpit dry_bones thrown_bone bony_beetle magikoopa flying_magikoopa world_koopatrol koopatrol hammer_bros bush_basic bush_blocky bush_dry bush_leafy bush_matted world_kammy battle_kammy goomba_bros goomba_king spiky_goomnut dark_toad koopa_bros buzzar tutankoopa chain_chomp world_tubba battle_tubba tubbas_heart big_lantern_ghost shy_squad_guy marshal_guy stilt_guy stilt_guy_unfold shy_stack_guy shy_stack_unfold shy_stack_damage shy_stack_rock general_guy general_guy_bomb tank_guy lava_piranha_head petit_piranha lava_bud huff_n_puff tuff_puff monstar crystal_king world_bowser battle_bowser luigi toad three_sisters vanna_t toad_kid toad_guard harry_t toad_minister postmaster conductor_toad train_station_toad fishmael artist_toad koopa koopa_without_shell world_bob_omb whacka dryite mouser boo yoshi yoshi_kid raven bubulb penguin shiver_toad world_bandit goompa goombaria gooma goompapa goomama the_master chan lee merlon chet_rippo rowf minh_t russ_t tayce_t fice_t bartender chanterelle rip_cheato chuck_quizmo merluvlee merlar merlow star_kid kolorado_wife koopa_koot kolorado battle_kolorado archeologist nomadimouse world_merlee battle_merlee disguised_moustafa moustafa oaklie bootler yakkey gourmet_guy village_leader leaders_friend rafael_raven tolielup gate_flower petunia posie lily rosie sun lakilulu ninji mayor_penguin mayor_penguin_wife penguin_patrol herringway merle star_rod fire coin parade_peach parade_koopas parade_burnt_bowser parade_luigi parade_partners parade_yoshis parade_kolorados parade_chicks parade_ice_show parade_toads parade_batons parade_drums parade_flags parade_horns parade_tubba_balloon parade_wizards parade_mario parade_shy_guys parade_twink leaf - -# Image settings -$(BUILD_DIR)/img/battle/text_action_command_ratings.ia4.png: IMG_FLAGS = --flip-y