Update and move tools (#596)

* update scripts and move outdated ones to /old/

* updates

* resolve issues

tools/asm_sizes.py

@@ -3,13 +3,21 @@
 import json
 import glob
 import os
+import argparse
+from enum import IntEnum
 
-print_funcs = True
+
+script_dir = os.path.dirname(os.path.realpath(__file__))
+asm_dir = script_dir + "/../ver/current/asm/nonmatchings"
+
+modes = [ "min", "max", "avg", "total", "size" ]
 
 sizes = {}
 
 funcs = {}
 
+
+# Calculate the number of instructions in a .s file
 def calc_insns(f_path):
     ret = 0
     with open(f_path) as f:
@@ -20,9 +28,11 @@ def calc_insns(f_path):
     funcs[f_path.split("/")[-1][:-2]] = ret
     return ret
 
+
+# Calculate different data points for each .c files and store them as a Tuple
 def do_dir(root, dir):
     max = 0
-    min = None
+    min = 0
     total = 0
 
     files = glob.glob(os.path.join(root, dir) + "/*.s")
@@ -31,27 +41,39 @@ def do_dir(root, dir):
         amt = calc_insns(f)
         if amt > max:
             max = amt
-        if min is None or amt < min:
+        if min == 0 or amt < min:
             min = amt
         total += amt
 
     avg = 0 if len(files) == 0 else total / len(files)
 
-    sizes[dir] = ((min, max, total, avg, len(files)))
+    sizes[root + "/" + dir] = ((min, max, total, avg, len(files)))
 
 
-script_dir = os.path.dirname(os.path.realpath(__file__))
-asm_dir = script_dir + "/../ver/current/asm/nonmatchings"
+parser = argparse.ArgumentParser(description="A tool to receive information about the number of non-matching .s files "
+                                            +"per .c file, or the size of .s files, measured by their number of instructions. "
+                                            +"Option -p is used by default if no option is specified.")
+group = parser.add_mutually_exclusive_group()
+group.add_argument("-f", "--files", help="Default. Print the number of non-matching .s files per .c file, ordered by size.", action='store_true', required=False)
+group.add_argument("-a", "--alphabetical", help="Print the size of .s files, ordered by name.", action='store_true', required=False)
+group.add_argument("-s", "--size", help="Print the size of .s files, ordered by size.", action='store_true', required=False)
+parser.add_argument("-l", "--limit", help="Only print the .c --files that are greater than or equal to the value.", type=int, default=0, required=False)
+parser.add_argument("-m", "--mode", help="Switches between output modes for --files. Allowed values are: {min, max, avg, total, size}.", choices=modes, default="size", metavar='', required=False)
 
-for root, dirs, files in os.walk(asm_dir):
-    for asm_dir in dirs:
-        if "/os" not in root and "/world/" not in root:
+args = parser.parse_args()
+
+
+if __name__ == "__main__":
+    for root, dirs, files in os.walk(asm_dir):
+        for asm_dir in dirs:
             do_dir(root, asm_dir)
-
-for thing in sorted(sizes.keys(), key=lambda x: sizes[x][4]):
-    val = sizes[thing][4]
-    if val > 0:
-        print(thing.ljust(25) + str(val))
-
-if print_funcs:
-    print(json.dumps(dict(sorted(funcs.items(), key=lambda f: f[1])), indent=4))
+    
+    if args.alphabetical:
+        print(json.dumps(dict(sorted(funcs.items(), key=lambda f: f[0])), indent=4))
+    elif args.size:
+        print(json.dumps(dict(sorted(funcs.items(), key=lambda f: f[1])), indent=4))
+    else:
+        for thing in sorted(sizes.keys(), key=lambda x: sizes[x][modes.index(args.mode)]):
+            val = sizes[thing][modes.index(args.mode)]
+            if val > args.limit:
+                print(thing.split("nonmatchings/")[1].ljust(50) + str(val))

tools/disasm_hud_element_animation.py

@@ -1,6 +1,7 @@
 #! /usr/bin/python3
 
 import struct
+import argparse
 
 def fmt_size(val):
     if val == 0:
@@ -56,6 +57,7 @@ def fmt_size(val):
     else:
         return val
 
+
 class HudElementScript():
     def __init__(self, symbol):
         self.symbol = symbol
@@ -141,9 +143,8 @@ class HudElementScript():
 
         print("};\n")
 
-if __name__ == "__main__":
-    import argparse
 
+if __name__ == "__main__":
     parser = argparse.ArgumentParser()
     parser.add_argument("file", type=str, help=".data.s file to dissassemble")
 

tools/find_duplicates.py

@@ -294,35 +294,42 @@ def do_cross_query():
     print(ccount.most_common(100))
 
 
-parser = argparse.ArgumentParser(description="Tools to assist with decomp")
-parser.add_argument("query", help="function or file")
-parser.add_argument("--threshold", help="score threshold between 0 and 1 (higher is more restrictive)", type=float, default=0.9, required=False)
-parser.add_argument("--num-out", help="number of functions to display", type=int, default=100, required=False)
-parser.add_argument("--generate-templates", help="automatically generate templates in `all` and `short` mode", action='store_true', required=False)
+parser = argparse.ArgumentParser(description="Tool to find duplicates for a specific function or to find all duplicates across the codebase.")
+group = parser.add_mutually_exclusive_group()
+group.add_argument("-a", "--all", help="find ALL duplicates and output them into a file", action='store_true', required=False)
+group.add_argument("-c", "--cross", help="do a cross query over the codebase", action='store_true', required=False)
+group.add_argument("-s", "--short", help="find MOST duplicates besides some very small duplicates. Cuts the runtime in half with minimal loss", action='store_true', required=False)
+parser.add_argument("query", help="function or file", nargs='?', default=None)
+parser.add_argument("-t", "--threshold", help="score threshold between 0 and 1 (higher is more restrictive)", type=float, default=0.9, required=False)
+parser.add_argument("-n", "--num-out", help="number of functions to display", type=int, default=100, required=False)
 
 args = parser.parse_args()
 
-rom_bytes = read_rom()
-map_syms = parse_map(os.path.join(root_dir, "ver", "current", "build", "papermario.map"))
-map_offsets = get_map_offsets(map_syms)
-
-s_files = get_all_s_files()
-
-query_dir = find_dir(args.query)
-
-if query_dir is not None:
-    files = os.listdir(query_dir)
-    for f_name in files:
-        do_query(f_name[:-2])
-else:
-    if args.query == "cross":
-        args.threshold = 0.985
-        do_cross_query()
-    elif args.query == "all":
-        args.threshold = 0.985
-        all_matches(True)
-    elif args.query == "short":
-        args.threshold = 0.985
-        all_matches(False)
+if __name__ == "__main__":
+    rom_bytes = read_rom()
+    map_syms = parse_map(os.path.join(root_dir, "ver", "current", "build", "papermario.map"))
+    map_offsets = get_map_offsets(map_syms)
+    
+    s_files = get_all_s_files()
+    
+    query_dir = find_dir(args.query)
+    
+    if query_dir is not None:
+        files = os.listdir(query_dir)
+        for f_name in files:
+            do_query(f_name[:-2])
     else:
-        do_query(args.query)
+        if args.cross:
+            args.threshold = 0.985
+            do_cross_query()
+        elif args.all:
+            args.threshold = 0.985
+            all_matches(True)
+        elif args.short:
+            args.threshold = 0.985
+            all_matches(False)
+        else:
+            if args.query is None:
+                parser.print_help()
+            else:
+                do_query(args.query)

tools/get_variable.py

@@ -1,5 +1,7 @@
 #!/usr/bin/python3
 
+import sys
+
 def get_variable(arg):
     v = arg - 2**32 # convert to s32
     if v > -250000000:
@@ -25,5 +27,7 @@ def get_variable(arg):
         return f"{arg}"
 
 if __name__ == "__main__":
-    import sys
-    print(get_variable(int(sys.argv[1], 0)))
+    try:
+        print(get_variable(int(sys.argv[1], 0)))
+    except:
+        print("Invalid literal for numeric operation. Please input a hex/dec number.")

tools/old/bootstrap_effects.py

@@ -5,7 +5,7 @@ import re
 from pathlib import Path
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
-root_dir = script_dir + "/../"
+root_dir = script_dir + "/../../"
 asm_dir = root_dir + "ver/current/asm/"
 asm_effects_dir = asm_dir + "nonmatchings/effects/"
 

tools/old/codescan.py

@@ -8,7 +8,7 @@ import subprocess
 from pathlib import Path
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
-root_dir = os.path.abspath(os.path.join(script_dir, ".."))
+root_dir = os.path.abspath(os.path.join(script_dir, "../.."))
 
 import glob, os
 os.chdir(root_dir)

tools/old/fix_s_filenames.py

@@ -7,7 +7,7 @@ import os
 import re
 
 script_dir = os.path.dirname(os.path.realpath(__file__))
-root_dir = script_dir + "/../"
+root_dir = script_dir + "/../../"
 asm_dir = root_dir + "ver/current/asm/nonmatchings/"
 
 for root, dirs, files in os.walk(asm_dir):

tools/rename.py

@@ -29,7 +29,7 @@ def handle_file(f_path, try_rename_file=False):
         f.write(f_text)
 
 
-# Read Star Rod's output file
+# Read Star Rod's output file (one rename per line, old and new, delimited by a space)
 with open(os.path.join(script_dir, "to_rename.txt")) as f:
     renames_text = f.readlines()
 

tools/star_rod_enum_to_decomp.py

@@ -6,13 +6,6 @@ import re
 import os
 from glob import glob
 
-parser = argparse.ArgumentParser(description='Convert a StarRod enum into an enum that is in decomp format')
-parser.add_argument("query", help="StarRod enum file")
-#parser.add_argument("-p", "--prefix", help="specify a prefix for the enum elements", type=str, required=False)
-parser.add_argument("-r", "--recursive", help="recursively convert all files to enums", type=bool, required=False)
-
-args = parser.parse_args()
-
 
 def create_enum(file_content, prefix, ordering):
     ret = ""
@@ -34,11 +27,11 @@ def create_enum(file_content, prefix, ordering):
 
     for (key, value) in re.findall(r'(\S+)\s+=\s+(\S+)', file_content):
         if ordering:
-            key = '_'.join(re.sub(r'([A-Z])', r' \1', key).split()).replace("H_P", "HP").replace("F_P", "FP").replace("J_P", "JP").replace("N_P_C", "NPC").replace("__", "_").replace("-", "")
+            key = '_'.join(re.sub(r'([A-Z]{1,2})', r' \1', key).split()).replace("N_P_C", "NPC").replace("__", "_").replace("-", "")
             key = prefix.upper() + '_{:<{width}}'.format(key, width=max_size + 2).upper()
             ret += "    " + key + " = 0x" + '{:>{fill}{width}}'.format(value, fill=0, width=8) + ",\n"
         else:
-            value = '_'.join(re.sub(r'([A-Z])', r' \1', value).split()).replace("H_P", "HP").replace("F_P", "FP").replace("J_P", "JP").replace("N_P_C", "NPC").replace("__", "_").replace("-", "")
+            value = '_'.join(re.sub(r'([A-Z]{1,2})', r' \1', value).split()).replace("N_P_C", "NPC").replace("__", "_").replace("-", "")
             value = prefix.upper() + '_{:<{width}}'.format(value, width=max_size + 2).upper()
             ret += "    " + value + " = 0x" + '{:>{fill}{width}}'.format(key, fill=0, width=8) + ",\n"
 
@@ -113,5 +106,12 @@ def main(args):
     file.close()
 
 
+parser = argparse.ArgumentParser(description='Convert a StarRod enum into an enum that is in a decomp compatible format')
+parser.add_argument("query", help="StarRod enum file or folder")
+parser.add_argument("-r", "--recursive", help="recursively convert all files to enums", type=bool, required=False)
+
+args = parser.parse_args()
+
+
 if __name__ == "__main__":
     main(parser.parse_args())