SJIS escape convert for battle/area/tik2/area.c (#1133)

* SHIFT-JIS escape convert

* scrap that ninja rule

* yet another attempt to fix the warning madness once and for all

* oops

---------

Co-authored-by: Ethan Roseman <ethteck@gmail.com>

Jenkinsfile

@@ -40,7 +40,7 @@ pipeline {
                         def jp_progress = sh(returnStdout: true, script: "python3 progress.py jp --pr-comment").trim()
                         def ique_progress = sh(returnStdout: true, script: "python3 progress.py ique --pr-comment").trim()
                         def pal_progress = sh(returnStdout: true, script: "python3 progress.py pal --pr-comment").trim()
-                        def warnings = sh(returnStdout: true, script: "./tools/warnings_count/check_new_warnings.sh --jenkins").trim()
+                        def warnings = sh(returnStdout: true, script: "./tools/warnings_count/compare_warnings.py --pr-message").trim()
                         def comment_id = -1
 
                         for (comment in pullRequest.comments) {

src/battle/area/tik2/area.c

@@ -24,12 +24,10 @@ Formation A(Formation_02) = {
     ACTOR_BY_POS(A(super_blooper), A(BlooperPos), 10),
 };
 
-// ソ is encoded as 0x83 0x5C, so it needs another backslash to be interpreted as valid escape sequence
-// TODO modify iconv instead of strings in this file
 BattleList A(Formations) = {
-    BATTLE(A(Formation_00), A(tik_01), "ゲッソ\ー"),
+    BATTLE(A(Formation_00), A(tik_01), "ゲッソー"),
-    BATTLE(A(Formation_01), A(tik_01), "ビリビリゲッソ\ー"),
+    BATTLE(A(Formation_01), A(tik_01), "ビリビリゲッソー"),
-    BATTLE(A(Formation_02), A(tik_01), "ビッグゲッソ\ー　チビゲッソ\ー"),
+    BATTLE(A(Formation_02), A(tik_01), "ビッグゲッソー　チビゲッソー"),
     {},
 };
 

tools/build/configure.py

@@ -62,8 +62,6 @@ def write_ninja_rules(
     cc_egcs = f"{cc_egcs_dir}/gcc"
     cxx = f"{BUILD_TOOLS}/cc/gcc/g++"
 
-    ICONV = "iconv --from UTF-8 --to $encoding"
-
     CPPFLAGS_COMMON = (
         "-Iver/$version/include -Iver/$version/build/include -Iinclude -Isrc -Iassets/$version -D_LANGUAGE_C -D_FINALROM "
         "-DVERSION=$version -DF3DEX_GBI_2 -D_MIPS_SZLONG=32"
@@ -141,7 +139,7 @@ def write_ninja_rules(
     ninja.rule(
         "cc",
         description="gcc $in",
-        command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} -DOLD_GCC $cppflags -MD -MF $out.d $in -o - | {ICONV} | {ccache}{cc} {cflags} $cflags - -o $out'",
+        command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} -DOLD_GCC $cppflags -MD -MF $out.d $in -o - | $iconv | {ccache}{cc} {cflags} $cflags - -o $out'",
         depfile="$out.d",
         deps="gcc",
     )
@@ -149,7 +147,7 @@ def write_ninja_rules(
     ninja.rule(
         "cc_modern",
         description="gcc $in",
-        command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | {ICONV} | {ccache}{cc_modern} {cflags_modern} $cflags - -o $out'",
+        command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | $iconv | {ccache}{cc_modern} {cflags_modern} $cflags - -o $out'",
         depfile="$out.d",
         deps="gcc",
     )
@@ -175,7 +173,7 @@ def write_ninja_rules(
     ninja.rule(
         "cxx",
         description="cxx $in",
-        command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | {ICONV} | {ccache}{cxx} {cflags} $cflags - -o $out'",
+        command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | $iconv | {ccache}{cxx} {cflags} $cflags - -o $out'",
         depfile="$out.d",
         deps="gcc",
     )
@@ -690,6 +688,12 @@ class Configure:
                 if version == "ique":
                     encoding = "EUC-JP"
 
+                iconv = f"iconv --from UTF-8 --to {encoding}"
+
+                # use tools/sjis-escape.py for src/battle/area/tik2/area.c
+                if version != "ique" and seg.dir.parts[-3:] == ("battle", "area", "tik2") and seg.name == "area":
+                    iconv += " | tools/sjis-escape.py"
+
                 # Dead cod
                 if isinstance(seg.parent.yaml, dict) and seg.parent.yaml.get("dead_code", False):
                     obj_path = str(entry.object_path)
@@ -701,7 +705,7 @@ class Configure:
                         variables={
                             "cflags": cflags,
                             "cppflags": f"-DVERSION_{self.version.upper()}",
-                            "encoding": encoding,
+                            "iconv": iconv,
                         },
                     )
                     build(
@@ -724,7 +728,7 @@ class Configure:
                         variables={
                             "cflags": cflags,
                             "cppflags": f"-DVERSION_{self.version.upper()}",
-                            "encoding": encoding,
+                            "iconv": iconv,
                         },
                     )
 
@@ -1077,7 +1081,7 @@ class Configure:
                                 variables={
                                     "cflags": "",
                                     "cppflags": f"-DVERSION_{self.version.upper()}",
-                                    "encoding": "CP932",  # similar to SHIFT-JIS, but includes backslash and tilde
+                                    "iconv": "iconv --from UTF-8 --to CP932",  # similar to SHIFT-JIS, but includes backslash and tilde
                                 },
                             )
                             build(elf_path, [o_path], "shape_ld")
@@ -1176,7 +1180,7 @@ class Configure:
                     variables={
                         "cflags": "",
                         "cppflags": f"-DVERSION_{self.version.upper()}",
-                        "encoding": "CP932",  # similar to SHIFT-JIS, but includes backslash and tilde
+                        "iconv": "iconv --from UTF-8 --to CP932",  # similar to SHIFT-JIS, but includes backslash and tilde
                     },
                 )
             else:

tools/sjis-escape.py

@@ -0,0 +1,36 @@
+#!/usr/bin/env python3
+
+import argparse
+import sys
+
+
+def is_sjis_2byte(c):
+    return (c >= 0x81 and c <= 0x9F) or (c >= 0xE0 and c <= 0xFC)
+
+
+parser = argparse.ArgumentParser(description="")
+parser.add_argument("infile", nargs="?", type=argparse.FileType("rb"), default=sys.stdin.buffer)
+parser.add_argument("outfile", nargs="?", type=argparse.FileType("wb"), default=sys.stdout.buffer)
+
+args = parser.parse_args()
+
+
+def main():
+    for line in args.infile:
+        newline = bytearray()
+        i = 0
+        lsize = len(line)
+        while i < lsize:
+            c = line[i]
+            if is_sjis_2byte(c):
+                i += 1
+                c2 = line[i]
+                newline += "\\x{:02x}\\x{:02x}".format(c, c2).encode("ascii")
+            else:
+                newline.append(c)
+            i += 1
+        args.outfile.write(newline)
+
+
+if __name__ == "__main__":
+    main()

tools/warnings_count/check_new_warnings.sh

@@ -1,30 +0,0 @@
-#!/usr/bin/env bash
-set -e
-set -o pipefail
-
-# This script can be used when you want to test locally the amount of warnings produced by your changes before doing a PR.
-
-pyflags=""
-rebuild=false
-
-if [[ $1 == "--jenkins" ]]
-  then
-    pyflags="--pr-message"
-    rebuild=false
-  else
-    rebuild=true
-fi
-
-if $rebuild
-then
-    echo "Rebuilding to collect warnings..."
-    rm -rf ver/*/build/src
-    ninja | grep warning | sort > tools/warnings_count/warnings_new.txt
-else
-    touch tools/warnings_count/warnings_new.txt
-    cat build_log.txt | grep warning | sort > tools/warnings_count/warnings_new.txt
-fi
-
-curl -L https://papermar.io/reports/warnings.txt > tools/warnings_count/warnings.txt
-
-python3 tools/warnings_count/compare_warnings.py tools/warnings_count/warnings.txt tools/warnings_count/warnings_new.txt $pyflags

tools/warnings_count/compare_warnings.py

@@ -1,33 +1,48 @@
 #!/usr/bin/env python3
 
 import argparse
+import os
+from pathlib import Path
 import sys
 
+import requests
 
-def countFileLines(filename: str) -> int:
+script_dir = Path(os.path.dirname(os.path.realpath(__file__)))
-    with open(filename) as f:
+root_dir = script_dir / "../.."
-        return len(f.readlines())
 
 
 def main():
     parser = argparse.ArgumentParser()
-    parser.add_argument(
-        "currentwarnings",
-        help="Name of file which contains the current warnings of the repo.",
-    )
-    parser.add_argument(
-        "newwarnings",
-        help="Name of file which contains the *new* warnings of the repo.",
-    )
     parser.add_argument("--pr-message", action="store_true")
     args = parser.parse_args()
 
-    currentLines = countFileLines(args.currentwarnings)
+    # Download the latest warnings.txt
-    newLines = countFileLines(args.newwarnings)
+    response = requests.get("https://papermar.io/reports/warnings.txt")
-    if newLines > currentLines:
+    current_warnings = response.content.decode("utf-8").strip().split("\n")
+
+    # Write the current warnings to a file
+    with open(script_dir / "warnings.txt", "w") as f:
+        for line in current_warnings:
+            f.write(line + "\n")
+
+    # Get the new warnings from the build log
+    if not (root_dir / "build_log.txt").exists():
+        print("build_log.txt not found. Exiting.")
+        sys.exit(1)
+
+    with open(root_dir / "build_log.txt") as f:
+        new_warnings = [line for line in f.readlines() if "warning" in line]
+
+    # Write the new warnings to a file
+    with open(script_dir / "warnings_new.txt", "w") as f:
+        f.writelines(new_warnings)
+
+    num_current_warnings = len(current_warnings)
+    num_new_warnings = len(new_warnings)
+    if num_new_warnings > num_current_warnings:
         stderr = False
         if args.pr_message:
-            delta = newLines - currentLines
+            delta = num_new_warnings - num_current_warnings
 
             if delta == 1:
                 print(f"⚠️ This PR introduces a warning:")
@@ -40,25 +55,21 @@ def main():
         else:
             print()
             print("There are more warnings now. Go fix them!")
-            print("\tCurrent warnings: " + str(currentLines))
+            print("\tCurrent warnings: " + str(num_current_warnings))
-            print("\tNew warnings: " + str(newLines))
+            print("\tNew warnings: " + str(num_new_warnings))
             print()
 
-        with open(args.newwarnings) as new:
+        for newLine in new_warnings:
-            new = new.readlines()
+            if "warning: previous declaration of" in newLine:
-            with open(args.currentwarnings) as current:
+                continue
-                current = current.readlines()
-                for newLine in new:
-                    if "warning: previous declaration of" in newLine:
-                        continue
 
-                    if newLine not in current:
+            if newLine not in current_warnings:
-                        if stderr:
+                if stderr:
-                            print(newLine.strip(), file=sys.stderr)
+                    print(newLine.strip(), file=sys.stderr)
-                        else:
+                else:
-                            print("- " + newLine.strip())
+                    print("- " + newLine.strip())
-    elif newLines < currentLines:
+    elif num_new_warnings < num_current_warnings:
-        delta = currentLines - newLines
+        delta = num_current_warnings - num_new_warnings
 
         if args.pr_message:
             print(f"✅ This PR fixes {delta} warnings!")