From 3637dae7495be0450d790e3aef2cd3bf6ea63665 Mon Sep 17 00:00:00 2001 From: AltoXorg <56553686+Alto1772@users.noreply.github.com> Date: Sun, 10 Dec 2023 00:50:54 +0800 Subject: [PATCH] SJIS escape convert for battle/area/tik2/area.c (#1133) * SHIFT-JIS escape convert * scrap that ninja rule * yet another attempt to fix the warning madness once and for all * oops --------- Co-authored-by: Ethan Roseman --- Jenkinsfile | 2 +- src/battle/area/tik2/area.c | 8 +-- tools/build/configure.py | 22 ++++--- tools/sjis-escape.py | 36 +++++++++++ tools/warnings_count/check_new_warnings.sh | 30 --------- tools/warnings_count/compare_warnings.py | 73 +++++++++++++--------- 6 files changed, 95 insertions(+), 76 deletions(-) create mode 100755 tools/sjis-escape.py delete mode 100755 tools/warnings_count/check_new_warnings.sh diff --git a/Jenkinsfile b/Jenkinsfile index 256e17e419..e3e46167ec 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -40,7 +40,7 @@ pipeline { def jp_progress = sh(returnStdout: true, script: "python3 progress.py jp --pr-comment").trim() def ique_progress = sh(returnStdout: true, script: "python3 progress.py ique --pr-comment").trim() def pal_progress = sh(returnStdout: true, script: "python3 progress.py pal --pr-comment").trim() - def warnings = sh(returnStdout: true, script: "./tools/warnings_count/check_new_warnings.sh --jenkins").trim() + def warnings = sh(returnStdout: true, script: "./tools/warnings_count/compare_warnings.py --pr-message").trim() def comment_id = -1 for (comment in pullRequest.comments) { diff --git a/src/battle/area/tik2/area.c b/src/battle/area/tik2/area.c index 0e35113779..973bfc1607 100644 --- a/src/battle/area/tik2/area.c +++ b/src/battle/area/tik2/area.c @@ -24,12 +24,10 @@ Formation A(Formation_02) = { ACTOR_BY_POS(A(super_blooper), A(BlooperPos), 10), }; -// ソ is encoded as 0x83 0x5C, so it needs another backslash to be interpreted as valid escape sequence -// TODO modify iconv instead of strings in this file BattleList A(Formations) = { - BATTLE(A(Formation_00), A(tik_01), "ゲッソ\ー"), - BATTLE(A(Formation_01), A(tik_01), "ビリビリゲッソ\ー"), - BATTLE(A(Formation_02), A(tik_01), "ビッグゲッソ\ー　チビゲッソ\ー"), + BATTLE(A(Formation_00), A(tik_01), "ゲッソー"), + BATTLE(A(Formation_01), A(tik_01), "ビリビリゲッソー"), + BATTLE(A(Formation_02), A(tik_01), "ビッグゲッソー　チビゲッソー"), {}, }; diff --git a/tools/build/configure.py b/tools/build/configure.py index 0714408951..a0e045fc15 100755 --- a/tools/build/configure.py +++ b/tools/build/configure.py @@ -62,8 +62,6 @@ def write_ninja_rules( cc_egcs = f"{cc_egcs_dir}/gcc" cxx = f"{BUILD_TOOLS}/cc/gcc/g++" - ICONV = "iconv --from UTF-8 --to $encoding" - CPPFLAGS_COMMON = ( "-Iver/$version/include -Iver/$version/build/include -Iinclude -Isrc -Iassets/$version -D_LANGUAGE_C -D_FINALROM " "-DVERSION=$version -DF3DEX_GBI_2 -D_MIPS_SZLONG=32" @@ -141,7 +139,7 @@ def write_ninja_rules( ninja.rule( "cc", description="gcc $in", - command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} -DOLD_GCC $cppflags -MD -MF $out.d $in -o - | {ICONV} | {ccache}{cc} {cflags} $cflags - -o $out'", + command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} -DOLD_GCC $cppflags -MD -MF $out.d $in -o - | $iconv | {ccache}{cc} {cflags} $cflags - -o $out'", depfile="$out.d", deps="gcc", ) @@ -149,7 +147,7 @@ def write_ninja_rules( ninja.rule( "cc_modern", description="gcc $in", - command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | {ICONV} | {ccache}{cc_modern} {cflags_modern} $cflags - -o $out'", + command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | $iconv | {ccache}{cc_modern} {cflags_modern} $cflags - -o $out'", depfile="$out.d", deps="gcc", ) @@ -175,7 +173,7 @@ def write_ninja_rules( ninja.rule( "cxx", description="cxx $in", - command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | {ICONV} | {ccache}{cxx} {cflags} $cflags - -o $out'", + command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | $iconv | {ccache}{cxx} {cflags} $cflags - -o $out'", depfile="$out.d", deps="gcc", ) @@ -690,6 +688,12 @@ class Configure: if version == "ique": encoding = "EUC-JP" + iconv = f"iconv --from UTF-8 --to {encoding}" + + # use tools/sjis-escape.py for src/battle/area/tik2/area.c + if version != "ique" and seg.dir.parts[-3:] == ("battle", "area", "tik2") and seg.name == "area": + iconv += " | tools/sjis-escape.py" + # Dead cod if isinstance(seg.parent.yaml, dict) and seg.parent.yaml.get("dead_code", False): obj_path = str(entry.object_path) @@ -701,7 +705,7 @@ class Configure: variables={ "cflags": cflags, "cppflags": f"-DVERSION_{self.version.upper()}", - "encoding": encoding, + "iconv": iconv, }, ) build( @@ -724,7 +728,7 @@ class Configure: variables={ "cflags": cflags, "cppflags": f"-DVERSION_{self.version.upper()}", - "encoding": encoding, + "iconv": iconv, }, ) @@ -1077,7 +1081,7 @@ class Configure: variables={ "cflags": "", "cppflags": f"-DVERSION_{self.version.upper()}", - "encoding": "CP932", # similar to SHIFT-JIS, but includes backslash and tilde + "iconv": "iconv --from UTF-8 --to CP932", # similar to SHIFT-JIS, but includes backslash and tilde }, ) build(elf_path, [o_path], "shape_ld") @@ -1176,7 +1180,7 @@ class Configure: variables={ "cflags": "", "cppflags": f"-DVERSION_{self.version.upper()}", - "encoding": "CP932", # similar to SHIFT-JIS, but includes backslash and tilde + "iconv": "iconv --from UTF-8 --to CP932", # similar to SHIFT-JIS, but includes backslash and tilde }, ) else: diff --git a/tools/sjis-escape.py b/tools/sjis-escape.py new file mode 100755 index 0000000000..88dc5c221a --- /dev/null +++ b/tools/sjis-escape.py @@ -0,0 +1,36 @@ +#!/usr/bin/env python3 + +import argparse +import sys + + +def is_sjis_2byte(c): + return (c >= 0x81 and c <= 0x9F) or (c >= 0xE0 and c <= 0xFC) + + +parser = argparse.ArgumentParser(description="") +parser.add_argument("infile", nargs="?", type=argparse.FileType("rb"), default=sys.stdin.buffer) +parser.add_argument("outfile", nargs="?", type=argparse.FileType("wb"), default=sys.stdout.buffer) + +args = parser.parse_args() + + +def main(): + for line in args.infile: + newline = bytearray() + i = 0 + lsize = len(line) + while i < lsize: + c = line[i] + if is_sjis_2byte(c): + i += 1 + c2 = line[i] + newline += "\\x{:02x}\\x{:02x}".format(c, c2).encode("ascii") + else: + newline.append(c) + i += 1 + args.outfile.write(newline) + + +if __name__ == "__main__": + main() diff --git a/tools/warnings_count/check_new_warnings.sh b/tools/warnings_count/check_new_warnings.sh deleted file mode 100755 index 4406bedb13..0000000000 --- a/tools/warnings_count/check_new_warnings.sh +++ /dev/null @@ -1,30 +0,0 @@ -#!/usr/bin/env bash -set -e -set -o pipefail - -# This script can be used when you want to test locally the amount of warnings produced by your changes before doing a PR. - -pyflags="" -rebuild=false - -if [[ $1 == "--jenkins" ]] - then - pyflags="--pr-message" - rebuild=false - else - rebuild=true -fi - -if $rebuild -then - echo "Rebuilding to collect warnings..." - rm -rf ver/*/build/src - ninja | grep warning | sort > tools/warnings_count/warnings_new.txt -else - touch tools/warnings_count/warnings_new.txt - cat build_log.txt | grep warning | sort > tools/warnings_count/warnings_new.txt -fi - -curl -L https://papermar.io/reports/warnings.txt > tools/warnings_count/warnings.txt - -python3 tools/warnings_count/compare_warnings.py tools/warnings_count/warnings.txt tools/warnings_count/warnings_new.txt $pyflags diff --git a/tools/warnings_count/compare_warnings.py b/tools/warnings_count/compare_warnings.py index cfd2008e9d..62d32aa491 100755 --- a/tools/warnings_count/compare_warnings.py +++ b/tools/warnings_count/compare_warnings.py @@ -1,33 +1,48 @@ #!/usr/bin/env python3 import argparse +import os +from pathlib import Path import sys +import requests -def countFileLines(filename: str) -> int: - with open(filename) as f: - return len(f.readlines()) +script_dir = Path(os.path.dirname(os.path.realpath(__file__))) +root_dir = script_dir / "../.." def main(): parser = argparse.ArgumentParser() - parser.add_argument( - "currentwarnings", - help="Name of file which contains the current warnings of the repo.", - ) - parser.add_argument( - "newwarnings", - help="Name of file which contains the *new* warnings of the repo.", - ) parser.add_argument("--pr-message", action="store_true") args = parser.parse_args() - currentLines = countFileLines(args.currentwarnings) - newLines = countFileLines(args.newwarnings) - if newLines > currentLines: + # Download the latest warnings.txt + response = requests.get("https://papermar.io/reports/warnings.txt") + current_warnings = response.content.decode("utf-8").strip().split("\n") + + # Write the current warnings to a file + with open(script_dir / "warnings.txt", "w") as f: + for line in current_warnings: + f.write(line + "\n") + + # Get the new warnings from the build log + if not (root_dir / "build_log.txt").exists(): + print("build_log.txt not found. Exiting.") + sys.exit(1) + + with open(root_dir / "build_log.txt") as f: + new_warnings = [line for line in f.readlines() if "warning" in line] + + # Write the new warnings to a file + with open(script_dir / "warnings_new.txt", "w") as f: + f.writelines(new_warnings) + + num_current_warnings = len(current_warnings) + num_new_warnings = len(new_warnings) + if num_new_warnings > num_current_warnings: stderr = False if args.pr_message: - delta = newLines - currentLines + delta = num_new_warnings - num_current_warnings if delta == 1: print(f"⚠️ This PR introduces a warning:") @@ -40,25 +55,21 @@ def main(): else: print() print("There are more warnings now. Go fix them!") - print("\tCurrent warnings: " + str(currentLines)) - print("\tNew warnings: " + str(newLines)) + print("\tCurrent warnings: " + str(num_current_warnings)) + print("\tNew warnings: " + str(num_new_warnings)) print() - with open(args.newwarnings) as new: - new = new.readlines() - with open(args.currentwarnings) as current: - current = current.readlines() - for newLine in new: - if "warning: previous declaration of" in newLine: - continue + for newLine in new_warnings: + if "warning: previous declaration of" in newLine: + continue - if newLine not in current: - if stderr: - print(newLine.strip(), file=sys.stderr) - else: - print("- " + newLine.strip()) - elif newLines < currentLines: - delta = currentLines - newLines + if newLine not in current_warnings: + if stderr: + print(newLine.strip(), file=sys.stderr) + else: + print("- " + newLine.strip()) + elif num_new_warnings < num_current_warnings: + delta = num_current_warnings - num_new_warnings if args.pr_message: print(f"✅ This PR fixes {delta} warnings!")