mirror of
https://github.com/pmret/papermario.git
synced 2024-09-16 14:32:37 +02:00
SJIS escape convert for battle/area/tik2/area.c (#1133)
* SHIFT-JIS escape convert * scrap that ninja rule * yet another attempt to fix the warning madness once and for all * oops --------- Co-authored-by: Ethan Roseman <ethteck@gmail.com>
This commit is contained in:
parent
746a42af64
commit
3637dae749
2
Jenkinsfile
vendored
2
Jenkinsfile
vendored
@ -40,7 +40,7 @@ pipeline {
|
|||||||
def jp_progress = sh(returnStdout: true, script: "python3 progress.py jp --pr-comment").trim()
|
def jp_progress = sh(returnStdout: true, script: "python3 progress.py jp --pr-comment").trim()
|
||||||
def ique_progress = sh(returnStdout: true, script: "python3 progress.py ique --pr-comment").trim()
|
def ique_progress = sh(returnStdout: true, script: "python3 progress.py ique --pr-comment").trim()
|
||||||
def pal_progress = sh(returnStdout: true, script: "python3 progress.py pal --pr-comment").trim()
|
def pal_progress = sh(returnStdout: true, script: "python3 progress.py pal --pr-comment").trim()
|
||||||
def warnings = sh(returnStdout: true, script: "./tools/warnings_count/check_new_warnings.sh --jenkins").trim()
|
def warnings = sh(returnStdout: true, script: "./tools/warnings_count/compare_warnings.py --pr-message").trim()
|
||||||
def comment_id = -1
|
def comment_id = -1
|
||||||
|
|
||||||
for (comment in pullRequest.comments) {
|
for (comment in pullRequest.comments) {
|
||||||
|
@ -24,12 +24,10 @@ Formation A(Formation_02) = {
|
|||||||
ACTOR_BY_POS(A(super_blooper), A(BlooperPos), 10),
|
ACTOR_BY_POS(A(super_blooper), A(BlooperPos), 10),
|
||||||
};
|
};
|
||||||
|
|
||||||
// ソ is encoded as 0x83 0x5C, so it needs another backslash to be interpreted as valid escape sequence
|
|
||||||
// TODO modify iconv instead of strings in this file
|
|
||||||
BattleList A(Formations) = {
|
BattleList A(Formations) = {
|
||||||
BATTLE(A(Formation_00), A(tik_01), "ゲッソ\ー"),
|
BATTLE(A(Formation_00), A(tik_01), "ゲッソー"),
|
||||||
BATTLE(A(Formation_01), A(tik_01), "ビリビリゲッソ\ー"),
|
BATTLE(A(Formation_01), A(tik_01), "ビリビリゲッソー"),
|
||||||
BATTLE(A(Formation_02), A(tik_01), "ビッグゲッソ\ー チビゲッソ\ー"),
|
BATTLE(A(Formation_02), A(tik_01), "ビッグゲッソー チビゲッソー"),
|
||||||
{},
|
{},
|
||||||
};
|
};
|
||||||
|
|
||||||
|
@ -62,8 +62,6 @@ def write_ninja_rules(
|
|||||||
cc_egcs = f"{cc_egcs_dir}/gcc"
|
cc_egcs = f"{cc_egcs_dir}/gcc"
|
||||||
cxx = f"{BUILD_TOOLS}/cc/gcc/g++"
|
cxx = f"{BUILD_TOOLS}/cc/gcc/g++"
|
||||||
|
|
||||||
ICONV = "iconv --from UTF-8 --to $encoding"
|
|
||||||
|
|
||||||
CPPFLAGS_COMMON = (
|
CPPFLAGS_COMMON = (
|
||||||
"-Iver/$version/include -Iver/$version/build/include -Iinclude -Isrc -Iassets/$version -D_LANGUAGE_C -D_FINALROM "
|
"-Iver/$version/include -Iver/$version/build/include -Iinclude -Isrc -Iassets/$version -D_LANGUAGE_C -D_FINALROM "
|
||||||
"-DVERSION=$version -DF3DEX_GBI_2 -D_MIPS_SZLONG=32"
|
"-DVERSION=$version -DF3DEX_GBI_2 -D_MIPS_SZLONG=32"
|
||||||
@ -141,7 +139,7 @@ def write_ninja_rules(
|
|||||||
ninja.rule(
|
ninja.rule(
|
||||||
"cc",
|
"cc",
|
||||||
description="gcc $in",
|
description="gcc $in",
|
||||||
command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} -DOLD_GCC $cppflags -MD -MF $out.d $in -o - | {ICONV} | {ccache}{cc} {cflags} $cflags - -o $out'",
|
command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} -DOLD_GCC $cppflags -MD -MF $out.d $in -o - | $iconv | {ccache}{cc} {cflags} $cflags - -o $out'",
|
||||||
depfile="$out.d",
|
depfile="$out.d",
|
||||||
deps="gcc",
|
deps="gcc",
|
||||||
)
|
)
|
||||||
@ -149,7 +147,7 @@ def write_ninja_rules(
|
|||||||
ninja.rule(
|
ninja.rule(
|
||||||
"cc_modern",
|
"cc_modern",
|
||||||
description="gcc $in",
|
description="gcc $in",
|
||||||
command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | {ICONV} | {ccache}{cc_modern} {cflags_modern} $cflags - -o $out'",
|
command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | $iconv | {ccache}{cc_modern} {cflags_modern} $cflags - -o $out'",
|
||||||
depfile="$out.d",
|
depfile="$out.d",
|
||||||
deps="gcc",
|
deps="gcc",
|
||||||
)
|
)
|
||||||
@ -175,7 +173,7 @@ def write_ninja_rules(
|
|||||||
ninja.rule(
|
ninja.rule(
|
||||||
"cxx",
|
"cxx",
|
||||||
description="cxx $in",
|
description="cxx $in",
|
||||||
command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | {ICONV} | {ccache}{cxx} {cflags} $cflags - -o $out'",
|
command=f"bash -o pipefail -c '{cpp} {CPPFLAGS} {extra_cppflags} $cppflags -MD -MF $out.d $in -o - | $iconv | {ccache}{cxx} {cflags} $cflags - -o $out'",
|
||||||
depfile="$out.d",
|
depfile="$out.d",
|
||||||
deps="gcc",
|
deps="gcc",
|
||||||
)
|
)
|
||||||
@ -690,6 +688,12 @@ class Configure:
|
|||||||
if version == "ique":
|
if version == "ique":
|
||||||
encoding = "EUC-JP"
|
encoding = "EUC-JP"
|
||||||
|
|
||||||
|
iconv = f"iconv --from UTF-8 --to {encoding}"
|
||||||
|
|
||||||
|
# use tools/sjis-escape.py for src/battle/area/tik2/area.c
|
||||||
|
if version != "ique" and seg.dir.parts[-3:] == ("battle", "area", "tik2") and seg.name == "area":
|
||||||
|
iconv += " | tools/sjis-escape.py"
|
||||||
|
|
||||||
# Dead cod
|
# Dead cod
|
||||||
if isinstance(seg.parent.yaml, dict) and seg.parent.yaml.get("dead_code", False):
|
if isinstance(seg.parent.yaml, dict) and seg.parent.yaml.get("dead_code", False):
|
||||||
obj_path = str(entry.object_path)
|
obj_path = str(entry.object_path)
|
||||||
@ -701,7 +705,7 @@ class Configure:
|
|||||||
variables={
|
variables={
|
||||||
"cflags": cflags,
|
"cflags": cflags,
|
||||||
"cppflags": f"-DVERSION_{self.version.upper()}",
|
"cppflags": f"-DVERSION_{self.version.upper()}",
|
||||||
"encoding": encoding,
|
"iconv": iconv,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
build(
|
build(
|
||||||
@ -724,7 +728,7 @@ class Configure:
|
|||||||
variables={
|
variables={
|
||||||
"cflags": cflags,
|
"cflags": cflags,
|
||||||
"cppflags": f"-DVERSION_{self.version.upper()}",
|
"cppflags": f"-DVERSION_{self.version.upper()}",
|
||||||
"encoding": encoding,
|
"iconv": iconv,
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
|
|
||||||
@ -1077,7 +1081,7 @@ class Configure:
|
|||||||
variables={
|
variables={
|
||||||
"cflags": "",
|
"cflags": "",
|
||||||
"cppflags": f"-DVERSION_{self.version.upper()}",
|
"cppflags": f"-DVERSION_{self.version.upper()}",
|
||||||
"encoding": "CP932", # similar to SHIFT-JIS, but includes backslash and tilde
|
"iconv": "iconv --from UTF-8 --to CP932", # similar to SHIFT-JIS, but includes backslash and tilde
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
build(elf_path, [o_path], "shape_ld")
|
build(elf_path, [o_path], "shape_ld")
|
||||||
@ -1176,7 +1180,7 @@ class Configure:
|
|||||||
variables={
|
variables={
|
||||||
"cflags": "",
|
"cflags": "",
|
||||||
"cppflags": f"-DVERSION_{self.version.upper()}",
|
"cppflags": f"-DVERSION_{self.version.upper()}",
|
||||||
"encoding": "CP932", # similar to SHIFT-JIS, but includes backslash and tilde
|
"iconv": "iconv --from UTF-8 --to CP932", # similar to SHIFT-JIS, but includes backslash and tilde
|
||||||
},
|
},
|
||||||
)
|
)
|
||||||
else:
|
else:
|
||||||
|
36
tools/sjis-escape.py
Executable file
36
tools/sjis-escape.py
Executable file
@ -0,0 +1,36 @@
|
|||||||
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
|
import argparse
|
||||||
|
import sys
|
||||||
|
|
||||||
|
|
||||||
|
def is_sjis_2byte(c):
|
||||||
|
return (c >= 0x81 and c <= 0x9F) or (c >= 0xE0 and c <= 0xFC)
|
||||||
|
|
||||||
|
|
||||||
|
parser = argparse.ArgumentParser(description="")
|
||||||
|
parser.add_argument("infile", nargs="?", type=argparse.FileType("rb"), default=sys.stdin.buffer)
|
||||||
|
parser.add_argument("outfile", nargs="?", type=argparse.FileType("wb"), default=sys.stdout.buffer)
|
||||||
|
|
||||||
|
args = parser.parse_args()
|
||||||
|
|
||||||
|
|
||||||
|
def main():
|
||||||
|
for line in args.infile:
|
||||||
|
newline = bytearray()
|
||||||
|
i = 0
|
||||||
|
lsize = len(line)
|
||||||
|
while i < lsize:
|
||||||
|
c = line[i]
|
||||||
|
if is_sjis_2byte(c):
|
||||||
|
i += 1
|
||||||
|
c2 = line[i]
|
||||||
|
newline += "\\x{:02x}\\x{:02x}".format(c, c2).encode("ascii")
|
||||||
|
else:
|
||||||
|
newline.append(c)
|
||||||
|
i += 1
|
||||||
|
args.outfile.write(newline)
|
||||||
|
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
@ -1,30 +0,0 @@
|
|||||||
#!/usr/bin/env bash
|
|
||||||
set -e
|
|
||||||
set -o pipefail
|
|
||||||
|
|
||||||
# This script can be used when you want to test locally the amount of warnings produced by your changes before doing a PR.
|
|
||||||
|
|
||||||
pyflags=""
|
|
||||||
rebuild=false
|
|
||||||
|
|
||||||
if [[ $1 == "--jenkins" ]]
|
|
||||||
then
|
|
||||||
pyflags="--pr-message"
|
|
||||||
rebuild=false
|
|
||||||
else
|
|
||||||
rebuild=true
|
|
||||||
fi
|
|
||||||
|
|
||||||
if $rebuild
|
|
||||||
then
|
|
||||||
echo "Rebuilding to collect warnings..."
|
|
||||||
rm -rf ver/*/build/src
|
|
||||||
ninja | grep warning | sort > tools/warnings_count/warnings_new.txt
|
|
||||||
else
|
|
||||||
touch tools/warnings_count/warnings_new.txt
|
|
||||||
cat build_log.txt | grep warning | sort > tools/warnings_count/warnings_new.txt
|
|
||||||
fi
|
|
||||||
|
|
||||||
curl -L https://papermar.io/reports/warnings.txt > tools/warnings_count/warnings.txt
|
|
||||||
|
|
||||||
python3 tools/warnings_count/compare_warnings.py tools/warnings_count/warnings.txt tools/warnings_count/warnings_new.txt $pyflags
|
|
@ -1,33 +1,48 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
|
|
||||||
import argparse
|
import argparse
|
||||||
|
import os
|
||||||
|
from pathlib import Path
|
||||||
import sys
|
import sys
|
||||||
|
|
||||||
|
import requests
|
||||||
|
|
||||||
def countFileLines(filename: str) -> int:
|
script_dir = Path(os.path.dirname(os.path.realpath(__file__)))
|
||||||
with open(filename) as f:
|
root_dir = script_dir / "../.."
|
||||||
return len(f.readlines())
|
|
||||||
|
|
||||||
|
|
||||||
def main():
|
def main():
|
||||||
parser = argparse.ArgumentParser()
|
parser = argparse.ArgumentParser()
|
||||||
parser.add_argument(
|
|
||||||
"currentwarnings",
|
|
||||||
help="Name of file which contains the current warnings of the repo.",
|
|
||||||
)
|
|
||||||
parser.add_argument(
|
|
||||||
"newwarnings",
|
|
||||||
help="Name of file which contains the *new* warnings of the repo.",
|
|
||||||
)
|
|
||||||
parser.add_argument("--pr-message", action="store_true")
|
parser.add_argument("--pr-message", action="store_true")
|
||||||
args = parser.parse_args()
|
args = parser.parse_args()
|
||||||
|
|
||||||
currentLines = countFileLines(args.currentwarnings)
|
# Download the latest warnings.txt
|
||||||
newLines = countFileLines(args.newwarnings)
|
response = requests.get("https://papermar.io/reports/warnings.txt")
|
||||||
if newLines > currentLines:
|
current_warnings = response.content.decode("utf-8").strip().split("\n")
|
||||||
|
|
||||||
|
# Write the current warnings to a file
|
||||||
|
with open(script_dir / "warnings.txt", "w") as f:
|
||||||
|
for line in current_warnings:
|
||||||
|
f.write(line + "\n")
|
||||||
|
|
||||||
|
# Get the new warnings from the build log
|
||||||
|
if not (root_dir / "build_log.txt").exists():
|
||||||
|
print("build_log.txt not found. Exiting.")
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
with open(root_dir / "build_log.txt") as f:
|
||||||
|
new_warnings = [line for line in f.readlines() if "warning" in line]
|
||||||
|
|
||||||
|
# Write the new warnings to a file
|
||||||
|
with open(script_dir / "warnings_new.txt", "w") as f:
|
||||||
|
f.writelines(new_warnings)
|
||||||
|
|
||||||
|
num_current_warnings = len(current_warnings)
|
||||||
|
num_new_warnings = len(new_warnings)
|
||||||
|
if num_new_warnings > num_current_warnings:
|
||||||
stderr = False
|
stderr = False
|
||||||
if args.pr_message:
|
if args.pr_message:
|
||||||
delta = newLines - currentLines
|
delta = num_new_warnings - num_current_warnings
|
||||||
|
|
||||||
if delta == 1:
|
if delta == 1:
|
||||||
print(f"⚠️ This PR introduces a warning:")
|
print(f"⚠️ This PR introduces a warning:")
|
||||||
@ -40,25 +55,21 @@ def main():
|
|||||||
else:
|
else:
|
||||||
print()
|
print()
|
||||||
print("There are more warnings now. Go fix them!")
|
print("There are more warnings now. Go fix them!")
|
||||||
print("\tCurrent warnings: " + str(currentLines))
|
print("\tCurrent warnings: " + str(num_current_warnings))
|
||||||
print("\tNew warnings: " + str(newLines))
|
print("\tNew warnings: " + str(num_new_warnings))
|
||||||
print()
|
print()
|
||||||
|
|
||||||
with open(args.newwarnings) as new:
|
for newLine in new_warnings:
|
||||||
new = new.readlines()
|
if "warning: previous declaration of" in newLine:
|
||||||
with open(args.currentwarnings) as current:
|
continue
|
||||||
current = current.readlines()
|
|
||||||
for newLine in new:
|
|
||||||
if "warning: previous declaration of" in newLine:
|
|
||||||
continue
|
|
||||||
|
|
||||||
if newLine not in current:
|
if newLine not in current_warnings:
|
||||||
if stderr:
|
if stderr:
|
||||||
print(newLine.strip(), file=sys.stderr)
|
print(newLine.strip(), file=sys.stderr)
|
||||||
else:
|
else:
|
||||||
print("- " + newLine.strip())
|
print("- " + newLine.strip())
|
||||||
elif newLines < currentLines:
|
elif num_new_warnings < num_current_warnings:
|
||||||
delta = currentLines - newLines
|
delta = num_current_warnings - num_new_warnings
|
||||||
|
|
||||||
if args.pr_message:
|
if args.pr_message:
|
||||||
print(f"✅ This PR fixes {delta} warnings!")
|
print(f"✅ This PR fixes {delta} warnings!")
|
||||||
|
Loading…
Reference in New Issue
Block a user