Use crunch64 for compression (#1189)

This commit is contained in:
Ethan Roseman 2024-06-02 19:48:57 +09:00 committed by GitHub
parent f2ef0713ff
commit 5f1f696bdc
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 40 additions and 306 deletions

1
.gitignore vendored
View File

@ -57,5 +57,4 @@ build/
*.backup
*.crash
/tools/Yay0compress
/tools/n64crc

1
Jenkinsfile vendored
View File

@ -19,6 +19,7 @@ pipeline {
sh 'curl -L "https://github.com/decompals/mips-gcc-egcs-2.91.66/releases/latest/download/mips-gcc-egcs-2.91.66-linux.tar.gz" | tar zx -C tools/build/cc/egcs'
sh 'pip install -U -r requirements.txt'
sh 'cargo install pigment64'
sh 'cargo install crunch64-cli'
sh './configure'
}
}

View File

@ -20,11 +20,15 @@ if ROOT.is_absolute():
ROOT = ROOT.relative_to(Path.cwd())
BUILD_TOOLS = Path("tools/build")
YAY0_COMPRESS_TOOL = f"{BUILD_TOOLS}/yay0/Yay0compress"
CRC_TOOL = f"{BUILD_TOOLS}/rom/n64crc"
PIGMENT = "pigment64"
PIGMENT_REQ_VERSION = "0.4.2"
PIGMENT64 = "pigment64"
CRUNCH64 = "crunch64"
RUST_TOOLS = [
(PIGMENT64, "pigment64", "0.4.2"),
(CRUNCH64, "crunch64-cli", "0.3.1"),
]
def exec_shell(command: List[str]) -> str:
@ -212,7 +216,7 @@ def write_ninja_rules(
ninja.rule(
"pigment",
description="img($img_type) $in",
command=f"{PIGMENT} to-bin $img_flags -f $img_type -o $out $in",
command=f"{PIGMENT64} to-bin $img_flags -f $img_type -o $out $in",
)
ninja.rule(
@ -224,7 +228,7 @@ def write_ninja_rules(
ninja.rule(
"yay0",
description="yay0 $in",
command=f"{BUILD_TOOLS}/yay0/Yay0compress $in $out",
command=f"crunch64 compress yay0 $in $out",
)
ninja.rule(
@ -354,7 +358,6 @@ def write_ninja_for_tools(ninja: ninja_syntax.Writer):
command=f"cc -w $in -O3 -o $out",
)
ninja.build(YAY0_COMPRESS_TOOL, "cc_tool", f"{BUILD_TOOLS}/yay0/Yay0compress.c")
ninja.build(CRC_TOOL, "cc_tool", f"{BUILD_TOOLS}/rom/n64crc.c")
@ -549,9 +552,7 @@ class Configure:
implicit = []
order_only = []
if task == "yay0":
implicit.append(YAY0_COMPRESS_TOOL)
elif task in ["cc", "cxx", "cc_modern"]:
if task in ["cc", "cxx", "cc_modern"]:
order_only.append("generated_code_" + self.version)
order_only.append("inc_img_bins_" + self.version)
@ -1334,16 +1335,33 @@ if __name__ == "__main__":
print(f" ./configure --cpp {gcc_cpps[0]}")
exit(1)
try:
version = exec_shell([PIGMENT, "--version"]).split(" ")[1].strip()
version_err_msg = ""
missing_tools = []
version_old_tools = []
for tool, crate_name, req_version in RUST_TOOLS:
try:
version = exec_shell([tool, "--version"]).split(" ")[1].strip()
if version < PIGMENT_REQ_VERSION:
print(f"error: {PIGMENT} version {PIGMENT_REQ_VERSION} or newer is required, system version is {version}\n")
exit(1)
except (FileNotFoundError, PermissionError):
print(f"error: {PIGMENT} is not installed\n")
print("To build and install it, obtain cargo:\n\tcurl https://sh.rustup.rs -sSf | sh")
print(f"and then run:\n\tcargo install {PIGMENT}")
if version < req_version:
version_err_msg += (
f"error: {tool} version {req_version} or newer is required, system version is {version}"
)
version_old_tools.append(crate_name)
except (FileNotFoundError, PermissionError):
missing_tools.append(crate_name)
if version_old_tools or missing_tools:
if version_err_msg:
print(version_err_msg)
if missing_tools:
print(f"error: cannot find required Rust tool(s): {', '.join(missing_tools)}")
print()
print("To install/update dependencies, obtain cargo:\n\tcurl https://sh.rustup.rs -sSf | sh")
print(f"and then run:")
for tool in missing_tools:
print(f"\tcargo install {tool}")
for tool in version_old_tools:
print(f"\tcargo install {tool}")
exit(1)
# default version behaviour is to only do those that exist

View File

@ -457,7 +457,9 @@ def build_player_sprites(sprite_order: List[str], build_dir: Path, asset_stack:
subprocess.run(
[
str(TOOLS_DIR / "build/yay0/Yay0compress"),
str("crunch64"),
"compress",
"yay0",
yay0_in_path,
yay0_out_path,
]

View File

@ -1 +0,0 @@
Yay0compress

View File

@ -1,285 +0,0 @@
#include <stdio.h>
#include <stdlib.h>
#include <string.h>
int main(int argc, const char** argv, const char** envp);
void encode(FILE* fp, int insize, unsigned char* bz);
void search(unsigned int a1, int insize, int* a3, unsigned int* a4, unsigned char* bz);
int mischarsearch(unsigned char* pattern, int patternlen, unsigned char* data, int datalen);
void initskip(unsigned short* skip, unsigned char* a1, int a2);
void writeshort(FILE* fp, short a1);
void writeint4(FILE* fp, int a1);
int main(int argc, const char** argv, const char** envp) {
FILE* fp; // idb
char src[999];
char dest[999];
int insize; // idb
unsigned char* bz;
if (argc < 3) {
fprintf(stderr, "Yay0compress [infile] [outfile]\n");
return 1;
}
strcpy(src, argv[1]);
strcpy(dest, argv[2]);
if ((fp = fopen(src, "rb")) == NULL) {
fprintf(stderr, "FILE OPEN ERROR![%s]\n", src);
return 1;
}
fseek(fp, 0, SEEK_END);
insize = ftell(fp);
fseek(fp, 0, SEEK_SET);
bz = malloc(insize);
fread(bz, 1, insize, fp);
fclose(fp);
for (int i = 0; src[i]; i++) {
if (src[i] == '.') {
src[i] = 0;
break;
}
}
if ((fp = fopen(dest, "wb")) == NULL) {
fprintf(stderr, "FILE CREATE ERROR![%s]\n", dest);
exit(1);
}
encode(fp, insize, bz);
return 0;
}
void encode(FILE* fp, int insize, unsigned char* bz) {
int pp; // weak
int dp; // idb
unsigned int* cmd;
unsigned char* def;
unsigned short* pol;
unsigned int v0; // esi
unsigned int v1 = 0; // edi
int v2 = 0; // edx
int v3 = 0; // ebx
int v4 = 0; // edx
unsigned int v6; // [esp+10h] [ebp-14h]
unsigned int v7 = 0; // [esp+14h] [ebp-10h]
int v8 = 0; // [esp+18h] [ebp-Ch]
unsigned int a4 = 0; // [esp+1Ch] [ebp-8h]
int a3 = 0; // [esp+20h] [ebp-4h]
int ncp; // weak
int npp; // weak
int ndp; // weak
int cp;
dp = 0;
pp = 0;
cp = 0;
npp = 0x1000;
ndp = 0x1000;
ncp = 0x1000;
cmd = calloc(0x4000, 1);
pol = malloc(2 * npp);
def = malloc(4 * ndp);
v0 = 0;
v6 = 1024;
v1 = 0x80000000;
while (v0 < insize) {
if (dp == 73) {
int dog = 5;
}
if (v6 < v0) {
v6 += 1024;
}
search(v0, insize, &a3, &a4, bz);
if (a4 <= 2) {
cmd[cp] |= v1;
def[dp++] = bz[v0++];
if (ndp == dp) {
ndp = dp + 0x1000;
def = realloc(def, dp + 0x1000);
}
} else {
search(v0 + 1, insize, &v8, &v7, bz);
if (v7 > a4 + 1) {
cmd[cp] |= v1;
def[dp++] = bz[v0++];
if (ndp == dp) {
ndp = dp + 0x1000;
def = realloc(def, dp + 0x1000);
}
v1 >>= 1;
if (!v1) {
v1 = 0x80000000;
v2 = cp++;
if (cp == ncp) {
ncp = v2 + 1025;
cmd = realloc(cmd, 4 * (v2 + 1025));
}
cmd[cp] = 0;
}
a4 = v7;
a3 = v8;
}
v3 = v0 - a3 - 1;
a3 = v0 - a3 - 1;
if (a4 > 0x11) {
pol[pp++] = v3;
def[dp++] = a4 - 18;
if (ndp == dp) {
ndp = dp + 0x1000;
def = realloc(def, dp + 0x1000);
}
} else {
pol[pp++] = v3 | (((short)a4 - 2) << 12);
}
if (npp == pp) {
npp += 0x1000;
pol = realloc(pol, 2 * (pp + 0x1000));
}
v0 += a4;
}
v1 >>= 1;
if (!v1) {
v1 = 0x80000000;
v4 = cp++;
if (cp == ncp) {
ncp = v4 + 1025;
cmd = realloc(cmd, 4 * (v4 + 1025));
}
cmd[cp] = 0;
}
}
if (v1 != 0x80000000) {
cp++;
}
fprintf(fp, "Yay0");
writeint4(fp, insize);
writeint4(fp, 4 * cp + 16);
writeint4(fp, 2 * pp + 4 * cp + 16);
for (int i = 0; i < cp; i++)
writeint4(fp, cmd[i]);
for (int i = 0; i < pp; i++)
writeshort(fp, pol[i]);
fwrite(def, 1, dp, fp);
fclose(fp);
}
void search(unsigned int a1, int insize, int* a3, unsigned int* a4, unsigned char* bz) {
unsigned int patternlen; // ebx
unsigned int v5; // esi
unsigned int* v6 = 0; // edi
unsigned int v7 = 0; // [esp+Ch] [ebp-10h]
int v8 = 0; // [esp+14h] [ebp-8h]
unsigned int v9; // [esp+18h] [ebp-4h]
patternlen = 3;
v5 = 0;
if (a1 > 0x1000) {
v5 = a1 - 0x1000;
}
v9 = 273;
if (insize - a1 <= 273) {
v9 = insize - a1;
}
if (v9 > 2) {
while (a1 > v5) {
v7 = mischarsearch(&bz[a1], patternlen, &bz[v5], patternlen + a1 - v5);
if (v7 >= a1 - v5) {
break;
}
for (; patternlen < v9; patternlen++) {
if (bz[patternlen + v5 + v7] != bz[patternlen + a1]) {
break;
}
}
if (v9 == patternlen) {
*a3 = v7 + v5;
*a4 = patternlen;
return;
}
v8 = v5 + v7;
patternlen++;
v5 += v7 + 1;
}
*a3 = v8;
if (patternlen > 3) {
patternlen--;
*a4 = patternlen;
return;
}
v6 = a4;
} else {
*a4 = 0;
v6 = a3;
}
*v6 = 0;
}
int mischarsearch(unsigned char* pattern, int patternlen, unsigned char* data, int datalen) {
unsigned short skip[256]; // idb
int i; // ebx
int v6; // eax
int j; // ecx
if (patternlen <= datalen) {
// initskip
for (int k = 0; k < 256; k++) {
skip[k] = patternlen;
}
for (int k = 0; k < patternlen; k++) {
skip[pattern[k]] = patternlen - k - 1;
}
for (i = patternlen - 1;; i += v6) {
if (pattern[patternlen - 1] == data[i]) {
i--;
j = patternlen - 2;
if (j < 0) {
return i + 1;
}
while (pattern[j] == data[i]) {
i--;
j--;
if (j < 0) {
return i + 1;
}
}
if (skip[data[i]] <= patternlen - j) {
v6 = patternlen - j;
continue;
}
}
v6 = skip[data[i]];
}
}
return datalen;
}
void writeshort(FILE* fp, short val) {
fputc((val & 0xff00) >> 8, fp);
fputc((val & 0x00ff) >> 0, fp);
}
void writeint4(FILE* fp, int val) {
fputc((val & 0x00ff000000) >> 24, fp);
fputc((val & 0x0000ff0000) >> 16, fp);
fputc((val & 0x000000ff00) >> 8, fp);
fputc((val & 0x00000000ff) >> 0, fp);
}