From 2774ebd2b498fb9885ee306189ca80d3711cc613 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Mon, 21 Nov 2022 23:31:48 +0100 Subject: [PATCH 01/44] starting work on backup verification --- .../core/CompressionStatus.java | 16 +++++++ .../core/create/FileInputStreamSupplier.java | 47 +++++++++++++++++++ .../core/create/HashingInputStream.java | 43 +++++++++++++++++ .../core/create/InputSupplier.java | 14 ++++++ .../compressors/AbstractCompressor.java | 35 ++++++++++++-- .../compressors/ParallelZipCompressor.java | 29 ++++-------- .../create/compressors/ZipCompressor.java | 15 +++--- .../compressors/tar/AbstractTarArchiver.java | 9 ++-- 8 files changed, 171 insertions(+), 37 deletions(-) create mode 100644 src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java create mode 100644 src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java create mode 100644 src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java create mode 100644 src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java new file mode 100644 index 0000000..0f7a459 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -0,0 +1,16 @@ +package net.szum123321.textile_backup.core; + +import java.io.Serializable; +import java.nio.file.Path; +import java.time.LocalDateTime; + +public record CompressionStatus(long[] treeHash, LocalDateTime date, long startTimestamp, long finishTimestamp, boolean ok, Path[] brokenFiles) implements Serializable { + + public static class Builder { + public synchronized void update(Path path, long hash, Exception error) { throw new RuntimeException("UNIMPLEMENTED!"); } + public synchronized void update(Path path, Exception error) { throw new RuntimeException("UNIMPLEMENTED!"); } + public synchronized void update(Path path, long hash) { throw new RuntimeException("UNIMPLEMENTED!"); } + + public CompressionStatus build() { throw new RuntimeException("UNIMPLEMENTED!"); } + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java new file mode 100644 index 0000000..465c004 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java @@ -0,0 +1,47 @@ +package net.szum123321.textile_backup.core.create; + +import net.szum123321.textile_backup.TextileBackup; +import net.szum123321.textile_backup.TextileLogger; +import net.szum123321.textile_backup.core.CompressionStatus; + +import java.io.IOException; +import java.io.InputStream; + +import java.nio.file.Files; +import java.nio.file.Path; + +public record FileInputStreamSupplier(Path path, String name, CompressionStatus.Builder builder) implements InputSupplier { + private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); + + @Override + public InputStream getInputStream() throws IOException { + try { + //TODO: put in hasher + return new HashingInputStream(Files.newInputStream(path), path, null, builder); + } catch (IOException e) { + builder.update(path, e); + throw e; + } + } + + @Override + public Path getPath() { + return path; + } + + @Override + public String getName() { + return name; + } + + @Override + public InputStream get() { + try { + return getInputStream(); + } catch (IOException e) { + log.error("An exception occurred while trying to create an input stream from file: {}!", path.toString(), e); + } + + return null; + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java b/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java new file mode 100644 index 0000000..612b625 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java @@ -0,0 +1,43 @@ +package net.szum123321.textile_backup.core.create; + +import net.szum123321.textile_backup.core.CompressionStatus; +import org.jetbrains.annotations.NotNull; + +import java.io.*; +import java.nio.file.Path; +import java.util.zip.Checksum; + +public class HashingInputStream extends FilterInputStream { + + private final Path path; + private final Checksum hasher; + private final CompressionStatus.Builder statusBuilder; + + public HashingInputStream(InputStream in, Path path, Checksum hasher, CompressionStatus.Builder statusBuilder) { + super(in); + this.hasher = hasher; + this.statusBuilder = statusBuilder; + this.path = path; + } + + @Override + public int read(byte @NotNull [] b, int off, int len) throws IOException { + int i = in.read(b, off, len); + if(i > -1) hasher.update(b, off, i); + return i; + } + + @Override + public int read() throws IOException { + int i = in.read(); + if(i > -1) hasher.update(i); + return i; + } + + @Override + public void close() throws IOException { + if(in.available() == 0) statusBuilder.update(path, hasher.getValue()); + else statusBuilder.update(path, hasher.getValue(), new RuntimeException("AAAaa")); + super.close(); + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java new file mode 100644 index 0000000..246df54 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java @@ -0,0 +1,14 @@ +package net.szum123321.textile_backup.core.create; + +import org.apache.commons.compress.parallel.InputStreamSupplier; + +import java.io.IOException; +import java.io.InputStream; +import java.nio.file.Path; + +public interface InputSupplier extends InputStreamSupplier { + InputStream getInputStream() throws IOException; + Path getPath(); + + String getName(); +} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index 45e5f65..696bada 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -21,9 +21,12 @@ package net.szum123321.textile_backup.core.create.compressors; import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileLogger; import net.szum123321.textile_backup.core.ActionInitiator; +import net.szum123321.textile_backup.core.CompressionStatus; import net.szum123321.textile_backup.core.NoSpaceLeftOnDeviceException; import net.szum123321.textile_backup.core.Utilities; import net.szum123321.textile_backup.core.create.BackupContext; +import net.szum123321.textile_backup.core.create.FileInputStreamSupplier; +import net.szum123321.textile_backup.core.create.InputSupplier; import java.io.*; import java.nio.file.Files; @@ -47,12 +50,13 @@ public abstract class AbstractCompressor { OutputStream arc = createArchiveOutputStream(bufferedOutputStream, ctx, coreLimit); Stream fileStream = Files.walk(inputFile)) { + CompressionStatus.Builder statusBuilder = new CompressionStatus.Builder(); + fileStream .filter(path -> !Utilities.isBlacklisted(inputFile.relativize(path))) .filter(Files::isRegularFile).forEach(file -> { try { - //hopefully one broken file won't spoil the whole archive - addEntry(file, inputFile.relativize(file).toString(), arc); + addEntry(new FileInputStreamSupplier(file, inputFile.relativize(file).toString(), statusBuilder), arc); } catch (IOException e) { log.error("An exception occurred while trying to compress: {}", inputFile.relativize(file).toString(), e); @@ -61,6 +65,13 @@ public abstract class AbstractCompressor { } }); + //Serialize using gson? + ByteArrayOutputStream bo = new ByteArrayOutputStream(); + ObjectOutputStream o = new ObjectOutputStream(bo); + o.writeObject(statusBuilder.build()); + + addEntry(new StatusFileInputSupplier(bo.toByteArray(), bo.size()), arc); + finish(arc); } catch(NoSpaceLeftOnDeviceException e) { log.error(""" @@ -88,7 +99,7 @@ public abstract class AbstractCompressor { } protected abstract OutputStream createArchiveOutputStream(OutputStream stream, BackupContext ctx, int coreLimit) throws IOException; - protected abstract void addEntry(Path file, String entryName, OutputStream arc) throws IOException; + protected abstract void addEntry(InputSupplier inputSupplier, OutputStream arc) throws IOException; protected void finish(OutputStream arc) throws InterruptedException, ExecutionException, IOException { //This function is only needed for the ParallelZipCompressor to write out ParallelScatterZipCreator @@ -97,4 +108,20 @@ public abstract class AbstractCompressor { protected void close() { //Same as above, just for ParallelGzipCompressor to shut down ExecutorService } -} + + private record StatusFileInputSupplier(byte[] data, int len) implements InputSupplier { + private final static String NAME = "textile_status.data"; + + @Override + public InputStream getInputStream() { return new ByteArrayInputStream(data, 0, len); } + + @Override + public Path getPath() { return Path.of(NAME); } + + @Override + public String getName() { return NAME; } + + @Override + public InputStream get() { return new ByteArrayInputStream(data, 0, len); } + } + } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java index 7933067..8d6f941 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java @@ -22,12 +22,11 @@ import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileLogger; import net.szum123321.textile_backup.core.NoSpaceLeftOnDeviceException; import net.szum123321.textile_backup.core.create.BackupContext; +import net.szum123321.textile_backup.core.create.InputSupplier; import org.apache.commons.compress.archivers.zip.*; -import org.apache.commons.compress.parallel.InputStreamSupplier; import java.io.*; import java.nio.file.Files; -import java.nio.file.Path; import java.util.Objects; import java.util.concurrent.*; import java.util.zip.ZipEntry; @@ -67,19 +66,19 @@ public class ParallelZipCompressor extends ZipCompressor { } @Override - protected void addEntry(Path file, String entryName, OutputStream arc) throws IOException { - ZipArchiveEntry entry = (ZipArchiveEntry)((ZipArchiveOutputStream)arc).createArchiveEntry(file, entryName); + protected void addEntry(InputSupplier input, OutputStream arc) throws IOException { + ZipArchiveEntry entry = (ZipArchiveEntry)((ZipArchiveOutputStream)arc).createArchiveEntry(input.getPath(), input.getName()); - if(ZipCompressor.isDotDat(file.getFileName().toString())) { + if(ZipCompressor.isDotDat(input.getPath().getFileName().toString())) { entry.setMethod(ZipEntry.STORED); - entry.setSize(Files.size(file)); - entry.setCompressedSize(Files.size(file)); - entry.setCrc(getCRC(file)); + entry.setSize(Files.size(input.getPath())); + entry.setCompressedSize(Files.size(input.getPath())); + entry.setCrc(getCRC(input.getPath())); } else entry.setMethod(ZipEntry.DEFLATED); entry.setTime(System.currentTimeMillis()); - scatterZipCreator.addArchiveEntry(entry, new FileInputStreamSupplier(file)); + scatterZipCreator.addArchiveEntry(entry, input); } @Override @@ -127,16 +126,4 @@ public class ParallelZipCompressor extends ZipCompressor { return isNative == that.isNative && Objects.equals(className, that.className) && Objects.equals(methodName, that.methodName); } } - - record FileInputStreamSupplier(Path sourceFile) implements InputStreamSupplier { - public InputStream get() { - try { - return Files.newInputStream(sourceFile); - } catch (IOException e) { - log.error("An exception occurred while trying to create an input stream from file: {}!", sourceFile.toString(), e); - } - - return null; - } - } } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java index 0210708..5b784ac 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java @@ -21,6 +21,7 @@ package net.szum123321.textile_backup.core.create.compressors; import net.szum123321.textile_backup.config.ConfigHelper; import net.szum123321.textile_backup.core.Utilities; import net.szum123321.textile_backup.core.create.BackupContext; +import net.szum123321.textile_backup.core.create.InputSupplier; import org.apache.commons.compress.archivers.zip.Zip64Mode; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; @@ -54,15 +55,15 @@ public class ZipCompressor extends AbstractCompressor { } @Override - protected void addEntry(Path file, String entryName, OutputStream arc) throws IOException { - try (InputStream fileInputStream = Files.newInputStream(file)){ - ZipArchiveEntry entry = (ZipArchiveEntry)((ZipArchiveOutputStream)arc).createArchiveEntry(file, entryName); + protected void addEntry(InputSupplier input, OutputStream arc) throws IOException { + try (InputStream fileInputStream = input.getInputStream()) { + ZipArchiveEntry entry = (ZipArchiveEntry)((ZipArchiveOutputStream)arc).createArchiveEntry(input.getPath(), input.getName()); - if(isDotDat(file.getFileName().toString())) { + if(isDotDat(input.getPath().getFileName().toString())) { entry.setMethod(ZipEntry.STORED); - entry.setSize(Files.size(file)); - entry.setCompressedSize(Files.size(file)); - entry.setCrc(getCRC(file)); + entry.setSize(Files.size(input.getPath())); + entry.setCompressedSize(Files.size(input.getPath())); + entry.setCrc(getCRC(input.getPath())); } ((ZipArchiveOutputStream)arc).putArchiveEntry(entry); diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java index 04489b4..0cae730 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java @@ -20,13 +20,12 @@ package net.szum123321.textile_backup.core.create.compressors.tar; import net.szum123321.textile_backup.core.create.BackupContext; import net.szum123321.textile_backup.core.create.compressors.AbstractCompressor; +import net.szum123321.textile_backup.core.create.InputSupplier; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.utils.IOUtils; import java.io.*; -import java.nio.file.Files; -import java.nio.file.Path; public class AbstractTarArchiver extends AbstractCompressor { protected OutputStream getCompressorOutputStream(OutputStream stream, BackupContext ctx, int coreLimit) throws IOException { @@ -43,9 +42,9 @@ public class AbstractTarArchiver extends AbstractCompressor { } @Override - protected void addEntry(Path file, String entryName, OutputStream arc) throws IOException { - try (InputStream fileInputStream = Files.newInputStream(file)){ - TarArchiveEntry entry = (TarArchiveEntry)((TarArchiveOutputStream) arc).createArchiveEntry(file, entryName); + protected void addEntry(InputSupplier in, OutputStream arc) throws IOException { + try (InputStream fileInputStream = in.getInputStream()) { + TarArchiveEntry entry = (TarArchiveEntry)((TarArchiveOutputStream) arc).createArchiveEntry(in.getPath(), in.getName()); ((TarArchiveOutputStream)arc).putArchiveEntry(entry); IOUtils.copy(fileInputStream, arc); From c816c70a6b0178447cd1b5de3971eacece9fe898 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 22 Nov 2022 14:16:12 +0100 Subject: [PATCH 02/44] added FileTreeHashBuilder for intelligently building single hash of file tree --- .../core/FileTreeHashBuilder.java | 66 +++++++++++++++++++ .../core/create/FileInputStreamSupplier.java | 19 +++++- .../core/create/HashingInputStream.java | 18 +++++ .../core/create/InputSupplier.java | 20 +++++- .../core/restore/HashingOutputStream.java | 55 ++++++++++++++++ 5 files changed, 175 insertions(+), 3 deletions(-) create mode 100644 src/main/java/net/szum123321/textile_backup/core/FileTreeHashBuilder.java create mode 100644 src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java diff --git a/src/main/java/net/szum123321/textile_backup/core/FileTreeHashBuilder.java b/src/main/java/net/szum123321/textile_backup/core/FileTreeHashBuilder.java new file mode 100644 index 0000000..a6084e6 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/FileTreeHashBuilder.java @@ -0,0 +1,66 @@ +/* + A simple backup mod for Fabric + Copyright (C) 2022 Szum123321 + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +package net.szum123321.textile_backup.core; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.function.Supplier; +import java.util.zip.Checksum; + +public class FileTreeHashBuilder { + private final static ThreadLocal buff = + ThreadLocal.withInitial(() -> new byte[Long.BYTES]); + private final Object lock = new Object(); + private final Supplier hasherProvider; + private long hash = 0, filesProcessed = 0, filesTotalSize = 0; + + public FileTreeHashBuilder(Supplier provider) { hasherProvider = provider; } + + public void update(Path path, long newHash) throws IOException { + byte[] raw = buff.get(); + var hasher = hasherProvider.get(); + + long size = Files.size(path); + + hasher.update(ByteBuffer.wrap(raw).putLong(size).array()); + hasher.update(path.toString().getBytes(StandardCharsets.UTF_8)); + hasher.update(ByteBuffer.wrap(raw).putLong(hash).array()); + + synchronized (lock) { + //This way exact order of files processed doesn't matter. + this.hash ^= hasher.getValue(); + filesProcessed++; + filesTotalSize += size; + } + } + + public long getValue() { + var hasher = hasherProvider.get(); + byte[] raw = buff.get(); + + hasher.update(ByteBuffer.wrap(raw).putLong(hash).array()); + hasher.update(ByteBuffer.wrap(raw).putLong(filesProcessed).array()); + hasher.update(ByteBuffer.wrap(raw).putLong(filesTotalSize).array()); + + return hasher.getValue(); + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java index 465c004..1f9daf8 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java @@ -1,3 +1,21 @@ +/* + A simple backup mod for Fabric + Copyright (C) 2022 Szum123321 + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + package net.szum123321.textile_backup.core.create; import net.szum123321.textile_backup.TextileBackup; @@ -16,7 +34,6 @@ public record FileInputStreamSupplier(Path path, String name, CompressionStatus. @Override public InputStream getInputStream() throws IOException { try { - //TODO: put in hasher return new HashingInputStream(Files.newInputStream(path), path, null, builder); } catch (IOException e) { builder.update(path, e); diff --git a/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java b/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java index 612b625..0b50904 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java @@ -1,3 +1,21 @@ +/* + A simple backup mod for Fabric + Copyright (C) 2022 Szum123321 + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + package net.szum123321.textile_backup.core.create; import net.szum123321.textile_backup.core.CompressionStatus; diff --git a/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java index 246df54..e90e8d4 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java @@ -1,3 +1,21 @@ +/* + A simple backup mod for Fabric + Copyright (C) 2022 Szum123321 + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + package net.szum123321.textile_backup.core.create; import org.apache.commons.compress.parallel.InputStreamSupplier; @@ -5,10 +23,8 @@ import org.apache.commons.compress.parallel.InputStreamSupplier; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; - public interface InputSupplier extends InputStreamSupplier { InputStream getInputStream() throws IOException; Path getPath(); - String getName(); } diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java b/src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java new file mode 100644 index 0000000..6757fa4 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java @@ -0,0 +1,55 @@ +/* + A simple backup mod for Fabric + Copyright (C) 2022 Szum123321 + + This program is free software: you can redistribute it and/or modify + it under the terms of the GNU General Public License as published by + the Free Software Foundation, either version 3 of the License, or + (at your option) any later version. + + This program is distributed in the hope that it will be useful, + but WITHOUT ANY WARRANTY; without even the implied warranty of + MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + GNU General Public License for more details. + + You should have received a copy of the GNU General Public License + along with this program. If not, see . +*/ + +package net.szum123321.textile_backup.core.restore; + +import org.jetbrains.annotations.NotNull; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; +import java.util.zip.Checksum; + +public class HashingOutputStream extends FilterOutputStream { + private final Path path; + private final Checksum hasher; + + public HashingOutputStream(OutputStream out, Path path, Checksum hasher) { + super(out); + this.path = path; + this.hasher = hasher; + } + + @Override + public void write(int b) throws IOException { + super.write(b); + hasher.update(b); + } + + @Override + public void write(byte[] @NotNull b, int off, int len) throws IOException { + super.write(b, off, len); + hasher.update(b, off, len); + } + + @Override + public void close() throws IOException { + super.close(); + } +} From 85452efbcafbcd0ed3e42e1d84ccc21628551824 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 22 Nov 2022 15:09:21 +0100 Subject: [PATCH 03/44] , --- .../core/CompressionStatus.java | 3 +- .../compressors/AbstractCompressor.java | 6 +-- .../core/restore/RestoreBackupRunnable.java | 48 +++++++++++++++---- 3 files changed, 43 insertions(+), 14 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java index 0f7a459..a3481e3 100644 --- a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -5,7 +5,8 @@ import java.nio.file.Path; import java.time.LocalDateTime; public record CompressionStatus(long[] treeHash, LocalDateTime date, long startTimestamp, long finishTimestamp, boolean ok, Path[] brokenFiles) implements Serializable { - + public static final String DATA_FILENAME = "textile_status.data"; + public boolean isValid(long decompressedHash) { return true; } public static class Builder { public synchronized void update(Path path, long hash, Exception error) { throw new RuntimeException("UNIMPLEMENTED!"); } public synchronized void update(Path path, Exception error) { throw new RuntimeException("UNIMPLEMENTED!"); } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index 696bada..7904234 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -110,16 +110,14 @@ public abstract class AbstractCompressor { } private record StatusFileInputSupplier(byte[] data, int len) implements InputSupplier { - private final static String NAME = "textile_status.data"; - @Override public InputStream getInputStream() { return new ByteArrayInputStream(data, 0, len); } @Override - public Path getPath() { return Path.of(NAME); } + public Path getPath() { return Path.of(CompressionStatus.DATA_FILENAME); } @Override - public String getName() { return NAME; } + public String getName() { return CompressionStatus.DATA_FILENAME; } @Override public InputStream get() { return new ByteArrayInputStream(data, 0, len); } diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java index f2dd22d..c3f5ccf 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java @@ -24,6 +24,7 @@ import net.szum123321.textile_backup.TextileLogger; import net.szum123321.textile_backup.config.ConfigHelper; import net.szum123321.textile_backup.config.ConfigPOJO; import net.szum123321.textile_backup.core.ActionInitiator; +import net.szum123321.textile_backup.core.CompressionStatus; import net.szum123321.textile_backup.core.LivingServer; import net.szum123321.textile_backup.core.Utilities; import net.szum123321.textile_backup.core.create.BackupContext; @@ -32,6 +33,8 @@ import net.szum123321.textile_backup.core.restore.decompressors.GenericTarDecomp import net.szum123321.textile_backup.core.restore.decompressors.ZipDecompressor; import java.io.IOException; +import java.io.InputStream; +import java.io.ObjectInputStream; import java.nio.file.Files; import java.nio.file.Path; @@ -66,13 +69,22 @@ public class RestoreBackupRunnable implements Runnable { ).run(); } - Path worldFile = Utilities.getWorldFolder(ctx.server()); + Path worldFile = Utilities.getWorldFolder(ctx.server()), tmp = null; try { - Path tmp = Files.createTempDirectory( - worldFile.getParent(), + tmp = Files.createTempDirectory( + ctx.server().getRunDirectory().toPath(), ctx.restoreableFile().getFile().getFileName().toString()); + } catch (IOException e) { + throw new RuntimeException(e); + } + if(tmp == null) { + //TODO: log error! + return; + } + + try { log.info("Starting decompression..."); if (ctx.restoreableFile().getArchiveFormat() == ConfigPOJO.ArchiveFormat.ZIP) @@ -80,18 +92,36 @@ public class RestoreBackupRunnable implements Runnable { else GenericTarDecompressor.decompress(ctx.restoreableFile().getFile(), tmp); - log.info("Deleting old world..."); + CompressionStatus status = null; - Utilities.deleteDirectory(worldFile); - Files.move(tmp, worldFile); + try (InputStream in = Files.newInputStream(tmp.resolve(CompressionStatus.DATA_FILENAME))) { + ObjectInputStream objectInputStream = new ObjectInputStream(in); + status = (CompressionStatus)objectInputStream.readObject(); + } catch (ClassNotFoundException e) { + throw new RuntimeException(e); + } - if (config.get().deleteOldBackupAfterRestore) { - log.info("Deleting old backup"); + if(status.isValid(0)) { + log.info("Deleting old world..."); - Files.delete(ctx.restoreableFile().getFile()); + Utilities.deleteDirectory(worldFile); + Files.move(tmp, worldFile); + + if (config.get().deleteOldBackupAfterRestore) { + log.info("Deleting old backup"); + Files.delete(ctx.restoreableFile().getFile()); + } } } catch (IOException e) { log.error("An exception occurred while trying to restore a backup!", e); + } finally { + if(Files.exists(tmp)) { + try { + Utilities.deleteDirectory(tmp); + } catch (IOException e) { + //TODO: Log error! + } + } } //in case we're playing on client From 4007d8f86d117f1c7e0f0c15333423b6fd93a8be Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Thu, 24 Nov 2022 00:38:55 +0100 Subject: [PATCH 04/44] Do I really need hashes? idk.... Either way it has to get ugly before it gets beautiful ... I think --- .../textile_backup/config/ConfigPOJO.java | 14 ++++ .../core/CompressionStatus.java | 27 ++++-- .../core/DataLeftException.java | 25 ++++++ .../core/create/BrokenFileHandler.java | 36 ++++++++ .../core/create/FileInputStreamSupplier.java | 41 +++++----- .../core/create/HashingInputStream.java | 51 ++++++------ .../core/create/MakeBackupRunnable.java | 34 ++++---- .../compressors/AbstractCompressor.java | 82 +++++++++++-------- .../compressors/ParallelZipCompressor.java | 4 +- .../assets/textile_backup/lang/en_us.json | 3 + src/main/resources/fabric.mod.json | 2 +- 11 files changed, 217 insertions(+), 102 deletions(-) create mode 100644 src/main/java/net/szum123321/textile_backup/core/DataLeftException.java create mode 100644 src/main/java/net/szum123321/textile_backup/core/create/BrokenFileHandler.java diff --git a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java index d6ad376..be45b40 100644 --- a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java +++ b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java @@ -123,6 +123,14 @@ public class ConfigPOJO implements ConfigData { @ConfigEntry.Gui.EnumHandler(option = ConfigEntry.Gui.EnumHandler.EnumDisplayOption.BUTTON) public ArchiveFormat format = ArchiveFormat.ZIP; + @Comment(""" + The Strict mode (default) aborts backup creation in case of any problem and deletes the created files + Permissible mode keeps partial/damaged backup but won't allow to restore it + Very Permissible mode will skip the verification process. THIS MOST CERTAINLY WILL LEAD TO DATA LOSS OR CORRUPTION + """) + @ConfigEntry.Gui.EnumHandler(option = ConfigEntry.Gui.EnumHandler.EnumDisplayOption.BUTTON) + public ErrorHandlingMode errorErrorHandlingMode = ErrorHandlingMode.STRICT; + @Comment("\nMinimal permission level required to run commands\n") @ConfigEntry.Category("Manage") @ConfigEntry.Gui.NoTooltip() @@ -177,6 +185,12 @@ public class ConfigPOJO implements ConfigData { } } + public enum ErrorHandlingMode { + STRICT, + PERMISSIBLE, + VERY_PERMISSIBLE + } + public enum ArchiveFormat { ZIP("zip"), GZIP("tar", "gz"), diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java index a3481e3..c16fb90 100644 --- a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -1,17 +1,30 @@ +/* + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + package net.szum123321.textile_backup.core; import java.io.Serializable; import java.nio.file.Path; import java.time.LocalDateTime; +import java.util.Map; -public record CompressionStatus(long[] treeHash, LocalDateTime date, long startTimestamp, long finishTimestamp, boolean ok, Path[] brokenFiles) implements Serializable { +public record CompressionStatus(long treeHash, LocalDateTime date, long startTimestamp, long finishTimestamp, Map brokenFiles) implements Serializable { public static final String DATA_FILENAME = "textile_status.data"; public boolean isValid(long decompressedHash) { return true; } - public static class Builder { - public synchronized void update(Path path, long hash, Exception error) { throw new RuntimeException("UNIMPLEMENTED!"); } - public synchronized void update(Path path, Exception error) { throw new RuntimeException("UNIMPLEMENTED!"); } - public synchronized void update(Path path, long hash) { throw new RuntimeException("UNIMPLEMENTED!"); } - public CompressionStatus build() { throw new RuntimeException("UNIMPLEMENTED!"); } - } } diff --git a/src/main/java/net/szum123321/textile_backup/core/DataLeftException.java b/src/main/java/net/szum123321/textile_backup/core/DataLeftException.java new file mode 100644 index 0000000..cc5c7b9 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/DataLeftException.java @@ -0,0 +1,25 @@ +/* + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package net.szum123321.textile_backup.core; + +import java.io.IOException; + +public class DataLeftException extends IOException { + public DataLeftException(long n) { super("Input stream closed with " + n + " bytes left!"); } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/BrokenFileHandler.java b/src/main/java/net/szum123321/textile_backup/core/create/BrokenFileHandler.java new file mode 100644 index 0000000..be24aa9 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/create/BrokenFileHandler.java @@ -0,0 +1,36 @@ +/* + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package net.szum123321.textile_backup.core.create; + +import org.spongepowered.include.com.google.common.collect.Maps; + +import java.nio.file.Path; +import java.util.HashMap; +import java.util.Map; + +public class BrokenFileHandler { + private final HashMap store = Maps.newHashMap(); + public void handle(Path file, Exception e) { store.put(file, e); } + + public boolean valid() { return store.isEmpty(); } + + public Map get() { + return store; + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java index 1f9daf8..695294e 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java @@ -1,26 +1,26 @@ /* - A simple backup mod for Fabric - Copyright (C) 2022 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ package net.szum123321.textile_backup.core.create; import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileLogger; -import net.szum123321.textile_backup.core.CompressionStatus; +import net.szum123321.textile_backup.core.FileTreeHashBuilder; import java.io.IOException; import java.io.InputStream; @@ -28,15 +28,16 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; -public record FileInputStreamSupplier(Path path, String name, CompressionStatus.Builder builder) implements InputSupplier { +public record FileInputStreamSupplier(Path path, String name, FileTreeHashBuilder hashTreeBuilder, BrokenFileHandler brokenFileHandler) implements InputSupplier { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); @Override public InputStream getInputStream() throws IOException { try { - return new HashingInputStream(Files.newInputStream(path), path, null, builder); + //TODO: select hashing algorithm! + return new HashingInputStream(Files.newInputStream(path), path, null, hashTreeBuilder, brokenFileHandler); } catch (IOException e) { - builder.update(path, e); + brokenFileHandler.handle(path, e); throw e; } } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java b/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java index 0b50904..532abca 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java @@ -1,41 +1,46 @@ /* - A simple backup mod for Fabric - Copyright (C) 2022 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ package net.szum123321.textile_backup.core.create; -import net.szum123321.textile_backup.core.CompressionStatus; +import net.szum123321.textile_backup.core.DataLeftException; +import net.szum123321.textile_backup.core.FileTreeHashBuilder; import org.jetbrains.annotations.NotNull; import java.io.*; import java.nio.file.Path; import java.util.zip.Checksum; +//This class calculates a hash of the file on the input stream, submits it to FileTreeHashBuilder. +//In case the whole underlying stream hasn't been read, also puts it into BrokeFileHandler public class HashingInputStream extends FilterInputStream { private final Path path; private final Checksum hasher; - private final CompressionStatus.Builder statusBuilder; + private final FileTreeHashBuilder hashBuilder; + private final BrokenFileHandler brokenFileHandler; - public HashingInputStream(InputStream in, Path path, Checksum hasher, CompressionStatus.Builder statusBuilder) { + public HashingInputStream(InputStream in, Path path, Checksum hasher, FileTreeHashBuilder hashBuilder, BrokenFileHandler brokenFileHandler) { super(in); - this.hasher = hasher; - this.statusBuilder = statusBuilder; this.path = path; + this.hasher = hasher; + this.hashBuilder = hashBuilder; + this.brokenFileHandler = brokenFileHandler; } @Override @@ -54,8 +59,8 @@ public class HashingInputStream extends FilterInputStream { @Override public void close() throws IOException { - if(in.available() == 0) statusBuilder.update(path, hasher.getValue()); - else statusBuilder.update(path, hasher.getValue(), new RuntimeException("AAAaa")); + if(in.available() == 0) hashBuilder.update(path, hasher.getValue()); + else brokenFileHandler.handle(path, new DataLeftException(in.available())); super.close(); } } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java index f2184b9..e142173 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java @@ -1,20 +1,20 @@ /* - A simple backup mod for Fabric - Copyright (C) 2020 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ package net.szum123321.textile_backup.core.create; @@ -104,7 +104,7 @@ public class MakeBackupRunnable implements Runnable { case TAR -> new AbstractTarArchiver().createArchive(world, outFile, context, coreCount); } - new Cleanup(context.commandSource(), Utilities.getLevelName(context.server())).call(); + Globals.INSTANCE.getQueueExecutor().submit(new Cleanup(context.commandSource(), Utilities.getLevelName(context.server()))); if(config.get().broadcastBackupDone) { Utilities.notifyPlayers( diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index 7904234..f27a358 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,11 +20,11 @@ package net.szum123321.textile_backup.core.create.compressors; import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileLogger; -import net.szum123321.textile_backup.core.ActionInitiator; -import net.szum123321.textile_backup.core.CompressionStatus; -import net.szum123321.textile_backup.core.NoSpaceLeftOnDeviceException; -import net.szum123321.textile_backup.core.Utilities; +import net.szum123321.textile_backup.config.ConfigHelper; +import net.szum123321.textile_backup.config.ConfigPOJO; +import net.szum123321.textile_backup.core.*; import net.szum123321.textile_backup.core.create.BackupContext; +import net.szum123321.textile_backup.core.create.BrokenFileHandler; import net.szum123321.textile_backup.core.create.FileInputStreamSupplier; import net.szum123321.textile_backup.core.create.InputSupplier; @@ -45,32 +45,53 @@ public abstract class AbstractCompressor { public void createArchive(Path inputFile, Path outputFile, BackupContext ctx, int coreLimit) { Instant start = Instant.now(); + FileTreeHashBuilder fileHashBuilder = new FileTreeHashBuilder(() -> null); //TODO: select hashing algorithm + BrokenFileHandler brokenFileHandler = new BrokenFileHandler(); + + boolean keep = true; + try (OutputStream outStream = Files.newOutputStream(outputFile); BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream); OutputStream arc = createArchiveOutputStream(bufferedOutputStream, ctx, coreLimit); Stream fileStream = Files.walk(inputFile)) { - - CompressionStatus.Builder statusBuilder = new CompressionStatus.Builder(); - - fileStream + var it = fileStream .filter(path -> !Utilities.isBlacklisted(inputFile.relativize(path))) - .filter(Files::isRegularFile).forEach(file -> { - try { - addEntry(new FileInputStreamSupplier(file, inputFile.relativize(file).toString(), statusBuilder), arc); - } catch (IOException e) { - log.error("An exception occurred while trying to compress: {}", inputFile.relativize(file).toString(), e); + .filter(Files::isRegularFile).iterator(); - if (ctx.initiator() == ActionInitiator.Player) - log.sendError(ctx, "Something went wrong while compressing files!"); - } - }); + while(it.hasNext()) { + Path file = it.next(); + + try { + addEntry(new FileInputStreamSupplier(file, inputFile.relativize(file).toString(), fileHashBuilder, brokenFileHandler), arc); + } catch (Exception e) { + if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode == ConfigPOJO.ErrorHandlingMode.STRICT) { + keep = false; + break; + } + brokenFileHandler.handle(file, e); + } + } + + //If there are still files left in the stream, we should handle them + while(it.hasNext()) { + Path file = it.next(); + brokenFileHandler.handle(file, new DataLeftException(Files.size(file))); + } + + Instant now = Instant.now(); + + CompressionStatus status = new CompressionStatus ( + fileHashBuilder.getValue(), + null, start.toEpochMilli(), now.toEpochMilli(), + brokenFileHandler.get() + ); //Serialize using gson? - ByteArrayOutputStream bo = new ByteArrayOutputStream(); - ObjectOutputStream o = new ObjectOutputStream(bo); - o.writeObject(statusBuilder.build()); - - addEntry(new StatusFileInputSupplier(bo.toByteArray(), bo.size()), arc); + try (ByteArrayOutputStream bo = new ByteArrayOutputStream(); + ObjectOutputStream o = new ObjectOutputStream(bo)) { + o.writeObject(status); + addEntry(new StatusFileInputSupplier(bo.toByteArray()), arc); + } finish(arc); } catch(NoSpaceLeftOnDeviceException e) { @@ -85,16 +106,17 @@ public abstract class AbstractCompressor { log.sendError(ctx, "Backup failed. The file is corrupt."); log.error("For help see: https://github.com/Szum123321/textile_backup/wiki/ZIP-Problems"); } + if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode == ConfigPOJO.ErrorHandlingMode.STRICT) keep = false; } catch (IOException | InterruptedException | ExecutionException e) { log.error("An exception occurred!", e); if(ctx.initiator() == ActionInitiator.Player) log.sendError(ctx, "Something went wrong while compressing files!"); + if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode == ConfigPOJO.ErrorHandlingMode.STRICT) keep = false; + } finally { close(); } - // close(); - log.sendInfoAL(ctx, "Compression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now()))); } @@ -109,17 +131,13 @@ public abstract class AbstractCompressor { //Same as above, just for ParallelGzipCompressor to shut down ExecutorService } - private record StatusFileInputSupplier(byte[] data, int len) implements InputSupplier { - @Override - public InputStream getInputStream() { return new ByteArrayInputStream(data, 0, len); } + private record StatusFileInputSupplier(byte[] data) implements InputSupplier { + public InputStream getInputStream() { return new ByteArrayInputStream(data); } - @Override public Path getPath() { return Path.of(CompressionStatus.DATA_FILENAME); } - @Override public String getName() { return CompressionStatus.DATA_FILENAME; } - @Override - public InputStream get() { return new ByteArrayInputStream(data, 0, len); } + public InputStream get() { return getInputStream(); } } } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java index 8d6f941..7ca5f67 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -99,7 +99,7 @@ public class ParallelZipCompressor extends ZipCompressor { if(!STACKTRACE_NO_SPACE_ON_LEFT_ON_DEVICE[i].equals(cause.getStackTrace()[i])) match = false; - //For clarity' sake let's not throw the ExecutionException itself rather only the cause, as the EE is just the wrapper + //For clarity's sake let's not throw the ExecutionException itself rather only the cause, as the EE is just the wrapper if(match) throw new NoSpaceLeftOnDeviceException(cause); } } diff --git a/src/main/resources/assets/textile_backup/lang/en_us.json b/src/main/resources/assets/textile_backup/lang/en_us.json index 7f92d1b..0c9ae5e 100644 --- a/src/main/resources/assets/textile_backup/lang/en_us.json +++ b/src/main/resources/assets/textile_backup/lang/en_us.json @@ -47,6 +47,9 @@ "text.autoconfig.textile_backup.option.format": "Archive and compression format", "text.autoconfig.textile_backup.option.format.@Tooltip": "See: https://github.com/Szum123321/textile_backup/wiki/Configuration#format", + "text.autoconfig.textile_backup.option.errorErrorHandlingMode": "Compression error handling mode", + "text.autoconfig.textile_backup.option.errorErrorHandlingMode.@Tooltip": "DO NOT ALTER unless fully certain", + "text.autoconfig.textile_backup.option.permissionLevel": "Min permission level", "text.autoconfig.textile_backup.option.alwaysSingleplayerAllowed": "Always allow on single-player", diff --git a/src/main/resources/fabric.mod.json b/src/main/resources/fabric.mod.json index 2366667..a8b5d65 100644 --- a/src/main/resources/fabric.mod.json +++ b/src/main/resources/fabric.mod.json @@ -38,7 +38,7 @@ ], "depends": { - "fabricloader": ">=0.14.6", + "fabricloader": ">=0.14.0", "fabric": "*", "minecraft": ">=1.19.1", "cloth-config2": "*", From dbb9a71749f31d18a83f4ea5b3e4daf1b398e8a6 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Fri, 25 Nov 2022 09:54:44 +0100 Subject: [PATCH 05/44] Moved around error handling. LocalDateTime is now passed with BackupContext. Replaced equals method with matches in ParallelZipCompressor.SimpleStackTraceElement to avert warning. --- .../szum123321/textile_backup/Globals.java | 4 +- .../textile_backup/config/ConfigPOJO.java | 4 +- .../core/CompressionStatus.java | 18 ++++++- .../core/create/BackupContext.java | 7 ++- .../core/create/MakeBackupRunnable.java | 26 ++++++---- .../compressors/AbstractCompressor.java | 47 ++++++++----------- .../compressors/ParallelZipCompressor.java | 16 ++----- 7 files changed, 66 insertions(+), 56 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/Globals.java b/src/main/java/net/szum123321/textile_backup/Globals.java index 8d33251..b906076 100644 --- a/src/main/java/net/szum123321/textile_backup/Globals.java +++ b/src/main/java/net/szum123321/textile_backup/Globals.java @@ -84,8 +84,8 @@ public class Globals { public Optional getLockedFile() { return Optional.ofNullable(lockedPath); } public void setLockedFile(Path p) { lockedPath = p; } - public boolean disableTMPFS() { return disableTMPFiles; } - public void updateTMPFSFlag(MinecraftServer server) { + public synchronized boolean disableTMPFS() { return disableTMPFiles; } + public synchronized void updateTMPFSFlag(MinecraftServer server) { disableTMPFiles = false; Path tmp_dir = Path.of(System.getProperty("java.io.tmpdir")); if( diff --git a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java index be45b40..439b1d2 100644 --- a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java +++ b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java @@ -188,7 +188,9 @@ public class ConfigPOJO implements ConfigData { public enum ErrorHandlingMode { STRICT, PERMISSIBLE, - VERY_PERMISSIBLE + VERY_PERMISSIBLE; + + public boolean isStrict() { return this == STRICT; } } public enum ArchiveFormat { diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java index c16fb90..c21245c 100644 --- a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -18,7 +18,8 @@ package net.szum123321.textile_backup.core; -import java.io.Serializable; +import java.io.*; +import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; import java.util.Map; @@ -27,4 +28,19 @@ public record CompressionStatus(long treeHash, LocalDateTime date, long startTim public static final String DATA_FILENAME = "textile_status.data"; public boolean isValid(long decompressedHash) { return true; } + public static CompressionStatus readFromFile(Path f) throws IOException, ClassNotFoundException { + try(InputStream i = Files.newInputStream(f); + ObjectInputStream obj = new ObjectInputStream(i)) { + return (CompressionStatus) obj.readObject(); + } + } + + public byte[] serialize() throws IOException { + try (ByteArrayOutputStream bo = new ByteArrayOutputStream(); + ObjectOutputStream o = new ObjectOutputStream(bo)) { + o.writeObject(this); + return bo.toByteArray(); + } + } + } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java b/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java index 63a3e3e..99e1474 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java @@ -24,11 +24,14 @@ import net.minecraft.server.command.ServerCommandSource; import net.szum123321.textile_backup.core.ActionInitiator; import org.jetbrains.annotations.NotNull; +import java.time.LocalDateTime; + public record BackupContext(@NotNull MinecraftServer server, ServerCommandSource commandSource, ActionInitiator initiator, boolean save, - String comment) { + String comment, + LocalDateTime startDate) { public boolean startedByPlayer() { return initiator == ActionInitiator.Player; @@ -103,7 +106,7 @@ public record BackupContext(@NotNull MinecraftServer server, else throw new RuntimeException("Neither MinecraftServer or ServerCommandSource were provided!"); } - return new BackupContext(server, commandSource, initiator, save, comment); + return new BackupContext(server, commandSource, initiator, save, comment, LocalDateTime.now()); } } } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java index e142173..0c79657 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java @@ -36,7 +36,6 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; -import java.time.LocalDateTime; /** * The actual object responsible for creating the backup @@ -53,7 +52,14 @@ public class MakeBackupRunnable implements Runnable { @Override public void run() { + Path outFile = Utilities + .getBackupRootPath(Utilities.getLevelName(context.server())) + .resolve(getFileName()); + + log.trace("Outfile is: {}", outFile); + try { + //I think I should synchronise those two next calls... Utilities.disableWorldSaving(context.server()); Globals.INSTANCE.disableWatchdog = true; @@ -65,12 +71,6 @@ public class MakeBackupRunnable implements Runnable { log.trace("Minecraft world is: {}", world); - Path outFile = Utilities - .getBackupRootPath(Utilities.getLevelName(context.server())) - .resolve(getFileName()); - - log.trace("Outfile is: {}", outFile); - Files.createDirectories(outFile.getParent()); Files.createFile(outFile); @@ -118,6 +118,14 @@ public class MakeBackupRunnable implements Runnable { //ExecutorService swallows exception, so I need to catch everything log.error("An exception occurred when trying to create new backup file!", e); + if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode.isStrict()) { + try { + Files.delete(outFile); + } catch (IOException ex) { + log.error("An exception occurred while tryin go delete: {}", outFile, ex); + } + } + if(context.initiator() == ActionInitiator.Player) log.sendError(context, "An exception occurred when trying to create new backup file!"); } finally { @@ -127,9 +135,7 @@ public class MakeBackupRunnable implements Runnable { } private String getFileName(){ - LocalDateTime now = LocalDateTime.now(); - - return Utilities.getDateTimeFormatter().format(now) + + return Utilities.getDateTimeFormatter().format(context.startDate()) + (context.comment() != null ? "#" + context.comment().replaceAll("[\\\\/:*?\"<>|#]", "") : "") + config.get().format.getCompleteString(); } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index f27a358..b689ba7 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -21,7 +21,6 @@ package net.szum123321.textile_backup.core.create.compressors; import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileLogger; import net.szum123321.textile_backup.config.ConfigHelper; -import net.szum123321.textile_backup.config.ConfigPOJO; import net.szum123321.textile_backup.core.*; import net.szum123321.textile_backup.core.create.BackupContext; import net.szum123321.textile_backup.core.create.BrokenFileHandler; @@ -42,14 +41,12 @@ import java.util.stream.Stream; public abstract class AbstractCompressor { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); - public void createArchive(Path inputFile, Path outputFile, BackupContext ctx, int coreLimit) { + public void createArchive(Path inputFile, Path outputFile, BackupContext ctx, int coreLimit) throws IOException, ExecutionException, InterruptedException { Instant start = Instant.now(); FileTreeHashBuilder fileHashBuilder = new FileTreeHashBuilder(() -> null); //TODO: select hashing algorithm BrokenFileHandler brokenFileHandler = new BrokenFileHandler(); - boolean keep = true; - try (OutputStream outStream = Files.newOutputStream(outputFile); BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream); OutputStream arc = createArchiveOutputStream(bufferedOutputStream, ctx, coreLimit); @@ -62,39 +59,35 @@ public abstract class AbstractCompressor { Path file = it.next(); try { - addEntry(new FileInputStreamSupplier(file, inputFile.relativize(file).toString(), fileHashBuilder, brokenFileHandler), arc); - } catch (Exception e) { - if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode == ConfigPOJO.ErrorHandlingMode.STRICT) { - keep = false; - break; - } + addEntry( + new FileInputStreamSupplier( + file, + inputFile.relativize(file).toString(), + fileHashBuilder, + brokenFileHandler), + arc + ); + } catch (IOException e) { brokenFileHandler.handle(file, e); + if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode.isStrict()) throw e; + else log.sendErrorAL(ctx, "An exception occurred while trying to compress: {}", + inputFile.relativize(file).toString(), e + ); } } - //If there are still files left in the stream, we should handle them - while(it.hasNext()) { - Path file = it.next(); - brokenFileHandler.handle(file, new DataLeftException(Files.size(file))); - } - Instant now = Instant.now(); CompressionStatus status = new CompressionStatus ( fileHashBuilder.getValue(), - null, start.toEpochMilli(), now.toEpochMilli(), + ctx.startDate(), start.toEpochMilli(), now.toEpochMilli(), brokenFileHandler.get() ); - //Serialize using gson? - try (ByteArrayOutputStream bo = new ByteArrayOutputStream(); - ObjectOutputStream o = new ObjectOutputStream(bo)) { - o.writeObject(status); - addEntry(new StatusFileInputSupplier(bo.toByteArray()), arc); - } + addEntry(new StatusFileInputSupplier(status.serialize()), arc); finish(arc); - } catch(NoSpaceLeftOnDeviceException e) { + } /*catch(NoSpaceLeftOnDeviceException e) { log.error(""" CRITICAL ERROR OCCURRED! The backup is corrupt! @@ -106,14 +99,14 @@ public abstract class AbstractCompressor { log.sendError(ctx, "Backup failed. The file is corrupt."); log.error("For help see: https://github.com/Szum123321/textile_backup/wiki/ZIP-Problems"); } - if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode == ConfigPOJO.ErrorHandlingMode.STRICT) keep = false; + if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode.isStrict()) keep = false; } catch (IOException | InterruptedException | ExecutionException e) { log.error("An exception occurred!", e); if(ctx.initiator() == ActionInitiator.Player) log.sendError(ctx, "Something went wrong while compressing files!"); - if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode == ConfigPOJO.ErrorHandlingMode.STRICT) keep = false; + if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode.isStrict()) keep = false; - } finally { + } */finally { close(); } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java index 7ca5f67..f41f0b8 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java @@ -96,8 +96,7 @@ public class ParallelZipCompressor extends ZipCompressor { boolean match = (cause.getStackTrace().length >= STACKTRACE_NO_SPACE_ON_LEFT_ON_DEVICE.length); if(match) { for(int i = 0; i < STACKTRACE_NO_SPACE_ON_LEFT_ON_DEVICE.length && match; i++) - if(!STACKTRACE_NO_SPACE_ON_LEFT_ON_DEVICE[i].equals(cause.getStackTrace()[i])) match = false; - + if(!STACKTRACE_NO_SPACE_ON_LEFT_ON_DEVICE[i].matches(cause.getStackTrace()[i])) match = false; //For clarity's sake let's not throw the ExecutionException itself rather only the cause, as the EE is just the wrapper if(match) throw new NoSpaceLeftOnDeviceException(cause); @@ -113,17 +112,8 @@ public class ParallelZipCompressor extends ZipCompressor { String methodName, boolean isNative ) { - @Override - public boolean equals(Object o) { - if (this == o) return true; - if (o == null) return false; - if(o.getClass() == StackTraceElement.class) { - StackTraceElement that = (StackTraceElement) o; - return (isNative == that.isNativeMethod()) && Objects.equals(className, that.getClassName()) && Objects.equals(methodName, that.getMethodName()); - } - if(getClass() != o.getClass()) return false; - SimpleStackTraceElement that = (SimpleStackTraceElement) o; - return isNative == that.isNative && Objects.equals(className, that.className) && Objects.equals(methodName, that.methodName); + public boolean matches(StackTraceElement o) { + return (isNative == o.isNativeMethod()) && Objects.equals(className, o.getClassName()) && Objects.equals(methodName, o.getMethodName()); } } } From 9c37affacd4b06d41fd773a1339090ee92eb7834 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Fri, 25 Nov 2022 14:00:18 +0100 Subject: [PATCH 06/44] Backup integrity is now checked --- .../textile_backup/TextileBackup.java | 18 ++-- .../commands/create/StartBackupCommand.java | 27 +++--- .../textile_backup/config/ConfigPOJO.java | 2 + .../core/CompressionStatus.java | 4 +- .../core/create/MakeBackupRunnable.java | 13 ++- .../create/MakeBackupRunnableFactory.java | 11 +-- .../core/restore/HashingOutputStream.java | 8 +- .../core/restore/RestoreBackupRunnable.java | 86 ++++++++----------- .../decompressors/GenericTarDecompressor.java | 16 ++-- .../decompressors/ZipDecompressor.java | 11 ++- 10 files changed, 101 insertions(+), 95 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/TextileBackup.java b/src/main/java/net/szum123321/textile_backup/TextileBackup.java index d1e313b..60d7e11 100644 --- a/src/main/java/net/szum123321/textile_backup/TextileBackup.java +++ b/src/main/java/net/szum123321/textile_backup/TextileBackup.java @@ -68,14 +68,16 @@ public class TextileBackup implements ModInitializer { Globals.INSTANCE.shutdownQueueExecutor(60000); if (config.get().shutdownBackup && Globals.INSTANCE.globalShutdownBackupFlag.get()) { - MakeBackupRunnableFactory.create( - BackupContext.Builder - .newBackupContextBuilder() - .setServer(server) - .setInitiator(ActionInitiator.Shutdown) - .setComment("shutdown") - .build() - ).run(); + try { + MakeBackupRunnableFactory.create( + BackupContext.Builder + .newBackupContextBuilder() + .setServer(server) + .setInitiator(ActionInitiator.Shutdown) + .setComment("shutdown") + .build() + ).call(); + } catch (Exception ignored) {} } }); diff --git a/src/main/java/net/szum123321/textile_backup/commands/create/StartBackupCommand.java b/src/main/java/net/szum123321/textile_backup/commands/create/StartBackupCommand.java index 830a88a..03ddb6a 100644 --- a/src/main/java/net/szum123321/textile_backup/commands/create/StartBackupCommand.java +++ b/src/main/java/net/szum123321/textile_backup/commands/create/StartBackupCommand.java @@ -41,22 +41,17 @@ public class StartBackupCommand { } private static int execute(ServerCommandSource source, @Nullable String comment) { - try { - Globals.INSTANCE.getQueueExecutor().submit( - MakeBackupRunnableFactory.create( - BackupContext.Builder - .newBackupContextBuilder() - .setCommandSource(source) - .setComment(comment) - .guessInitiator() - .saveServer() - .build() - ) - ); - } catch (Exception e) { - log.error("Something went wrong while executing command!", e); - throw e; - } + Globals.INSTANCE.getQueueExecutor().submit( + MakeBackupRunnableFactory.create( + BackupContext.Builder + .newBackupContextBuilder() + .setCommandSource(source) + .setComment(comment) + .guessInitiator() + .saveServer() + .build() + ) + ); return 1; } diff --git a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java index 439b1d2..690f653 100644 --- a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java +++ b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java @@ -191,6 +191,8 @@ public class ConfigPOJO implements ConfigData { VERY_PERMISSIBLE; public boolean isStrict() { return this == STRICT; } + + public boolean verify() { return this != VERY_PERMISSIBLE; } } public enum ArchiveFormat { diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java index c21245c..1e9803f 100644 --- a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -28,8 +28,8 @@ public record CompressionStatus(long treeHash, LocalDateTime date, long startTim public static final String DATA_FILENAME = "textile_status.data"; public boolean isValid(long decompressedHash) { return true; } - public static CompressionStatus readFromFile(Path f) throws IOException, ClassNotFoundException { - try(InputStream i = Files.newInputStream(f); + public static CompressionStatus readFromFile(Path folder) throws IOException, ClassNotFoundException { + try(InputStream i = Files.newInputStream(folder.resolve(DATA_FILENAME)); ObjectInputStream obj = new ObjectInputStream(i)) { return (CompressionStatus) obj.readObject(); } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java index 0c79657..fd3472f 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java @@ -36,11 +36,13 @@ import java.io.IOException; import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; /** * The actual object responsible for creating the backup */ -public class MakeBackupRunnable implements Runnable { +public class MakeBackupRunnable implements Callable { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); private final static ConfigHelper config = ConfigHelper.INSTANCE; @@ -49,9 +51,8 @@ public class MakeBackupRunnable implements Runnable { public MakeBackupRunnable(BackupContext context) { this.context = context; } - @Override - public void run() { + public Void call() throws IOException, ExecutionException, InterruptedException { Path outFile = Utilities .getBackupRootPath(Utilities.getLevelName(context.server())) .resolve(getFileName()); @@ -114,7 +115,7 @@ public class MakeBackupRunnable implements Runnable { } else { log.sendInfoAL(context, "Done!"); } - } catch (Throwable e) { + } catch (InterruptedException | ExecutionException | IOException e) { //ExecutorService swallows exception, so I need to catch everything log.error("An exception occurred when trying to create new backup file!", e); @@ -128,10 +129,14 @@ public class MakeBackupRunnable implements Runnable { if(context.initiator() == ActionInitiator.Player) log.sendError(context, "An exception occurred when trying to create new backup file!"); + + throw e; } finally { Utilities.enableWorldSaving(context.server()); Globals.INSTANCE.disableWatchdog = false; } + + return null; } private String getFileName(){ diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnableFactory.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnableFactory.java index 36d8eb6..2bcbb2e 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnableFactory.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnableFactory.java @@ -25,12 +25,13 @@ import net.szum123321.textile_backup.config.ConfigHelper; import net.szum123321.textile_backup.core.Utilities; import java.time.LocalDateTime; +import java.util.concurrent.Callable; public class MakeBackupRunnableFactory { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); private final static ConfigHelper config = ConfigHelper.INSTANCE; - public static Runnable create(BackupContext ctx) { + public static Callable create(BackupContext ctx) { if(config.get().broadcastBackupStart) { Utilities.notifyPlayers(ctx.server(), "Warning! Server backup will begin shortly. You may experience some lag." @@ -58,13 +59,7 @@ public class MakeBackupRunnableFactory { if (ctx.shouldSave()) { log.sendInfoAL(ctx, "Saving server..."); - ctx.server().getPlayerManager().saveAllPlayerData(); - - try { - ctx.server().save(false, true, true); - } catch (Exception e) { - log.sendErrorAL(ctx,"An exception occurred when trying to save the world!"); - } + ctx.server().saveAll(true, true, false); } return new MakeBackupRunnable(ctx); diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java b/src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java index 6757fa4..6f9b3c8 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java @@ -18,6 +18,7 @@ package net.szum123321.textile_backup.core.restore; +import net.szum123321.textile_backup.core.FileTreeHashBuilder; import org.jetbrains.annotations.NotNull; import java.io.FilterOutputStream; @@ -30,12 +31,16 @@ public class HashingOutputStream extends FilterOutputStream { private final Path path; private final Checksum hasher; - public HashingOutputStream(OutputStream out, Path path, Checksum hasher) { + private final FileTreeHashBuilder hashBuilder; + + public HashingOutputStream(OutputStream out, Path path, Checksum hasher, FileTreeHashBuilder hashBuilder) { super(out); this.path = path; this.hasher = hasher; + this.hashBuilder = hashBuilder; } + @Override public void write(int b) throws IOException { super.write(b); @@ -51,5 +56,6 @@ public class HashingOutputStream extends FilterOutputStream { @Override public void close() throws IOException { super.close(); + hashBuilder.update(path, hasher.getValue()); } } diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java index c3f5ccf..5821bbc 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java @@ -25,7 +25,6 @@ import net.szum123321.textile_backup.config.ConfigHelper; import net.szum123321.textile_backup.config.ConfigPOJO; import net.szum123321.textile_backup.core.ActionInitiator; import net.szum123321.textile_backup.core.CompressionStatus; -import net.szum123321.textile_backup.core.LivingServer; import net.szum123321.textile_backup.core.Utilities; import net.szum123321.textile_backup.core.create.BackupContext; import net.szum123321.textile_backup.core.create.MakeBackupRunnableFactory; @@ -33,12 +32,11 @@ import net.szum123321.textile_backup.core.restore.decompressors.GenericTarDecomp import net.szum123321.textile_backup.core.restore.decompressors.ZipDecompressor; import java.io.IOException; -import java.io.InputStream; -import java.io.ObjectInputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.concurrent.ExecutionException; +import java.util.concurrent.FutureTask; -//TODO: Verify backup's validity? public class RestoreBackupRunnable implements Runnable { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); private final static ConfigHelper config = ConfigHelper.INSTANCE; @@ -56,71 +54,73 @@ public class RestoreBackupRunnable implements Runnable { log.info("Shutting down server..."); ctx.server().stop(false); - awaitServerShutdown(); - if(config.get().backupOldWorlds) { - MakeBackupRunnableFactory.create( - BackupContext.Builder - .newBackupContextBuilder() - .setServer(ctx.server()) - .setInitiator(ActionInitiator.Restore) - .setComment("Old_World" + (ctx.comment() != null ? "_" + ctx.comment() : "")) - .build() - ).run(); - } - - Path worldFile = Utilities.getWorldFolder(ctx.server()), tmp = null; + Path worldFile = Utilities.getWorldFolder(ctx.server()), tmp; try { tmp = Files.createTempDirectory( ctx.server().getRunDirectory().toPath(), ctx.restoreableFile().getFile().getFileName().toString()); } catch (IOException e) { - throw new RuntimeException(e); - } - - if(tmp == null) { - //TODO: log error! + log.error("An exception occurred while unpacking backup", e); return; } + FutureTask waitForShutdown = new FutureTask<>(() -> { + ctx.server().getThread().join(); //wait for server to die and save all its state + if(config.get().backupOldWorlds) { + return MakeBackupRunnableFactory.create( + BackupContext.Builder + .newBackupContextBuilder() + .setServer(ctx.server()) + .setInitiator(ActionInitiator.Restore) + .setComment("Old_World" + (ctx.comment() != null ? "_" + ctx.comment() : "")) + .build() + ).call(); + } + return null; + }); + + new Thread(waitForShutdown).start(); + try { log.info("Starting decompression..."); + long hash; + if (ctx.restoreableFile().getArchiveFormat() == ConfigPOJO.ArchiveFormat.ZIP) - ZipDecompressor.decompress(ctx.restoreableFile().getFile(), tmp); + hash = ZipDecompressor.decompress(ctx.restoreableFile().getFile(), tmp); else - GenericTarDecompressor.decompress(ctx.restoreableFile().getFile(), tmp); + hash = GenericTarDecompressor.decompress(ctx.restoreableFile().getFile(), tmp); - CompressionStatus status = null; + CompressionStatus status = CompressionStatus.readFromFile(tmp); + Files.delete(tmp.resolve(CompressionStatus.DATA_FILENAME)); - try (InputStream in = Files.newInputStream(tmp.resolve(CompressionStatus.DATA_FILENAME))) { - ObjectInputStream objectInputStream = new ObjectInputStream(in); - status = (CompressionStatus)objectInputStream.readObject(); - } catch (ClassNotFoundException e) { - throw new RuntimeException(e); - } + //locks until the backup is finished + waitForShutdown.get(); - if(status.isValid(0)) { - log.info("Deleting old world..."); + if(status.isValid(hash) || !config.get().errorErrorHandlingMode.verify()) { + if(status.isValid(hash)) log.info("Backup valid. Restoring"); + else log.info("Backup is damaged, but verification is disabled. Restoring"); Utilities.deleteDirectory(worldFile); Files.move(tmp, worldFile); if (config.get().deleteOldBackupAfterRestore) { - log.info("Deleting old backup"); + log.info("Deleting restored backup file"); Files.delete(ctx.restoreableFile().getFile()); } + } else { + log.error("File tree hash mismatch! Got: {}, Expected {}. Aborting", hash, status.treeHash()); } - } catch (IOException e) { + } catch (ExecutionException | InterruptedException | ClassNotFoundException | IOException e) { log.error("An exception occurred while trying to restore a backup!", e); } finally { + //Regardless of what happended, we shiuld still clean up if(Files.exists(tmp)) { try { Utilities.deleteDirectory(tmp); - } catch (IOException e) { - //TODO: Log error! - } + } catch (IOException ignored) {} } } @@ -129,14 +129,4 @@ public class RestoreBackupRunnable implements Runnable { log.info("Done!"); } - - private void awaitServerShutdown() { - while(((LivingServer)ctx.server()).isAlive()) { - try { - Thread.sleep(100); - } catch (InterruptedException e) { - log.error("Exception occurred!", e); - } - } - } } \ No newline at end of file diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java index de4696c..c6fe061 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java @@ -20,7 +20,9 @@ package net.szum123321.textile_backup.core.restore.decompressors; import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileLogger; +import net.szum123321.textile_backup.core.FileTreeHashBuilder; import net.szum123321.textile_backup.core.Utilities; +import net.szum123321.textile_backup.core.restore.HashingOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.CompressorException; @@ -36,8 +38,9 @@ import java.time.Instant; public class GenericTarDecompressor { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); - public static void decompress(Path input, Path target) throws IOException { + public static long decompress(Path input, Path target) throws IOException { Instant start = Instant.now(); + FileTreeHashBuilder treeBuilder = new FileTreeHashBuilder(() -> null); try (InputStream fileInputStream = Files.newInputStream(input); InputStream bufferedInputStream = new BufferedInputStream(fileInputStream); @@ -46,10 +49,8 @@ public class GenericTarDecompressor { TarArchiveEntry entry; while ((entry = archiveInputStream.getNextTarEntry()) != null) { - if(!archiveInputStream.canReadEntryData(entry)) { - log.error("Something when wrong while trying to decompress {}", entry.getName()); - continue; - } + if(!archiveInputStream.canReadEntryData(entry)) + throw new IOException("Couldn't read archive entry! " + entry.getName()); Path file = target.resolve(entry.getName()); @@ -58,7 +59,8 @@ public class GenericTarDecompressor { } else { Files.createDirectories(file.getParent()); try (OutputStream outputStream = Files.newOutputStream(file); - BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outputStream)) { + HashingOutputStream hashingStream = new HashingOutputStream(outputStream, file, null, treeBuilder); + BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(hashingStream)) { IOUtils.copy(archiveInputStream, bufferedOutputStream); } } @@ -68,6 +70,8 @@ public class GenericTarDecompressor { } log.info("Decompression took {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now()))); + + return treeBuilder.getValue(); } private static InputStream getCompressorInputStream(InputStream inputStream) throws CompressorException { diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java index 0e12347..ad1e9be 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java @@ -20,7 +20,9 @@ package net.szum123321.textile_backup.core.restore.decompressors; import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileLogger; +import net.szum123321.textile_backup.core.FileTreeHashBuilder; import net.szum123321.textile_backup.core.Utilities; +import net.szum123321.textile_backup.core.restore.HashingOutputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipFile; import org.apache.commons.compress.utils.IOUtils; @@ -35,9 +37,11 @@ import java.util.Iterator; public class ZipDecompressor { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); - public static void decompress(Path inputFile, Path target) throws IOException { + public static long decompress(Path inputFile, Path target) throws IOException { Instant start = Instant.now(); + FileTreeHashBuilder hashBuilder = new FileTreeHashBuilder(() -> null); + try(ZipFile zipFile = new ZipFile(inputFile.toFile())) { for (Iterator it = zipFile.getEntries().asIterator(); it.hasNext(); ) { ZipArchiveEntry entry = it.next(); @@ -48,7 +52,8 @@ public class ZipDecompressor { } else { Files.createDirectories(file.getParent()); try (OutputStream outputStream = Files.newOutputStream(file); - BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outputStream)) { + HashingOutputStream hashingStream = new HashingOutputStream(outputStream, file, null, hashBuilder); + BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(hashingStream)) { IOUtils.copy(zipFile.getInputStream(entry), bufferedOutputStream); } } @@ -56,5 +61,7 @@ public class ZipDecompressor { } log.info("Decompression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now()))); + + return hashBuilder.getValue(); } } From f6cd361fff081276b52b4bd9db07058dee077920 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Sun, 27 Nov 2022 13:55:34 +0100 Subject: [PATCH 07/44] Selected hashing algorithm. It's a custom job which merges SeaHash with Xoroshift64*. Should be fast and correct enough to for this use case. hope I will be able to speed it up with SIMD, as java is scheduled to soon include Vector API (a part of project Panama) --- .../szum123321/textile_backup/Globals.java | 8 +- .../core/FileTreeHashBuilder.java | 28 ++--- .../szum123321/textile_backup/core/Hash.java | 30 +++++ .../textile_backup/core/XorSeaHash.java | 103 ++++++++++++++++++ .../core/create/FileInputStreamSupplier.java | 3 +- .../core/create/HashingInputStream.java | 9 +- .../compressors/AbstractCompressor.java | 2 +- .../core/restore/HashingOutputStream.java | 12 +- .../decompressors/GenericTarDecompressor.java | 2 +- .../decompressors/ZipDecompressor.java | 2 +- 10 files changed, 161 insertions(+), 38 deletions(-) create mode 100644 src/main/java/net/szum123321/textile_backup/core/Hash.java create mode 100644 src/main/java/net/szum123321/textile_backup/core/XorSeaHash.java diff --git a/src/main/java/net/szum123321/textile_backup/Globals.java b/src/main/java/net/szum123321/textile_backup/Globals.java index b906076..7c779ce 100644 --- a/src/main/java/net/szum123321/textile_backup/Globals.java +++ b/src/main/java/net/szum123321/textile_backup/Globals.java @@ -20,7 +20,9 @@ package net.szum123321.textile_backup; import net.minecraft.server.MinecraftServer; +import net.szum123321.textile_backup.core.Hash; import net.szum123321.textile_backup.core.Utilities; +import net.szum123321.textile_backup.core.XorSeaHash; import net.szum123321.textile_backup.core.create.MakeBackupRunnable; import net.szum123321.textile_backup.core.restore.AwaitThread; import org.apache.commons.io.FileUtils; @@ -34,11 +36,13 @@ import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; +import java.util.function.Supplier; public class Globals { public static final Globals INSTANCE = new Globals(); - private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); - public final static DateTimeFormatter defaultDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH.mm.ss"); + private static final TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); + public static final DateTimeFormatter defaultDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH.mm.ss"); + public static final Supplier CHECKSUM_SUPPLIER = XorSeaHash::new; private ExecutorService executorService = null;// = Executors.newSingleThreadExecutor(); public final AtomicBoolean globalShutdownBackupFlag = new AtomicBoolean(true); diff --git a/src/main/java/net/szum123321/textile_backup/core/FileTreeHashBuilder.java b/src/main/java/net/szum123321/textile_backup/core/FileTreeHashBuilder.java index a6084e6..7af0f61 100644 --- a/src/main/java/net/szum123321/textile_backup/core/FileTreeHashBuilder.java +++ b/src/main/java/net/szum123321/textile_backup/core/FileTreeHashBuilder.java @@ -18,35 +18,26 @@ package net.szum123321.textile_backup.core; +import net.szum123321.textile_backup.Globals; + import java.io.IOException; -import java.nio.ByteBuffer; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; -import java.util.function.Supplier; -import java.util.zip.Checksum; - public class FileTreeHashBuilder { - private final static ThreadLocal buff = - ThreadLocal.withInitial(() -> new byte[Long.BYTES]); private final Object lock = new Object(); - private final Supplier hasherProvider; private long hash = 0, filesProcessed = 0, filesTotalSize = 0; - public FileTreeHashBuilder(Supplier provider) { hasherProvider = provider; } - public void update(Path path, long newHash) throws IOException { - byte[] raw = buff.get(); - var hasher = hasherProvider.get(); + var hasher = Globals.CHECKSUM_SUPPLIER.get(); long size = Files.size(path); - hasher.update(ByteBuffer.wrap(raw).putLong(size).array()); hasher.update(path.toString().getBytes(StandardCharsets.UTF_8)); - hasher.update(ByteBuffer.wrap(raw).putLong(hash).array()); + hasher.update(newHash); synchronized (lock) { - //This way exact order of files processed doesn't matter. + //This way, the exact order of files processed doesn't matter. this.hash ^= hasher.getValue(); filesProcessed++; filesTotalSize += size; @@ -54,12 +45,11 @@ public class FileTreeHashBuilder { } public long getValue() { - var hasher = hasherProvider.get(); - byte[] raw = buff.get(); + var hasher = Globals.CHECKSUM_SUPPLIER.get(); - hasher.update(ByteBuffer.wrap(raw).putLong(hash).array()); - hasher.update(ByteBuffer.wrap(raw).putLong(filesProcessed).array()); - hasher.update(ByteBuffer.wrap(raw).putLong(filesTotalSize).array()); + hasher.update(hash); + hasher.update(filesProcessed); + hasher.update(filesTotalSize); return hasher.getValue(); } diff --git a/src/main/java/net/szum123321/textile_backup/core/Hash.java b/src/main/java/net/szum123321/textile_backup/core/Hash.java new file mode 100644 index 0000000..e562cb4 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/Hash.java @@ -0,0 +1,30 @@ +/* + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package net.szum123321.textile_backup.core; + +public interface Hash { + void update(byte b); + void update(long b); + default void update(byte[] b) { + update(b, 0, b.length); + } + void update(byte[] b, int off, int len); + + long getValue(); +} diff --git a/src/main/java/net/szum123321/textile_backup/core/XorSeaHash.java b/src/main/java/net/szum123321/textile_backup/core/XorSeaHash.java new file mode 100644 index 0000000..1b74483 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/XorSeaHash.java @@ -0,0 +1,103 @@ +/* + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package net.szum123321.textile_backup.core; + +import java.nio.ByteBuffer; +import java.nio.ByteOrder; + +/* + This algorithm copies construction of SeaHash (https://ticki.github.io/blog/seahash-explained/) including its IV + What it differs in is that it uses Xoroshift64* instead of PCG. Although it might lower the output quality, + I don't think it matters that much, honestly. One advantage the xoroshift has is that it should be + easier to implement with AVX. Java should soon ship its vector api by default. + */ +public class XorSeaHash implements Hash { + //SeaHash IV + private final long[] state = { 0x16f11fe89b0d677cL, 0xb480a793d8e6c86cL, 0x6fe2e5aaf078ebc9L, 0x14f994a4c5259381L}; + private final int buffer_size = (state.length + 1) * Long.BYTES; + private final int buffer_limit = state.length * Long.BYTES; + private final byte[] _byte_buffer = new byte[buffer_size]; + //Enforce endianness + private final ByteBuffer buffer = ByteBuffer.wrap(_byte_buffer).order(ByteOrder.LITTLE_ENDIAN); + + private long hashed_data_length = 0; + + @Override + public void update(byte b) { + buffer.put(b); + hashed_data_length += 1; + if (buffer.position() >= buffer_limit) round(); + } + + @Override + public void update(long b) { + buffer.putLong(b); + hashed_data_length += Long.BYTES; + if(buffer.position() >= buffer_limit) round(); + } + + public void update(byte [] data) { update(data, 0, data.length); } + + public void update(byte[] data, int off, int len) { + int pos = off; + while(pos < len) { + int n = Math.min(len - pos, buffer_limit - buffer.position()); + System.arraycopy(data, pos, _byte_buffer, buffer.position(), n); + pos += n; + buffer.position(buffer.position() + n); + if(buffer.position() >= buffer_limit) round(); + } + + hashed_data_length += len; + } + + @Override + public long getValue() { + if(buffer.position() != 0) round(); + + long result = state[0]; + result ^= state[1]; + result ^= state[2]; + result ^= state[3]; + result ^= hashed_data_length; + + return xorshift64star(result); + } + + private void round() { + while(buffer.position() < buffer_limit) buffer.put((byte)0); + int p = buffer.position(); + buffer.rewind(); + + for(int i = 0; i < 4; i++) state[i] ^= buffer.getLong(); + for(int i = 0; i < 4; i++) state[i] = xorshift64star(state[i]); + + if(p > buffer_limit) { + System.arraycopy(_byte_buffer, buffer_limit, _byte_buffer, 0, buffer.limit() - p); + buffer.position(buffer.limit() - p); + } + } + + long xorshift64star(long s) { + s ^= (s >> 12); + s ^= (s << 25); + s ^= (s >> 27); + return s * 0x2545F4914F6CDD1DL; + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java index 695294e..2ee1795 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java @@ -34,8 +34,7 @@ public record FileInputStreamSupplier(Path path, String name, FileTreeHashBuilde @Override public InputStream getInputStream() throws IOException { try { - //TODO: select hashing algorithm! - return new HashingInputStream(Files.newInputStream(path), path, null, hashTreeBuilder, brokenFileHandler); + return new HashingInputStream(Files.newInputStream(path), path, hashTreeBuilder, brokenFileHandler); } catch (IOException e) { brokenFileHandler.handle(path, e); throw e; diff --git a/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java b/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java index 532abca..8577de8 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java @@ -18,27 +18,26 @@ package net.szum123321.textile_backup.core.create; +import net.szum123321.textile_backup.Globals; import net.szum123321.textile_backup.core.DataLeftException; import net.szum123321.textile_backup.core.FileTreeHashBuilder; +import net.szum123321.textile_backup.core.Hash; import org.jetbrains.annotations.NotNull; import java.io.*; import java.nio.file.Path; -import java.util.zip.Checksum; //This class calculates a hash of the file on the input stream, submits it to FileTreeHashBuilder. //In case the whole underlying stream hasn't been read, also puts it into BrokeFileHandler public class HashingInputStream extends FilterInputStream { - private final Path path; - private final Checksum hasher; + private final Hash hasher = Globals.CHECKSUM_SUPPLIER.get(); private final FileTreeHashBuilder hashBuilder; private final BrokenFileHandler brokenFileHandler; - public HashingInputStream(InputStream in, Path path, Checksum hasher, FileTreeHashBuilder hashBuilder, BrokenFileHandler brokenFileHandler) { + public HashingInputStream(InputStream in, Path path, FileTreeHashBuilder hashBuilder, BrokenFileHandler brokenFileHandler) { super(in); this.path = path; - this.hasher = hasher; this.hashBuilder = hashBuilder; this.brokenFileHandler = brokenFileHandler; } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index b689ba7..3d6eee8 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -44,7 +44,7 @@ public abstract class AbstractCompressor { public void createArchive(Path inputFile, Path outputFile, BackupContext ctx, int coreLimit) throws IOException, ExecutionException, InterruptedException { Instant start = Instant.now(); - FileTreeHashBuilder fileHashBuilder = new FileTreeHashBuilder(() -> null); //TODO: select hashing algorithm + FileTreeHashBuilder fileHashBuilder = new FileTreeHashBuilder(); BrokenFileHandler brokenFileHandler = new BrokenFileHandler(); try (OutputStream outStream = Files.newOutputStream(outputFile); diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java b/src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java index 6f9b3c8..e8375f5 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java @@ -18,29 +18,27 @@ package net.szum123321.textile_backup.core.restore; +import net.szum123321.textile_backup.Globals; import net.szum123321.textile_backup.core.FileTreeHashBuilder; +import net.szum123321.textile_backup.core.Hash; import org.jetbrains.annotations.NotNull; import java.io.FilterOutputStream; import java.io.IOException; import java.io.OutputStream; import java.nio.file.Path; -import java.util.zip.Checksum; public class HashingOutputStream extends FilterOutputStream { private final Path path; - private final Checksum hasher; - + private final Hash hasher = Globals.CHECKSUM_SUPPLIER.get(); private final FileTreeHashBuilder hashBuilder; - public HashingOutputStream(OutputStream out, Path path, Checksum hasher, FileTreeHashBuilder hashBuilder) { + public HashingOutputStream(OutputStream out, Path path, FileTreeHashBuilder hashBuilder) { super(out); this.path = path; - this.hasher = hasher; this.hashBuilder = hashBuilder; } - @Override public void write(int b) throws IOException { super.write(b); @@ -48,7 +46,7 @@ public class HashingOutputStream extends FilterOutputStream { } @Override - public void write(byte[] @NotNull b, int off, int len) throws IOException { + public void write(byte @NotNull [] b, int off, int len) throws IOException { super.write(b, off, len); hasher.update(b, off, len); } diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java index c6fe061..216c2e8 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java @@ -40,7 +40,7 @@ public class GenericTarDecompressor { public static long decompress(Path input, Path target) throws IOException { Instant start = Instant.now(); - FileTreeHashBuilder treeBuilder = new FileTreeHashBuilder(() -> null); + FileTreeHashBuilder treeBuilder = new FileTreeHashBuilder(); try (InputStream fileInputStream = Files.newInputStream(input); InputStream bufferedInputStream = new BufferedInputStream(fileInputStream); diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java index ad1e9be..9ad58cf 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java @@ -40,7 +40,7 @@ public class ZipDecompressor { public static long decompress(Path inputFile, Path target) throws IOException { Instant start = Instant.now(); - FileTreeHashBuilder hashBuilder = new FileTreeHashBuilder(() -> null); + FileTreeHashBuilder hashBuilder = new FileTreeHashBuilder(); try(ZipFile zipFile = new ZipFile(inputFile.toFile())) { for (Iterator it = zipFile.getEntries().asIterator(); it.hasNext(); ) { From afe9c8d051928580d5b61b7c8b875bb88fa6b1cf Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Sun, 27 Nov 2022 13:57:02 +0100 Subject: [PATCH 08/44] Removed redundant BufferStreams and LivingServer subsystem --- .../textile_backup/core/LivingServer.java | 23 ---------- .../decompressors/GenericTarDecompressor.java | 5 +-- .../decompressors/ZipDecompressor.java | 5 +-- .../mixin/MinecraftServerMixin.java | 44 ------------------- src/main/resources/textile_backup.mixins.json | 1 - 5 files changed, 4 insertions(+), 74 deletions(-) delete mode 100644 src/main/java/net/szum123321/textile_backup/core/LivingServer.java delete mode 100644 src/main/java/net/szum123321/textile_backup/mixin/MinecraftServerMixin.java diff --git a/src/main/java/net/szum123321/textile_backup/core/LivingServer.java b/src/main/java/net/szum123321/textile_backup/core/LivingServer.java deleted file mode 100644 index 0011d6b..0000000 --- a/src/main/java/net/szum123321/textile_backup/core/LivingServer.java +++ /dev/null @@ -1,23 +0,0 @@ -/* - * A simple backup mod for Fabric - * Copyright (C) 2021 Szum123321 - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package net.szum123321.textile_backup.core; - -public interface LivingServer { - boolean isAlive(); -} diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java index 216c2e8..f87f829 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java @@ -59,9 +59,8 @@ public class GenericTarDecompressor { } else { Files.createDirectories(file.getParent()); try (OutputStream outputStream = Files.newOutputStream(file); - HashingOutputStream hashingStream = new HashingOutputStream(outputStream, file, null, treeBuilder); - BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(hashingStream)) { - IOUtils.copy(archiveInputStream, bufferedOutputStream); + HashingOutputStream out = new HashingOutputStream(outputStream, file, treeBuilder)) { + IOUtils.copy(archiveInputStream, out); } } } diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java index 9ad58cf..f67ec96 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java @@ -52,9 +52,8 @@ public class ZipDecompressor { } else { Files.createDirectories(file.getParent()); try (OutputStream outputStream = Files.newOutputStream(file); - HashingOutputStream hashingStream = new HashingOutputStream(outputStream, file, null, hashBuilder); - BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(hashingStream)) { - IOUtils.copy(zipFile.getInputStream(entry), bufferedOutputStream); + HashingOutputStream out = new HashingOutputStream(outputStream, file, hashBuilder)) { + IOUtils.copy(zipFile.getInputStream(entry), out); } } } diff --git a/src/main/java/net/szum123321/textile_backup/mixin/MinecraftServerMixin.java b/src/main/java/net/szum123321/textile_backup/mixin/MinecraftServerMixin.java deleted file mode 100644 index 758ba84..0000000 --- a/src/main/java/net/szum123321/textile_backup/mixin/MinecraftServerMixin.java +++ /dev/null @@ -1,44 +0,0 @@ -/* - A simple backup mod for Fabric - Copyright (C) 2020 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ - -package net.szum123321.textile_backup.mixin; - -import net.minecraft.server.MinecraftServer; -import net.szum123321.textile_backup.core.LivingServer; -import org.spongepowered.asm.mixin.Mixin; -import org.spongepowered.asm.mixin.Unique; -import org.spongepowered.asm.mixin.injection.At; -import org.spongepowered.asm.mixin.injection.Inject; -import org.spongepowered.asm.mixin.injection.callback.CallbackInfo; - -@Mixin(MinecraftServer.class) -public class MinecraftServerMixin implements LivingServer { - @Unique - private boolean isAlive = true; - - @Inject(method = "shutdown", at = @At("TAIL")) - public void onFinalWorldSave(CallbackInfo ci) { - isAlive = false; - } - - @Unique - @Override - public boolean isAlive() { - return isAlive; - } -} diff --git a/src/main/resources/textile_backup.mixins.json b/src/main/resources/textile_backup.mixins.json index 82c2d00..6222ec9 100644 --- a/src/main/resources/textile_backup.mixins.json +++ b/src/main/resources/textile_backup.mixins.json @@ -4,7 +4,6 @@ "compatibilityLevel": "JAVA_16", "mixins": [ "DedicatedServerWatchdogMixin", - "MinecraftServerMixin", "MinecraftServerSessionAccessor" ], "client": [ From c040d05bd86430926c4e219136c39aa705d4e724 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Sun, 27 Nov 2022 13:58:45 +0100 Subject: [PATCH 09/44] Added toString method to CompressionStatus, version bump to 3.0.0-a, few small tweaks --- gradle.properties | 4 +-- .../textile_backup/TextileBackup.java | 2 +- .../core/CompressionStatus.java | 32 ++++++++++++++++++- .../core/NoSpaceLeftOnDeviceException.java | 2 +- .../core/create/MakeBackupRunnable.java | 28 ++++++---------- .../core/restore/RestoreBackupRunnable.java | 12 ++++--- 6 files changed, 53 insertions(+), 27 deletions(-) diff --git a/gradle.properties b/gradle.properties index 3ed2600..993b3d3 100644 --- a/gradle.properties +++ b/gradle.properties @@ -6,7 +6,7 @@ yarn_mappings=1.19.2+build.28 loader_version=0.14.10 #Fabric api -fabric_version=0.64.0+1.19.2 +fabric_version=0.67.1+1.19.2 #Cloth Config cloth_version=8.2.88 @@ -21,6 +21,6 @@ lazydfu_version=v0.1.3 pgzip_commit_hash=af5f5c297e735f3f2df7aa4eb0e19a5810b8aff6 # Mod Properties -mod_version = 2.5.0 +mod_version = 3.0.0-a maven_group = net.szum123321 archives_base_name = textile_backup \ No newline at end of file diff --git a/src/main/java/net/szum123321/textile_backup/TextileBackup.java b/src/main/java/net/szum123321/textile_backup/TextileBackup.java index 60d7e11..99c0833 100644 --- a/src/main/java/net/szum123321/textile_backup/TextileBackup.java +++ b/src/main/java/net/szum123321/textile_backup/TextileBackup.java @@ -63,7 +63,7 @@ public class TextileBackup implements ModInitializer { Globals.INSTANCE.updateTMPFSFlag(server); }); - //Wait 60s for already submited backups to finish. After that kill the bastards and run the one last if required + //Wait 60s for already submitted backups to finish. After that kill the bastards and run the one last if required ServerLifecycleEvents.SERVER_STOPPED.register(server -> { Globals.INSTANCE.shutdownQueueExecutor(60000); diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java index 1e9803f..58cb8e2 100644 --- a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -22,11 +22,14 @@ import java.io.*; import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; +import java.time.format.DateTimeFormatter; import java.util.Map; public record CompressionStatus(long treeHash, LocalDateTime date, long startTimestamp, long finishTimestamp, Map brokenFiles) implements Serializable { public static final String DATA_FILENAME = "textile_status.data"; - public boolean isValid(long decompressedHash) { return true; } + public boolean isValid(long decompressedHash) { + return decompressedHash == treeHash; + } public static CompressionStatus readFromFile(Path folder) throws IOException, ClassNotFoundException { try(InputStream i = Files.newInputStream(folder.resolve(DATA_FILENAME)); @@ -43,4 +46,31 @@ public record CompressionStatus(long treeHash, LocalDateTime date, long startTim } } + @Override + public String toString() { + StringBuilder builder = new StringBuilder(); + builder.append("Hash: ") + .append(treeHash) + .append(", Date: ") + .append(date.format(DateTimeFormatter.ISO_DATE_TIME)) + .append(", start time stamp: ").append(startTimestamp) + .append(", finish time stamp: ").append(finishTimestamp); + + builder.append(", broken files: "); + if(brokenFiles.isEmpty()) builder.append("[]"); + else { + builder.append("[\n"); + for(Path i: brokenFiles.keySet()) { + builder.append(i.toString()) + .append(":"); + + ByteArrayOutputStream o = new ByteArrayOutputStream(); + brokenFiles.get(i).printStackTrace(new PrintStream(o)); + builder.append(o).append("\n"); + } + builder.append("]"); + } + + return builder.toString(); + } } diff --git a/src/main/java/net/szum123321/textile_backup/core/NoSpaceLeftOnDeviceException.java b/src/main/java/net/szum123321/textile_backup/core/NoSpaceLeftOnDeviceException.java index 3f71e4b..044cdb9 100644 --- a/src/main/java/net/szum123321/textile_backup/core/NoSpaceLeftOnDeviceException.java +++ b/src/main/java/net/szum123321/textile_backup/core/NoSpaceLeftOnDeviceException.java @@ -25,6 +25,6 @@ import java.io.IOException; */ public class NoSpaceLeftOnDeviceException extends IOException { public NoSpaceLeftOnDeviceException(Throwable cause) { - super(cause); + super("The underlying filesystem has ran out of available space.\nSee: https://github.com/Szum123321/textile_backup/wiki/ZIP-Problems", cause); } } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java index fd3472f..e550a0a 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java @@ -51,11 +51,10 @@ public class MakeBackupRunnable implements Callable { public MakeBackupRunnable(BackupContext context) { this.context = context; } + @Override public Void call() throws IOException, ExecutionException, InterruptedException { - Path outFile = Utilities - .getBackupRootPath(Utilities.getLevelName(context.server())) - .resolve(getFileName()); + Path outFile = Utilities.getBackupRootPath(Utilities.getLevelName(context.server())).resolve(getFileName()); log.trace("Outfile is: {}", outFile); @@ -77,11 +76,9 @@ public class MakeBackupRunnable implements Callable { int coreCount; - if(config.get().compressionCoreCountLimit <= 0) { - coreCount = Runtime.getRuntime().availableProcessors(); - } else { + if (config.get().compressionCoreCountLimit <= 0) coreCount = Runtime.getRuntime().availableProcessors(); + else coreCount = Math.min(config.get().compressionCoreCountLimit, Runtime.getRuntime().availableProcessors()); - } log.trace("Running compression on {} threads. Available cores: {}", coreCount, Runtime.getRuntime().availableProcessors()); @@ -107,11 +104,8 @@ public class MakeBackupRunnable implements Callable { Globals.INSTANCE.getQueueExecutor().submit(new Cleanup(context.commandSource(), Utilities.getLevelName(context.server()))); - if(config.get().broadcastBackupDone) { - Utilities.notifyPlayers( - context.server(), - "Done!" - ); + if (config.get().broadcastBackupDone) { + Utilities.notifyPlayers(context.server(), "Done!"); } else { log.sendInfoAL(context, "Done!"); } @@ -119,7 +113,7 @@ public class MakeBackupRunnable implements Callable { //ExecutorService swallows exception, so I need to catch everything log.error("An exception occurred when trying to create new backup file!", e); - if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode.isStrict()) { + if (ConfigHelper.INSTANCE.get().errorErrorHandlingMode.isStrict()) { try { Files.delete(outFile); } catch (IOException ex) { @@ -127,7 +121,7 @@ public class MakeBackupRunnable implements Callable { } } - if(context.initiator() == ActionInitiator.Player) + if (context.initiator() == ActionInitiator.Player) log.sendError(context, "An exception occurred when trying to create new backup file!"); throw e; @@ -139,9 +133,7 @@ public class MakeBackupRunnable implements Callable { return null; } - private String getFileName(){ - return Utilities.getDateTimeFormatter().format(context.startDate()) + - (context.comment() != null ? "#" + context.comment().replaceAll("[\\\\/:*?\"<>|#]", "") : "") + - config.get().format.getCompleteString(); + private String getFileName() { + return Utilities.getDateTimeFormatter().format(context.startDate()) + (context.comment() != null ? "#" + context.comment().replaceAll("[\\\\/:*?\"<>|#]", "") : "") + config.get().format.getCompleteString(); } } diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java index 5821bbc..230c27c 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java @@ -55,7 +55,8 @@ public class RestoreBackupRunnable implements Runnable { ctx.server().stop(false); - Path worldFile = Utilities.getWorldFolder(ctx.server()), tmp; + Path worldFile = Utilities.getWorldFolder(ctx.server()), + tmp; try { tmp = Files.createTempDirectory( @@ -99,8 +100,11 @@ public class RestoreBackupRunnable implements Runnable { //locks until the backup is finished waitForShutdown.get(); - if(status.isValid(hash) || !config.get().errorErrorHandlingMode.verify()) { - if(status.isValid(hash)) log.info("Backup valid. Restoring"); + log.info("Status: {}", status); + + boolean valid = status.isValid(hash); + if(valid || !config.get().errorErrorHandlingMode.verify()) { + if(valid) log.info("Backup valid. Restoring"); else log.info("Backup is damaged, but verification is disabled. Restoring"); Utilities.deleteDirectory(worldFile); @@ -116,7 +120,7 @@ public class RestoreBackupRunnable implements Runnable { } catch (ExecutionException | InterruptedException | ClassNotFoundException | IOException e) { log.error("An exception occurred while trying to restore a backup!", e); } finally { - //Regardless of what happended, we shiuld still clean up + //Regardless of what happened, we should still clean up if(Files.exists(tmp)) { try { Utilities.deleteDirectory(tmp); From 2f96f7728eb78a4f481f65cce6886ee34f0c499b Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Sun, 27 Nov 2022 23:04:21 +0100 Subject: [PATCH 10/44] Updated all copyright headers --- .../szum123321/textile_backup/Globals.java | 25 ++-- .../textile_backup/TextileBackup.java | 32 ++--- .../textile_backup/TextileLogger.java | 2 +- .../textile_backup/client/ModMenuEntry.java | 25 ++-- .../commands/CommandExceptions.java | 2 +- .../commands/FileSuggestionProvider.java | 2 +- .../commands/create/CleanupCommand.java | 32 ++--- .../commands/create/StartBackupCommand.java | 32 ++--- .../commands/manage/BlacklistCommand.java | 2 +- .../commands/manage/DeleteCommand.java | 2 +- .../commands/manage/ListBackupsCommand.java | 2 +- .../commands/manage/WhitelistCommand.java | 2 +- .../commands/restore/KillRestoreCommand.java | 2 +- .../restore/RestoreBackupCommand.java | 2 +- .../textile_backup/config/ConfigHelper.java | 2 +- .../textile_backup/config/ConfigPOJO.java | 2 +- .../textile_backup/core/ActionInitiator.java | 2 +- .../textile_backup/core/Cleanup.java | 25 ++-- .../core/CompressionStatus.java | 22 ++-- .../core/DataLeftException.java | 2 +- .../core/NoSpaceLeftOnDeviceException.java | 2 +- .../textile_backup/core/RestoreableFile.java | 25 ++-- .../textile_backup/core/Utilities.java | 32 ++--- .../core/create/BackupContext.java | 2 +- .../core/create/BackupScheduler.java | 2 +- .../core/create/BrokenFileHandler.java | 2 +- .../core/create/FileInputStreamSupplier.java | 2 +- .../core/create/InputSupplier.java | 32 ++--- .../create/MakeBackupRunnableFactory.java | 25 ++-- .../compressors/AbstractCompressor.java | 2 +- .../compressors/ParallelZipCompressor.java | 2 +- .../create/compressors/ZipCompressor.java | 2 +- .../compressors/tar/AbstractTarArchiver.java | 2 +- .../tar/ParallelBZip2Compressor.java | 2 +- .../tar/ParallelGzipCompressor.java | 2 +- .../{ => digest}/FileTreeHashBuilder.java | 112 ++++++++--------- .../HashingInputStream.java | 2 +- .../HashingOutputStream.java | 118 +++++++++--------- .../core/restore/AwaitThread.java | 2 +- .../core/restore/RestoreBackupRunnable.java | 32 ++--- .../core/restore/RestoreContext.java | 2 +- .../core/restore/RestoreHelper.java | 32 ++--- .../decompressors/GenericTarDecompressor.java | 32 ++--- .../decompressors/ZipDecompressor.java | 32 ++--- .../mixin/DedicatedServerWatchdogMixin.java | 2 +- .../mixin/MinecraftServerSessionAccessor.java | 2 +- 46 files changed, 359 insertions(+), 364 deletions(-) rename src/main/java/net/szum123321/textile_backup/core/{ => digest}/FileTreeHashBuilder.java (58%) rename src/main/java/net/szum123321/textile_backup/core/{create => digest}/HashingInputStream.java (98%) rename src/main/java/net/szum123321/textile_backup/core/{restore => digest}/HashingOutputStream.java (60%) diff --git a/src/main/java/net/szum123321/textile_backup/Globals.java b/src/main/java/net/szum123321/textile_backup/Globals.java index 7c779ce..cf4b0eb 100644 --- a/src/main/java/net/szum123321/textile_backup/Globals.java +++ b/src/main/java/net/szum123321/textile_backup/Globals.java @@ -1,20 +1,19 @@ /* - * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package net.szum123321.textile_backup; diff --git a/src/main/java/net/szum123321/textile_backup/TextileBackup.java b/src/main/java/net/szum123321/textile_backup/TextileBackup.java index 99c0833..d2625db 100644 --- a/src/main/java/net/szum123321/textile_backup/TextileBackup.java +++ b/src/main/java/net/szum123321/textile_backup/TextileBackup.java @@ -1,20 +1,20 @@ /* - A simple backup mod for Fabric - Copyright (C) 2020 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ package net.szum123321.textile_backup; diff --git a/src/main/java/net/szum123321/textile_backup/TextileLogger.java b/src/main/java/net/szum123321/textile_backup/TextileLogger.java index 6d2815a..2e77da9 100644 --- a/src/main/java/net/szum123321/textile_backup/TextileLogger.java +++ b/src/main/java/net/szum123321/textile_backup/TextileLogger.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2021 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/client/ModMenuEntry.java b/src/main/java/net/szum123321/textile_backup/client/ModMenuEntry.java index 75017a2..ddb2c04 100644 --- a/src/main/java/net/szum123321/textile_backup/client/ModMenuEntry.java +++ b/src/main/java/net/szum123321/textile_backup/client/ModMenuEntry.java @@ -1,20 +1,19 @@ /* - * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package net.szum123321.textile_backup.client; diff --git a/src/main/java/net/szum123321/textile_backup/commands/CommandExceptions.java b/src/main/java/net/szum123321/textile_backup/commands/CommandExceptions.java index 3c31542..e0009d0 100644 --- a/src/main/java/net/szum123321/textile_backup/commands/CommandExceptions.java +++ b/src/main/java/net/szum123321/textile_backup/commands/CommandExceptions.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/commands/FileSuggestionProvider.java b/src/main/java/net/szum123321/textile_backup/commands/FileSuggestionProvider.java index 319e341..da2ec52 100644 --- a/src/main/java/net/szum123321/textile_backup/commands/FileSuggestionProvider.java +++ b/src/main/java/net/szum123321/textile_backup/commands/FileSuggestionProvider.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/commands/create/CleanupCommand.java b/src/main/java/net/szum123321/textile_backup/commands/create/CleanupCommand.java index c0567ee..765e4ae 100644 --- a/src/main/java/net/szum123321/textile_backup/commands/create/CleanupCommand.java +++ b/src/main/java/net/szum123321/textile_backup/commands/create/CleanupCommand.java @@ -1,20 +1,20 @@ /* - A simple backup mod for Fabric - Copyright (C) 2020 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ package net.szum123321.textile_backup.commands.create; diff --git a/src/main/java/net/szum123321/textile_backup/commands/create/StartBackupCommand.java b/src/main/java/net/szum123321/textile_backup/commands/create/StartBackupCommand.java index 03ddb6a..1ec744f 100644 --- a/src/main/java/net/szum123321/textile_backup/commands/create/StartBackupCommand.java +++ b/src/main/java/net/szum123321/textile_backup/commands/create/StartBackupCommand.java @@ -1,20 +1,20 @@ /* - A simple backup mod for Fabric - Copyright (C) 2020 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ package net.szum123321.textile_backup.commands.create; diff --git a/src/main/java/net/szum123321/textile_backup/commands/manage/BlacklistCommand.java b/src/main/java/net/szum123321/textile_backup/commands/manage/BlacklistCommand.java index 5baaedd..6e138be 100644 --- a/src/main/java/net/szum123321/textile_backup/commands/manage/BlacklistCommand.java +++ b/src/main/java/net/szum123321/textile_backup/commands/manage/BlacklistCommand.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2021 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/commands/manage/DeleteCommand.java b/src/main/java/net/szum123321/textile_backup/commands/manage/DeleteCommand.java index 5fc0279..f1c4d0c 100644 --- a/src/main/java/net/szum123321/textile_backup/commands/manage/DeleteCommand.java +++ b/src/main/java/net/szum123321/textile_backup/commands/manage/DeleteCommand.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/commands/manage/ListBackupsCommand.java b/src/main/java/net/szum123321/textile_backup/commands/manage/ListBackupsCommand.java index 962709e..c9b4656 100644 --- a/src/main/java/net/szum123321/textile_backup/commands/manage/ListBackupsCommand.java +++ b/src/main/java/net/szum123321/textile_backup/commands/manage/ListBackupsCommand.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/commands/manage/WhitelistCommand.java b/src/main/java/net/szum123321/textile_backup/commands/manage/WhitelistCommand.java index c5da2f4..728e62b 100644 --- a/src/main/java/net/szum123321/textile_backup/commands/manage/WhitelistCommand.java +++ b/src/main/java/net/szum123321/textile_backup/commands/manage/WhitelistCommand.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2021 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/commands/restore/KillRestoreCommand.java b/src/main/java/net/szum123321/textile_backup/commands/restore/KillRestoreCommand.java index f0578a0..f63efc8 100644 --- a/src/main/java/net/szum123321/textile_backup/commands/restore/KillRestoreCommand.java +++ b/src/main/java/net/szum123321/textile_backup/commands/restore/KillRestoreCommand.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/commands/restore/RestoreBackupCommand.java b/src/main/java/net/szum123321/textile_backup/commands/restore/RestoreBackupCommand.java index b53f18a..b65e166 100644 --- a/src/main/java/net/szum123321/textile_backup/commands/restore/RestoreBackupCommand.java +++ b/src/main/java/net/szum123321/textile_backup/commands/restore/RestoreBackupCommand.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/config/ConfigHelper.java b/src/main/java/net/szum123321/textile_backup/config/ConfigHelper.java index 07332bd..704c082 100644 --- a/src/main/java/net/szum123321/textile_backup/config/ConfigHelper.java +++ b/src/main/java/net/szum123321/textile_backup/config/ConfigHelper.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2021 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java index 690f653..61b4458 100644 --- a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java +++ b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2021 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/ActionInitiator.java b/src/main/java/net/szum123321/textile_backup/core/ActionInitiator.java index 07375b9..e3c01b7 100644 --- a/src/main/java/net/szum123321/textile_backup/core/ActionInitiator.java +++ b/src/main/java/net/szum123321/textile_backup/core/ActionInitiator.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/Cleanup.java b/src/main/java/net/szum123321/textile_backup/core/Cleanup.java index 56fd97d..6347b61 100644 --- a/src/main/java/net/szum123321/textile_backup/core/Cleanup.java +++ b/src/main/java/net/szum123321/textile_backup/core/Cleanup.java @@ -1,20 +1,19 @@ /* - * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package net.szum123321.textile_backup.core; diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java index 58cb8e2..022c880 100644 --- a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -1,19 +1,19 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 + * Copyright (C) 2022 Szum123321 * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package net.szum123321.textile_backup.core; diff --git a/src/main/java/net/szum123321/textile_backup/core/DataLeftException.java b/src/main/java/net/szum123321/textile_backup/core/DataLeftException.java index cc5c7b9..02ac7f0 100644 --- a/src/main/java/net/szum123321/textile_backup/core/DataLeftException.java +++ b/src/main/java/net/szum123321/textile_backup/core/DataLeftException.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/NoSpaceLeftOnDeviceException.java b/src/main/java/net/szum123321/textile_backup/core/NoSpaceLeftOnDeviceException.java index 044cdb9..a487f89 100644 --- a/src/main/java/net/szum123321/textile_backup/core/NoSpaceLeftOnDeviceException.java +++ b/src/main/java/net/szum123321/textile_backup/core/NoSpaceLeftOnDeviceException.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2021 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/RestoreableFile.java b/src/main/java/net/szum123321/textile_backup/core/RestoreableFile.java index b44d9e5..bb3a8c8 100644 --- a/src/main/java/net/szum123321/textile_backup/core/RestoreableFile.java +++ b/src/main/java/net/szum123321/textile_backup/core/RestoreableFile.java @@ -1,20 +1,19 @@ /* - * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package net.szum123321.textile_backup.core; diff --git a/src/main/java/net/szum123321/textile_backup/core/Utilities.java b/src/main/java/net/szum123321/textile_backup/core/Utilities.java index 65a6f9f..1989993 100644 --- a/src/main/java/net/szum123321/textile_backup/core/Utilities.java +++ b/src/main/java/net/szum123321/textile_backup/core/Utilities.java @@ -1,20 +1,20 @@ /* - A simple backup mod for Fabric - Copyright (C) 2020 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ package net.szum123321.textile_backup.core; diff --git a/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java b/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java index 99e1474..6eb6a57 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/create/BackupScheduler.java b/src/main/java/net/szum123321/textile_backup/core/create/BackupScheduler.java index fb54d37..3359db9 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/BackupScheduler.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/BackupScheduler.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/create/BrokenFileHandler.java b/src/main/java/net/szum123321/textile_backup/core/create/BrokenFileHandler.java index be24aa9..8e9b314 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/BrokenFileHandler.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/BrokenFileHandler.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java index 2ee1795..46a302f 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java index e90e8d4..74960b2 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java @@ -1,20 +1,20 @@ /* - A simple backup mod for Fabric - Copyright (C) 2022 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ package net.szum123321.textile_backup.core.create; diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnableFactory.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnableFactory.java index 2bcbb2e..099d17c 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnableFactory.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnableFactory.java @@ -1,20 +1,19 @@ /* - * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . */ package net.szum123321.textile_backup.core.create; diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index 3d6eee8..5c6db3d 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java index f41f0b8..8acf42f 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java index 5b784ac..5a87a97 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java index 0cae730..5aa8a3c 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelBZip2Compressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelBZip2Compressor.java index e3e9e05..76d96fe 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelBZip2Compressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelBZip2Compressor.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelGzipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelGzipCompressor.java index dcf9f1c..0a59638 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelGzipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelGzipCompressor.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/FileTreeHashBuilder.java b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java similarity index 58% rename from src/main/java/net/szum123321/textile_backup/core/FileTreeHashBuilder.java rename to src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java index 7af0f61..e0bb1ba 100644 --- a/src/main/java/net/szum123321/textile_backup/core/FileTreeHashBuilder.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java @@ -1,56 +1,56 @@ -/* - A simple backup mod for Fabric - Copyright (C) 2022 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ - -package net.szum123321.textile_backup.core; - -import net.szum123321.textile_backup.Globals; - -import java.io.IOException; -import java.nio.charset.StandardCharsets; -import java.nio.file.Files; -import java.nio.file.Path; -public class FileTreeHashBuilder { - private final Object lock = new Object(); - private long hash = 0, filesProcessed = 0, filesTotalSize = 0; - - public void update(Path path, long newHash) throws IOException { - var hasher = Globals.CHECKSUM_SUPPLIER.get(); - - long size = Files.size(path); - - hasher.update(path.toString().getBytes(StandardCharsets.UTF_8)); - hasher.update(newHash); - - synchronized (lock) { - //This way, the exact order of files processed doesn't matter. - this.hash ^= hasher.getValue(); - filesProcessed++; - filesTotalSize += size; - } - } - - public long getValue() { - var hasher = Globals.CHECKSUM_SUPPLIER.get(); - - hasher.update(hash); - hasher.update(filesProcessed); - hasher.update(filesTotalSize); - - return hasher.getValue(); - } -} +/* + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package net.szum123321.textile_backup.core; + +import net.szum123321.textile_backup.Globals; + +import java.io.IOException; +import java.nio.charset.StandardCharsets; +import java.nio.file.Files; +import java.nio.file.Path; +public class FileTreeHashBuilder { + private final Object lock = new Object(); + private long hash = 0, filesProcessed = 0, filesTotalSize = 0; + + public void update(Path path, long newHash) throws IOException { + var hasher = Globals.CHECKSUM_SUPPLIER.get(); + + long size = Files.size(path); + + hasher.update(path.toString().getBytes(StandardCharsets.UTF_8)); + hasher.update(newHash); + + synchronized (lock) { + //This way, the exact order of files processed doesn't matter. + this.hash ^= hasher.getValue(); + filesProcessed++; + filesTotalSize += size; + } + } + + public long getValue() { + var hasher = Globals.CHECKSUM_SUPPLIER.get(); + + hasher.update(hash); + hasher.update(filesProcessed); + hasher.update(filesTotalSize); + + return hasher.getValue(); + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java similarity index 98% rename from src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java rename to src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java index 8577de8..becfb3d 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java similarity index 60% rename from src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java rename to src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java index e8375f5..4ce9887 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/HashingOutputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java @@ -1,59 +1,59 @@ -/* - A simple backup mod for Fabric - Copyright (C) 2022 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ - -package net.szum123321.textile_backup.core.restore; - -import net.szum123321.textile_backup.Globals; -import net.szum123321.textile_backup.core.FileTreeHashBuilder; -import net.szum123321.textile_backup.core.Hash; -import org.jetbrains.annotations.NotNull; - -import java.io.FilterOutputStream; -import java.io.IOException; -import java.io.OutputStream; -import java.nio.file.Path; - -public class HashingOutputStream extends FilterOutputStream { - private final Path path; - private final Hash hasher = Globals.CHECKSUM_SUPPLIER.get(); - private final FileTreeHashBuilder hashBuilder; - - public HashingOutputStream(OutputStream out, Path path, FileTreeHashBuilder hashBuilder) { - super(out); - this.path = path; - this.hashBuilder = hashBuilder; - } - - @Override - public void write(int b) throws IOException { - super.write(b); - hasher.update(b); - } - - @Override - public void write(byte @NotNull [] b, int off, int len) throws IOException { - super.write(b, off, len); - hasher.update(b, off, len); - } - - @Override - public void close() throws IOException { - super.close(); - hashBuilder.update(path, hasher.getValue()); - } -} +/* + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package net.szum123321.textile_backup.core.restore; + +import net.szum123321.textile_backup.Globals; +import net.szum123321.textile_backup.core.FileTreeHashBuilder; +import net.szum123321.textile_backup.core.Hash; +import org.jetbrains.annotations.NotNull; + +import java.io.FilterOutputStream; +import java.io.IOException; +import java.io.OutputStream; +import java.nio.file.Path; + +public class HashingOutputStream extends FilterOutputStream { + private final Path path; + private final Hash hasher = Globals.CHECKSUM_SUPPLIER.get(); + private final FileTreeHashBuilder hashBuilder; + + public HashingOutputStream(OutputStream out, Path path, FileTreeHashBuilder hashBuilder) { + super(out); + this.path = path; + this.hashBuilder = hashBuilder; + } + + @Override + public void write(int b) throws IOException { + super.write(b); + hasher.update(b); + } + + @Override + public void write(byte @NotNull [] b, int off, int len) throws IOException { + super.write(b, off, len); + hasher.update(b, off, len); + } + + @Override + public void close() throws IOException { + super.close(); + hashBuilder.update(path, hasher.getValue()); + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/AwaitThread.java b/src/main/java/net/szum123321/textile_backup/core/restore/AwaitThread.java index c022092..c728d5e 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/AwaitThread.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/AwaitThread.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java index 230c27c..4aeb58b 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java @@ -1,20 +1,20 @@ /* - A simple backup mod for Fabric - Copyright (C) 2020 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ package net.szum123321.textile_backup.core.restore; diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreContext.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreContext.java index f5391a5..e0ac528 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreContext.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreContext.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2020 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreHelper.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreHelper.java index 075e9fd..a0bf092 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreHelper.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreHelper.java @@ -1,20 +1,20 @@ /* - A simple backup mod for Fabric - Copyright (C) 2020 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ package net.szum123321.textile_backup.core.restore; diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java index f87f829..65878d3 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java @@ -1,20 +1,20 @@ /* - A simple backup mod for Fabric - Copyright (C) 2020 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ package net.szum123321.textile_backup.core.restore.decompressors; diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java index f67ec96..e085cd0 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java @@ -1,20 +1,20 @@ /* - A simple backup mod for Fabric - Copyright (C) 2020 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ package net.szum123321.textile_backup.core.restore.decompressors; diff --git a/src/main/java/net/szum123321/textile_backup/mixin/DedicatedServerWatchdogMixin.java b/src/main/java/net/szum123321/textile_backup/mixin/DedicatedServerWatchdogMixin.java index 415939c..5c15842 100644 --- a/src/main/java/net/szum123321/textile_backup/mixin/DedicatedServerWatchdogMixin.java +++ b/src/main/java/net/szum123321/textile_backup/mixin/DedicatedServerWatchdogMixin.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2021 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by diff --git a/src/main/java/net/szum123321/textile_backup/mixin/MinecraftServerSessionAccessor.java b/src/main/java/net/szum123321/textile_backup/mixin/MinecraftServerSessionAccessor.java index 91a8696..36f099b 100644 --- a/src/main/java/net/szum123321/textile_backup/mixin/MinecraftServerSessionAccessor.java +++ b/src/main/java/net/szum123321/textile_backup/mixin/MinecraftServerSessionAccessor.java @@ -1,6 +1,6 @@ /* * A simple backup mod for Fabric - * Copyright (C) 2021 Szum123321 + * Copyright (C) 2022 Szum123321 * * This program is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by From d871fc14cb57ca221dc2e46aba21d86690ae4e7d Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Sun, 27 Nov 2022 23:09:21 +0100 Subject: [PATCH 11/44] moved hash-related stuff core.digest package. Renamed XorSeaHash to BalticHash :). --- src/main/java/net/szum123321/textile_backup/Globals.java | 2 +- .../core/create/FileInputStreamSupplier.java | 3 ++- .../core/create/compressors/AbstractCompressor.java | 1 + .../core/{XorSeaHash.java => digest/BalticHash.java} | 2 +- .../textile_backup/core/digest/FileTreeHashBuilder.java | 2 +- .../szum123321/textile_backup/core/{ => digest}/Hash.java | 2 +- .../textile_backup/core/digest/HashingInputStream.java | 7 +++---- .../textile_backup/core/digest/HashingOutputStream.java | 2 +- .../core/restore/decompressors/GenericTarDecompressor.java | 4 ++-- .../core/restore/decompressors/ZipDecompressor.java | 4 ++-- 10 files changed, 15 insertions(+), 14 deletions(-) rename src/main/java/net/szum123321/textile_backup/core/{XorSeaHash.java => digest/BalticHash.java} (98%) rename src/main/java/net/szum123321/textile_backup/core/{ => digest}/Hash.java (94%) diff --git a/src/main/java/net/szum123321/textile_backup/Globals.java b/src/main/java/net/szum123321/textile_backup/Globals.java index cf4b0eb..10fc9ec 100644 --- a/src/main/java/net/szum123321/textile_backup/Globals.java +++ b/src/main/java/net/szum123321/textile_backup/Globals.java @@ -19,7 +19,7 @@ package net.szum123321.textile_backup; import net.minecraft.server.MinecraftServer; -import net.szum123321.textile_backup.core.Hash; +import net.szum123321.textile_backup.core.digest.Hash; import net.szum123321.textile_backup.core.Utilities; import net.szum123321.textile_backup.core.XorSeaHash; import net.szum123321.textile_backup.core.create.MakeBackupRunnable; diff --git a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java index 46a302f..2e2ac40 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java @@ -20,7 +20,8 @@ package net.szum123321.textile_backup.core.create; import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileLogger; -import net.szum123321.textile_backup.core.FileTreeHashBuilder; +import net.szum123321.textile_backup.core.digest.FileTreeHashBuilder; +import net.szum123321.textile_backup.core.digest.HashingInputStream; import java.io.IOException; import java.io.InputStream; diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index 5c6db3d..38c742a 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -26,6 +26,7 @@ import net.szum123321.textile_backup.core.create.BackupContext; import net.szum123321.textile_backup.core.create.BrokenFileHandler; import net.szum123321.textile_backup.core.create.FileInputStreamSupplier; import net.szum123321.textile_backup.core.create.InputSupplier; +import net.szum123321.textile_backup.core.digest.FileTreeHashBuilder; import java.io.*; import java.nio.file.Files; diff --git a/src/main/java/net/szum123321/textile_backup/core/XorSeaHash.java b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java similarity index 98% rename from src/main/java/net/szum123321/textile_backup/core/XorSeaHash.java rename to src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java index 1b74483..99feb74 100644 --- a/src/main/java/net/szum123321/textile_backup/core/XorSeaHash.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -package net.szum123321.textile_backup.core; +package net.szum123321.textile_backup.core.digest; import java.nio.ByteBuffer; import java.nio.ByteOrder; diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java index e0bb1ba..90ce907 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -package net.szum123321.textile_backup.core; +package net.szum123321.textile_backup.core.digest; import net.szum123321.textile_backup.Globals; diff --git a/src/main/java/net/szum123321/textile_backup/core/Hash.java b/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java similarity index 94% rename from src/main/java/net/szum123321/textile_backup/core/Hash.java rename to src/main/java/net/szum123321/textile_backup/core/digest/Hash.java index e562cb4..f662d95 100644 --- a/src/main/java/net/szum123321/textile_backup/core/Hash.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -package net.szum123321.textile_backup.core; +package net.szum123321.textile_backup.core.digest; public interface Hash { void update(byte b); diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java index becfb3d..7240f0a 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java @@ -16,19 +16,18 @@ * along with this program. If not, see . */ -package net.szum123321.textile_backup.core.create; +package net.szum123321.textile_backup.core.digest; import net.szum123321.textile_backup.Globals; import net.szum123321.textile_backup.core.DataLeftException; -import net.szum123321.textile_backup.core.FileTreeHashBuilder; -import net.szum123321.textile_backup.core.Hash; +import net.szum123321.textile_backup.core.create.BrokenFileHandler; import org.jetbrains.annotations.NotNull; import java.io.*; import java.nio.file.Path; //This class calculates a hash of the file on the input stream, submits it to FileTreeHashBuilder. -//In case the whole underlying stream hasn't been read, also puts it into BrokeFileHandler +//In case the underlying stream hasn't been read completely in, puts it into BrokeFileHandler public class HashingInputStream extends FilterInputStream { private final Path path; private final Hash hasher = Globals.CHECKSUM_SUPPLIER.get(); diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java index 4ce9887..33545ec 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java @@ -16,7 +16,7 @@ * along with this program. If not, see . */ -package net.szum123321.textile_backup.core.restore; +package net.szum123321.textile_backup.core.digest; import net.szum123321.textile_backup.Globals; import net.szum123321.textile_backup.core.FileTreeHashBuilder; diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java index 65878d3..cbd50af 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java @@ -20,9 +20,9 @@ package net.szum123321.textile_backup.core.restore.decompressors; import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileLogger; -import net.szum123321.textile_backup.core.FileTreeHashBuilder; +import net.szum123321.textile_backup.core.digest.FileTreeHashBuilder; import net.szum123321.textile_backup.core.Utilities; -import net.szum123321.textile_backup.core.restore.HashingOutputStream; +import net.szum123321.textile_backup.core.digest.HashingOutputStream; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; import org.apache.commons.compress.archivers.tar.TarArchiveInputStream; import org.apache.commons.compress.compressors.CompressorException; diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java index e085cd0..9475716 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java @@ -20,9 +20,9 @@ package net.szum123321.textile_backup.core.restore.decompressors; import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileLogger; -import net.szum123321.textile_backup.core.FileTreeHashBuilder; +import net.szum123321.textile_backup.core.digest.FileTreeHashBuilder; import net.szum123321.textile_backup.core.Utilities; -import net.szum123321.textile_backup.core.restore.HashingOutputStream; +import net.szum123321.textile_backup.core.digest.HashingOutputStream; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipFile; import org.apache.commons.compress.utils.IOUtils; From 300fe18b10c9163bbc2e7411bf786022d689511b Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Sun, 27 Nov 2022 23:11:41 +0100 Subject: [PATCH 12/44] Added support of virtual file (in this case textile_status.data). --- .../core/create/FileInputStreamSupplier.java | 8 +++++-- .../core/create/InputSupplier.java | 8 ++++++- .../compressors/AbstractCompressor.java | 5 ++++- .../compressors/ParallelZipCompressor.java | 22 +++++++++++++------ .../create/compressors/ZipCompressor.java | 18 ++++++++++----- .../compressors/tar/AbstractTarArchiver.java | 12 +++++++--- 6 files changed, 54 insertions(+), 19 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java index 2e2ac40..30c2556 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java @@ -28,6 +28,7 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Optional; public record FileInputStreamSupplier(Path path, String name, FileTreeHashBuilder hashTreeBuilder, BrokenFileHandler brokenFileHandler) implements InputSupplier { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); @@ -43,8 +44,11 @@ public record FileInputStreamSupplier(Path path, String name, FileTreeHashBuilde } @Override - public Path getPath() { - return path; + public Optional getPath() { return Optional.of(path); } + + @Override + public long size() throws IOException { + return Files.size(path); } @Override diff --git a/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java index 74960b2..6a5ec6a 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java @@ -23,8 +23,14 @@ import org.apache.commons.compress.parallel.InputStreamSupplier; import java.io.IOException; import java.io.InputStream; import java.nio.file.Path; +import java.util.Optional; + + public interface InputSupplier extends InputStreamSupplier { InputStream getInputStream() throws IOException; - Path getPath(); + //If an entry is virtual (a.k.a there is no actual file to open, only input stream) + Optional getPath(); String getName(); + + long size() throws IOException; } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index 38c742a..596ec48 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -33,6 +33,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.time.Duration; import java.time.Instant; +import java.util.Optional; import java.util.concurrent.ExecutionException; import java.util.stream.Stream; @@ -128,10 +129,12 @@ public abstract class AbstractCompressor { private record StatusFileInputSupplier(byte[] data) implements InputSupplier { public InputStream getInputStream() { return new ByteArrayInputStream(data); } - public Path getPath() { return Path.of(CompressionStatus.DATA_FILENAME); } + public Optional getPath() { return Optional.empty(); } public String getName() { return CompressionStatus.DATA_FILENAME; } + public long size() { return data.length; } + public InputStream get() { return getInputStream(); } } } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java index 8acf42f..3632179 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java @@ -27,6 +27,7 @@ import org.apache.commons.compress.archivers.zip.*; import java.io.*; import java.nio.file.Files; +import java.nio.file.Path; import java.util.Objects; import java.util.concurrent.*; import java.util.zip.ZipEntry; @@ -67,14 +68,21 @@ public class ParallelZipCompressor extends ZipCompressor { @Override protected void addEntry(InputSupplier input, OutputStream arc) throws IOException { - ZipArchiveEntry entry = (ZipArchiveEntry)((ZipArchiveOutputStream)arc).createArchiveEntry(input.getPath(), input.getName()); - - if(ZipCompressor.isDotDat(input.getPath().getFileName().toString())) { + ZipArchiveEntry entry; + if(input.getPath().isEmpty()) { + entry = new ZipArchiveEntry(input.getName()); entry.setMethod(ZipEntry.STORED); - entry.setSize(Files.size(input.getPath())); - entry.setCompressedSize(Files.size(input.getPath())); - entry.setCrc(getCRC(input.getPath())); - } else entry.setMethod(ZipEntry.DEFLATED); + entry.setSize(input.size()); + } else { + Path file = input.getPath().get(); + entry = (ZipArchiveEntry) ((ZipArchiveOutputStream) arc).createArchiveEntry(file, input.getName()); + if (ZipCompressor.isDotDat(file.toString())) { + entry.setMethod(ZipEntry.STORED); + entry.setSize(Files.size(file)); + entry.setCompressedSize(Files.size(file)); + entry.setCrc(getCRC(file)); + } else entry.setMethod(ZipEntry.DEFLATED); + } entry.setTime(System.currentTimeMillis()); diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java index 5a87a97..5a39bab 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java @@ -57,13 +57,21 @@ public class ZipCompressor extends AbstractCompressor { @Override protected void addEntry(InputSupplier input, OutputStream arc) throws IOException { try (InputStream fileInputStream = input.getInputStream()) { - ZipArchiveEntry entry = (ZipArchiveEntry)((ZipArchiveOutputStream)arc).createArchiveEntry(input.getPath(), input.getName()); + ZipArchiveEntry entry; - if(isDotDat(input.getPath().getFileName().toString())) { + if(input.getPath().isEmpty()) { + entry = new ZipArchiveEntry(input.getName()); entry.setMethod(ZipEntry.STORED); - entry.setSize(Files.size(input.getPath())); - entry.setCompressedSize(Files.size(input.getPath())); - entry.setCrc(getCRC(input.getPath())); + entry.setSize(input.size()); + } else { + Path file = input.getPath().get(); + entry = (ZipArchiveEntry) ((ZipArchiveOutputStream) arc).createArchiveEntry(file, input.getName()); + if (isDotDat(file.toString())) { + entry.setMethod(ZipEntry.STORED); + entry.setSize(Files.size(file)); + entry.setCompressedSize(Files.size(file)); + entry.setCrc(getCRC(file)); + } else entry.setMethod(ZipEntry.DEFLATED); } ((ZipArchiveOutputStream)arc).putArchiveEntry(entry); diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java index 5aa8a3c..94d0ad2 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java @@ -42,9 +42,15 @@ public class AbstractTarArchiver extends AbstractCompressor { } @Override - protected void addEntry(InputSupplier in, OutputStream arc) throws IOException { - try (InputStream fileInputStream = in.getInputStream()) { - TarArchiveEntry entry = (TarArchiveEntry)((TarArchiveOutputStream) arc).createArchiveEntry(in.getPath(), in.getName()); + protected void addEntry(InputSupplier input, OutputStream arc) throws IOException { + try (InputStream fileInputStream = input.getInputStream()) { + TarArchiveEntry entry; + if(input.getPath().isEmpty()) {//Virtual entry + entry = new TarArchiveEntry(input.getName()); + entry.setSize(input.size()); + } else + entry = (TarArchiveEntry)((TarArchiveOutputStream) arc).createArchiveEntry(input.getPath().get(), input.getName()); + ((TarArchiveOutputStream)arc).putArchiveEntry(entry); IOUtils.copy(fileInputStream, arc); From febbb95b97c4b6ca2a34dd87a162e15ee22e7cdb Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Sun, 27 Nov 2022 23:12:25 +0100 Subject: [PATCH 13/44] A bit of cleanup of BalticHash. Added wip simd version --- .../core/digest/BalticHash.java | 31 +++--- .../core/digest/BalticHashSIMD.java | 95 +++++++++++++++++++ 2 files changed, 110 insertions(+), 16 deletions(-) create mode 100644 src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java index 99feb74..0b0b108 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java @@ -20,6 +20,7 @@ package net.szum123321.textile_backup.core.digest; import java.nio.ByteBuffer; import java.nio.ByteOrder; +import java.util.Arrays; /* This algorithm copies construction of SeaHash (https://ticki.github.io/blog/seahash-explained/) including its IV @@ -27,33 +28,30 @@ import java.nio.ByteOrder; I don't think it matters that much, honestly. One advantage the xoroshift has is that it should be easier to implement with AVX. Java should soon ship its vector api by default. */ -public class XorSeaHash implements Hash { +public class BalticHash implements Hash { + protected final static long[] IV = { 0x16f11fe89b0d677cL, 0xb480a793d8e6c86cL, 0x6fe2e5aaf078ebc9L, 0x14f994a4c5259381L}; + //SeaHash IV - private final long[] state = { 0x16f11fe89b0d677cL, 0xb480a793d8e6c86cL, 0x6fe2e5aaf078ebc9L, 0x14f994a4c5259381L}; - private final int buffer_size = (state.length + 1) * Long.BYTES; - private final int buffer_limit = state.length * Long.BYTES; - private final byte[] _byte_buffer = new byte[buffer_size]; + private final long[] state = Arrays.copyOf(IV, IV.length); + protected final int buffer_limit = state.length * Long.BYTES; + protected final byte[] _byte_buffer = new byte[(state.length + 1) * Long.BYTES]; //Enforce endianness - private final ByteBuffer buffer = ByteBuffer.wrap(_byte_buffer).order(ByteOrder.LITTLE_ENDIAN); + protected final ByteBuffer buffer = ByteBuffer.wrap(_byte_buffer).order(ByteOrder.LITTLE_ENDIAN); - private long hashed_data_length = 0; + protected long hashed_data_length = 0; - @Override public void update(byte b) { buffer.put(b); hashed_data_length += 1; if (buffer.position() >= buffer_limit) round(); } - @Override public void update(long b) { buffer.putLong(b); hashed_data_length += Long.BYTES; if(buffer.position() >= buffer_limit) round(); } - public void update(byte [] data) { update(data, 0, data.length); } - public void update(byte[] data, int off, int len) { int pos = off; while(pos < len) { @@ -67,9 +65,11 @@ public class XorSeaHash implements Hash { hashed_data_length += len; } - @Override public long getValue() { - if(buffer.position() != 0) round(); + if(buffer.position() != 0) { + while(buffer.position() < buffer_limit) buffer.put((byte)0); + round(); + } long result = state[0]; result ^= state[1]; @@ -80,8 +80,7 @@ public class XorSeaHash implements Hash { return xorshift64star(result); } - private void round() { - while(buffer.position() < buffer_limit) buffer.put((byte)0); + protected void round() { int p = buffer.position(); buffer.rewind(); @@ -91,7 +90,7 @@ public class XorSeaHash implements Hash { if(p > buffer_limit) { System.arraycopy(_byte_buffer, buffer_limit, _byte_buffer, 0, buffer.limit() - p); buffer.position(buffer.limit() - p); - } + } else buffer.rewind(); } long xorshift64star(long s) { diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java new file mode 100644 index 0000000..8159f43 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java @@ -0,0 +1,95 @@ +/* + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package net.szum123321.textile_backup.core.digest; + +//import jdk.incubator.vector.*; + +import net.szum123321.textile_backup.core.digest.BalticHash; + +/*Mostly working XorSeaHash impl using SIMD. Should speed up calculation on most systems currently in use + +It's actually slower. I tested it by comparing runtimes while hashing a directly opened FileInputStream. +My cpu is AMD Ryzen 5 3500U + +There are two reasons I can think of: either vector construction simply takes so much time or jvm auto-vectorizes better than me + +It's still probably far from being the slowest part of code, so I don't expect any major slowdowns + +I will keep this code here for future work perhaps + */ +public class BalticHashSIMD extends BalticHash { + public BalticHashSIMD() { throw new UnsupportedOperationException(); } //For safety + + /* private LongVector state = LongVector.fromArray(LongVector.SPECIES_256, IV, 0); + + @Override + public long getValue() { + if(buffer.position() != 0) { + while(buffer.position() < buffer_limit) buffer.put((byte)0); + round(); + } + + long result = state.reduceLanesToLong(VectorOperators.XOR); + result ^= hashed_data_length; + + return xorshift64star(result); + } + + @Override + public void update(byte[] data, int off, int len) { + int pos = off; + while(pos < len) { + int n = Math.min(len - pos, buffer_limit - buffer.position()); + if(n == 32) { + var v = ByteVector.fromArray(ByteVector.SPECIES_256, data, pos).reinterpretAsLongs(); + state = state.lanewise(VectorOperators.XOR, v); + state = xorshift64star(state); + } else { + System.arraycopy(data, pos, _byte_buffer, buffer.position(), n); + buffer.position(buffer.position() + n); + if(buffer.position() == buffer_limit) round(); + } + pos += n; + } + + hashed_data_length += len; + } + + @Override + protected void round() { + var s = ByteVector.fromArray(ByteVector.SPECIES_256, _byte_buffer, 0).reinterpretAsLongs(); + state = state.lanewise(VectorOperators.XOR, s); + state = xorshift64star(state); + + int p = buffer.position(); + + if(p > buffer_limit) { + System.arraycopy(_byte_buffer, buffer_limit, _byte_buffer, 0, buffer.limit() - p); + buffer.position(buffer.limit() - p); + } else buffer.rewind(); + } + + LongVector xorshift64star(LongVector v) { + v = v.lanewise(VectorOperators.XOR, v.lanewise(VectorOperators.ASHR, 12)); + v = v.lanewise(VectorOperators.XOR, v.lanewise(VectorOperators.LSHL, 25)); + v = v.lanewise(VectorOperators.XOR, v.lanewise(VectorOperators.ASHR, 27)); + v = v.lanewise(VectorOperators.MUL, 0x2545F4914F6CDD1DL); + return v; + }*/ +} \ No newline at end of file From 7734c16e068da648aefc4457f766f4628919d4f0 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Sun, 27 Nov 2022 23:19:37 +0100 Subject: [PATCH 14/44] those didnt work --- .../core/create/MakeBackupRunnable.java | 13 ++++++------- .../core/digest/FileTreeHashBuilder.java | 5 ++++- .../core/digest/HashingOutputStream.java | 6 ++---- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java index e550a0a..917c68b 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java @@ -102,14 +102,13 @@ public class MakeBackupRunnable implements Callable { case TAR -> new AbstractTarArchiver().createArchive(world, outFile, context, coreCount); } - Globals.INSTANCE.getQueueExecutor().submit(new Cleanup(context.commandSource(), Utilities.getLevelName(context.server()))); + if(!Globals.INSTANCE.getQueueExecutor().isShutdown()) + Globals.INSTANCE.getQueueExecutor().submit(new Cleanup(context.commandSource(), Utilities.getLevelName(context.server()))); - if (config.get().broadcastBackupDone) { - Utilities.notifyPlayers(context.server(), "Done!"); - } else { - log.sendInfoAL(context, "Done!"); - } - } catch (InterruptedException | ExecutionException | IOException e) { + if (config.get().broadcastBackupDone) Utilities.notifyPlayers(context.server(), "Done!"); + else log.sendInfoAL(context, "Done!"); + + } catch (Throwable e) { //ExecutorService swallows exception, so I need to catch everything log.error("An exception occurred when trying to create new backup file!", e); diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java index 90ce907..ec657ec 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java @@ -19,6 +19,7 @@ package net.szum123321.textile_backup.core.digest; import net.szum123321.textile_backup.Globals; +import net.szum123321.textile_backup.core.CompressionStatus; import java.io.IOException; import java.nio.charset.StandardCharsets; @@ -29,11 +30,13 @@ public class FileTreeHashBuilder { private long hash = 0, filesProcessed = 0, filesTotalSize = 0; public void update(Path path, long newHash) throws IOException { + if(path.getFileName().toString().equals(CompressionStatus.DATA_FILENAME)) return; + var hasher = Globals.CHECKSUM_SUPPLIER.get(); long size = Files.size(path); - hasher.update(path.toString().getBytes(StandardCharsets.UTF_8)); + hasher.update(path.getFileName().toString().getBytes(StandardCharsets.UTF_8)); hasher.update(newHash); synchronized (lock) { diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java index 33545ec..f4ea178 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java @@ -19,8 +19,6 @@ package net.szum123321.textile_backup.core.digest; import net.szum123321.textile_backup.Globals; -import net.szum123321.textile_backup.core.FileTreeHashBuilder; -import net.szum123321.textile_backup.core.Hash; import org.jetbrains.annotations.NotNull; import java.io.FilterOutputStream; @@ -41,14 +39,14 @@ public class HashingOutputStream extends FilterOutputStream { @Override public void write(int b) throws IOException { - super.write(b); hasher.update(b); + super.write(b); } @Override public void write(byte @NotNull [] b, int off, int len) throws IOException { - super.write(b, off, len); hasher.update(b, off, len); + super.write(b, off, len); } @Override From 993d6359adb92b7020abbbe7cc130d9783bd4b5c Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Sun, 27 Nov 2022 23:20:05 +0100 Subject: [PATCH 15/44] typos --- .../java/net/szum123321/textile_backup/config/ConfigPOJO.java | 2 +- .../textile_backup/core/create/MakeBackupRunnable.java | 4 ++-- .../textile_backup/core/restore/RestoreBackupRunnable.java | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java index 61b4458..fab5f4c 100644 --- a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java +++ b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java @@ -124,7 +124,7 @@ public class ConfigPOJO implements ConfigData { public ArchiveFormat format = ArchiveFormat.ZIP; @Comment(""" - The Strict mode (default) aborts backup creation in case of any problem and deletes the created files + \nThe Strict mode (default) aborts backup creation in case of any problem and deletes created files Permissible mode keeps partial/damaged backup but won't allow to restore it Very Permissible mode will skip the verification process. THIS MOST CERTAINLY WILL LEAD TO DATA LOSS OR CORRUPTION """) diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java index 917c68b..c9d1528 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java @@ -59,7 +59,7 @@ public class MakeBackupRunnable implements Callable { log.trace("Outfile is: {}", outFile); try { - //I think I should synchronise those two next calls... + //I think I should synchronise these two next calls... Utilities.disableWorldSaving(context.server()); Globals.INSTANCE.disableWatchdog = true; @@ -116,7 +116,7 @@ public class MakeBackupRunnable implements Callable { try { Files.delete(outFile); } catch (IOException ex) { - log.error("An exception occurred while tryin go delete: {}", outFile, ex); + log.error("An exception occurred while trying go delete: {}", outFile, ex); } } diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java index 4aeb58b..f1741e2 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java @@ -70,7 +70,7 @@ public class RestoreBackupRunnable implements Runnable { FutureTask waitForShutdown = new FutureTask<>(() -> { ctx.server().getThread().join(); //wait for server to die and save all its state if(config.get().backupOldWorlds) { - return MakeBackupRunnableFactory.create( + return MakeBackupRunnableFactory.create ( BackupContext.Builder .newBackupContextBuilder() .setServer(ctx.server()) From 2053df7311c98284e3aa34db0e46539329f2db57 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Mon, 28 Nov 2022 18:53:55 +0100 Subject: [PATCH 16/44] debugging in progress... --- build.gradle | 4 +-- .../szum123321/textile_backup/Globals.java | 31 ++++++++++++++-- .../core/digest/FileTreeHashBuilder.java | 4 +++ .../textile_backup/core/digest/Hash.java | 4 +++ .../core/digest/HashingInputStream.java | 36 ++++++++++++++++--- .../core/digest/HashingOutputStream.java | 20 ++++++++++- .../core/restore/RestoreBackupRunnable.java | 5 +-- .../decompressors/ZipDecompressor.java | 18 ++++++++-- 8 files changed, 109 insertions(+), 13 deletions(-) diff --git a/build.gradle b/build.gradle index 72e04cd..b838b88 100644 --- a/build.gradle +++ b/build.gradle @@ -46,8 +46,8 @@ dependencies { modImplementation("com.terraformersmc:modmenu:${project.modmenu_version}") //General compression library - implementation "org.apache.commons:commons-compress:1.21" - include "org.apache.commons:commons-compress:1.21" + implementation "org.apache.commons:commons-compress:1.22" + include "org.apache.commons:commons-compress:1.22" //LZMA support implementation 'org.tukaani:xz:1.9' diff --git a/src/main/java/net/szum123321/textile_backup/Globals.java b/src/main/java/net/szum123321/textile_backup/Globals.java index 10fc9ec..6e425b7 100644 --- a/src/main/java/net/szum123321/textile_backup/Globals.java +++ b/src/main/java/net/szum123321/textile_backup/Globals.java @@ -21,11 +21,14 @@ package net.szum123321.textile_backup; import net.minecraft.server.MinecraftServer; import net.szum123321.textile_backup.core.digest.Hash; import net.szum123321.textile_backup.core.Utilities; -import net.szum123321.textile_backup.core.XorSeaHash; + import net.szum123321.textile_backup.core.create.MakeBackupRunnable; import net.szum123321.textile_backup.core.restore.AwaitThread; import org.apache.commons.io.FileUtils; +import org.tukaani.xz.check.CRC64; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.file.Files; import java.nio.file.Path; import java.time.format.DateTimeFormatter; @@ -41,7 +44,31 @@ public class Globals { public static final Globals INSTANCE = new Globals(); private static final TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); public static final DateTimeFormatter defaultDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH.mm.ss"); - public static final Supplier CHECKSUM_SUPPLIER = XorSeaHash::new; + public static final Supplier CHECKSUM_SUPPLIER = () -> new Hash() { + private final CRC64 crc = new CRC64(); + @Override + public void update(byte b) { + crc.update(new byte[]{b}); + } + + @Override + public void update(long b) { + ByteBuffer v = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + v.putLong(b); + crc.update(v.array()); + } + + @Override + public void update(byte[] b, int off, int len) { + crc.update(b, off, len); + } + + @Override + public long getValue() { + ByteBuffer b = ByteBuffer.wrap(crc.finish()); + return b.getLong(); + } + }; private ExecutorService executorService = null;// = Executors.newSingleThreadExecutor(); public final AtomicBoolean globalShutdownBackupFlag = new AtomicBoolean(true); diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java index ec657ec..d735ece 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java @@ -19,6 +19,8 @@ package net.szum123321.textile_backup.core.digest; import net.szum123321.textile_backup.Globals; +import net.szum123321.textile_backup.TextileBackup; +import net.szum123321.textile_backup.TextileLogger; import net.szum123321.textile_backup.core.CompressionStatus; import java.io.IOException; @@ -26,11 +28,13 @@ import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; public class FileTreeHashBuilder { + private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); private final Object lock = new Object(); private long hash = 0, filesProcessed = 0, filesTotalSize = 0; public void update(Path path, long newHash) throws IOException { if(path.getFileName().toString().equals(CompressionStatus.DATA_FILENAME)) return; + //log.info("Putting: {}, {}", path, newHash); var hasher = Globals.CHECKSUM_SUPPLIER.get(); diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java b/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java index f662d95..d05533a 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java @@ -20,6 +20,10 @@ package net.szum123321.textile_backup.core.digest; public interface Hash { void update(byte b); + + default void update(int b) { + update((byte)b); + } void update(long b); default void update(byte[] b) { update(b, 0, b.length); diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java index 7240f0a..8c8d2e9 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java @@ -19,6 +19,8 @@ package net.szum123321.textile_backup.core.digest; import net.szum123321.textile_backup.Globals; +import net.szum123321.textile_backup.TextileBackup; +import net.szum123321.textile_backup.TextileLogger; import net.szum123321.textile_backup.core.DataLeftException; import net.szum123321.textile_backup.core.create.BrokenFileHandler; import org.jetbrains.annotations.NotNull; @@ -29,11 +31,24 @@ import java.nio.file.Path; //This class calculates a hash of the file on the input stream, submits it to FileTreeHashBuilder. //In case the underlying stream hasn't been read completely in, puts it into BrokeFileHandler public class HashingInputStream extends FilterInputStream { + private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); private final Path path; private final Hash hasher = Globals.CHECKSUM_SUPPLIER.get(); private final FileTreeHashBuilder hashBuilder; private final BrokenFileHandler brokenFileHandler; + private int cnt = 0; + + @Override + public synchronized void reset() throws IOException { + log.info("Called reset! {}", path); + } + + @Override + public boolean markSupported() { + return false; + } + public HashingInputStream(InputStream in, Path path, FileTreeHashBuilder hashBuilder, BrokenFileHandler brokenFileHandler) { super(in); this.path = path; @@ -44,21 +59,34 @@ public class HashingInputStream extends FilterInputStream { @Override public int read(byte @NotNull [] b, int off, int len) throws IOException { int i = in.read(b, off, len); - if(i > -1) hasher.update(b, off, i); + if(i > -1) { + hasher.update(b, off, i); + cnt += i; + } return i; } @Override public int read() throws IOException { int i = in.read(); - if(i > -1) hasher.update(i); + if(i > -1) { + hasher.update(i); + cnt++; + } return i; } @Override public void close() throws IOException { - if(in.available() == 0) hashBuilder.update(path, hasher.getValue()); - else brokenFileHandler.handle(path, new DataLeftException(in.available())); + if(in.available() == 0) { + long val = hasher.getValue(); + hashBuilder.update(path, val); + log.info("Read in {}, of {}, with hash {}", path, cnt, val); + } + else { + brokenFileHandler.handle(path, new DataLeftException(in.available())); + //log.info("bad file {} {}",path, cnt); + } super.close(); } } diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java index f4ea178..1ca8042 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java @@ -19,6 +19,8 @@ package net.szum123321.textile_backup.core.digest; import net.szum123321.textile_backup.Globals; +import net.szum123321.textile_backup.TextileBackup; +import net.szum123321.textile_backup.TextileLogger; import org.jetbrains.annotations.NotNull; import java.io.FilterOutputStream; @@ -27,31 +29,47 @@ import java.io.OutputStream; import java.nio.file.Path; public class HashingOutputStream extends FilterOutputStream { + private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); + private final Path path; private final Hash hasher = Globals.CHECKSUM_SUPPLIER.get(); private final FileTreeHashBuilder hashBuilder; + private long cnt = 0; + public HashingOutputStream(OutputStream out, Path path, FileTreeHashBuilder hashBuilder) { super(out); this.path = path; this.hashBuilder = hashBuilder; } + @Override + public void flush() throws IOException { + //log.info("Called flush! {}", path); + super.flush(); + } + @Override public void write(int b) throws IOException { hasher.update(b); + cnt++; super.write(b); } @Override public void write(byte @NotNull [] b, int off, int len) throws IOException { + cnt += len; + log.info("Called: {} with {}", path, len); hasher.update(b, off, len); super.write(b, off, len); } @Override public void close() throws IOException { + long h = hasher.getValue(); + log.info("Read in: {}, of {}, with hash {}", path, cnt, h); + hashBuilder.update(path, h); super.close(); - hashBuilder.update(path, hasher.getValue()); + } } diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java index f1741e2..8caa282 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java @@ -121,11 +121,12 @@ public class RestoreBackupRunnable implements Runnable { log.error("An exception occurred while trying to restore a backup!", e); } finally { //Regardless of what happened, we should still clean up - if(Files.exists(tmp)) { + /* if(Files.exists(tmp)) { try { Utilities.deleteDirectory(tmp); } catch (IOException ignored) {} - } + }*/ + //TODO: uncomment } //in case we're playing on client diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java index 9475716..234fd34 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java @@ -47,13 +47,27 @@ public class ZipDecompressor { ZipArchiveEntry entry = it.next(); Path file = target.resolve(entry.getName()); + byte[] buff = new byte[4096]; + + log.info("Unpacking {} uncompressed {} compressed {}", entry.getName(), entry.getSize(), entry.getCompressedSize()); + if(entry.isDirectory()) { Files.createDirectories(file); } else { Files.createDirectories(file.getParent()); try (OutputStream outputStream = Files.newOutputStream(file); - HashingOutputStream out = new HashingOutputStream(outputStream, file, hashBuilder)) { - IOUtils.copy(zipFile.getInputStream(entry), out); + HashingOutputStream out = new HashingOutputStream(outputStream, file, hashBuilder); + InputStream in = zipFile.getInputStream(entry)) { + + int n; + long count = 0; + while((n = in.read(buff, 0, buff.length)) >= 1) { + out.write(buff, 0, n); + count += n; + } + + log.info("File {}, in size {}, copied {}", entry.getName(), in.available(), count); + } } } From 5367a00cdce9432b346b4e7c4fd19dd82d1092c8 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Mon, 28 Nov 2022 19:43:30 +0100 Subject: [PATCH 17/44] individual file hashing works. filetree still fails --- .../core/digest/HashingOutputStream.java | 20 ++----------------- .../decompressors/ZipDecompressor.java | 14 +------------ 2 files changed, 3 insertions(+), 31 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java index 1ca8042..909d2ef 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java @@ -19,8 +19,6 @@ package net.szum123321.textile_backup.core.digest; import net.szum123321.textile_backup.Globals; -import net.szum123321.textile_backup.TextileBackup; -import net.szum123321.textile_backup.TextileLogger; import org.jetbrains.annotations.NotNull; import java.io.FilterOutputStream; @@ -29,45 +27,31 @@ import java.io.OutputStream; import java.nio.file.Path; public class HashingOutputStream extends FilterOutputStream { - private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); - private final Path path; private final Hash hasher = Globals.CHECKSUM_SUPPLIER.get(); private final FileTreeHashBuilder hashBuilder; - private long cnt = 0; - public HashingOutputStream(OutputStream out, Path path, FileTreeHashBuilder hashBuilder) { super(out); this.path = path; this.hashBuilder = hashBuilder; } - @Override - public void flush() throws IOException { - //log.info("Called flush! {}", path); - super.flush(); - } - @Override public void write(int b) throws IOException { hasher.update(b); - cnt++; - super.write(b); + out.write(b); } @Override public void write(byte @NotNull [] b, int off, int len) throws IOException { - cnt += len; - log.info("Called: {} with {}", path, len); hasher.update(b, off, len); - super.write(b, off, len); + out.write(b, off, len); } @Override public void close() throws IOException { long h = hasher.getValue(); - log.info("Read in: {}, of {}, with hash {}", path, cnt, h); hashBuilder.update(path, h); super.close(); diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java index 234fd34..2bae84a 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java @@ -47,10 +47,6 @@ public class ZipDecompressor { ZipArchiveEntry entry = it.next(); Path file = target.resolve(entry.getName()); - byte[] buff = new byte[4096]; - - log.info("Unpacking {} uncompressed {} compressed {}", entry.getName(), entry.getSize(), entry.getCompressedSize()); - if(entry.isDirectory()) { Files.createDirectories(file); } else { @@ -59,15 +55,7 @@ public class ZipDecompressor { HashingOutputStream out = new HashingOutputStream(outputStream, file, hashBuilder); InputStream in = zipFile.getInputStream(entry)) { - int n; - long count = 0; - while((n = in.read(buff, 0, buff.length)) >= 1) { - out.write(buff, 0, n); - count += n; - } - - log.info("File {}, in size {}, copied {}", entry.getName(), in.available(), count); - + IOUtils.copy(in, out); } } } From 2efe1121579e2c5997b4600afbf663a027f83396 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 29 Nov 2022 00:55:18 +0100 Subject: [PATCH 18/44] Repaired race condition with ParallelZip. Hashing now works! --- .../szum123321/textile_backup/Globals.java | 32 ++----------- .../core/create/BackupContext.java | 10 +++- .../core/create/FileInputStreamSupplier.java | 12 +++-- .../core/create/MakeBackupRunnable.java | 4 +- .../compressors/AbstractCompressor.java | 42 ++++++++-------- .../core/digest/FileTreeHashBuilder.java | 22 ++++++--- .../core/digest/HashingInputStream.java | 48 +++++++------------ .../core/restore/RestoreBackupRunnable.java | 13 +++-- 8 files changed, 80 insertions(+), 103 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/Globals.java b/src/main/java/net/szum123321/textile_backup/Globals.java index 6e425b7..37d620a 100644 --- a/src/main/java/net/szum123321/textile_backup/Globals.java +++ b/src/main/java/net/szum123321/textile_backup/Globals.java @@ -19,16 +19,14 @@ package net.szum123321.textile_backup; import net.minecraft.server.MinecraftServer; +import net.szum123321.textile_backup.core.digest.BalticHash; import net.szum123321.textile_backup.core.digest.Hash; import net.szum123321.textile_backup.core.Utilities; import net.szum123321.textile_backup.core.create.MakeBackupRunnable; import net.szum123321.textile_backup.core.restore.AwaitThread; import org.apache.commons.io.FileUtils; -import org.tukaani.xz.check.CRC64; -import java.nio.ByteBuffer; -import java.nio.ByteOrder; import java.nio.file.Files; import java.nio.file.Path; import java.time.format.DateTimeFormatter; @@ -44,33 +42,9 @@ public class Globals { public static final Globals INSTANCE = new Globals(); private static final TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); public static final DateTimeFormatter defaultDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH.mm.ss"); - public static final Supplier CHECKSUM_SUPPLIER = () -> new Hash() { - private final CRC64 crc = new CRC64(); - @Override - public void update(byte b) { - crc.update(new byte[]{b}); - } + public static final Supplier CHECKSUM_SUPPLIER = BalticHash::new; - @Override - public void update(long b) { - ByteBuffer v = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); - v.putLong(b); - crc.update(v.array()); - } - - @Override - public void update(byte[] b, int off, int len) { - crc.update(b, off, len); - } - - @Override - public long getValue() { - ByteBuffer b = ByteBuffer.wrap(crc.finish()); - return b.getLong(); - } - }; - - private ExecutorService executorService = null;// = Executors.newSingleThreadExecutor(); + private ExecutorService executorService = null;//TODO: AAAAAAAAAAAAAAA MEMORY LEAK!!!!!!!!! public final AtomicBoolean globalShutdownBackupFlag = new AtomicBoolean(true); public boolean disableWatchdog = false; private boolean disableTMPFiles = false; diff --git a/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java b/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java index 6eb6a57..c3469be 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java @@ -30,6 +30,7 @@ public record BackupContext(@NotNull MinecraftServer server, ServerCommandSource commandSource, ActionInitiator initiator, boolean save, + boolean cleanup, String comment, LocalDateTime startDate) { @@ -46,6 +47,7 @@ public record BackupContext(@NotNull MinecraftServer server, private ServerCommandSource commandSource; private ActionInitiator initiator; private boolean save; + private boolean cleanup; private String comment; private boolean guessInitiator; @@ -55,6 +57,7 @@ public record BackupContext(@NotNull MinecraftServer server, this.commandSource = null; this.initiator = null; this.save = false; + cleanup = true; //defaults this.comment = null; guessInitiator = false; @@ -94,6 +97,11 @@ public record BackupContext(@NotNull MinecraftServer server, return this; } + public Builder dontCleanup() { + this.cleanup = false; + return this; + } + public BackupContext build() { if (guessInitiator) { initiator = commandSource.getEntity() instanceof PlayerEntity ? ActionInitiator.Player : ActionInitiator.ServerConsole; @@ -106,7 +114,7 @@ public record BackupContext(@NotNull MinecraftServer server, else throw new RuntimeException("Neither MinecraftServer or ServerCommandSource were provided!"); } - return new BackupContext(server, commandSource, initiator, save, comment, LocalDateTime.now()); + return new BackupContext(server, commandSource, initiator, save, cleanup, comment, LocalDateTime.now()); } } } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java index 30c2556..90ca134 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java @@ -29,15 +29,19 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.Optional; +import java.util.concurrent.CountDownLatch; -public record FileInputStreamSupplier(Path path, String name, FileTreeHashBuilder hashTreeBuilder, BrokenFileHandler brokenFileHandler) implements InputSupplier { +public record FileInputStreamSupplier(Path path, String name, FileTreeHashBuilder hashTreeBuilder, BrokenFileHandler brokenFileHandler, CountDownLatch latch) implements InputSupplier { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); @Override public InputStream getInputStream() throws IOException { try { - return new HashingInputStream(Files.newInputStream(path), path, hashTreeBuilder, brokenFileHandler); + return new HashingInputStream(Files.newInputStream(path), path, hashTreeBuilder, brokenFileHandler, latch); } catch (IOException e) { + //Probably good idea to just put it here. In the case an exception is thrown here, it would be probable + //The latch would never be lifted + latch.countDown(); brokenFileHandler.handle(path, e); throw e; } @@ -47,9 +51,7 @@ public record FileInputStreamSupplier(Path path, String name, FileTreeHashBuilde public Optional getPath() { return Optional.of(path); } @Override - public long size() throws IOException { - return Files.size(path); - } + public long size() throws IOException { return Files.size(path); } @Override public String getName() { diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java index c9d1528..c07e9fc 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java @@ -102,8 +102,8 @@ public class MakeBackupRunnable implements Callable { case TAR -> new AbstractTarArchiver().createArchive(world, outFile, context, coreCount); } - if(!Globals.INSTANCE.getQueueExecutor().isShutdown()) - Globals.INSTANCE.getQueueExecutor().submit(new Cleanup(context.commandSource(), Utilities.getLevelName(context.server()))); + if(context.cleanup()) + new Cleanup(context.commandSource(), Utilities.getLevelName(context.server())).call(); if (config.get().broadcastBackupDone) Utilities.notifyPlayers(context.server(), "Done!"); else log.sendInfoAL(context, "Done!"); diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index 596ec48..87f91d3 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -34,7 +34,9 @@ import java.nio.file.Path; import java.time.Duration; import java.time.Instant; import java.util.Optional; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; +import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; /** @@ -53,9 +55,18 @@ public abstract class AbstractCompressor { BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream); OutputStream arc = createArchiveOutputStream(bufferedOutputStream, ctx, coreLimit); Stream fileStream = Files.walk(inputFile)) { + + AtomicInteger fileCounter = new AtomicInteger(0); + var it = fileStream .filter(path -> !Utilities.isBlacklisted(inputFile.relativize(path))) - .filter(Files::isRegularFile).iterator(); + .filter(Files::isRegularFile) + .peek(x -> fileCounter.incrementAndGet()).toList() + .iterator(); + + log.info("File count: {}", fileCounter.get()); + + CountDownLatch latch = new CountDownLatch(fileCounter.get()); while(it.hasNext()) { Path file = it.next(); @@ -66,7 +77,8 @@ public abstract class AbstractCompressor { file, inputFile.relativize(file).toString(), fileHashBuilder, - brokenFileHandler), + brokenFileHandler, + latch), arc ); } catch (IOException e) { @@ -78,37 +90,21 @@ public abstract class AbstractCompressor { } } + latch.await(); + Instant now = Instant.now(); CompressionStatus status = new CompressionStatus ( fileHashBuilder.getValue(), + brokenFileHandler.get(), ctx.startDate(), start.toEpochMilli(), now.toEpochMilli(), - brokenFileHandler.get() + TextileBackup.VERSION ); addEntry(new StatusFileInputSupplier(status.serialize()), arc); finish(arc); - } /*catch(NoSpaceLeftOnDeviceException e) { - log.error(""" - CRITICAL ERROR OCCURRED! - The backup is corrupt! - Don't panic! This is a known issue! - For help see: https://github.com/Szum123321/textile_backup/wiki/ZIP-Problems - In case this isn't it here's also the exception itself""", e); - - if(ctx.initiator() == ActionInitiator.Player) { - log.sendError(ctx, "Backup failed. The file is corrupt."); - log.error("For help see: https://github.com/Szum123321/textile_backup/wiki/ZIP-Problems"); - } - if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode.isStrict()) keep = false; - } catch (IOException | InterruptedException | ExecutionException e) { - log.error("An exception occurred!", e); - if(ctx.initiator() == ActionInitiator.Player) - log.sendError(ctx, "Something went wrong while compressing files!"); - if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode.isStrict()) keep = false; - - } */finally { + } finally { close(); } diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java index d735ece..5722d7b 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java @@ -27,19 +27,23 @@ import java.io.IOException; import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; +import java.util.concurrent.atomic.AtomicBoolean; + public class FileTreeHashBuilder { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); private final Object lock = new Object(); private long hash = 0, filesProcessed = 0, filesTotalSize = 0; + private final AtomicBoolean closed = new AtomicBoolean(false); public void update(Path path, long newHash) throws IOException { - if(path.getFileName().toString().equals(CompressionStatus.DATA_FILENAME)) return; - //log.info("Putting: {}, {}", path, newHash); + if(closed.get()) throw new RuntimeException("Hash Builder already closed!"); - var hasher = Globals.CHECKSUM_SUPPLIER.get(); + if(path.getFileName().toString().equals(CompressionStatus.DATA_FILENAME)) return; long size = Files.size(path); + var hasher = Globals.CHECKSUM_SUPPLIER.get(); + hasher.update(path.getFileName().toString().getBytes(StandardCharsets.UTF_8)); hasher.update(newHash); @@ -53,11 +57,15 @@ public class FileTreeHashBuilder { public long getValue() { var hasher = Globals.CHECKSUM_SUPPLIER.get(); + closed.set(true); - hasher.update(hash); - hasher.update(filesProcessed); - hasher.update(filesTotalSize); + synchronized (lock) { + log.debug("Closing: files: {}, bytes {}, raw hash {}", filesProcessed, filesTotalSize, hash); + hasher.update(hash); + hasher.update(filesProcessed); + hasher.update(filesTotalSize); - return hasher.getValue(); + return hasher.getValue(); + } } } diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java index 8c8d2e9..501935f 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java @@ -19,39 +19,28 @@ package net.szum123321.textile_backup.core.digest; import net.szum123321.textile_backup.Globals; -import net.szum123321.textile_backup.TextileBackup; -import net.szum123321.textile_backup.TextileLogger; import net.szum123321.textile_backup.core.DataLeftException; import net.szum123321.textile_backup.core.create.BrokenFileHandler; import org.jetbrains.annotations.NotNull; import java.io.*; import java.nio.file.Path; +import java.util.concurrent.CountDownLatch; //This class calculates a hash of the file on the input stream, submits it to FileTreeHashBuilder. //In case the underlying stream hasn't been read completely in, puts it into BrokeFileHandler public class HashingInputStream extends FilterInputStream { - private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); private final Path path; private final Hash hasher = Globals.CHECKSUM_SUPPLIER.get(); private final FileTreeHashBuilder hashBuilder; private final BrokenFileHandler brokenFileHandler; + private final CountDownLatch latch; - private int cnt = 0; - @Override - public synchronized void reset() throws IOException { - log.info("Called reset! {}", path); - } - - @Override - public boolean markSupported() { - return false; - } - - public HashingInputStream(InputStream in, Path path, FileTreeHashBuilder hashBuilder, BrokenFileHandler brokenFileHandler) { + public HashingInputStream(InputStream in, Path path, FileTreeHashBuilder hashBuilder, BrokenFileHandler brokenFileHandler, CountDownLatch latch) { super(in); this.path = path; + this.latch = latch; this.hashBuilder = hashBuilder; this.brokenFileHandler = brokenFileHandler; } @@ -59,34 +48,29 @@ public class HashingInputStream extends FilterInputStream { @Override public int read(byte @NotNull [] b, int off, int len) throws IOException { int i = in.read(b, off, len); - if(i > -1) { - hasher.update(b, off, i); - cnt += i; - } + if(i != -1) hasher.update(b, off, i); return i; } @Override public int read() throws IOException { int i = in.read(); - if(i > -1) { - hasher.update(i); - cnt++; - } + if(i != -1) hasher.update(i); return i; } + @Override + public boolean markSupported() { + return false; + } + @Override public void close() throws IOException { - if(in.available() == 0) { - long val = hasher.getValue(); - hashBuilder.update(path, val); - log.info("Read in {}, of {}, with hash {}", path, cnt, val); - } - else { - brokenFileHandler.handle(path, new DataLeftException(in.available())); - //log.info("bad file {} {}",path, cnt); - } + if(in.available() == 0) hashBuilder.update(path, hasher.getValue()); + else brokenFileHandler.handle(path, new DataLeftException(in.available())); + + latch.countDown(); + super.close(); } } diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java index 8caa282..806a4f5 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java @@ -67,6 +67,8 @@ public class RestoreBackupRunnable implements Runnable { return; } + //By making a separate thread we can start unpacking an old backup instantly + //Let the server shut down gracefully, and wait for the old world backup to complete FutureTask waitForShutdown = new FutureTask<>(() -> { ctx.server().getThread().join(); //wait for server to die and save all its state if(config.get().backupOldWorlds) { @@ -75,6 +77,7 @@ public class RestoreBackupRunnable implements Runnable { .newBackupContextBuilder() .setServer(ctx.server()) .setInitiator(ActionInitiator.Restore) + .dontCleanup() .setComment("Old_World" + (ctx.comment() != null ? "_" + ctx.comment() : "")) .build() ).call(); @@ -82,7 +85,7 @@ public class RestoreBackupRunnable implements Runnable { return null; }); - new Thread(waitForShutdown).start(); + new Thread(waitForShutdown, "Server shutdown wait thread").start(); try { log.info("Starting decompression..."); @@ -97,11 +100,14 @@ public class RestoreBackupRunnable implements Runnable { CompressionStatus status = CompressionStatus.readFromFile(tmp); Files.delete(tmp.resolve(CompressionStatus.DATA_FILENAME)); + log.info("Waiting for server to fully terminate..."); + //locks until the backup is finished waitForShutdown.get(); log.info("Status: {}", status); + //TODO: check broken file array boolean valid = status.isValid(hash); if(valid || !config.get().errorErrorHandlingMode.verify()) { if(valid) log.info("Backup valid. Restoring"); @@ -121,12 +127,11 @@ public class RestoreBackupRunnable implements Runnable { log.error("An exception occurred while trying to restore a backup!", e); } finally { //Regardless of what happened, we should still clean up - /* if(Files.exists(tmp)) { + if(Files.exists(tmp)) { try { Utilities.deleteDirectory(tmp); } catch (IOException ignored) {} - }*/ - //TODO: uncomment + } } //in case we're playing on client From 86ae95b02e1a4810f1bd206be065ab71a49c87b5 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 29 Nov 2022 00:55:41 +0100 Subject: [PATCH 19/44] Added mod version info to metadata --- .../java/net/szum123321/textile_backup/TextileBackup.java | 8 ++++++++ .../szum123321/textile_backup/core/CompressionStatus.java | 7 +++++-- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/TextileBackup.java b/src/main/java/net/szum123321/textile_backup/TextileBackup.java index d2625db..4376430 100644 --- a/src/main/java/net/szum123321/textile_backup/TextileBackup.java +++ b/src/main/java/net/szum123321/textile_backup/TextileBackup.java @@ -26,6 +26,8 @@ import net.fabricmc.api.ModInitializer; import net.fabricmc.fabric.api.command.v2.CommandRegistrationCallback; import net.fabricmc.fabric.api.event.lifecycle.v1.ServerLifecycleEvents; import net.fabricmc.fabric.api.event.lifecycle.v1.ServerTickEvents; +import net.fabricmc.loader.api.FabricLoader; +import net.fabricmc.loader.api.Version; import net.minecraft.server.command.ServerCommandSource; import net.szum123321.textile_backup.commands.create.CleanupCommand; import net.szum123321.textile_backup.commands.create.StartBackupCommand; @@ -45,6 +47,7 @@ import net.szum123321.textile_backup.core.create.MakeBackupRunnableFactory; public class TextileBackup implements ModInitializer { public static final String MOD_NAME = "Textile Backup"; public static final String MOD_ID = "textile_backup"; + public static final Version VERSION; private final static TextileLogger log = new TextileLogger(MOD_NAME); private final static ConfigHelper config = ConfigHelper.INSTANCE; @@ -105,4 +108,9 @@ public class TextileBackup implements ModInitializer { .then(KillRestoreCommand.register()) )); } + + static { + VERSION = FabricLoader.getInstance().getModContainer(MOD_ID).orElseThrow().getMetadata().getVersion(); + FabricLoader.getInstance().getModContainer("minecraft").orElseThrow().getMetadata().getVersion(); + } } diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java index 022c880..e4a723a 100644 --- a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -18,6 +18,8 @@ package net.szum123321.textile_backup.core; +import net.fabricmc.loader.api.Version; + import java.io.*; import java.nio.file.Files; import java.nio.file.Path; @@ -25,7 +27,7 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.Map; -public record CompressionStatus(long treeHash, LocalDateTime date, long startTimestamp, long finishTimestamp, Map brokenFiles) implements Serializable { +public record CompressionStatus(long treeHash, Map brokenFiles, LocalDateTime date, long startTimestamp, long finishTimestamp, Version modVersion) implements Serializable { public static final String DATA_FILENAME = "textile_status.data"; public boolean isValid(long decompressedHash) { return decompressedHash == treeHash; @@ -54,7 +56,8 @@ public record CompressionStatus(long treeHash, LocalDateTime date, long startTim .append(", Date: ") .append(date.format(DateTimeFormatter.ISO_DATE_TIME)) .append(", start time stamp: ").append(startTimestamp) - .append(", finish time stamp: ").append(finishTimestamp); + .append(", finish time stamp: ").append(finishTimestamp) + .append(", Mod Version:").append(modVersion.getFriendlyString()); builder.append(", broken files: "); if(brokenFiles.isEmpty()) builder.append("[]"); From 4622f3fd0d34c5242eb7474778d8fe4f879dbae0 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 29 Nov 2022 15:05:51 +0100 Subject: [PATCH 20/44] all works now*. more debugging still needed. --- .../core/CompressionStatus.java | 8 +- .../compressors/AbstractCompressor.java | 4 +- .../core/digest/BalticHash.java | 1 - .../core/digest/BalticHashSIMD.java | 98 +++++++++---------- 4 files changed, 54 insertions(+), 57 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java index e4a723a..db3f5d2 100644 --- a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -18,8 +18,6 @@ package net.szum123321.textile_backup.core; -import net.fabricmc.loader.api.Version; - import java.io.*; import java.nio.file.Files; import java.nio.file.Path; @@ -27,10 +25,10 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.Map; -public record CompressionStatus(long treeHash, Map brokenFiles, LocalDateTime date, long startTimestamp, long finishTimestamp, Version modVersion) implements Serializable { +public record CompressionStatus(long treeHash, Map brokenFiles, LocalDateTime date, long startTimestamp, long finishTimestamp) implements Serializable { public static final String DATA_FILENAME = "textile_status.data"; public boolean isValid(long decompressedHash) { - return decompressedHash == treeHash; + return decompressedHash == treeHash && brokenFiles.isEmpty(); } public static CompressionStatus readFromFile(Path folder) throws IOException, ClassNotFoundException { @@ -57,7 +55,7 @@ public record CompressionStatus(long treeHash, Map brokenFiles, .append(date.format(DateTimeFormatter.ISO_DATE_TIME)) .append(", start time stamp: ").append(startTimestamp) .append(", finish time stamp: ").append(finishTimestamp) - .append(", Mod Version:").append(modVersion.getFriendlyString()); + ;//.append(", Mod Version: ").append(modVersion.getFriendlyString()); builder.append(", broken files: "); if(brokenFiles.isEmpty()) builder.append("[]"); diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index 87f91d3..a0cf75e 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -97,8 +97,8 @@ public abstract class AbstractCompressor { CompressionStatus status = new CompressionStatus ( fileHashBuilder.getValue(), brokenFileHandler.get(), - ctx.startDate(), start.toEpochMilli(), now.toEpochMilli(), - TextileBackup.VERSION + ctx.startDate(), start.toEpochMilli(), now.toEpochMilli()//, + //TextileBackup.VERSION ); addEntry(new StatusFileInputSupplier(status.serialize()), arc); diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java index 0b0b108..3ad5280 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java @@ -37,7 +37,6 @@ public class BalticHash implements Hash { protected final byte[] _byte_buffer = new byte[(state.length + 1) * Long.BYTES]; //Enforce endianness protected final ByteBuffer buffer = ByteBuffer.wrap(_byte_buffer).order(ByteOrder.LITTLE_ENDIAN); - protected long hashed_data_length = 0; public void update(byte b) { diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java index 8159f43..ed633a2 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java @@ -33,63 +33,63 @@ It's still probably far from being the slowest part of code, so I don't expect a I will keep this code here for future work perhaps */ -public class BalticHashSIMD extends BalticHash { +public class BalticHashSIMD extends BalticHash {/* public BalticHashSIMD() { throw new UnsupportedOperationException(); } //For safety - /* private LongVector state = LongVector.fromArray(LongVector.SPECIES_256, IV, 0); + private LongVector state = LongVector.fromArray(LongVector.SPECIES_256, IV, 0); - @Override - public long getValue() { - if(buffer.position() != 0) { - while(buffer.position() < buffer_limit) buffer.put((byte)0); - round(); + @Override + public long getValue() { + if (buffer.position() != 0) { + while (buffer.position() < buffer_limit) buffer.put((byte) 0); + round(); + } + + long result = state.reduceLanesToLong(VectorOperators.XOR); + result ^= hashed_data_length; + + return xorshift64star(result); + } + + @Override + public void update(byte[] data, int off, int len) { + int pos = off; + while (pos < len) { + int n = Math.min(len - pos, buffer_limit - buffer.position()); + if (n == 32) { + var v = ByteVector.fromArray(ByteVector.SPECIES_256, data, pos).reinterpretAsLongs(); + state = state.lanewise(VectorOperators.XOR, v); + state = xorshift64star(state); + } else { + System.arraycopy(data, pos, _byte_buffer, buffer.position(), n); + buffer.position(buffer.position() + n); + if (buffer.position() == buffer_limit) round(); } - - long result = state.reduceLanesToLong(VectorOperators.XOR); - result ^= hashed_data_length; - - return xorshift64star(result); + pos += n; } - @Override - public void update(byte[] data, int off, int len) { - int pos = off; - while(pos < len) { - int n = Math.min(len - pos, buffer_limit - buffer.position()); - if(n == 32) { - var v = ByteVector.fromArray(ByteVector.SPECIES_256, data, pos).reinterpretAsLongs(); - state = state.lanewise(VectorOperators.XOR, v); - state = xorshift64star(state); - } else { - System.arraycopy(data, pos, _byte_buffer, buffer.position(), n); - buffer.position(buffer.position() + n); - if(buffer.position() == buffer_limit) round(); - } - pos += n; - } + hashed_data_length += len; + } - hashed_data_length += len; - } + @Override + protected void round() { + var s = ByteVector.fromArray(ByteVector.SPECIES_256, _byte_buffer, 0).reinterpretAsLongs(); + state = state.lanewise(VectorOperators.XOR, s); + state = xorshift64star(state); - @Override - protected void round() { - var s = ByteVector.fromArray(ByteVector.SPECIES_256, _byte_buffer, 0).reinterpretAsLongs(); - state = state.lanewise(VectorOperators.XOR, s); - state = xorshift64star(state); + int p = buffer.position(); - int p = buffer.position(); + if (p > buffer_limit) { + System.arraycopy(_byte_buffer, buffer_limit, _byte_buffer, 0, buffer.limit() - p); + buffer.position(buffer.limit() - p); + } else buffer.rewind(); + } - if(p > buffer_limit) { - System.arraycopy(_byte_buffer, buffer_limit, _byte_buffer, 0, buffer.limit() - p); - buffer.position(buffer.limit() - p); - } else buffer.rewind(); - } - - LongVector xorshift64star(LongVector v) { - v = v.lanewise(VectorOperators.XOR, v.lanewise(VectorOperators.ASHR, 12)); - v = v.lanewise(VectorOperators.XOR, v.lanewise(VectorOperators.LSHL, 25)); - v = v.lanewise(VectorOperators.XOR, v.lanewise(VectorOperators.ASHR, 27)); - v = v.lanewise(VectorOperators.MUL, 0x2545F4914F6CDD1DL); - return v; - }*/ + LongVector xorshift64star(LongVector v) { + v = v.lanewise(VectorOperators.XOR, v.lanewise(VectorOperators.ASHR, 12)); + v = v.lanewise(VectorOperators.XOR, v.lanewise(VectorOperators.LSHL, 25)); + v = v.lanewise(VectorOperators.XOR, v.lanewise(VectorOperators.ASHR, 27)); + v = v.lanewise(VectorOperators.MUL, 0x2545F4914F6CDD1DL); + return v; + }*/ } \ No newline at end of file From 8385044154dbaeb3fb68dc6fe0fe43da15df1245 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 29 Nov 2022 15:06:30 +0100 Subject: [PATCH 21/44] typos --- .../net/szum123321/textile_backup/config/ConfigPOJO.java | 2 +- .../szum123321/textile_backup/core/ActionInitiator.java | 4 +--- .../szum123321/textile_backup/core/RestoreableFile.java | 2 +- .../textile_backup/core/create/BackupContext.java | 9 ++++----- 4 files changed, 7 insertions(+), 10 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java index fab5f4c..cdc4a3f 100644 --- a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java +++ b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java @@ -90,7 +90,7 @@ public class ConfigPOJO implements ConfigData { public long maxAge = 0; @Comment(""" - \nMaximum size of backup folder in kilo bytes (1024). + \nMaximum size of backup folder in kibi bytes (1024). If set to 0 then backups will not be deleted """) @ConfigEntry.Gui.Tooltip() diff --git a/src/main/java/net/szum123321/textile_backup/core/ActionInitiator.java b/src/main/java/net/szum123321/textile_backup/core/ActionInitiator.java index e3c01b7..21e58aa 100644 --- a/src/main/java/net/szum123321/textile_backup/core/ActionInitiator.java +++ b/src/main/java/net/szum123321/textile_backup/core/ActionInitiator.java @@ -26,9 +26,7 @@ public enum ActionInitiator { ServerConsole("Server Console", "from"), //some/ting typed a command and it was not a player (command blocks and server console count) Timer("Timer", "by"), //a.k.a scheduler Shutdown("Server Shutdown", "by"), - Restore("Backup Restoration", "because of"), - Null("Null (That shouldn't have happened)", "form"); - + Restore("Backup Restoration", "because of"); private final String name; private final String prefix; diff --git a/src/main/java/net/szum123321/textile_backup/core/RestoreableFile.java b/src/main/java/net/szum123321/textile_backup/core/RestoreableFile.java index bb3a8c8..5db2d24 100644 --- a/src/main/java/net/szum123321/textile_backup/core/RestoreableFile.java +++ b/src/main/java/net/szum123321/textile_backup/core/RestoreableFile.java @@ -56,7 +56,7 @@ public class RestoreableFile implements Comparable { } //removes repetition of the files stream thingy with awfully large lambdas - public static T applyOnFiles(Path root, T def, Consumer errorConsumer, Function, T> streamConsumer) { + public static T applyOnFiles(Path root, T def, Consumer errorConsumer, Function, T> streamConsumer) { try (Stream stream = Files.list(root)) { return streamConsumer.apply(stream.flatMap(f -> RestoreableFile.build(f).stream())); } catch (IOException e) { diff --git a/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java b/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java index c3469be..d626695 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java @@ -18,13 +18,14 @@ package net.szum123321.textile_backup.core.create; -import net.minecraft.entity.player.PlayerEntity; import net.minecraft.server.MinecraftServer; import net.minecraft.server.command.ServerCommandSource; import net.szum123321.textile_backup.core.ActionInitiator; +import net.szum123321.textile_backup.core.Utilities; import org.jetbrains.annotations.NotNull; import java.time.LocalDateTime; +import java.util.NoSuchElementException; public record BackupContext(@NotNull MinecraftServer server, ServerCommandSource commandSource, @@ -104,10 +105,8 @@ public record BackupContext(@NotNull MinecraftServer server, public BackupContext build() { if (guessInitiator) { - initiator = commandSource.getEntity() instanceof PlayerEntity ? ActionInitiator.Player : ActionInitiator.ServerConsole; - } else if (initiator == null) { - initiator = ActionInitiator.Null; - } + initiator = Utilities.wasSentByPlayer(commandSource) ? ActionInitiator.Player : ActionInitiator.ServerConsole; + } else if (initiator == null) throw new NoSuchElementException("No initiator provided!"); if (server == null) { if (commandSource != null) setServer(commandSource.getServer()); From 472aeda1841a2010a4763b7e9d5ef350b3a7b849 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 29 Nov 2022 22:31:04 +0100 Subject: [PATCH 22/44] improved comments and formatting --- .../net/szum123321/textile_backup/Globals.java | 10 ++++++++++ .../core/create/InputSupplier.java | 3 +-- .../create/compressors/AbstractCompressor.java | 9 ++++++--- .../compressors/tar/AbstractTarArchiver.java | 2 +- .../textile_backup/core/digest/BalticHash.java | 13 ++++++------- .../core/digest/BalticHashSIMD.java | 16 ++++++++-------- .../core/digest/FileTreeHashBuilder.java | 4 ++++ .../textile_backup/core/digest/Hash.java | 11 +++++------ .../core/digest/HashingInputStream.java | 11 ++++++++--- .../core/restore/RestoreBackupRunnable.java | 3 +++ 10 files changed, 52 insertions(+), 30 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/Globals.java b/src/main/java/net/szum123321/textile_backup/Globals.java index 37d620a..62f04d3 100644 --- a/src/main/java/net/szum123321/textile_backup/Globals.java +++ b/src/main/java/net/szum123321/textile_backup/Globals.java @@ -51,6 +51,8 @@ public class Globals { private AwaitThread restoreAwaitThread = null; private Path lockedPath = null; + private String combinedVersionString; + private Globals() {} public ExecutorService getQueueExecutor() { return executorService; } @@ -107,4 +109,12 @@ public class Globals { if(disableTMPFiles) log.error("Might cause: https://github.com/Szum123321/textile_backup/wiki/ZIP-Problems"); } + + public String getCombinedVersionString() { + return combinedVersionString; + } + + public void setCombinedVersionString(String combinedVersionString) { + this.combinedVersionString = combinedVersionString; + } } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java index 6a5ec6a..76a40e7 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/InputSupplier.java @@ -25,10 +25,9 @@ import java.io.InputStream; import java.nio.file.Path; import java.util.Optional; - public interface InputSupplier extends InputStreamSupplier { InputStream getInputStream() throws IOException; - //If an entry is virtual (a.k.a there is no actual file to open, only input stream) + //If an entry is virtual (a.k.a. there is no actual file to open, only input stream) Optional getPath(); String getName(); diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index a0cf75e..9866626 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -40,7 +40,7 @@ import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; /** - * Basic abstract class representing directory compressor + * Basic abstract class representing directory compressor with all the bells and whistles */ public abstract class AbstractCompressor { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); @@ -49,14 +49,14 @@ public abstract class AbstractCompressor { Instant start = Instant.now(); FileTreeHashBuilder fileHashBuilder = new FileTreeHashBuilder(); - BrokenFileHandler brokenFileHandler = new BrokenFileHandler(); + BrokenFileHandler brokenFileHandler = new BrokenFileHandler(); //Basically a hashmap storing files and their respective exceptions try (OutputStream outStream = Files.newOutputStream(outputFile); BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream); OutputStream arc = createArchiveOutputStream(bufferedOutputStream, ctx, coreLimit); Stream fileStream = Files.walk(inputFile)) { - AtomicInteger fileCounter = new AtomicInteger(0); + AtomicInteger fileCounter = new AtomicInteger(0); //number of files to compress var it = fileStream .filter(path -> !Utilities.isBlacklisted(inputFile.relativize(path))) @@ -66,6 +66,7 @@ public abstract class AbstractCompressor { log.info("File count: {}", fileCounter.get()); + //will be used in conjunction with ParallelZip to avoid race condition CountDownLatch latch = new CountDownLatch(fileCounter.get()); while(it.hasNext()) { @@ -83,6 +84,7 @@ public abstract class AbstractCompressor { ); } catch (IOException e) { brokenFileHandler.handle(file, e); + //In Permissive mode we allow partial backups if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode.isStrict()) throw e; else log.sendErrorAL(ctx, "An exception occurred while trying to compress: {}", inputFile.relativize(file).toString(), e @@ -90,6 +92,7 @@ public abstract class AbstractCompressor { } } + //wait for all the InputStreams to close/fail with InputSupplier latch.await(); Instant now = Instant.now(); diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java index 94d0ad2..4ae84a3 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java @@ -45,7 +45,7 @@ public class AbstractTarArchiver extends AbstractCompressor { protected void addEntry(InputSupplier input, OutputStream arc) throws IOException { try (InputStream fileInputStream = input.getInputStream()) { TarArchiveEntry entry; - if(input.getPath().isEmpty()) {//Virtual entry + if(input.getPath().isEmpty()) { //Virtual entry entry = new TarArchiveEntry(input.getName()); entry.setSize(input.size()); } else diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java index 3ad5280..b1892b9 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java @@ -22,16 +22,15 @@ import java.nio.ByteBuffer; import java.nio.ByteOrder; import java.util.Arrays; -/* - This algorithm copies construction of SeaHash (https://ticki.github.io/blog/seahash-explained/) including its IV - What it differs in is that it uses Xoroshift64* instead of PCG. Although it might lower the output quality, - I don't think it matters that much, honestly. One advantage the xoroshift has is that it should be - easier to implement with AVX. Java should soon ship its vector api by default. +/** + * This algorithm copies construction of SeaHash including its IV. + * What it differs in is that it uses Xoroshift64* instead of PCG as its pseudo-random function. Although it might lower + * the output quality, I don't think it matters that much, honestly. One advantage of xoroshift is that it should be + * easier to implement with AVX. Java should soon ship its vector api by default. */ public class BalticHash implements Hash { - protected final static long[] IV = { 0x16f11fe89b0d677cL, 0xb480a793d8e6c86cL, 0x6fe2e5aaf078ebc9L, 0x14f994a4c5259381L}; - //SeaHash IV + protected final static long[] IV = { 0x16f11fe89b0d677cL, 0xb480a793d8e6c86cL, 0x6fe2e5aaf078ebc9L, 0x14f994a4c5259381L }; private final long[] state = Arrays.copyOf(IV, IV.length); protected final int buffer_limit = state.length * Long.BYTES; protected final byte[] _byte_buffer = new byte[(state.length + 1) * Long.BYTES]; diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java index ed633a2..2e15c56 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java @@ -22,16 +22,16 @@ package net.szum123321.textile_backup.core.digest; import net.szum123321.textile_backup.core.digest.BalticHash; -/*Mostly working XorSeaHash impl using SIMD. Should speed up calculation on most systems currently in use +/** + * Mostly working XorSeaHash impl using SIMD. Should speed up calculation on most systems currently in use -It's actually slower. I tested it by comparing runtimes while hashing a directly opened FileInputStream. -My cpu is AMD Ryzen 5 3500U +
...
-There are two reasons I can think of: either vector construction simply takes so much time or jvm auto-vectorizes better than me - -It's still probably far from being the slowest part of code, so I don't expect any major slowdowns - -I will keep this code here for future work perhaps + * It's actually slower. I tested it by comparing runtimes while hashing a directly opened FileInputStream. + * My cpu is AMD Ryzen 5 3500U + * There are two reasons I can think of: either vector construction simply takes so much time or jvm auto-vectorizes better than I. + * It's still probably far from being the slowest part of code, so I don't expect any major slowdowns + * I will keep this code here for future work perhaps */ public class BalticHashSIMD extends BalticHash {/* public BalticHashSIMD() { throw new UnsupportedOperationException(); } //For safety diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java index 5722d7b..8c1e136 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java @@ -29,6 +29,10 @@ import java.nio.file.Files; import java.nio.file.Path; import java.util.concurrent.atomic.AtomicBoolean; +/** + * What this class does is it collects the hashed files and combines them into a single number, + * thus we can verify file tree integrity + */ public class FileTreeHashBuilder { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); private final Object lock = new Object(); diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java b/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java index d05533a..aa54578 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java @@ -21,13 +21,12 @@ package net.szum123321.textile_backup.core.digest; public interface Hash { void update(byte b); - default void update(int b) { - update((byte)b); - } + default void update(int b) { update((byte)b); } + void update(long b); - default void update(byte[] b) { - update(b, 0, b.length); - } + + default void update(byte[] b) { update(b, 0, b.length); } + void update(byte[] b, int off, int len); long getValue(); diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java index 501935f..2f51eb0 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java @@ -27,8 +27,14 @@ import java.io.*; import java.nio.file.Path; import java.util.concurrent.CountDownLatch; -//This class calculates a hash of the file on the input stream, submits it to FileTreeHashBuilder. -//In case the underlying stream hasn't been read completely in, puts it into BrokeFileHandler +/** + * This class calculates a hash of the file on the input stream, submits it to FileTreeHashBuilder. + * In case the underlying stream hasn't been read completely in, puts it into BrokeFileHandler + + * Futhermore, ParallelZip works by putting al the file requests into a queue and then compressing them + * with multiple threads. Thus, we have to make sure that all the files have been read before requesting the final value + * That is what CountDownLatch does + */ public class HashingInputStream extends FilterInputStream { private final Path path; private final Hash hasher = Globals.CHECKSUM_SUPPLIER.get(); @@ -36,7 +42,6 @@ public class HashingInputStream extends FilterInputStream { private final BrokenFileHandler brokenFileHandler; private final CountDownLatch latch; - public HashingInputStream(InputStream in, Path path, FileTreeHashBuilder hashBuilder, BrokenFileHandler brokenFileHandler, CountDownLatch latch) { super(in); this.path = path; diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java index 806a4f5..0eff593 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java @@ -37,6 +37,9 @@ import java.nio.file.Path; import java.util.concurrent.ExecutionException; import java.util.concurrent.FutureTask; +/** + * This class restores a file provided by RestoreContext. + */ public class RestoreBackupRunnable implements Runnable { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); private final static ConfigHelper config = ConfigHelper.INSTANCE; From 5f1706eed33fe423462dcdf818d31344892d369c Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 29 Nov 2022 22:34:02 +0100 Subject: [PATCH 23/44] added game&mod version to the status file --- .../java/net/szum123321/textile_backup/TextileBackup.java | 8 +++++++- .../szum123321/textile_backup/core/CompressionStatus.java | 4 ++-- .../core/create/compressors/AbstractCompressor.java | 5 +++-- 3 files changed, 12 insertions(+), 5 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/TextileBackup.java b/src/main/java/net/szum123321/textile_backup/TextileBackup.java index 4376430..f4b97b8 100644 --- a/src/main/java/net/szum123321/textile_backup/TextileBackup.java +++ b/src/main/java/net/szum123321/textile_backup/TextileBackup.java @@ -54,7 +54,13 @@ public class TextileBackup implements ModInitializer { @Override public void onInitialize() { - log.info("Starting Textile Backup by Szum123321"); + Globals.INSTANCE.setCombinedVersionString( + FabricLoader.getInstance().getModContainer(MOD_ID).orElseThrow().getMetadata().getVersion().getFriendlyString() + + ":" + + FabricLoader.getInstance().getModContainer("minecraft").orElseThrow().getMetadata().getVersion().getFriendlyString() + ); + + log.info("Starting Textile Backup {} by Szum123321", Globals.INSTANCE.getCombinedVersionString()); ConfigHelper.updateInstance(AutoConfig.register(ConfigPOJO.class, JanksonConfigSerializer::new)); diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java index db3f5d2..e9c618f 100644 --- a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -25,7 +25,7 @@ import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.Map; -public record CompressionStatus(long treeHash, Map brokenFiles, LocalDateTime date, long startTimestamp, long finishTimestamp) implements Serializable { +public record CompressionStatus(long treeHash, Map brokenFiles, LocalDateTime date, long startTimestamp, long finishTimestamp, String version) implements Serializable { public static final String DATA_FILENAME = "textile_status.data"; public boolean isValid(long decompressedHash) { return decompressedHash == treeHash && brokenFiles.isEmpty(); @@ -55,7 +55,7 @@ public record CompressionStatus(long treeHash, Map brokenFiles, .append(date.format(DateTimeFormatter.ISO_DATE_TIME)) .append(", start time stamp: ").append(startTimestamp) .append(", finish time stamp: ").append(finishTimestamp) - ;//.append(", Mod Version: ").append(modVersion.getFriendlyString()); + .append(", Mod Version: ").append(version); builder.append(", broken files: "); if(brokenFiles.isEmpty()) builder.append("[]"); diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index 9866626..65bf00a 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -18,6 +18,7 @@ package net.szum123321.textile_backup.core.create.compressors; +import net.szum123321.textile_backup.Globals; import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileLogger; import net.szum123321.textile_backup.config.ConfigHelper; @@ -100,8 +101,8 @@ public abstract class AbstractCompressor { CompressionStatus status = new CompressionStatus ( fileHashBuilder.getValue(), brokenFileHandler.get(), - ctx.startDate(), start.toEpochMilli(), now.toEpochMilli()//, - //TextileBackup.VERSION + ctx.startDate(), start.toEpochMilli(), now.toEpochMilli(), + Globals.INSTANCE.getCombinedVersionString() ); addEntry(new StatusFileInputSupplier(status.serialize()), arc); From 19cfb3cb277c4837062b9becb27b4dad83a7e90b Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 29 Nov 2022 22:34:47 +0100 Subject: [PATCH 24/44] finalized verification code --- .../core/CompressionStatus.java | 20 +++++++++++++++++-- .../core/restore/RestoreBackupRunnable.java | 15 +++++++------- 2 files changed, 25 insertions(+), 10 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java index e9c618f..e97831b 100644 --- a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -18,17 +18,33 @@ package net.szum123321.textile_backup.core; +import net.szum123321.textile_backup.core.restore.RestoreContext; + import java.io.*; import java.nio.file.Files; import java.nio.file.Path; import java.time.LocalDateTime; import java.time.format.DateTimeFormatter; import java.util.Map; +import java.util.Optional; public record CompressionStatus(long treeHash, Map brokenFiles, LocalDateTime date, long startTimestamp, long finishTimestamp, String version) implements Serializable { public static final String DATA_FILENAME = "textile_status.data"; - public boolean isValid(long decompressedHash) { - return decompressedHash == treeHash && brokenFiles.isEmpty(); + public Optional isValid(long hash, RestoreContext ctx) throws RuntimeException { + if(hash != treeHash) + return Optional.of("Tree Hash mismatch!\n Expected: " + treeHash + ", got: " + hash); + + if(!brokenFiles.isEmpty()) + return Optional.of("Damaged files present! ^"); + + if(ctx.restoreableFile().getCreationTime() != date) + return Optional.of( + "Creation date mismatch!\n Expected: " + + date.format(DateTimeFormatter.ISO_DATE_TIME) + ", got: " + + ctx.restoreableFile().getCreationTime().format(DateTimeFormatter.ISO_DATE_TIME) + ); + + return Optional.empty(); } public static CompressionStatus readFromFile(Path folder) throws IOException, ClassNotFoundException { diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java index 0eff593..9fd39ed 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java @@ -34,7 +34,6 @@ import net.szum123321.textile_backup.core.restore.decompressors.ZipDecompressor; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; -import java.util.concurrent.ExecutionException; import java.util.concurrent.FutureTask; /** @@ -110,11 +109,11 @@ public class RestoreBackupRunnable implements Runnable { log.info("Status: {}", status); - //TODO: check broken file array - boolean valid = status.isValid(hash); - if(valid || !config.get().errorErrorHandlingMode.verify()) { - if(valid) log.info("Backup valid. Restoring"); - else log.info("Backup is damaged, but verification is disabled. Restoring"); + var state = status.isValid(hash, ctx); + + if(state.isEmpty() || !config.get().errorErrorHandlingMode.verify()) { + if (state.isEmpty()) log.info("Backup valid. Restoring"); + else log.info("Backup is damaged, but verification is disabled [{}]. Restoring", state.get()); Utilities.deleteDirectory(worldFile); Files.move(tmp, worldFile); @@ -124,9 +123,9 @@ public class RestoreBackupRunnable implements Runnable { Files.delete(ctx.restoreableFile().getFile()); } } else { - log.error("File tree hash mismatch! Got: {}, Expected {}. Aborting", hash, status.treeHash()); + log.error(state.get()); } - } catch (ExecutionException | InterruptedException | ClassNotFoundException | IOException e) { + } catch (Exception e) { log.error("An exception occurred while trying to restore a backup!", e); } finally { //Regardless of what happened, we should still clean up From c2048c5039581256f5fb1876dcd48942e82efa56 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 29 Nov 2022 22:35:06 +0100 Subject: [PATCH 25/44] ehhhhhhhh --- .../textile_backup/core/create/MakeBackupRunnable.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java index c07e9fc..d0bbb8d 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java @@ -133,6 +133,8 @@ public class MakeBackupRunnable implements Callable { } private String getFileName() { - return Utilities.getDateTimeFormatter().format(context.startDate()) + (context.comment() != null ? "#" + context.comment().replaceAll("[\\\\/:*?\"<>|#]", "") : "") + config.get().format.getCompleteString(); + return Utilities.getDateTimeFormatter().format(context.startDate()) + + (context.comment() != null ? "#" + context.comment().replaceAll("[\\\\/:*?\"<>|#]", "") : "") + + config.get().format.getCompleteString(); } } From 458ab0182293e87a1a107e830c09b51830d8c585 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 29 Nov 2022 22:35:15 +0100 Subject: [PATCH 26/44] potential bug repair --- .../textile_backup/core/digest/HashingInputStream.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java index 2f51eb0..6a31343 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java @@ -71,11 +71,11 @@ public class HashingInputStream extends FilterInputStream { @Override public void close() throws IOException { + latch.countDown(); + if(in.available() == 0) hashBuilder.update(path, hasher.getValue()); else brokenFileHandler.handle(path, new DataLeftException(in.available())); - latch.countDown(); - super.close(); } } From 3af7d75042a4db93f0f5ee33985288982b9dac4f Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 29 Nov 2022 22:43:16 +0100 Subject: [PATCH 27/44] changed built fle names and moved errorErrorHandlingMode to the end of config, so it appears as far down as possible --- build.gradle | 2 +- .../textile_backup/config/ConfigPOJO.java | 16 ++++++++-------- 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/build.gradle b/build.gradle index b838b88..34131f2 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ sourceCompatibility = JavaVersion.VERSION_17 targetCompatibility = JavaVersion.VERSION_17 archivesBaseName = project.archives_base_name -version = "${project.mod_version}-${getMcMinor(project.minecraft_version)}" +version = "${project.mod_version}-${project.minecraft_version}" group = project.maven_group repositories { diff --git a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java index cdc4a3f..4cbc4b3 100644 --- a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java +++ b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java @@ -123,14 +123,6 @@ public class ConfigPOJO implements ConfigData { @ConfigEntry.Gui.EnumHandler(option = ConfigEntry.Gui.EnumHandler.EnumDisplayOption.BUTTON) public ArchiveFormat format = ArchiveFormat.ZIP; - @Comment(""" - \nThe Strict mode (default) aborts backup creation in case of any problem and deletes created files - Permissible mode keeps partial/damaged backup but won't allow to restore it - Very Permissible mode will skip the verification process. THIS MOST CERTAINLY WILL LEAD TO DATA LOSS OR CORRUPTION - """) - @ConfigEntry.Gui.EnumHandler(option = ConfigEntry.Gui.EnumHandler.EnumDisplayOption.BUTTON) - public ErrorHandlingMode errorErrorHandlingMode = ErrorHandlingMode.STRICT; - @Comment("\nMinimal permission level required to run commands\n") @ConfigEntry.Category("Manage") @ConfigEntry.Gui.NoTooltip() @@ -170,6 +162,14 @@ public class ConfigPOJO implements ConfigData { @ConfigEntry.Gui.Tooltip() public String dateTimeFormat = "yyyy.MM.dd_HH-mm-ss"; + @Comment(""" + \nThe Strict mode (default) aborts backup creation in case of any problem and deletes created files + Permissible mode keeps partial/damaged backup but won't allow to restore it + Very Permissible mode will skip the verification process. THIS MOST CERTAINLY WILL LEAD TO DATA LOSS OR CORRUPTION + """) + @ConfigEntry.Gui.EnumHandler(option = ConfigEntry.Gui.EnumHandler.EnumDisplayOption.BUTTON) + public ErrorHandlingMode errorErrorHandlingMode = ErrorHandlingMode.STRICT; + @Override public void validatePostLoad() throws ValidationException { if(compressionCoreCountLimit > Runtime.getRuntime().availableProcessors()) From cffc6590014931508d6ceecde6badb2c01936a2d Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Thu, 1 Dec 2022 07:45:57 +0100 Subject: [PATCH 28/44] don't re-pack Textile data --- src/main/java/net/szum123321/textile_backup/core/Utilities.java | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/main/java/net/szum123321/textile_backup/core/Utilities.java b/src/main/java/net/szum123321/textile_backup/core/Utilities.java index 1989993..8a3ff02 100644 --- a/src/main/java/net/szum123321/textile_backup/core/Utilities.java +++ b/src/main/java/net/szum123321/textile_backup/core/Utilities.java @@ -119,6 +119,8 @@ public class Utilities { if (path.getFileName().toString().equals("session.lock")) return true; } + if(path.getFileName().endsWith(CompressionStatus.DATA_FILENAME)) return true; + return config.get().fileBlacklist.stream().anyMatch(path::startsWith); } From 663089a7a3a6e661d6fb2bbcd25a7d7636701d4d Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 20 Dec 2022 00:46:03 +0100 Subject: [PATCH 29/44] improved naming --- .../net/szum123321/textile_backup/config/ConfigPOJO.java | 4 ++-- .../szum123321/textile_backup/core/CompressionStatus.java | 8 +++++--- .../textile_backup/core/create/MakeBackupRunnable.java | 2 +- .../core/create/compressors/AbstractCompressor.java | 2 +- src/main/resources/assets/textile_backup/lang/en_us.json | 4 ++-- 5 files changed, 11 insertions(+), 9 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java index 4cbc4b3..f13f2da 100644 --- a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java +++ b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java @@ -168,7 +168,7 @@ public class ConfigPOJO implements ConfigData { Very Permissible mode will skip the verification process. THIS MOST CERTAINLY WILL LEAD TO DATA LOSS OR CORRUPTION """) @ConfigEntry.Gui.EnumHandler(option = ConfigEntry.Gui.EnumHandler.EnumDisplayOption.BUTTON) - public ErrorHandlingMode errorErrorHandlingMode = ErrorHandlingMode.STRICT; + public IntegrityVerificationMode integrityVerificationMode = IntegrityVerificationMode.STRICT; @Override public void validatePostLoad() throws ValidationException { @@ -185,7 +185,7 @@ public class ConfigPOJO implements ConfigData { } } - public enum ErrorHandlingMode { + public enum IntegrityVerificationMode { STRICT, PERMISSIBLE, VERY_PERMISSIBLE; diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java index e97831b..fa1ffbe 100644 --- a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -30,7 +30,7 @@ import java.util.Optional; public record CompressionStatus(long treeHash, Map brokenFiles, LocalDateTime date, long startTimestamp, long finishTimestamp, String version) implements Serializable { public static final String DATA_FILENAME = "textile_status.data"; - public Optional isValid(long hash, RestoreContext ctx) throws RuntimeException { + public Optional validate(long hash, RestoreContext ctx) throws RuntimeException { if(hash != treeHash) return Optional.of("Tree Hash mismatch!\n Expected: " + treeHash + ", got: " + hash); @@ -47,8 +47,10 @@ public record CompressionStatus(long treeHash, Map brokenFiles, return Optional.empty(); } - public static CompressionStatus readFromFile(Path folder) throws IOException, ClassNotFoundException { - try(InputStream i = Files.newInputStream(folder.resolve(DATA_FILENAME)); + public static Path resolveStatusFilename(Path directory) { return directory.resolve(DATA_FILENAME); } + + public static CompressionStatus readFromFile(Path directory) throws IOException, ClassNotFoundException { + try(InputStream i = Files.newInputStream(directory.resolve(DATA_FILENAME)); ObjectInputStream obj = new ObjectInputStream(i)) { return (CompressionStatus) obj.readObject(); } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java index d0bbb8d..13553a4 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java @@ -112,7 +112,7 @@ public class MakeBackupRunnable implements Callable { //ExecutorService swallows exception, so I need to catch everything log.error("An exception occurred when trying to create new backup file!", e); - if (ConfigHelper.INSTANCE.get().errorErrorHandlingMode.isStrict()) { + if (ConfigHelper.INSTANCE.get().integrityVerificationMode.isStrict()) { try { Files.delete(outFile); } catch (IOException ex) { diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index 65bf00a..c50c1d3 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -86,7 +86,7 @@ public abstract class AbstractCompressor { } catch (IOException e) { brokenFileHandler.handle(file, e); //In Permissive mode we allow partial backups - if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode.isStrict()) throw e; + if (ConfigHelper.INSTANCE.get().integrityVerificationMode.isStrict()) throw e; else log.sendErrorAL(ctx, "An exception occurred while trying to compress: {}", inputFile.relativize(file).toString(), e ); diff --git a/src/main/resources/assets/textile_backup/lang/en_us.json b/src/main/resources/assets/textile_backup/lang/en_us.json index 0c9ae5e..9846353 100644 --- a/src/main/resources/assets/textile_backup/lang/en_us.json +++ b/src/main/resources/assets/textile_backup/lang/en_us.json @@ -47,8 +47,8 @@ "text.autoconfig.textile_backup.option.format": "Archive and compression format", "text.autoconfig.textile_backup.option.format.@Tooltip": "See: https://github.com/Szum123321/textile_backup/wiki/Configuration#format", - "text.autoconfig.textile_backup.option.errorErrorHandlingMode": "Compression error handling mode", - "text.autoconfig.textile_backup.option.errorErrorHandlingMode.@Tooltip": "DO NOT ALTER unless fully certain", + "text.autoconfig.textile_backup.option.integrityVerificationMode": "Verify the backup file has not been damaged", + "text.autoconfig.textile_backup.option.integrityVerificationMode.@Tooltip": "DO NOT ALTER unless fully certain", "text.autoconfig.textile_backup.option.permissionLevel": "Min permission level", From 9b3b908d0ad6b8dfcd9804e1b45fe550593d5696 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 20 Dec 2022 00:46:46 +0100 Subject: [PATCH 30/44] removed redundant counter --- .../create/compressors/AbstractCompressor.java | 16 ++++------------ 1 file changed, 4 insertions(+), 12 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index c50c1d3..559688d 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -37,7 +37,6 @@ import java.time.Instant; import java.util.Optional; import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; -import java.util.concurrent.atomic.AtomicInteger; import java.util.stream.Stream; /** @@ -57,22 +56,15 @@ public abstract class AbstractCompressor { OutputStream arc = createArchiveOutputStream(bufferedOutputStream, ctx, coreLimit); Stream fileStream = Files.walk(inputFile)) { - AtomicInteger fileCounter = new AtomicInteger(0); //number of files to compress - - var it = fileStream + var fileList = fileStream .filter(path -> !Utilities.isBlacklisted(inputFile.relativize(path))) .filter(Files::isRegularFile) - .peek(x -> fileCounter.incrementAndGet()).toList() - .iterator(); - - log.info("File count: {}", fileCounter.get()); + .toList(); //will be used in conjunction with ParallelZip to avoid race condition - CountDownLatch latch = new CountDownLatch(fileCounter.get()); - - while(it.hasNext()) { - Path file = it.next(); + CountDownLatch latch = new CountDownLatch(fileList.size()); + for (Path file : fileList) { try { addEntry( new FileInputStreamSupplier( From ed9c9a16fc6b5f57d3b4709cb74f25b2f02457b9 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 20 Dec 2022 00:47:57 +0100 Subject: [PATCH 31/44] moved name hashing to streams --- .../core/digest/FileTreeHashBuilder.java | 8 +---- .../core/digest/HashingInputStream.java | 7 ++-- .../core/digest/HashingOutputStream.java | 7 ++-- .../core/restore/RestoreBackupRunnable.java | 33 ++++++++++++------- 4 files changed, 32 insertions(+), 23 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java index 8c1e136..4abfde0 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java @@ -24,7 +24,6 @@ import net.szum123321.textile_backup.TextileLogger; import net.szum123321.textile_backup.core.CompressionStatus; import java.io.IOException; -import java.nio.charset.StandardCharsets; import java.nio.file.Files; import java.nio.file.Path; import java.util.concurrent.atomic.AtomicBoolean; @@ -46,14 +45,9 @@ public class FileTreeHashBuilder { long size = Files.size(path); - var hasher = Globals.CHECKSUM_SUPPLIER.get(); - - hasher.update(path.getFileName().toString().getBytes(StandardCharsets.UTF_8)); - hasher.update(newHash); - synchronized (lock) { //This way, the exact order of files processed doesn't matter. - this.hash ^= hasher.getValue(); + this.hash ^= newHash; filesProcessed++; filesTotalSize += size; } diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java index 6a31343..ea88076 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java @@ -24,6 +24,7 @@ import net.szum123321.textile_backup.core.create.BrokenFileHandler; import org.jetbrains.annotations.NotNull; import java.io.*; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; import java.util.concurrent.CountDownLatch; @@ -31,7 +32,7 @@ import java.util.concurrent.CountDownLatch; * This class calculates a hash of the file on the input stream, submits it to FileTreeHashBuilder. * In case the underlying stream hasn't been read completely in, puts it into BrokeFileHandler - * Futhermore, ParallelZip works by putting al the file requests into a queue and then compressing them + * Furthermore, ParallelZip works by putting all the file requests into a queue and then compressing them * with multiple threads. Thus, we have to make sure that all the files have been read before requesting the final value * That is what CountDownLatch does */ @@ -60,7 +61,7 @@ public class HashingInputStream extends FilterInputStream { @Override public int read() throws IOException { int i = in.read(); - if(i != -1) hasher.update(i); + if(i != -1) hasher.update((byte)i); return i; } @@ -71,6 +72,8 @@ public class HashingInputStream extends FilterInputStream { @Override public void close() throws IOException { + hasher.update(path.getFileName().toString().getBytes(StandardCharsets.UTF_8)); + latch.countDown(); if(in.available() == 0) hashBuilder.update(path, hasher.getValue()); diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java index 909d2ef..3b293ba 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java @@ -24,6 +24,7 @@ import org.jetbrains.annotations.NotNull; import java.io.FilterOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.nio.charset.StandardCharsets; import java.nio.file.Path; public class HashingOutputStream extends FilterOutputStream { @@ -39,21 +40,21 @@ public class HashingOutputStream extends FilterOutputStream { @Override public void write(int b) throws IOException { - hasher.update(b); out.write(b); + hasher.update((byte)b); } @Override public void write(byte @NotNull [] b, int off, int len) throws IOException { - hasher.update(b, off, len); out.write(b, off, len); + hasher.update(b, off, len); } @Override public void close() throws IOException { + hasher.update(path.getFileName().toString().getBytes(StandardCharsets.UTF_8)); long h = hasher.getValue(); hashBuilder.update(path, h); super.close(); - } } diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java index 9fd39ed..cd1135b 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java @@ -34,6 +34,7 @@ import net.szum123321.textile_backup.core.restore.decompressors.ZipDecompressor; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.Optional; import java.util.concurrent.FutureTask; /** @@ -63,7 +64,8 @@ public class RestoreBackupRunnable implements Runnable { try { tmp = Files.createTempDirectory( ctx.server().getRunDirectory().toPath(), - ctx.restoreableFile().getFile().getFileName().toString()); + ctx.restoreableFile().getFile().getFileName().toString() + ); } catch (IOException e) { log.error("An exception occurred while unpacking backup", e); return; @@ -72,7 +74,8 @@ public class RestoreBackupRunnable implements Runnable { //By making a separate thread we can start unpacking an old backup instantly //Let the server shut down gracefully, and wait for the old world backup to complete FutureTask waitForShutdown = new FutureTask<>(() -> { - ctx.server().getThread().join(); //wait for server to die and save all its state + ctx.server().getThread().join(); //wait for server thread to die and save all its state + if(config.get().backupOldWorlds) { return MakeBackupRunnableFactory.create ( BackupContext.Builder @@ -84,6 +87,7 @@ public class RestoreBackupRunnable implements Runnable { .build() ).call(); } + return null; }); @@ -99,21 +103,28 @@ public class RestoreBackupRunnable implements Runnable { else hash = GenericTarDecompressor.decompress(ctx.restoreableFile().getFile(), tmp); - CompressionStatus status = CompressionStatus.readFromFile(tmp); - Files.delete(tmp.resolve(CompressionStatus.DATA_FILENAME)); - log.info("Waiting for server to fully terminate..."); //locks until the backup is finished waitForShutdown.get(); - log.info("Status: {}", status); + Optional errorMsg; - var state = status.isValid(hash, ctx); + if(Files.notExists(CompressionStatus.resolveStatusFilename(tmp))) { + errorMsg = Optional.of("Status file not found!"); + } else { + CompressionStatus status = CompressionStatus.readFromFile(tmp); - if(state.isEmpty() || !config.get().errorErrorHandlingMode.verify()) { - if (state.isEmpty()) log.info("Backup valid. Restoring"); - else log.info("Backup is damaged, but verification is disabled [{}]. Restoring", state.get()); + log.info("Status: {}", status); + + Files.delete(tmp.resolve(CompressionStatus.DATA_FILENAME)); + + errorMsg = status.validate(hash, ctx); + } + + if(errorMsg.isEmpty() || !config.get().integrityVerificationMode.verify()) { + if (errorMsg.isEmpty()) log.info("Backup valid. Restoring"); + else log.info("Backup is damaged, but verification is disabled [{}]. Restoring", errorMsg.get()); Utilities.deleteDirectory(worldFile); Files.move(tmp, worldFile); @@ -123,7 +134,7 @@ public class RestoreBackupRunnable implements Runnable { Files.delete(ctx.restoreableFile().getFile()); } } else { - log.error(state.get()); + log.error(errorMsg.get()); } } catch (Exception e) { log.error("An exception occurred while trying to restore a backup!", e); From 97c607f9b20f6b2d21ab1983e319a9e7ad9da0f7 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 20 Dec 2022 00:48:20 +0100 Subject: [PATCH 32/44] formatting&spelling --- .../java/net/szum123321/textile_backup/config/ConfigPOJO.java | 1 + .../net/szum123321/textile_backup/core/CompressionStatus.java | 4 ++-- .../textile_backup/core/create/FileInputStreamSupplier.java | 4 ++-- 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java index f13f2da..af9d410 100644 --- a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java +++ b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java @@ -28,6 +28,7 @@ import net.szum123321.textile_backup.TextileBackup; import java.time.format.DateTimeFormatter; import java.util.*; +//TODO: Remove BZIP2 and LZMA compressors. As for the popular vote @Config(name = TextileBackup.MOD_ID) public class ConfigPOJO implements ConfigData { @Comment("\nShould every world have its own backup folder?\n") diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java index fa1ffbe..7ca0cfc 100644 --- a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -71,8 +71,8 @@ public record CompressionStatus(long treeHash, Map brokenFiles, .append(treeHash) .append(", Date: ") .append(date.format(DateTimeFormatter.ISO_DATE_TIME)) - .append(", start time stamp: ").append(startTimestamp) - .append(", finish time stamp: ").append(finishTimestamp) + .append(", start timestamp: ").append(startTimestamp) + .append(", finish timestamp: ").append(finishTimestamp) .append(", Mod Version: ").append(version); builder.append(", broken files: "); diff --git a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java index 90ca134..7cadc99 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java @@ -39,8 +39,8 @@ public record FileInputStreamSupplier(Path path, String name, FileTreeHashBuilde try { return new HashingInputStream(Files.newInputStream(path), path, hashTreeBuilder, brokenFileHandler, latch); } catch (IOException e) { - //Probably good idea to just put it here. In the case an exception is thrown here, it would be probable - //The latch would never be lifted + //Probably good idea to just put it here. In the case an exception is thrown here, it could be possble + //The latch would have never been lifted latch.countDown(); brokenFileHandler.handle(path, e); throw e; From 7d31e6710c6b49a6711907ebdc5cfd7685b1dc9f Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 27 Dec 2022 13:31:23 +0100 Subject: [PATCH 33/44] repaired hash & added tests --- build.gradle | 11 ---- .../textile_backup/TextileBackup.java | 13 ++-- .../core/digest/BalticHash.java | 4 +- .../core/digest/BalticHashSIMD.java | 3 +- .../textile_backup/test/BalticHashTest.java | 63 +++++++++++++++++++ .../textile_backup/TextileBackupTest.java | 10 --- src/test/resources/fabric.mod.json | 35 ----------- .../resources/textile_backup-test.mixins.json | 12 ---- 8 files changed, 73 insertions(+), 78 deletions(-) create mode 100644 src/main/java/net/szum123321/textile_backup/test/BalticHashTest.java delete mode 100644 src/test/java/net/szum123321/test/textile_backup/TextileBackupTest.java delete mode 100644 src/test/resources/fabric.mod.json delete mode 100644 src/test/resources/textile_backup-test.mixins.json diff --git a/build.gradle b/build.gradle index 34131f2..5efb5cb 100644 --- a/build.gradle +++ b/build.gradle @@ -17,17 +17,6 @@ repositories { mavenCentral() } -loom { - runs { - testServer { - server() - ideConfigGenerated project.rootProject == project - name = "Testmod Server" - source sourceSets.test - } - } -} - dependencies { //to change the versions see the gradle.properties file minecraft "com.mojang:minecraft:${project.minecraft_version}" diff --git a/src/main/java/net/szum123321/textile_backup/TextileBackup.java b/src/main/java/net/szum123321/textile_backup/TextileBackup.java index f4b97b8..853aa3e 100644 --- a/src/main/java/net/szum123321/textile_backup/TextileBackup.java +++ b/src/main/java/net/szum123321/textile_backup/TextileBackup.java @@ -27,7 +27,6 @@ import net.fabricmc.fabric.api.command.v2.CommandRegistrationCallback; import net.fabricmc.fabric.api.event.lifecycle.v1.ServerLifecycleEvents; import net.fabricmc.fabric.api.event.lifecycle.v1.ServerTickEvents; import net.fabricmc.loader.api.FabricLoader; -import net.fabricmc.loader.api.Version; import net.minecraft.server.command.ServerCommandSource; import net.szum123321.textile_backup.commands.create.CleanupCommand; import net.szum123321.textile_backup.commands.create.StartBackupCommand; @@ -43,11 +42,11 @@ import net.szum123321.textile_backup.core.ActionInitiator; import net.szum123321.textile_backup.core.create.BackupContext; import net.szum123321.textile_backup.core.create.BackupScheduler; import net.szum123321.textile_backup.core.create.MakeBackupRunnableFactory; +import net.szum123321.textile_backup.test.BalticHashTest; public class TextileBackup implements ModInitializer { public static final String MOD_NAME = "Textile Backup"; public static final String MOD_ID = "textile_backup"; - public static final Version VERSION; private final static TextileLogger log = new TextileLogger(MOD_NAME); private final static ConfigHelper config = ConfigHelper.INSTANCE; @@ -62,6 +61,11 @@ public class TextileBackup implements ModInitializer { log.info("Starting Textile Backup {} by Szum123321", Globals.INSTANCE.getCombinedVersionString()); + if(FabricLoader.getInstance().isDevelopmentEnvironment()) { + //Run the tests + BalticHashTest.run(); + } + ConfigHelper.updateInstance(AutoConfig.register(ConfigPOJO.class, JanksonConfigSerializer::new)); ServerTickEvents.END_SERVER_TICK.register(BackupScheduler::tick); @@ -114,9 +118,4 @@ public class TextileBackup implements ModInitializer { .then(KillRestoreCommand.register()) )); } - - static { - VERSION = FabricLoader.getInstance().getModContainer(MOD_ID).orElseThrow().getMetadata().getVersion(); - FabricLoader.getInstance().getModContainer("minecraft").orElseThrow().getMetadata().getVersion(); - } } diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java index b1892b9..2be911e 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java @@ -51,10 +51,10 @@ public class BalticHash implements Hash { } public void update(byte[] data, int off, int len) { - int pos = off; + int pos = 0; while(pos < len) { int n = Math.min(len - pos, buffer_limit - buffer.position()); - System.arraycopy(data, pos, _byte_buffer, buffer.position(), n); + System.arraycopy(data, off + pos, _byte_buffer, buffer.position(), n); pos += n; buffer.position(buffer.position() + n); if(buffer.position() >= buffer_limit) round(); diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java index 2e15c56..5bd5753 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java @@ -51,9 +51,10 @@ public class BalticHashSIMD extends BalticHash {/* return xorshift64star(result); } +//This is wrong. will have to correct ( @Override public void update(byte[] data, int off, int len) { - int pos = off; + int pos = off; //should be = 0 while (pos < len) { int n = Math.min(len - pos, buffer_limit - buffer.position()); if (n == 32) { diff --git a/src/main/java/net/szum123321/textile_backup/test/BalticHashTest.java b/src/main/java/net/szum123321/textile_backup/test/BalticHashTest.java new file mode 100644 index 0000000..55300fd --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/test/BalticHashTest.java @@ -0,0 +1,63 @@ +/* + * A simple backup mod for Fabric + * Copyright (C) 2022 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package net.szum123321.textile_backup.test; + +import net.minecraft.util.math.random.Random; +import net.szum123321.textile_backup.TextileBackup; +import net.szum123321.textile_backup.TextileLogger; +import net.szum123321.textile_backup.core.digest.BalticHash; + +public class BalticHashTest { + private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); + final static int TEST_LEN = 21377; //simple prime + public static void run() throws RuntimeException { + log.info("Running hash test"); + Random r = Random.create(2137); + long x = 0; + + byte[] data = new byte[TEST_LEN]; + + for(int i = 0; i < TEST_LEN; i++) data[i] = (byte)r.nextInt(); + + //Test block mode + for(int i = 0; i < 5*2; i++) x ^= randomHash(data, r); + if(x != 0) throw new RuntimeException("Hash mismatch!"); + + log.info("Test passed"); + } + + static long randomHash(byte[] data, Random r) { + int n = data.length; + + BalticHash h = new BalticHash(); + + int m = r.nextBetween(1, n); + + int nn = n, p = 0; + + for(int i = 0; i < m; i++) { + int k = r.nextBetween(1, nn - (m - i - 1)); + h.update(data, p, k); + p += k; + nn -= k; + } + + return h.getValue(); + } +} diff --git a/src/test/java/net/szum123321/test/textile_backup/TextileBackupTest.java b/src/test/java/net/szum123321/test/textile_backup/TextileBackupTest.java deleted file mode 100644 index bd646f2..0000000 --- a/src/test/java/net/szum123321/test/textile_backup/TextileBackupTest.java +++ /dev/null @@ -1,10 +0,0 @@ -package net.szum123321.test.textile_backup; - -import net.fabricmc.api.ModInitializer; - -public class TextileBackupTest implements ModInitializer { - @Override - public void onInitialize() { - - } -} diff --git a/src/test/resources/fabric.mod.json b/src/test/resources/fabric.mod.json deleted file mode 100644 index 809dace..0000000 --- a/src/test/resources/fabric.mod.json +++ /dev/null @@ -1,35 +0,0 @@ -{ - "schemaVersion": 1, - "id": "textile_backup", - "version": "${version}", - - "name": "Textile Backup Test", - "authors": [ - "Szum123321" - ], - "contact": { - "homepage": "https://www.curseforge.com/minecraft/mc-mods/textile-backup", - "issues": "https://github.com/Szum123321/textile_backup/issues", - "sources": "https://github.com/Szum123321/textile_backup" - }, - - "license": "GPLv3", - - "environment": "*", - "entrypoints": { - "main": [ - "net.szum123321.test.textile_backup.TextileBackupTest" - ] - }, - "mixins": [ - ], - - "depends": { - "fabricloader": ">=0.14.6", - "fabric": "*", - "minecraft": ">=1.19.1", - "cloth-config2": "*", - "java": ">=16", - "textile_backup": "*" - } -} \ No newline at end of file diff --git a/src/test/resources/textile_backup-test.mixins.json b/src/test/resources/textile_backup-test.mixins.json deleted file mode 100644 index f858f63..0000000 --- a/src/test/resources/textile_backup-test.mixins.json +++ /dev/null @@ -1,12 +0,0 @@ -{ - "required": true, - "package": "net.szum123321.test.textile_backup.mixin", - "compatibilityLevel": "JAVA_16", - "mixins": [ - ], - "client": [ - ], - "injectors": { - "defaultRequire": 1 - } -} From cf3078259b4d4b3bf3e006d8845176789399848d Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Tue, 27 Dec 2022 15:35:03 +0100 Subject: [PATCH 34/44] bugfixes --- gradle.properties | 17 ++++++++--------- .../textile_backup/config/ConfigPOJO.java | 5 +++-- .../textile_backup/core/CompressionStatus.java | 2 +- .../core/restore/RestoreBackupRunnable.java | 5 +++++ .../assets/textile_backup/lang/en_us.json | 4 ++-- 5 files changed, 19 insertions(+), 14 deletions(-) diff --git a/gradle.properties b/gradle.properties index 993b3d3..eb61004 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,26 +1,25 @@ # Done to increase the memory available to gradle. org.gradle.jvmargs=-Xmx1G -minecraft_version=1.19.2 -yarn_mappings=1.19.2+build.28 -loader_version=0.14.10 +minecraft_version=1.19.3 +yarn_mappings=1.19.3+build.3 +loader_version=0.14.11 #Fabric api -fabric_version=0.67.1+1.19.2 +fabric_version=0.69.1+1.19.3 #Cloth Config -cloth_version=8.2.88 +cloth_version=9.0.94 #ModMenu -modmenu_version=4.1.0 +modmenu_version=5.0.2 -#Lazy DFU for faster dev start -lazydfu_version=v0.1.3 +databreaker_version=0.2.10 #Hash of commit form which parallel gzip will be build pgzip_commit_hash=af5f5c297e735f3f2df7aa4eb0e19a5810b8aff6 # Mod Properties -mod_version = 3.0.0-a +mod_version = 3.0.0-b maven_group = net.szum123321 archives_base_name = textile_backup \ No newline at end of file diff --git a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java index af9d410..0cec14a 100644 --- a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java +++ b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java @@ -24,6 +24,7 @@ import me.shedaniel.autoconfig.annotation.Config; import me.shedaniel.autoconfig.annotation.ConfigEntry; import me.shedaniel.cloth.clothconfig.shadowed.blue.endless.jankson.Comment; import net.szum123321.textile_backup.TextileBackup; +import net.szum123321.textile_backup.core.create.compressors.AbstractCompressor; import java.time.format.DateTimeFormatter; import java.util.*; @@ -199,8 +200,8 @@ public class ConfigPOJO implements ConfigData { public enum ArchiveFormat { ZIP("zip"), GZIP("tar", "gz"), - BZIP2("tar", "bz2"), - LZMA("tar", "xz"), + //BZIP2("tar", "bz2"), + //LZMA("tar", "xz"), TAR("tar"); private final List extensionPieces; diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java index 7ca0cfc..af2d3c9 100644 --- a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -37,7 +37,7 @@ public record CompressionStatus(long treeHash, Map brokenFiles, if(!brokenFiles.isEmpty()) return Optional.of("Damaged files present! ^"); - if(ctx.restoreableFile().getCreationTime() != date) + if(ctx.restoreableFile().getCreationTime().equals(date)) return Optional.of( "Creation date mismatch!\n Expected: " + date.format(DateTimeFormatter.ISO_DATE_TIME) + ", got: " + diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java index cd1135b..6f705eb 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java @@ -30,6 +30,7 @@ import net.szum123321.textile_backup.core.create.BackupContext; import net.szum123321.textile_backup.core.create.MakeBackupRunnableFactory; import net.szum123321.textile_backup.core.restore.decompressors.GenericTarDecompressor; import net.szum123321.textile_backup.core.restore.decompressors.ZipDecompressor; +import net.szum123321.textile_backup.mixin.MinecraftServerSessionAccessor; import java.io.IOException; import java.nio.file.Files; @@ -126,6 +127,9 @@ public class RestoreBackupRunnable implements Runnable { if (errorMsg.isEmpty()) log.info("Backup valid. Restoring"); else log.info("Backup is damaged, but verification is disabled [{}]. Restoring", errorMsg.get()); + ((MinecraftServerSessionAccessor) ctx.server()) + .getSession().close(); + Utilities.deleteDirectory(worldFile); Files.move(tmp, worldFile); @@ -136,6 +140,7 @@ public class RestoreBackupRunnable implements Runnable { } else { log.error(errorMsg.get()); } + } catch (Exception e) { log.error("An exception occurred while trying to restore a backup!", e); } finally { diff --git a/src/main/resources/assets/textile_backup/lang/en_us.json b/src/main/resources/assets/textile_backup/lang/en_us.json index 9846353..39b3aa3 100644 --- a/src/main/resources/assets/textile_backup/lang/en_us.json +++ b/src/main/resources/assets/textile_backup/lang/en_us.json @@ -47,8 +47,8 @@ "text.autoconfig.textile_backup.option.format": "Archive and compression format", "text.autoconfig.textile_backup.option.format.@Tooltip": "See: https://github.com/Szum123321/textile_backup/wiki/Configuration#format", - "text.autoconfig.textile_backup.option.integrityVerificationMode": "Verify the backup file has not been damaged", - "text.autoconfig.textile_backup.option.integrityVerificationMode.@Tooltip": "DO NOT ALTER unless fully certain", + "text.autoconfig.textile_backup.option.integrityVerificationMode": "Verify backup integrity", + "text.autoconfig.textile_backup.option.integrityVerificationMode.@Tooltip": "DO NOT ALTER unless fully aware of consequences", "text.autoconfig.textile_backup.option.permissionLevel": "Min permission level", From 9ffaff1a2d8db065ebb48a8ebcf3b372277078ae Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Thu, 9 Mar 2023 21:18:56 +0100 Subject: [PATCH 35/44] formatting --- .../core/CompressionStatus.java | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java index af2d3c9..de5a2b3 100644 --- a/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java +++ b/src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java @@ -30,12 +30,12 @@ import java.util.Optional; public record CompressionStatus(long treeHash, Map brokenFiles, LocalDateTime date, long startTimestamp, long finishTimestamp, String version) implements Serializable { public static final String DATA_FILENAME = "textile_status.data"; + public Optional validate(long hash, RestoreContext ctx) throws RuntimeException { if(hash != treeHash) - return Optional.of("Tree Hash mismatch!\n Expected: " + treeHash + ", got: " + hash); + return Optional.of("Tree Hash mismatch!\n Expected: " + hex(treeHash) + ", got: " + hex(hash)); - if(!brokenFiles.isEmpty()) - return Optional.of("Damaged files present! ^"); + if(!brokenFiles.isEmpty()) return Optional.of("Damaged files present! ^"); if(ctx.restoreableFile().getCreationTime().equals(date)) return Optional.of( @@ -67,15 +67,16 @@ public record CompressionStatus(long treeHash, Map brokenFiles, @Override public String toString() { StringBuilder builder = new StringBuilder(); + builder.append("{ "); builder.append("Hash: ") - .append(treeHash) + .append(hex(treeHash)) .append(", Date: ") .append(date.format(DateTimeFormatter.ISO_DATE_TIME)) - .append(", start timestamp: ").append(startTimestamp) - .append(", finish timestamp: ").append(finishTimestamp) + .append(", Start timestamp: ").append(startTimestamp) + .append(", Finish timestamp: ").append(finishTimestamp) .append(", Mod Version: ").append(version); - builder.append(", broken files: "); + builder.append(", Broken files: "); if(brokenFiles.isEmpty()) builder.append("[]"); else { builder.append("[\n"); @@ -90,6 +91,10 @@ public record CompressionStatus(long treeHash, Map brokenFiles, builder.append("]"); } + builder.append(" }"); + return builder.toString(); } + + private static String hex(long val) { return "0x" + Long.toHexString(val).toUpperCase(); } } From 27d6d68e9711f91020bc597068e3bd5d8cab2695 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Thu, 9 Mar 2023 21:46:25 +0100 Subject: [PATCH 36/44] moved catch incrementing into hash builder --- .../core/create/FileInputStreamSupplier.java | 7 ++-- .../core/create/MakeBackupRunnable.java | 6 +-- .../compressors/AbstractCompressor.java | 15 ++++---- .../core/digest/FileTreeHashBuilder.java | 38 ++++++++++++------- .../core/digest/HashingInputStream.java | 14 +++---- .../decompressors/GenericTarDecompressor.java | 8 +++- .../decompressors/ZipDecompressor.java | 8 +++- 7 files changed, 55 insertions(+), 41 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java index 7cadc99..abd7292 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java @@ -29,19 +29,18 @@ import java.io.InputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.Optional; -import java.util.concurrent.CountDownLatch; -public record FileInputStreamSupplier(Path path, String name, FileTreeHashBuilder hashTreeBuilder, BrokenFileHandler brokenFileHandler, CountDownLatch latch) implements InputSupplier { +public record FileInputStreamSupplier(Path path, String name, FileTreeHashBuilder hashTreeBuilder, BrokenFileHandler brokenFileHandler) implements InputSupplier { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); @Override public InputStream getInputStream() throws IOException { try { - return new HashingInputStream(Files.newInputStream(path), path, hashTreeBuilder, brokenFileHandler, latch); + return new HashingInputStream(Files.newInputStream(path), path, hashTreeBuilder, brokenFileHandler); } catch (IOException e) { //Probably good idea to just put it here. In the case an exception is thrown here, it could be possble //The latch would have never been lifted - latch.countDown(); + hashTreeBuilder.update(path, 0); brokenFileHandler.handle(path, e); throw e; } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java index 13553a4..03d98a6 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java @@ -30,10 +30,8 @@ import net.szum123321.textile_backup.core.create.compressors.ZipCompressor; import net.szum123321.textile_backup.core.create.compressors.tar.AbstractTarArchiver; import net.szum123321.textile_backup.core.create.compressors.tar.ParallelBZip2Compressor; import net.szum123321.textile_backup.core.create.compressors.tar.ParallelGzipCompressor; -import org.apache.commons.compress.compressors.lzma.LZMACompressorOutputStream; import java.io.IOException; -import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.util.concurrent.Callable; @@ -94,11 +92,11 @@ public class MakeBackupRunnable implements Callable { } case BZIP2 -> ParallelBZip2Compressor.getInstance().createArchive(world, outFile, context, coreCount); case GZIP -> ParallelGzipCompressor.getInstance().createArchive(world, outFile, context, coreCount); - case LZMA -> new AbstractTarArchiver() { + /* case LZMA -> new AbstractTarArchiver() { protected OutputStream getCompressorOutputStream(OutputStream stream, BackupContext ctx, int coreLimit) throws IOException { return new LZMACompressorOutputStream(stream); } - }.createArchive(world, outFile, context, coreCount); + }.createArchive(world, outFile, context, coreCount);*/ case TAR -> new AbstractTarArchiver().createArchive(world, outFile, context, coreCount); } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index 559688d..f5f6815 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -35,7 +35,6 @@ import java.nio.file.Path; import java.time.Duration; import java.time.Instant; import java.util.Optional; -import java.util.concurrent.CountDownLatch; import java.util.concurrent.ExecutionException; import java.util.stream.Stream; @@ -48,7 +47,6 @@ public abstract class AbstractCompressor { public void createArchive(Path inputFile, Path outputFile, BackupContext ctx, int coreLimit) throws IOException, ExecutionException, InterruptedException { Instant start = Instant.now(); - FileTreeHashBuilder fileHashBuilder = new FileTreeHashBuilder(); BrokenFileHandler brokenFileHandler = new BrokenFileHandler(); //Basically a hashmap storing files and their respective exceptions try (OutputStream outStream = Files.newOutputStream(outputFile); @@ -61,8 +59,7 @@ public abstract class AbstractCompressor { .filter(Files::isRegularFile) .toList(); - //will be used in conjunction with ParallelZip to avoid race condition - CountDownLatch latch = new CountDownLatch(fileList.size()); + FileTreeHashBuilder fileHashBuilder = new FileTreeHashBuilder(fileList.size()); for (Path file : fileList) { try { @@ -71,12 +68,12 @@ public abstract class AbstractCompressor { file, inputFile.relativize(file).toString(), fileHashBuilder, - brokenFileHandler, - latch), + brokenFileHandler), arc ); } catch (IOException e) { brokenFileHandler.handle(file, e); + fileHashBuilder.update(file, 0); //In Permissive mode we allow partial backups if (ConfigHelper.INSTANCE.get().integrityVerificationMode.isStrict()) throw e; else log.sendErrorAL(ctx, "An exception occurred while trying to compress: {}", @@ -85,13 +82,15 @@ public abstract class AbstractCompressor { } } + arc.flush(); + //wait for all the InputStreams to close/fail with InputSupplier - latch.await(); Instant now = Instant.now(); + long treeHash = fileHashBuilder.getValue(true); CompressionStatus status = new CompressionStatus ( - fileHashBuilder.getValue(), + treeHash, brokenFileHandler.get(), ctx.startDate(), start.toEpochMilli(), now.toEpochMilli(), Globals.INSTANCE.getCombinedVersionString() diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java index 4abfde0..aebb394 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java @@ -26,6 +26,7 @@ import net.szum123321.textile_backup.core.CompressionStatus; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; +import java.util.concurrent.CountDownLatch; import java.util.concurrent.atomic.AtomicBoolean; /** @@ -35,35 +36,44 @@ import java.util.concurrent.atomic.AtomicBoolean; public class FileTreeHashBuilder { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); private final Object lock = new Object(); - private long hash = 0, filesProcessed = 0, filesTotalSize = 0; + private long hash = 0, filesToProcess, filesTotalSize = 0; private final AtomicBoolean closed = new AtomicBoolean(false); - public void update(Path path, long newHash) throws IOException { - if(closed.get()) throw new RuntimeException("Hash Builder already closed!"); + private final CountDownLatch latch; + public FileTreeHashBuilder(int filesToProcess) { + this.filesToProcess = filesToProcess; + latch = new CountDownLatch(filesToProcess); + } + + public void update(Path path, long newHash) throws IOException { if(path.getFileName().toString().equals(CompressionStatus.DATA_FILENAME)) return; + latch.countDown(); + long size = Files.size(path); synchronized (lock) { - //This way, the exact order of files processed doesn't matter. this.hash ^= newHash; - filesProcessed++; + filesToProcess--; filesTotalSize += size; } } - public long getValue() { + public int getRemaining() { return (int) latch.getCount(); } + + synchronized public long getValue(boolean lock) throws InterruptedException { + long leftover = latch.getCount(); + if(lock) latch.await(); + else if(leftover != 0) log.warn("Finishing with {} files unprocessed!", leftover); + var hasher = Globals.CHECKSUM_SUPPLIER.get(); - closed.set(true); - synchronized (lock) { - log.debug("Closing: files: {}, bytes {}, raw hash {}", filesProcessed, filesTotalSize, hash); - hasher.update(hash); - hasher.update(filesProcessed); - hasher.update(filesTotalSize); + log.debug("Closing: files: {}, bytes {}, raw hash {}", filesToProcess, filesTotalSize, hash); + hasher.update(hash); + hasher.update(filesToProcess); + hasher.update(filesTotalSize); - return hasher.getValue(); - } + return hasher.getValue(); } } diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java index ea88076..df9ef98 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java @@ -19,6 +19,8 @@ package net.szum123321.textile_backup.core.digest; import net.szum123321.textile_backup.Globals; +import net.szum123321.textile_backup.TextileBackup; +import net.szum123321.textile_backup.TextileLogger; import net.szum123321.textile_backup.core.DataLeftException; import net.szum123321.textile_backup.core.create.BrokenFileHandler; import org.jetbrains.annotations.NotNull; @@ -26,7 +28,6 @@ import org.jetbrains.annotations.NotNull; import java.io.*; import java.nio.charset.StandardCharsets; import java.nio.file.Path; -import java.util.concurrent.CountDownLatch; /** * This class calculates a hash of the file on the input stream, submits it to FileTreeHashBuilder. @@ -37,16 +38,16 @@ import java.util.concurrent.CountDownLatch; * That is what CountDownLatch does */ public class HashingInputStream extends FilterInputStream { + private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); + private final Path path; private final Hash hasher = Globals.CHECKSUM_SUPPLIER.get(); private final FileTreeHashBuilder hashBuilder; private final BrokenFileHandler brokenFileHandler; - private final CountDownLatch latch; - public HashingInputStream(InputStream in, Path path, FileTreeHashBuilder hashBuilder, BrokenFileHandler brokenFileHandler, CountDownLatch latch) { + public HashingInputStream(InputStream in, Path path, FileTreeHashBuilder hashBuilder, BrokenFileHandler brokenFileHandler) { super(in); this.path = path; - this.latch = latch; this.hashBuilder = hashBuilder; this.brokenFileHandler = brokenFileHandler; } @@ -74,10 +75,9 @@ public class HashingInputStream extends FilterInputStream { public void close() throws IOException { hasher.update(path.getFileName().toString().getBytes(StandardCharsets.UTF_8)); - latch.countDown(); + hashBuilder.update(path, hasher.getValue()); - if(in.available() == 0) hashBuilder.update(path, hasher.getValue()); - else brokenFileHandler.handle(path, new DataLeftException(in.available())); + if(in.available() != 0) brokenFileHandler.handle(path, new DataLeftException(in.available())); super.close(); } diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java index cbd50af..8d06c34 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/GenericTarDecompressor.java @@ -40,7 +40,7 @@ public class GenericTarDecompressor { public static long decompress(Path input, Path target) throws IOException { Instant start = Instant.now(); - FileTreeHashBuilder treeBuilder = new FileTreeHashBuilder(); + FileTreeHashBuilder treeBuilder = new FileTreeHashBuilder(0); try (InputStream fileInputStream = Files.newInputStream(input); InputStream bufferedInputStream = new BufferedInputStream(fileInputStream); @@ -70,7 +70,11 @@ public class GenericTarDecompressor { log.info("Decompression took {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now()))); - return treeBuilder.getValue(); + try { + return treeBuilder.getValue(false); + } catch (InterruptedException ignored) { + return 0; + } } private static InputStream getCompressorInputStream(InputStream inputStream) throws CompressorException { diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java index 2bae84a..996f38b 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/decompressors/ZipDecompressor.java @@ -40,7 +40,7 @@ public class ZipDecompressor { public static long decompress(Path inputFile, Path target) throws IOException { Instant start = Instant.now(); - FileTreeHashBuilder hashBuilder = new FileTreeHashBuilder(); + FileTreeHashBuilder hashBuilder = new FileTreeHashBuilder(0); try(ZipFile zipFile = new ZipFile(inputFile.toFile())) { for (Iterator it = zipFile.getEntries().asIterator(); it.hasNext(); ) { @@ -63,6 +63,10 @@ public class ZipDecompressor { log.info("Decompression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now()))); - return hashBuilder.getValue(); + try { + return hashBuilder.getValue(false); + } catch (InterruptedException ignored) { + return 0; + } } } From 6782c4fe5f820b85240791cca780e6adcdc9bfec Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Wed, 10 May 2023 08:55:54 +0200 Subject: [PATCH 37/44] Merged BackuContext, MakeBackupRunnable, MakeBackupRunnableFactory into ExecutableBackup --- .../szum123321/textile_backup/Globals.java | 5 +- .../textile_backup/TextileBackup.java | 9 +- .../textile_backup/TextileLogger.java | 11 +- .../commands/create/StartBackupCommand.java | 19 +- .../core/create/BackupContext.java | 119 --------- .../core/create/BackupScheduler.java | 30 ++- .../core/create/ExecutableBackup.java | 232 ++++++++++++++++++ .../core/create/MakeBackupRunnable.java | 138 ----------- .../create/MakeBackupRunnableFactory.java | 66 ----- .../compressors/AbstractCompressor.java | 6 +- .../compressors/ParallelZipCompressor.java | 4 +- .../create/compressors/ZipCompressor.java | 4 +- .../compressors/tar/AbstractTarArchiver.java | 6 +- .../tar/ParallelBZip2Compressor.java | 4 +- .../tar/ParallelGzipCompressor.java | 4 +- .../core/restore/RestoreBackupRunnable.java | 20 +- 16 files changed, 289 insertions(+), 388 deletions(-) delete mode 100644 src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java create mode 100644 src/main/java/net/szum123321/textile_backup/core/create/ExecutableBackup.java delete mode 100644 src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java delete mode 100644 src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnableFactory.java diff --git a/src/main/java/net/szum123321/textile_backup/Globals.java b/src/main/java/net/szum123321/textile_backup/Globals.java index 62f04d3..26beea8 100644 --- a/src/main/java/net/szum123321/textile_backup/Globals.java +++ b/src/main/java/net/szum123321/textile_backup/Globals.java @@ -23,7 +23,6 @@ import net.szum123321.textile_backup.core.digest.BalticHash; import net.szum123321.textile_backup.core.digest.Hash; import net.szum123321.textile_backup.core.Utilities; -import net.szum123321.textile_backup.core.create.MakeBackupRunnable; import net.szum123321.textile_backup.core.restore.AwaitThread; import org.apache.commons.io.FileUtils; @@ -70,8 +69,8 @@ public class Globals { if(!executorService.awaitTermination(timeout, TimeUnit.MICROSECONDS)) { log.error("Timeout occurred while waiting for currently running backups to finish!"); executorService.shutdownNow().stream() - .filter(r -> r instanceof MakeBackupRunnable) - .map(r -> (MakeBackupRunnable)r) + // .filter(r -> r instanceof ExecutableBackup) + // .map(r -> (ExecutableBackup)r) .forEach(r -> log.error("Dropping: {}", r.toString())); if(!executorService.awaitTermination(1000, TimeUnit.MICROSECONDS)) log.error("Couldn't shut down the executor!"); diff --git a/src/main/java/net/szum123321/textile_backup/TextileBackup.java b/src/main/java/net/szum123321/textile_backup/TextileBackup.java index 853aa3e..4d10e03 100644 --- a/src/main/java/net/szum123321/textile_backup/TextileBackup.java +++ b/src/main/java/net/szum123321/textile_backup/TextileBackup.java @@ -39,9 +39,8 @@ import net.szum123321.textile_backup.commands.restore.RestoreBackupCommand; import net.szum123321.textile_backup.config.ConfigHelper; import net.szum123321.textile_backup.config.ConfigPOJO; import net.szum123321.textile_backup.core.ActionInitiator; -import net.szum123321.textile_backup.core.create.BackupContext; import net.szum123321.textile_backup.core.create.BackupScheduler; -import net.szum123321.textile_backup.core.create.MakeBackupRunnableFactory; +import net.szum123321.textile_backup.core.create.ExecutableBackup; import net.szum123321.textile_backup.test.BalticHashTest; public class TextileBackup implements ModInitializer { @@ -82,14 +81,14 @@ public class TextileBackup implements ModInitializer { if (config.get().shutdownBackup && Globals.INSTANCE.globalShutdownBackupFlag.get()) { try { - MakeBackupRunnableFactory.create( - BackupContext.Builder + ExecutableBackup.Builder .newBackupContextBuilder() .setServer(server) .setInitiator(ActionInitiator.Shutdown) .setComment("shutdown") + .announce() .build() - ).call(); + .call(); } catch (Exception ignored) {} } }); diff --git a/src/main/java/net/szum123321/textile_backup/TextileLogger.java b/src/main/java/net/szum123321/textile_backup/TextileLogger.java index 2e77da9..0a38389 100644 --- a/src/main/java/net/szum123321/textile_backup/TextileLogger.java +++ b/src/main/java/net/szum123321/textile_backup/TextileLogger.java @@ -23,7 +23,7 @@ import net.minecraft.text.Text; import net.minecraft.text.MutableText; import net.minecraft.util.Formatting; import net.szum123321.textile_backup.core.Utilities; -import net.szum123321.textile_backup.core.create.BackupContext; +import net.szum123321.textile_backup.core.create.ExecutableBackup; import org.apache.logging.log4j.Level; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -112,7 +112,7 @@ public class TextileLogger { sendFeedback(Level.INFO, source, msg, args); } - public void sendInfo(BackupContext context, String msg, Object... args) { + public void sendInfo(ExecutableBackup context, String msg, Object... args) { sendInfo(context.commandSource(), msg, args); } @@ -120,7 +120,8 @@ public class TextileLogger { sendFeedback(Level.ERROR, source, msg, args); } - public void sendError(BackupContext context, String msg, Object... args) { + + public void sendError(ExecutableBackup context, String msg, Object... args) { sendError(context.commandSource(), msg, args); } @@ -134,7 +135,7 @@ public class TextileLogger { sendToPlayerAndLog(Level.INFO, source, msg, args); } - public void sendInfoAL(BackupContext context, String msg, Object... args) { + public void sendInfoAL(ExecutableBackup context, String msg, Object... args) { sendInfoAL(context.commandSource(), msg, args); } @@ -142,7 +143,7 @@ public class TextileLogger { sendToPlayerAndLog(Level.ERROR, source, msg, args); } - public void sendErrorAL(BackupContext context, String msg, Object... args) { + public void sendErrorAL(ExecutableBackup context, String msg, Object... args) { sendErrorAL(context.commandSource(), msg, args); } } diff --git a/src/main/java/net/szum123321/textile_backup/commands/create/StartBackupCommand.java b/src/main/java/net/szum123321/textile_backup/commands/create/StartBackupCommand.java index 1ec744f..7964c5a 100644 --- a/src/main/java/net/szum123321/textile_backup/commands/create/StartBackupCommand.java +++ b/src/main/java/net/szum123321/textile_backup/commands/create/StartBackupCommand.java @@ -25,8 +25,7 @@ import net.minecraft.server.command.ServerCommandSource; import net.szum123321.textile_backup.Globals; import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileLogger; -import net.szum123321.textile_backup.core.create.BackupContext; -import net.szum123321.textile_backup.core.create.MakeBackupRunnableFactory; +import net.szum123321.textile_backup.core.create.ExecutableBackup; import javax.annotation.Nullable; @@ -42,15 +41,13 @@ public class StartBackupCommand { private static int execute(ServerCommandSource source, @Nullable String comment) { Globals.INSTANCE.getQueueExecutor().submit( - MakeBackupRunnableFactory.create( - BackupContext.Builder - .newBackupContextBuilder() - .setCommandSource(source) - .setComment(comment) - .guessInitiator() - .saveServer() - .build() - ) + ExecutableBackup.Builder + .newBackupContextBuilder() + .setCommandSource(source) + .setComment(comment) + .guessInitiator() + .saveServer() + .build() ); return 1; diff --git a/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java b/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java deleted file mode 100644 index d626695..0000000 --- a/src/main/java/net/szum123321/textile_backup/core/create/BackupContext.java +++ /dev/null @@ -1,119 +0,0 @@ -/* - * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package net.szum123321.textile_backup.core.create; - -import net.minecraft.server.MinecraftServer; -import net.minecraft.server.command.ServerCommandSource; -import net.szum123321.textile_backup.core.ActionInitiator; -import net.szum123321.textile_backup.core.Utilities; -import org.jetbrains.annotations.NotNull; - -import java.time.LocalDateTime; -import java.util.NoSuchElementException; - -public record BackupContext(@NotNull MinecraftServer server, - ServerCommandSource commandSource, - ActionInitiator initiator, - boolean save, - boolean cleanup, - String comment, - LocalDateTime startDate) { - - public boolean startedByPlayer() { - return initiator == ActionInitiator.Player; - } - - public boolean shouldSave() { - return save; - } - - public static class Builder { - private MinecraftServer server; - private ServerCommandSource commandSource; - private ActionInitiator initiator; - private boolean save; - private boolean cleanup; - private String comment; - - private boolean guessInitiator; - - public Builder() { - this.server = null; - this.commandSource = null; - this.initiator = null; - this.save = false; - cleanup = true; //defaults - this.comment = null; - - guessInitiator = false; - } - - public static Builder newBackupContextBuilder() { - return new Builder(); - } - - public Builder setCommandSource(ServerCommandSource commandSource) { - this.commandSource = commandSource; - return this; - } - - public Builder setServer(MinecraftServer server) { - this.server = server; - return this; - } - - public Builder setInitiator(ActionInitiator initiator) { - this.initiator = initiator; - return this; - } - - public Builder setComment(String comment) { - this.comment = comment; - return this; - } - - public Builder guessInitiator() { - this.guessInitiator = true; - return this; - } - - public Builder saveServer() { - this.save = true; - return this; - } - - public Builder dontCleanup() { - this.cleanup = false; - return this; - } - - public BackupContext build() { - if (guessInitiator) { - initiator = Utilities.wasSentByPlayer(commandSource) ? ActionInitiator.Player : ActionInitiator.ServerConsole; - } else if (initiator == null) throw new NoSuchElementException("No initiator provided!"); - - if (server == null) { - if (commandSource != null) setServer(commandSource.getServer()); - else throw new RuntimeException("Neither MinecraftServer or ServerCommandSource were provided!"); - } - - return new BackupContext(server, commandSource, initiator, save, cleanup, comment, LocalDateTime.now()); - } - } -} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/BackupScheduler.java b/src/main/java/net/szum123321/textile_backup/core/create/BackupScheduler.java index 3359db9..55a13f3 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/BackupScheduler.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/BackupScheduler.java @@ -53,14 +53,13 @@ public class BackupScheduler { if(nextBackup <= now) { //It's time to run Globals.INSTANCE.getQueueExecutor().submit( - MakeBackupRunnableFactory.create( - BackupContext.Builder - .newBackupContextBuilder() - .setServer(server) - .setInitiator(ActionInitiator.Timer) - .saveServer() - .build() - ) + ExecutableBackup.Builder + .newBackupContextBuilder() + .setServer(server) + .setInitiator(ActionInitiator.Timer) + .saveServer() + .announce() + .build() ); nextBackup = now + config.get().backupInterval; @@ -76,14 +75,13 @@ public class BackupScheduler { if(scheduled && nextBackup <= now) { //Verify we hadn't done the final one, and it's time to do so Globals.INSTANCE.getQueueExecutor().submit( - MakeBackupRunnableFactory.create( - BackupContext.Builder - .newBackupContextBuilder() - .setServer(server) - .setInitiator(ActionInitiator.Timer) - .saveServer() - .build() - ) + ExecutableBackup.Builder + .newBackupContextBuilder() + .setServer(server) + .setInitiator(ActionInitiator.Timer) + .saveServer() + .announce() + .build() ); scheduled = false; diff --git a/src/main/java/net/szum123321/textile_backup/core/create/ExecutableBackup.java b/src/main/java/net/szum123321/textile_backup/core/create/ExecutableBackup.java new file mode 100644 index 0000000..9f5071f --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/create/ExecutableBackup.java @@ -0,0 +1,232 @@ +package net.szum123321.textile_backup.core.create; + +import net.minecraft.server.MinecraftServer; +import net.minecraft.server.command.ServerCommandSource; +import net.szum123321.textile_backup.Globals; +import net.szum123321.textile_backup.TextileBackup; +import net.szum123321.textile_backup.TextileLogger; +import net.szum123321.textile_backup.config.ConfigHelper; +import net.szum123321.textile_backup.core.ActionInitiator; +import net.szum123321.textile_backup.core.Cleanup; +import net.szum123321.textile_backup.core.Utilities; +import net.szum123321.textile_backup.core.create.compressors.ParallelZipCompressor; +import net.szum123321.textile_backup.core.create.compressors.ZipCompressor; +import net.szum123321.textile_backup.core.create.compressors.tar.AbstractTarArchiver; +import net.szum123321.textile_backup.core.create.compressors.tar.ParallelGzipCompressor; +import org.jetbrains.annotations.NotNull; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.NoSuchElementException; +import java.util.concurrent.Callable; +import java.util.concurrent.ExecutionException; + +public record ExecutableBackup(@NotNull MinecraftServer server, + ServerCommandSource commandSource, + ActionInitiator initiator, + boolean save, + boolean cleanup, + String comment, + LocalDateTime startDate) implements Callable { + + private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); + private final static ConfigHelper config = ConfigHelper.INSTANCE; + + public boolean startedByPlayer() { + return initiator == ActionInitiator.Player; + } + + public void announce() { + if(config.get().broadcastBackupStart) { + Utilities.notifyPlayers(server, + "Warning! Server backup will begin shortly. You may experience some lag." + ); + } else { + log.sendInfoAL(this, "Warning! Server backup will begin shortly. You may experience some lag."); + } + + StringBuilder builder = new StringBuilder(); + + builder.append("Backup started "); + + builder.append(initiator.getPrefix()); + + if(startedByPlayer()) + builder.append(commandSource.getDisplayName().getString()); + else + builder.append(initiator.getName()); + + builder.append(" on: "); + builder.append(Utilities.getDateTimeFormatter().format(LocalDateTime.now())); + + log.info(builder.toString()); + } + @Override + public Void call() throws IOException, ExecutionException, InterruptedException { + if (save) { //save the world + log.sendInfoAL(this, "Saving server..."); + server.saveAll(true, true, false); + } + + Path outFile = Utilities.getBackupRootPath(Utilities.getLevelName(server)).resolve(getFileName()); + + log.trace("Outfile is: {}", outFile); + + try { + //I think I should synchronise these two next calls... + Utilities.disableWorldSaving(server); + Globals.INSTANCE.disableWatchdog = true; + + Globals.INSTANCE.updateTMPFSFlag(server); + + log.sendInfoAL(this, "Starting backup"); + + Path world = Utilities.getWorldFolder(server); + + log.trace("Minecraft world is: {}", world); + + Files.createDirectories(outFile.getParent()); + Files.createFile(outFile); + + int coreCount; + + if (config.get().compressionCoreCountLimit <= 0) coreCount = Runtime.getRuntime().availableProcessors(); + else + coreCount = Math.min(config.get().compressionCoreCountLimit, Runtime.getRuntime().availableProcessors()); + + log.trace("Running compression on {} threads. Available cores: {}", coreCount, Runtime.getRuntime().availableProcessors()); + + switch (config.get().format) { + case ZIP -> { + if (coreCount > 1 && !Globals.INSTANCE.disableTMPFS()) { + log.trace("Using PARALLEL Zip Compressor. Threads: {}", coreCount); + ParallelZipCompressor.getInstance().createArchive(world, outFile, this, coreCount); + } else { + log.trace("Using REGULAR Zip Compressor."); + ZipCompressor.getInstance().createArchive(world, outFile, this, coreCount); + } + } + case GZIP -> ParallelGzipCompressor.getInstance().createArchive(world, outFile, this, coreCount); + case TAR -> new AbstractTarArchiver().createArchive(world, outFile, this, coreCount); + } + + if(cleanup) new Cleanup(commandSource, Utilities.getLevelName(server)).call(); + + if (config.get().broadcastBackupDone) Utilities.notifyPlayers(server, "Done!"); + else log.sendInfoAL(this, "Done!"); + + } catch (Throwable e) { + //ExecutorService swallows exception, so I need to catch everything + log.error("An exception occurred when trying to create new backup file!", e); + + if (ConfigHelper.INSTANCE.get().integrityVerificationMode.isStrict()) { + try { + Files.delete(outFile); + } catch (IOException ex) { + log.error("An exception occurred while trying go delete: {}", outFile, ex); + } + } + + if (initiator == ActionInitiator.Player) + log.sendError(this, "An exception occurred when trying to create new backup file!"); + + throw e; + } finally { + Utilities.enableWorldSaving(server); + Globals.INSTANCE.disableWatchdog = false; + } + + return null; + } + + private String getFileName() { + return Utilities.getDateTimeFormatter().format(startDate) + + (comment != null ? "#" + comment.replaceAll("[\\\\/:*?\"<>|#]", "") : "") + + config.get().format.getCompleteString(); + } + public static class Builder { + private MinecraftServer server; + private ServerCommandSource commandSource; + private ActionInitiator initiator; + private boolean save; + private boolean cleanup; + private String comment; + private boolean announce; + + private boolean guessInitiator; + + public Builder() { + this.server = null; + this.commandSource = null; + this.initiator = null; + this.save = false; + cleanup = true; //defaults + this.comment = null; + this.announce = false; + + guessInitiator = false; + } + + public static ExecutableBackup.Builder newBackupContextBuilder() { + return new ExecutableBackup.Builder(); + } + + public ExecutableBackup.Builder setCommandSource(ServerCommandSource commandSource) { + this.commandSource = commandSource; + return this; + } + + public ExecutableBackup.Builder setServer(MinecraftServer server) { + this.server = server; + return this; + } + + public ExecutableBackup.Builder setInitiator(ActionInitiator initiator) { + this.initiator = initiator; + return this; + } + + public ExecutableBackup.Builder setComment(String comment) { + this.comment = comment; + return this; + } + + public ExecutableBackup.Builder guessInitiator() { + this.guessInitiator = true; + return this; + } + + public ExecutableBackup.Builder saveServer() { + this.save = true; + return this; + } + + public ExecutableBackup.Builder noCleanup() { + this.cleanup = false; + return this; + } + + public ExecutableBackup.Builder announce() { + this.announce = true; + return this; + } + + public ExecutableBackup build() { + if (guessInitiator) { + initiator = Utilities.wasSentByPlayer(commandSource) ? ActionInitiator.Player : ActionInitiator.ServerConsole; + } else if (initiator == null) throw new NoSuchElementException("No initiator provided!"); + + if (server == null) { + if (commandSource != null) setServer(commandSource.getServer()); + else throw new RuntimeException("Neither MinecraftServer or ServerCommandSource were provided!"); + } + + ExecutableBackup v = new ExecutableBackup(server, commandSource, initiator, save, cleanup, comment, LocalDateTime.now()); + + if(announce) v.announce(); + return v; + } + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java deleted file mode 100644 index 03d98a6..0000000 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java +++ /dev/null @@ -1,138 +0,0 @@ -/* - * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package net.szum123321.textile_backup.core.create; - -import net.szum123321.textile_backup.Globals; -import net.szum123321.textile_backup.TextileBackup; -import net.szum123321.textile_backup.TextileLogger; -import net.szum123321.textile_backup.config.ConfigHelper; -import net.szum123321.textile_backup.core.ActionInitiator; -import net.szum123321.textile_backup.core.Cleanup; -import net.szum123321.textile_backup.core.Utilities; -import net.szum123321.textile_backup.core.create.compressors.ParallelZipCompressor; -import net.szum123321.textile_backup.core.create.compressors.ZipCompressor; -import net.szum123321.textile_backup.core.create.compressors.tar.AbstractTarArchiver; -import net.szum123321.textile_backup.core.create.compressors.tar.ParallelBZip2Compressor; -import net.szum123321.textile_backup.core.create.compressors.tar.ParallelGzipCompressor; - -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; - -/** - * The actual object responsible for creating the backup - */ -public class MakeBackupRunnable implements Callable { - private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); - private final static ConfigHelper config = ConfigHelper.INSTANCE; - - private final BackupContext context; - - public MakeBackupRunnable(BackupContext context) { - this.context = context; - } - - @Override - public Void call() throws IOException, ExecutionException, InterruptedException { - Path outFile = Utilities.getBackupRootPath(Utilities.getLevelName(context.server())).resolve(getFileName()); - - log.trace("Outfile is: {}", outFile); - - try { - //I think I should synchronise these two next calls... - Utilities.disableWorldSaving(context.server()); - Globals.INSTANCE.disableWatchdog = true; - - Globals.INSTANCE.updateTMPFSFlag(context.server()); - - log.sendInfoAL(context, "Starting backup"); - - Path world = Utilities.getWorldFolder(context.server()); - - log.trace("Minecraft world is: {}", world); - - Files.createDirectories(outFile.getParent()); - Files.createFile(outFile); - - int coreCount; - - if (config.get().compressionCoreCountLimit <= 0) coreCount = Runtime.getRuntime().availableProcessors(); - else - coreCount = Math.min(config.get().compressionCoreCountLimit, Runtime.getRuntime().availableProcessors()); - - log.trace("Running compression on {} threads. Available cores: {}", coreCount, Runtime.getRuntime().availableProcessors()); - - switch (config.get().format) { - case ZIP -> { - if (coreCount > 1 && !Globals.INSTANCE.disableTMPFS()) { - log.trace("Using PARALLEL Zip Compressor. Threads: {}", coreCount); - ParallelZipCompressor.getInstance().createArchive(world, outFile, context, coreCount); - } else { - log.trace("Using REGULAR Zip Compressor."); - ZipCompressor.getInstance().createArchive(world, outFile, context, coreCount); - } - } - case BZIP2 -> ParallelBZip2Compressor.getInstance().createArchive(world, outFile, context, coreCount); - case GZIP -> ParallelGzipCompressor.getInstance().createArchive(world, outFile, context, coreCount); - /* case LZMA -> new AbstractTarArchiver() { - protected OutputStream getCompressorOutputStream(OutputStream stream, BackupContext ctx, int coreLimit) throws IOException { - return new LZMACompressorOutputStream(stream); - } - }.createArchive(world, outFile, context, coreCount);*/ - case TAR -> new AbstractTarArchiver().createArchive(world, outFile, context, coreCount); - } - - if(context.cleanup()) - new Cleanup(context.commandSource(), Utilities.getLevelName(context.server())).call(); - - if (config.get().broadcastBackupDone) Utilities.notifyPlayers(context.server(), "Done!"); - else log.sendInfoAL(context, "Done!"); - - } catch (Throwable e) { - //ExecutorService swallows exception, so I need to catch everything - log.error("An exception occurred when trying to create new backup file!", e); - - if (ConfigHelper.INSTANCE.get().integrityVerificationMode.isStrict()) { - try { - Files.delete(outFile); - } catch (IOException ex) { - log.error("An exception occurred while trying go delete: {}", outFile, ex); - } - } - - if (context.initiator() == ActionInitiator.Player) - log.sendError(context, "An exception occurred when trying to create new backup file!"); - - throw e; - } finally { - Utilities.enableWorldSaving(context.server()); - Globals.INSTANCE.disableWatchdog = false; - } - - return null; - } - - private String getFileName() { - return Utilities.getDateTimeFormatter().format(context.startDate()) + - (context.comment() != null ? "#" + context.comment().replaceAll("[\\\\/:*?\"<>|#]", "") : "") + - config.get().format.getCompleteString(); - } -} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnableFactory.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnableFactory.java deleted file mode 100644 index 099d17c..0000000 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnableFactory.java +++ /dev/null @@ -1,66 +0,0 @@ -/* - * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package net.szum123321.textile_backup.core.create; - -import net.szum123321.textile_backup.TextileBackup; -import net.szum123321.textile_backup.TextileLogger; -import net.szum123321.textile_backup.config.ConfigHelper; -import net.szum123321.textile_backup.core.Utilities; - -import java.time.LocalDateTime; -import java.util.concurrent.Callable; - -public class MakeBackupRunnableFactory { - private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); - private final static ConfigHelper config = ConfigHelper.INSTANCE; - - public static Callable create(BackupContext ctx) { - if(config.get().broadcastBackupStart) { - Utilities.notifyPlayers(ctx.server(), - "Warning! Server backup will begin shortly. You may experience some lag." - ); - } else { - log.sendInfoAL(ctx, "Warning! Server backup will begin shortly. You may experience some lag."); - } - - StringBuilder builder = new StringBuilder(); - - builder.append("Backup started "); - - builder.append(ctx.initiator().getPrefix()); - - if(ctx.startedByPlayer()) - builder.append(ctx.commandSource().getDisplayName().getString()); - else - builder.append(ctx.initiator().getName()); - - builder.append(" on: "); - builder.append(Utilities.getDateTimeFormatter().format(LocalDateTime.now())); - - log.info(builder.toString()); - - if (ctx.shouldSave()) { - log.sendInfoAL(ctx, "Saving server..."); - - ctx.server().saveAll(true, true, false); - } - - return new MakeBackupRunnable(ctx); - } -} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index f5f6815..108c6e5 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -23,8 +23,8 @@ import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileLogger; import net.szum123321.textile_backup.config.ConfigHelper; import net.szum123321.textile_backup.core.*; -import net.szum123321.textile_backup.core.create.BackupContext; import net.szum123321.textile_backup.core.create.BrokenFileHandler; +import net.szum123321.textile_backup.core.create.ExecutableBackup; import net.szum123321.textile_backup.core.create.FileInputStreamSupplier; import net.szum123321.textile_backup.core.create.InputSupplier; import net.szum123321.textile_backup.core.digest.FileTreeHashBuilder; @@ -44,7 +44,7 @@ import java.util.stream.Stream; public abstract class AbstractCompressor { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); - public void createArchive(Path inputFile, Path outputFile, BackupContext ctx, int coreLimit) throws IOException, ExecutionException, InterruptedException { + public void createArchive(Path inputFile, Path outputFile, ExecutableBackup ctx, int coreLimit) throws IOException, ExecutionException, InterruptedException { Instant start = Instant.now(); BrokenFileHandler brokenFileHandler = new BrokenFileHandler(); //Basically a hashmap storing files and their respective exceptions @@ -106,7 +106,7 @@ public abstract class AbstractCompressor { log.sendInfoAL(ctx, "Compression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now()))); } - protected abstract OutputStream createArchiveOutputStream(OutputStream stream, BackupContext ctx, int coreLimit) throws IOException; + protected abstract OutputStream createArchiveOutputStream(OutputStream stream, ExecutableBackup ctx, int coreLimit) throws IOException; protected abstract void addEntry(InputSupplier inputSupplier, OutputStream arc) throws IOException; protected void finish(OutputStream arc) throws InterruptedException, ExecutionException, IOException { diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java index 3632179..7ed67a1 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java @@ -21,7 +21,7 @@ package net.szum123321.textile_backup.core.create.compressors; import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.TextileLogger; import net.szum123321.textile_backup.core.NoSpaceLeftOnDeviceException; -import net.szum123321.textile_backup.core.create.BackupContext; +import net.szum123321.textile_backup.core.create.ExecutableBackup; import net.szum123321.textile_backup.core.create.InputSupplier; import org.apache.commons.compress.archivers.zip.*; @@ -61,7 +61,7 @@ public class ParallelZipCompressor extends ZipCompressor { } @Override - protected OutputStream createArchiveOutputStream(OutputStream stream, BackupContext ctx, int coreLimit) { + protected OutputStream createArchiveOutputStream(OutputStream stream, ExecutableBackup ctx, int coreLimit) { scatterZipCreator = new ParallelScatterZipCreator(Executors.newFixedThreadPool(coreLimit)); return super.createArchiveOutputStream(stream, ctx, coreLimit); } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java index 5a39bab..a1b224b 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ZipCompressor.java @@ -20,7 +20,7 @@ package net.szum123321.textile_backup.core.create.compressors; import net.szum123321.textile_backup.config.ConfigHelper; import net.szum123321.textile_backup.core.Utilities; -import net.szum123321.textile_backup.core.create.BackupContext; +import net.szum123321.textile_backup.core.create.ExecutableBackup; import net.szum123321.textile_backup.core.create.InputSupplier; import org.apache.commons.compress.archivers.zip.Zip64Mode; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; @@ -43,7 +43,7 @@ public class ZipCompressor extends AbstractCompressor { } @Override - protected OutputStream createArchiveOutputStream(OutputStream stream, BackupContext ctx, int coreLimit) { + protected OutputStream createArchiveOutputStream(OutputStream stream, ExecutableBackup ctx, int coreLimit) { ZipArchiveOutputStream arc = new ZipArchiveOutputStream(stream); arc.setMethod(ZipArchiveOutputStream.DEFLATED); diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java index 4ae84a3..52e86e5 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/AbstractTarArchiver.java @@ -18,7 +18,7 @@ package net.szum123321.textile_backup.core.create.compressors.tar; -import net.szum123321.textile_backup.core.create.BackupContext; +import net.szum123321.textile_backup.core.create.ExecutableBackup; import net.szum123321.textile_backup.core.create.compressors.AbstractCompressor; import net.szum123321.textile_backup.core.create.InputSupplier; import org.apache.commons.compress.archivers.tar.TarArchiveEntry; @@ -28,12 +28,12 @@ import org.apache.commons.compress.utils.IOUtils; import java.io.*; public class AbstractTarArchiver extends AbstractCompressor { - protected OutputStream getCompressorOutputStream(OutputStream stream, BackupContext ctx, int coreLimit) throws IOException { + protected OutputStream getCompressorOutputStream(OutputStream stream, ExecutableBackup ctx, int coreLimit) throws IOException { return stream; } @Override - protected OutputStream createArchiveOutputStream(OutputStream stream, BackupContext ctx, int coreLimit) throws IOException { + protected OutputStream createArchiveOutputStream(OutputStream stream, ExecutableBackup ctx, int coreLimit) throws IOException { TarArchiveOutputStream tar = new TarArchiveOutputStream(getCompressorOutputStream(stream, ctx, coreLimit)); tar.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); tar.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelBZip2Compressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelBZip2Compressor.java index 76d96fe..2bd2840 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelBZip2Compressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelBZip2Compressor.java @@ -18,7 +18,7 @@ package net.szum123321.textile_backup.core.create.compressors.tar; -import net.szum123321.textile_backup.core.create.BackupContext; +import net.szum123321.textile_backup.core.create.ExecutableBackup; import org.at4j.comp.bzip2.BZip2OutputStream; import org.at4j.comp.bzip2.BZip2OutputStreamSettings; @@ -30,7 +30,7 @@ public class ParallelBZip2Compressor extends AbstractTarArchiver { } @Override - protected OutputStream getCompressorOutputStream(OutputStream stream, BackupContext ctx, int coreLimit) throws IOException { + protected OutputStream getCompressorOutputStream(OutputStream stream, ExecutableBackup ctx, int coreLimit) throws IOException { return new BZip2OutputStream(stream, new BZip2OutputStreamSettings().setNumberOfEncoderThreads(coreLimit)); } } \ No newline at end of file diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelGzipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelGzipCompressor.java index 0a59638..94b9da6 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelGzipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/tar/ParallelGzipCompressor.java @@ -18,7 +18,7 @@ package net.szum123321.textile_backup.core.create.compressors.tar; -import net.szum123321.textile_backup.core.create.BackupContext; +import net.szum123321.textile_backup.core.create.ExecutableBackup; import org.anarres.parallelgzip.ParallelGZIPOutputStream; import java.io.*; @@ -33,7 +33,7 @@ public class ParallelGzipCompressor extends AbstractTarArchiver { } @Override - protected OutputStream getCompressorOutputStream(OutputStream stream, BackupContext ctx, int coreLimit) throws IOException { + protected OutputStream getCompressorOutputStream(OutputStream stream, ExecutableBackup ctx, int coreLimit) throws IOException { executorService = Executors.newFixedThreadPool(coreLimit); return new ParallelGZIPOutputStream(stream, executorService); diff --git a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java index 6f705eb..b3520fc 100644 --- a/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/restore/RestoreBackupRunnable.java @@ -26,8 +26,7 @@ import net.szum123321.textile_backup.config.ConfigPOJO; import net.szum123321.textile_backup.core.ActionInitiator; import net.szum123321.textile_backup.core.CompressionStatus; import net.szum123321.textile_backup.core.Utilities; -import net.szum123321.textile_backup.core.create.BackupContext; -import net.szum123321.textile_backup.core.create.MakeBackupRunnableFactory; +import net.szum123321.textile_backup.core.create.ExecutableBackup; import net.szum123321.textile_backup.core.restore.decompressors.GenericTarDecompressor; import net.szum123321.textile_backup.core.restore.decompressors.ZipDecompressor; import net.szum123321.textile_backup.mixin.MinecraftServerSessionAccessor; @@ -78,20 +77,19 @@ public class RestoreBackupRunnable implements Runnable { ctx.server().getThread().join(); //wait for server thread to die and save all its state if(config.get().backupOldWorlds) { - return MakeBackupRunnableFactory.create ( - BackupContext.Builder + return ExecutableBackup.Builder .newBackupContextBuilder() .setServer(ctx.server()) .setInitiator(ActionInitiator.Restore) - .dontCleanup() + .noCleanup() .setComment("Old_World" + (ctx.comment() != null ? "_" + ctx.comment() : "")) - .build() - ).call(); + .announce() + .build().call(); } - return null; }); + //run the thread. new Thread(waitForShutdown, "Server shutdown wait thread").start(); try { @@ -106,7 +104,7 @@ public class RestoreBackupRunnable implements Runnable { log.info("Waiting for server to fully terminate..."); - //locks until the backup is finished + //locks until the backup is finished and the server is dead waitForShutdown.get(); Optional errorMsg; @@ -127,8 +125,8 @@ public class RestoreBackupRunnable implements Runnable { if (errorMsg.isEmpty()) log.info("Backup valid. Restoring"); else log.info("Backup is damaged, but verification is disabled [{}]. Restoring", errorMsg.get()); - ((MinecraftServerSessionAccessor) ctx.server()) - .getSession().close(); + //Disables write lock to override world file + ((MinecraftServerSessionAccessor) ctx.server()).getSession().close(); Utilities.deleteDirectory(worldFile); Files.move(tmp, worldFile); From fe9f9f3e0d39593bd7b52e5dd16cdb28984b1d94 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Wed, 10 May 2023 08:56:37 +0200 Subject: [PATCH 38/44] 1.20 snapshot update --- build.gradle | 2 +- gradle.properties | 12 ++++++------ gradle/wrapper/gradle-wrapper.properties | 2 +- .../szum123321/textile_backup/config/ConfigPOJO.java | 5 ++--- src/main/resources/fabric.mod.json | 2 +- 5 files changed, 11 insertions(+), 12 deletions(-) diff --git a/build.gradle b/build.gradle index 5efb5cb..4c3b6b8 100644 --- a/build.gradle +++ b/build.gradle @@ -1,5 +1,5 @@ plugins { - id 'fabric-loom' version '1.0-SNAPSHOT' + id 'fabric-loom' version '1.2-SNAPSHOT' id 'maven-publish' } diff --git a/gradle.properties b/gradle.properties index eb61004..2c79c99 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,18 +1,18 @@ # Done to increase the memory available to gradle. org.gradle.jvmargs=-Xmx1G -minecraft_version=1.19.3 -yarn_mappings=1.19.3+build.3 -loader_version=0.14.11 +minecraft_version=23w18a +yarn_mappings=23w18a+build.5 +loader_version=0.14.19 #Fabric api -fabric_version=0.69.1+1.19.3 +fabric_version=0.80.1+1.20 #Cloth Config -cloth_version=9.0.94 +cloth_version=11.0.97 #ModMenu -modmenu_version=5.0.2 +modmenu_version=6.2.2 databreaker_version=0.2.10 diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index ae04661..fae0804 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,5 +1,5 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-7.5.1-bin.zip +distributionUrl=https\://services.gradle.org/distributions/gradle-8.1.1-bin.zip zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java index 0cec14a..ae6106a 100644 --- a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java +++ b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java @@ -18,13 +18,12 @@ package net.szum123321.textile_backup.config; -import blue.endless.jankson.annotation.SerializedName; +import me.shedaniel.cloth.clothconfig.shadowed.blue.endless.jankson.annotation.SerializedName; +import me.shedaniel.cloth.clothconfig.shadowed.blue.endless.jankson.Comment; import me.shedaniel.autoconfig.ConfigData; import me.shedaniel.autoconfig.annotation.Config; import me.shedaniel.autoconfig.annotation.ConfigEntry; -import me.shedaniel.cloth.clothconfig.shadowed.blue.endless.jankson.Comment; import net.szum123321.textile_backup.TextileBackup; -import net.szum123321.textile_backup.core.create.compressors.AbstractCompressor; import java.time.format.DateTimeFormatter; import java.util.*; diff --git a/src/main/resources/fabric.mod.json b/src/main/resources/fabric.mod.json index a8b5d65..fbd649f 100644 --- a/src/main/resources/fabric.mod.json +++ b/src/main/resources/fabric.mod.json @@ -40,7 +40,7 @@ "depends": { "fabricloader": ">=0.14.0", "fabric": "*", - "minecraft": ">=1.19.1", + "minecraft": "1.20-alpha.23.18.a", "cloth-config2": "*", "java": ">=16" }, From b9af3b37775c6d07f44c95b729f34ff21cfd6e7d Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Wed, 10 May 2023 23:32:54 +0200 Subject: [PATCH 39/44] YYYYEEEEEEEEEEEEEEEEEEe I finally debugged the hash!!!!! The problem was due to an error in usage o FileTreeHashBuilder during unpacking. --- gradle.properties | 8 ++++---- .../net/szum123321/textile_backup/TextileBackup.java | 5 ----- .../szum123321/textile_backup/core/Utilities.java | 4 +--- .../textile_backup/core/digest/BalticHash.java | 4 ++-- .../core/digest/FileTreeHashBuilder.java | 12 +++++------- .../szum123321/textile_backup/core/digest/Hash.java | 3 +-- .../core/digest/HashingInputStream.java | 2 -- .../core/digest/HashingOutputStream.java | 2 +- src/main/resources/fabric.mod.json | 2 +- 9 files changed, 15 insertions(+), 27 deletions(-) diff --git a/gradle.properties b/gradle.properties index 2c79c99..5ddf9f0 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,18 +1,18 @@ # Done to increase the memory available to gradle. org.gradle.jvmargs=-Xmx1G -minecraft_version=23w18a -yarn_mappings=23w18a+build.5 +minecraft_version=1.20-pre1 +yarn_mappings=1.20-pre1+build.2 loader_version=0.14.19 #Fabric api -fabric_version=0.80.1+1.20 +fabric_version=0.80.2+1.20 #Cloth Config cloth_version=11.0.97 #ModMenu -modmenu_version=6.2.2 +modmenu_version=7.0.0-beta.2 databreaker_version=0.2.10 diff --git a/src/main/java/net/szum123321/textile_backup/TextileBackup.java b/src/main/java/net/szum123321/textile_backup/TextileBackup.java index 4d10e03..876bc5b 100644 --- a/src/main/java/net/szum123321/textile_backup/TextileBackup.java +++ b/src/main/java/net/szum123321/textile_backup/TextileBackup.java @@ -60,11 +60,6 @@ public class TextileBackup implements ModInitializer { log.info("Starting Textile Backup {} by Szum123321", Globals.INSTANCE.getCombinedVersionString()); - if(FabricLoader.getInstance().isDevelopmentEnvironment()) { - //Run the tests - BalticHashTest.run(); - } - ConfigHelper.updateInstance(AutoConfig.register(ConfigPOJO.class, JanksonConfigSerializer::new)); ServerTickEvents.END_SERVER_TICK.register(BackupScheduler::tick); diff --git a/src/main/java/net/szum123321/textile_backup/core/Utilities.java b/src/main/java/net/szum123321/textile_backup/core/Utilities.java index 8a3ff02..7d82400 100644 --- a/src/main/java/net/szum123321/textile_backup/core/Utilities.java +++ b/src/main/java/net/szum123321/textile_backup/core/Utilities.java @@ -115,9 +115,7 @@ public class Utilities { } public static boolean isBlacklisted(Path path) { - if(isWindows()) { //hotfix! - if (path.getFileName().toString().equals("session.lock")) return true; - } + if (path.getFileName().equals("session.lock")) return true; if(path.getFileName().endsWith(CompressionStatus.DATA_FILENAME)) return true; diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java index 2be911e..1b6ab70 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java @@ -38,8 +38,8 @@ public class BalticHash implements Hash { protected final ByteBuffer buffer = ByteBuffer.wrap(_byte_buffer).order(ByteOrder.LITTLE_ENDIAN); protected long hashed_data_length = 0; - public void update(byte b) { - buffer.put(b); + public void update(int b) { + buffer.put((byte)b); hashed_data_length += 1; if (buffer.position() >= buffer_limit) round(); } diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java index aebb394..6e42232 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java @@ -36,13 +36,11 @@ import java.util.concurrent.atomic.AtomicBoolean; public class FileTreeHashBuilder { private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); private final Object lock = new Object(); - private long hash = 0, filesToProcess, filesTotalSize = 0; - private final AtomicBoolean closed = new AtomicBoolean(false); + private long hash = 0, filesProcessed = 0, filesTotalSize = 0; private final CountDownLatch latch; public FileTreeHashBuilder(int filesToProcess) { - this.filesToProcess = filesToProcess; latch = new CountDownLatch(filesToProcess); } @@ -55,23 +53,23 @@ public class FileTreeHashBuilder { synchronized (lock) { this.hash ^= newHash; - filesToProcess--; filesTotalSize += size; + filesProcessed++; } } public int getRemaining() { return (int) latch.getCount(); } - synchronized public long getValue(boolean lock) throws InterruptedException { + public long getValue(boolean lock) throws InterruptedException { long leftover = latch.getCount(); if(lock) latch.await(); else if(leftover != 0) log.warn("Finishing with {} files unprocessed!", leftover); var hasher = Globals.CHECKSUM_SUPPLIER.get(); - log.debug("Closing: files: {}, bytes {}, raw hash {}", filesToProcess, filesTotalSize, hash); + log.debug("Closing: files: {}, bytes {}, raw hash {}", filesProcessed, filesTotalSize, hash); hasher.update(hash); - hasher.update(filesToProcess); + hasher.update(filesProcessed); hasher.update(filesTotalSize); return hasher.getValue(); diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java b/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java index aa54578..8d67b83 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java @@ -19,9 +19,8 @@ package net.szum123321.textile_backup.core.digest; public interface Hash { - void update(byte b); - default void update(int b) { update((byte)b); } + void update(int b) ; void update(long b); diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java index df9ef98..409efa1 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java @@ -38,8 +38,6 @@ import java.nio.file.Path; * That is what CountDownLatch does */ public class HashingInputStream extends FilterInputStream { - private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); - private final Path path; private final Hash hasher = Globals.CHECKSUM_SUPPLIER.get(); private final FileTreeHashBuilder hashBuilder; diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java index 3b293ba..46b04a5 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java @@ -41,7 +41,7 @@ public class HashingOutputStream extends FilterOutputStream { @Override public void write(int b) throws IOException { out.write(b); - hasher.update((byte)b); + hasher.update(b); } @Override diff --git a/src/main/resources/fabric.mod.json b/src/main/resources/fabric.mod.json index fbd649f..cbf3e57 100644 --- a/src/main/resources/fabric.mod.json +++ b/src/main/resources/fabric.mod.json @@ -40,7 +40,7 @@ "depends": { "fabricloader": ">=0.14.0", "fabric": "*", - "minecraft": "1.20-alpha.23.18.a", + "minecraft": "1.20-beta.1", "cloth-config2": "*", "java": ">=16" }, From 41934c32cf79a2fb7f0ac3750ab355c23866ed89 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Fri, 12 May 2023 18:36:39 +0200 Subject: [PATCH 40/44] FileTreeHashBuilder::update now also takes the number of bytes as an argument. I/OHashingStreams count no. of bytes written. --- .../core/create/FileInputStreamSupplier.java | 4 +- .../compressors/AbstractCompressor.java | 2 +- .../core/digest/BalticHash.java | 2 +- .../core/digest/BalticHashSIMD.java | 96 ------------------- .../core/digest/FileTreeHashBuilder.java | 6 +- .../core/digest/HashingInputStream.java | 20 ++-- .../core/digest/HashingOutputStream.java | 15 +-- 7 files changed, 28 insertions(+), 117 deletions(-) delete mode 100644 src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java diff --git a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java index abd7292..2c43a25 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java @@ -38,9 +38,9 @@ public record FileInputStreamSupplier(Path path, String name, FileTreeHashBuilde try { return new HashingInputStream(Files.newInputStream(path), path, hashTreeBuilder, brokenFileHandler); } catch (IOException e) { - //Probably good idea to just put it here. In the case an exception is thrown here, it could be possble + //Probably good idea to just put it here. In the case an exception is thrown here, it could be possible //The latch would have never been lifted - hashTreeBuilder.update(path, 0); + hashTreeBuilder.update(path, 0, 0); brokenFileHandler.handle(path, e); throw e; } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java index 108c6e5..01d3de1 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java @@ -73,7 +73,7 @@ public abstract class AbstractCompressor { ); } catch (IOException e) { brokenFileHandler.handle(file, e); - fileHashBuilder.update(file, 0); + fileHashBuilder.update(file, 0, 0); //In Permissive mode we allow partial backups if (ConfigHelper.INSTANCE.get().integrityVerificationMode.isStrict()) throw e; else log.sendErrorAL(ctx, "An exception occurred while trying to compress: {}", diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java index 1b6ab70..14a6050 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHash.java @@ -23,7 +23,7 @@ import java.nio.ByteOrder; import java.util.Arrays; /** - * This algorithm copies construction of SeaHash including its IV. + * This algorithm copies the construction of SeaHash including its IV. * What it differs in is that it uses Xoroshift64* instead of PCG as its pseudo-random function. Although it might lower * the output quality, I don't think it matters that much, honestly. One advantage of xoroshift is that it should be * easier to implement with AVX. Java should soon ship its vector api by default. diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java b/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java deleted file mode 100644 index 5bd5753..0000000 --- a/src/main/java/net/szum123321/textile_backup/core/digest/BalticHashSIMD.java +++ /dev/null @@ -1,96 +0,0 @@ -/* - * A simple backup mod for Fabric - * Copyright (C) 2022 Szum123321 - * - * This program is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * This program is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with this program. If not, see . - */ - -package net.szum123321.textile_backup.core.digest; - -//import jdk.incubator.vector.*; - -import net.szum123321.textile_backup.core.digest.BalticHash; - -/** - * Mostly working XorSeaHash impl using SIMD. Should speed up calculation on most systems currently in use - -
...
- - * It's actually slower. I tested it by comparing runtimes while hashing a directly opened FileInputStream. - * My cpu is AMD Ryzen 5 3500U - * There are two reasons I can think of: either vector construction simply takes so much time or jvm auto-vectorizes better than I. - * It's still probably far from being the slowest part of code, so I don't expect any major slowdowns - * I will keep this code here for future work perhaps - */ -public class BalticHashSIMD extends BalticHash {/* - public BalticHashSIMD() { throw new UnsupportedOperationException(); } //For safety - - private LongVector state = LongVector.fromArray(LongVector.SPECIES_256, IV, 0); - - @Override - public long getValue() { - if (buffer.position() != 0) { - while (buffer.position() < buffer_limit) buffer.put((byte) 0); - round(); - } - - long result = state.reduceLanesToLong(VectorOperators.XOR); - result ^= hashed_data_length; - - return xorshift64star(result); - } - -//This is wrong. will have to correct ( - @Override - public void update(byte[] data, int off, int len) { - int pos = off; //should be = 0 - while (pos < len) { - int n = Math.min(len - pos, buffer_limit - buffer.position()); - if (n == 32) { - var v = ByteVector.fromArray(ByteVector.SPECIES_256, data, pos).reinterpretAsLongs(); - state = state.lanewise(VectorOperators.XOR, v); - state = xorshift64star(state); - } else { - System.arraycopy(data, pos, _byte_buffer, buffer.position(), n); - buffer.position(buffer.position() + n); - if (buffer.position() == buffer_limit) round(); - } - pos += n; - } - - hashed_data_length += len; - } - - @Override - protected void round() { - var s = ByteVector.fromArray(ByteVector.SPECIES_256, _byte_buffer, 0).reinterpretAsLongs(); - state = state.lanewise(VectorOperators.XOR, s); - state = xorshift64star(state); - - int p = buffer.position(); - - if (p > buffer_limit) { - System.arraycopy(_byte_buffer, buffer_limit, _byte_buffer, 0, buffer.limit() - p); - buffer.position(buffer.limit() - p); - } else buffer.rewind(); - } - - LongVector xorshift64star(LongVector v) { - v = v.lanewise(VectorOperators.XOR, v.lanewise(VectorOperators.ASHR, 12)); - v = v.lanewise(VectorOperators.XOR, v.lanewise(VectorOperators.LSHL, 25)); - v = v.lanewise(VectorOperators.XOR, v.lanewise(VectorOperators.ASHR, 27)); - v = v.lanewise(VectorOperators.MUL, 0x2545F4914F6CDD1DL); - return v; - }*/ -} \ No newline at end of file diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java index 6e42232..f3cad18 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java @@ -44,16 +44,14 @@ public class FileTreeHashBuilder { latch = new CountDownLatch(filesToProcess); } - public void update(Path path, long newHash) throws IOException { + public void update(Path path, long newHash, long bytes) throws IOException { if(path.getFileName().toString().equals(CompressionStatus.DATA_FILENAME)) return; latch.countDown(); - long size = Files.size(path); - synchronized (lock) { this.hash ^= newHash; - filesTotalSize += size; + filesTotalSize += bytes; filesProcessed++; } } diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java index 409efa1..a288b25 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingInputStream.java @@ -19,8 +19,6 @@ package net.szum123321.textile_backup.core.digest; import net.szum123321.textile_backup.Globals; -import net.szum123321.textile_backup.TextileBackup; -import net.szum123321.textile_backup.TextileLogger; import net.szum123321.textile_backup.core.DataLeftException; import net.szum123321.textile_backup.core.create.BrokenFileHandler; import org.jetbrains.annotations.NotNull; @@ -39,10 +37,12 @@ import java.nio.file.Path; */ public class HashingInputStream extends FilterInputStream { private final Path path; - private final Hash hasher = Globals.CHECKSUM_SUPPLIER.get(); + private final Hash hash = Globals.CHECKSUM_SUPPLIER.get(); private final FileTreeHashBuilder hashBuilder; private final BrokenFileHandler brokenFileHandler; + private long bytesWritten = 0; + public HashingInputStream(InputStream in, Path path, FileTreeHashBuilder hashBuilder, BrokenFileHandler brokenFileHandler) { super(in); this.path = path; @@ -53,14 +53,20 @@ public class HashingInputStream extends FilterInputStream { @Override public int read(byte @NotNull [] b, int off, int len) throws IOException { int i = in.read(b, off, len); - if(i != -1) hasher.update(b, off, i); + if(i != -1) { + hash.update(b, off, i); + bytesWritten += i; + } return i; } @Override public int read() throws IOException { int i = in.read(); - if(i != -1) hasher.update((byte)i); + if(i != -1) { + hash.update(i); + bytesWritten++; + } return i; } @@ -71,9 +77,9 @@ public class HashingInputStream extends FilterInputStream { @Override public void close() throws IOException { - hasher.update(path.getFileName().toString().getBytes(StandardCharsets.UTF_8)); + hash.update(path.getFileName().toString().getBytes(StandardCharsets.UTF_8)); - hashBuilder.update(path, hasher.getValue()); + hashBuilder.update(path, hash.getValue(), bytesWritten); if(in.available() != 0) brokenFileHandler.handle(path, new DataLeftException(in.available())); diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java index 46b04a5..0cb15ca 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/HashingOutputStream.java @@ -29,9 +29,11 @@ import java.nio.file.Path; public class HashingOutputStream extends FilterOutputStream { private final Path path; - private final Hash hasher = Globals.CHECKSUM_SUPPLIER.get(); + private final Hash hash = Globals.CHECKSUM_SUPPLIER.get(); private final FileTreeHashBuilder hashBuilder; + private long bytesWritten = 0; + public HashingOutputStream(OutputStream out, Path path, FileTreeHashBuilder hashBuilder) { super(out); this.path = path; @@ -41,20 +43,21 @@ public class HashingOutputStream extends FilterOutputStream { @Override public void write(int b) throws IOException { out.write(b); - hasher.update(b); + hash.update(b); + bytesWritten++; } @Override public void write(byte @NotNull [] b, int off, int len) throws IOException { out.write(b, off, len); - hasher.update(b, off, len); + hash.update(b, off, len); + bytesWritten += len; } @Override public void close() throws IOException { - hasher.update(path.getFileName().toString().getBytes(StandardCharsets.UTF_8)); - long h = hasher.getValue(); - hashBuilder.update(path, h); + hash.update(path.getFileName().toString().getBytes(StandardCharsets.UTF_8)); + hashBuilder.update(path, hash.getValue(), bytesWritten); super.close(); } } From be20e2b17af66b05e16ed04092aad92812c8022a Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Fri, 12 May 2023 18:47:54 +0200 Subject: [PATCH 41/44] some cleanup --- .../szum123321/textile_backup/Globals.java | 29 ++++++++++++++++++- .../core/create/BrokenFileHandler.java | 4 +-- .../core/create/ExecutableBackup.java | 5 ++-- .../textile_backup/core/digest/Hash.java | 2 +- 4 files changed, 32 insertions(+), 8 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/Globals.java b/src/main/java/net/szum123321/textile_backup/Globals.java index 26beea8..2567528 100644 --- a/src/main/java/net/szum123321/textile_backup/Globals.java +++ b/src/main/java/net/szum123321/textile_backup/Globals.java @@ -26,6 +26,8 @@ import net.szum123321.textile_backup.core.Utilities; import net.szum123321.textile_backup.core.restore.AwaitThread; import org.apache.commons.io.FileUtils; +import java.nio.ByteBuffer; +import java.nio.ByteOrder; import java.nio.file.Files; import java.nio.file.Path; import java.time.format.DateTimeFormatter; @@ -36,12 +38,37 @@ import java.util.concurrent.Executors; import java.util.concurrent.TimeUnit; import java.util.concurrent.atomic.AtomicBoolean; import java.util.function.Supplier; +import java.util.zip.CRC32; public class Globals { public static final Globals INSTANCE = new Globals(); private static final TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME); public static final DateTimeFormatter defaultDateTimeFormatter = DateTimeFormatter.ofPattern("yyyy-MM-dd_HH.mm.ss"); - public static final Supplier CHECKSUM_SUPPLIER = BalticHash::new; + public static final Supplier CHECKSUM_SUPPLIER = BalticHash::new;/*() -> new Hash() { + private final CRC32 crc = new CRC32(); + + @Override + public void update ( int b){ + crc.update(b); + } + + @Override + public void update ( long b) { + ByteBuffer v = ByteBuffer.allocate(8).order(ByteOrder.LITTLE_ENDIAN); + v.putLong(b); + crc.update(v.array()); + } + + @Override + public void update ( byte[] b, int off, int len){ + crc.update(b, off, len); + } + + @Override + public long getValue () { + return crc.getValue(); + } + };*/ private ExecutorService executorService = null;//TODO: AAAAAAAAAAAAAAA MEMORY LEAK!!!!!!!!! public final AtomicBoolean globalShutdownBackupFlag = new AtomicBoolean(true); diff --git a/src/main/java/net/szum123321/textile_backup/core/create/BrokenFileHandler.java b/src/main/java/net/szum123321/textile_backup/core/create/BrokenFileHandler.java index 8e9b314..8e06585 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/BrokenFileHandler.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/BrokenFileHandler.java @@ -18,14 +18,12 @@ package net.szum123321.textile_backup.core.create; -import org.spongepowered.include.com.google.common.collect.Maps; - import java.nio.file.Path; import java.util.HashMap; import java.util.Map; public class BrokenFileHandler { - private final HashMap store = Maps.newHashMap(); + private final Map store = new HashMap<>(); public void handle(Path file, Exception e) { store.put(file, e); } public boolean valid() { return store.isEmpty(); } diff --git a/src/main/java/net/szum123321/textile_backup/core/create/ExecutableBackup.java b/src/main/java/net/szum123321/textile_backup/core/create/ExecutableBackup.java index 9f5071f..f51e033 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/ExecutableBackup.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/ExecutableBackup.java @@ -21,7 +21,6 @@ import java.nio.file.Path; import java.time.LocalDateTime; import java.util.NoSuchElementException; import java.util.concurrent.Callable; -import java.util.concurrent.ExecutionException; public record ExecutableBackup(@NotNull MinecraftServer server, ServerCommandSource commandSource, @@ -64,7 +63,7 @@ public record ExecutableBackup(@NotNull MinecraftServer server, log.info(builder.toString()); } @Override - public Void call() throws IOException, ExecutionException, InterruptedException { + public Void call() throws Exception { if (save) { //save the world log.sendInfoAL(this, "Saving server..."); server.saveAll(true, true, false); @@ -119,7 +118,7 @@ public record ExecutableBackup(@NotNull MinecraftServer server, } catch (Throwable e) { //ExecutorService swallows exception, so I need to catch everything - log.error("An exception occurred when trying to create new backup file!", e); + log.error("An exception occurred when trying to create a new backup file!", e); if (ConfigHelper.INSTANCE.get().integrityVerificationMode.isStrict()) { try { diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java b/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java index 8d67b83..c601cb8 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/Hash.java @@ -20,7 +20,7 @@ package net.szum123321.textile_backup.core.digest; public interface Hash { - void update(int b) ; + void update(int b); void update(long b); From 1e70d605f18fe2352373b02ee42c39f05b5da47e Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Fri, 12 May 2023 18:48:58 +0200 Subject: [PATCH 42/44] removed unused imports --- .../textile_backup/core/digest/FileTreeHashBuilder.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java index f3cad18..64c03b4 100644 --- a/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java +++ b/src/main/java/net/szum123321/textile_backup/core/digest/FileTreeHashBuilder.java @@ -24,10 +24,8 @@ import net.szum123321.textile_backup.TextileLogger; import net.szum123321.textile_backup.core.CompressionStatus; import java.io.IOException; -import java.nio.file.Files; import java.nio.file.Path; import java.util.concurrent.CountDownLatch; -import java.util.concurrent.atomic.AtomicBoolean; /** * What this class does is it collects the hashed files and combines them into a single number, From bb34166ba356c6ae01b98675668237df361ae53f Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Fri, 12 May 2023 18:52:52 +0200 Subject: [PATCH 43/44] unretireing bzip lzma --- .../java/net/szum123321/textile_backup/config/ConfigPOJO.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java index ae6106a..e695347 100644 --- a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java +++ b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java @@ -199,8 +199,8 @@ public class ConfigPOJO implements ConfigData { public enum ArchiveFormat { ZIP("zip"), GZIP("tar", "gz"), - //BZIP2("tar", "bz2"), - //LZMA("tar", "xz"), + BZIP2("tar", "bz2"), + LZMA("tar", "xz"), TAR("tar"); private final List extensionPieces; From 4b4d42cda62f4c095d1635d286e459b070de2ac7 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Sun, 4 Jun 2023 11:54:36 +0200 Subject: [PATCH 44/44] final prod version fo 1.20 --- build.gradle | 4 ++-- gradle.properties | 12 ++++++------ .../net/szum123321/textile_backup/TextileLogger.java | 2 +- .../szum123321/textile_backup/config/ConfigPOJO.java | 2 -- src/main/resources/fabric.mod.json | 2 +- 5 files changed, 10 insertions(+), 12 deletions(-) diff --git a/build.gradle b/build.gradle index 4c3b6b8..5dcdb51 100644 --- a/build.gradle +++ b/build.gradle @@ -7,7 +7,7 @@ sourceCompatibility = JavaVersion.VERSION_17 targetCompatibility = JavaVersion.VERSION_17 archivesBaseName = project.archives_base_name -version = "${project.mod_version}-${project.minecraft_version}" +version = "${project.mod_version}-${getMcMinor(project.minecraft_version)}" group = project.maven_group repositories { @@ -105,7 +105,7 @@ publishing { } static def getMcMinor(ver) { - String[] arr = ((String)ver).split("\\.") + String[] arr = ((String)ver).split("[.-]") if(arr.length < 2) return ver diff --git a/gradle.properties b/gradle.properties index 5ddf9f0..0bbd976 100644 --- a/gradle.properties +++ b/gradle.properties @@ -1,15 +1,15 @@ # Done to increase the memory available to gradle. org.gradle.jvmargs=-Xmx1G -minecraft_version=1.20-pre1 -yarn_mappings=1.20-pre1+build.2 -loader_version=0.14.19 +minecraft_version=1.20-rc1 +yarn_mappings=1.20-rc1+build.2 +loader_version=0.14.21 #Fabric api -fabric_version=0.80.2+1.20 +fabric_version=0.83.0+1.20 #Cloth Config -cloth_version=11.0.97 +cloth_version=11.0.98 #ModMenu modmenu_version=7.0.0-beta.2 @@ -20,6 +20,6 @@ databreaker_version=0.2.10 pgzip_commit_hash=af5f5c297e735f3f2df7aa4eb0e19a5810b8aff6 # Mod Properties -mod_version = 3.0.0-b +mod_version = 3.0.0 maven_group = net.szum123321 archives_base_name = textile_backup \ No newline at end of file diff --git a/src/main/java/net/szum123321/textile_backup/TextileLogger.java b/src/main/java/net/szum123321/textile_backup/TextileLogger.java index 0a38389..08b5dc7 100644 --- a/src/main/java/net/szum123321/textile_backup/TextileLogger.java +++ b/src/main/java/net/szum123321/textile_backup/TextileLogger.java @@ -94,7 +94,7 @@ public class TextileLogger { else if(level.intLevel() <= Level.WARN.intLevel()) text.formatted(Formatting.RED); else text.formatted(Formatting.WHITE); - source.sendFeedback(prefixText.copy().append(text), false); + source.sendFeedback(() -> prefixText.copy().append(text), false); return true; } else { diff --git a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java index e695347..7ab41ca 100644 --- a/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java +++ b/src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java @@ -115,8 +115,6 @@ public class ConfigPOJO implements ConfigData { \nAvailable formats are: ZIP - normal zip archive using standard deflate compression GZIP - tar.gz using gzip compression - BZIP2 - tar.bz2 archive using bzip2 compression - LZMA - tar.xz using lzma compression TAR - .tar with no compression """) @ConfigEntry.Gui.Tooltip() diff --git a/src/main/resources/fabric.mod.json b/src/main/resources/fabric.mod.json index cbf3e57..3b349e6 100644 --- a/src/main/resources/fabric.mod.json +++ b/src/main/resources/fabric.mod.json @@ -40,7 +40,7 @@ "depends": { "fabricloader": ">=0.14.0", "fabric": "*", - "minecraft": "1.20-beta.1", + "minecraft": "^1.20-", "cloth-config2": "*", "java": ">=16" },