Alex/textile_backup
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Do I really need hashes? idk.... Either way it has to get ugly before it gets beautiful ... I think

Browse Source
This commit is contained in:
Szum123321 2022-11-24 00:38:55 +01:00
parent 85452efbca
commit 4007d8f86d
11 changed files with 217 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
src/main/java/net/szum123321/textile_backup/config/ConfigPOJO.java
View File

@ -123,6 +123,14 @@ public class ConfigPOJO implements ConfigData {
@ConfigEntry.Gui.EnumHandler(option = ConfigEntry.Gui.EnumHandler.EnumDisplayOption.BUTTON)
public ArchiveFormat format = ArchiveFormat.ZIP;
@Comment("""
The Strict mode (default) aborts backup creation in case of any problem and deletes the created files
Permissible mode keeps partial/damaged backup but won't allow to restore it
Very Permissible mode will skip the verification process. THIS MOST CERTAINLY WILL LEAD TO DATA LOSS OR CORRUPTION
""")
@ConfigEntry.Gui.EnumHandler(option = ConfigEntry.Gui.EnumHandler.EnumDisplayOption.BUTTON)
public ErrorHandlingMode errorErrorHandlingMode = ErrorHandlingMode.STRICT;
@Comment("\nMinimal permission level required to run commands\n")
@ConfigEntry.Category("Manage")
@ConfigEntry.Gui.NoTooltip()
@ -177,6 +185,12 @@ public class ConfigPOJO implements ConfigData {
}
}
public enum ErrorHandlingMode {
STRICT,
PERMISSIBLE,
VERY_PERMISSIBLE
}
public enum ArchiveFormat {
ZIP("zip"),
GZIP("tar", "gz"),

27
src/main/java/net/szum123321/textile_backup/core/CompressionStatus.java
View File

@ -1,17 +1,30 @@
/*
* A simple backup mod for Fabric
* Copyright (C) 2022 Szum123321
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package net.szum123321.textile_backup.core;
import java.io.Serializable;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.Map;
public record CompressionStatus(long[] treeHash, LocalDateTime date, long startTimestamp, long finishTimestamp, boolean ok, Path[] brokenFiles) implements Serializable {
public record CompressionStatus(long treeHash, LocalDateTime date, long startTimestamp, long finishTimestamp, Map<Path, Exception> brokenFiles) implements Serializable {
public static final String DATA_FILENAME = "textile_status.data";
public boolean isValid(long decompressedHash) { return true; }
public static class Builder {
public synchronized void update(Path path, long hash, Exception error) { throw new RuntimeException("UNIMPLEMENTED!"); }
public synchronized void update(Path path, Exception error) { throw new RuntimeException("UNIMPLEMENTED!"); }
public synchronized void update(Path path, long hash) { throw new RuntimeException("UNIMPLEMENTED!"); }
public CompressionStatus build() { throw new RuntimeException("UNIMPLEMENTED!"); }
}
}

25
src/main/java/net/szum123321/textile_backup/core/DataLeftException.java Normal file
View File

@ -0,0 +1,25 @@
/*
* A simple backup mod for Fabric
* Copyright (C) 2022 Szum123321
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package net.szum123321.textile_backup.core;
import java.io.IOException;
public class DataLeftException extends IOException {
public DataLeftException(long n) { super("Input stream closed with " + n + " bytes left!"); }
}

36
src/main/java/net/szum123321/textile_backup/core/create/BrokenFileHandler.java Normal file
View File

@ -0,0 +1,36 @@
/*
* A simple backup mod for Fabric
* Copyright (C) 2022 Szum123321
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package net.szum123321.textile_backup.core.create;
import org.spongepowered.include.com.google.common.collect.Maps;
import java.nio.file.Path;
import java.util.HashMap;
import java.util.Map;
public class BrokenFileHandler {
private final HashMap<Path, Exception> store = Maps.newHashMap();
public void handle(Path file, Exception e) { store.put(file, e); }
public boolean valid() { return store.isEmpty(); }
public Map<Path, Exception> get() {
return store;
}
}

41
src/main/java/net/szum123321/textile_backup/core/create/FileInputStreamSupplier.java
View File

@ -1,26 +1,26 @@
/*
A simple backup mod for Fabric
Copyright (C) 2022 Szum123321
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
* A simple backup mod for Fabric
* Copyright (C) 2022 Szum123321
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package net.szum123321.textile_backup.core.create;
import net.szum123321.textile_backup.TextileBackup;
import net.szum123321.textile_backup.TextileLogger;
import net.szum123321.textile_backup.core.CompressionStatus;
import net.szum123321.textile_backup.core.FileTreeHashBuilder;
import java.io.IOException;
import java.io.InputStream;
@ -28,15 +28,16 @@ import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
public record FileInputStreamSupplier(Path path, String name, CompressionStatus.Builder builder) implements InputSupplier {
public record FileInputStreamSupplier(Path path, String name, FileTreeHashBuilder hashTreeBuilder, BrokenFileHandler brokenFileHandler) implements InputSupplier {
private final static TextileLogger log = new TextileLogger(TextileBackup.MOD_NAME);
@Override
public InputStream getInputStream() throws IOException {
try {
return new HashingInputStream(Files.newInputStream(path), path, null, builder);
//TODO: select hashing algorithm!
return new HashingInputStream(Files.newInputStream(path), path, null, hashTreeBuilder, brokenFileHandler);
} catch (IOException e) {
builder.update(path, e);
brokenFileHandler.handle(path, e);
throw e;
}
}

51
src/main/java/net/szum123321/textile_backup/core/create/HashingInputStream.java
View File

@ -1,41 +1,46 @@
/*
A simple backup mod for Fabric
Copyright (C) 2022 Szum123321
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
* A simple backup mod for Fabric
* Copyright (C) 2022 Szum123321
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package net.szum123321.textile_backup.core.create;
import net.szum123321.textile_backup.core.CompressionStatus;
import net.szum123321.textile_backup.core.DataLeftException;
import net.szum123321.textile_backup.core.FileTreeHashBuilder;
import org.jetbrains.annotations.NotNull;
import java.io.*;
import java.nio.file.Path;
import java.util.zip.Checksum;
//This class calculates a hash of the file on the input stream, submits it to FileTreeHashBuilder.
//In case the whole underlying stream hasn't been read, also puts it into BrokeFileHandler
public class HashingInputStream extends FilterInputStream {
private final Path path;
private final Checksum hasher;
private final CompressionStatus.Builder statusBuilder;
private final FileTreeHashBuilder hashBuilder;
private final BrokenFileHandler brokenFileHandler;
public HashingInputStream(InputStream in, Path path, Checksum hasher, CompressionStatus.Builder statusBuilder) {
public HashingInputStream(InputStream in, Path path, Checksum hasher, FileTreeHashBuilder hashBuilder, BrokenFileHandler brokenFileHandler) {
super(in);
this.hasher = hasher;
this.statusBuilder = statusBuilder;
this.path = path;
this.hasher = hasher;
this.hashBuilder = hashBuilder;
this.brokenFileHandler = brokenFileHandler;
}
@Override
@ -54,8 +59,8 @@ public class HashingInputStream extends FilterInputStream {
@Override
public void close() throws IOException {
if(in.available() == 0) statusBuilder.update(path, hasher.getValue());
else statusBuilder.update(path, hasher.getValue(), new RuntimeException("AAAaa"));
if(in.available() == 0) hashBuilder.update(path, hasher.getValue());
else brokenFileHandler.handle(path, new DataLeftException(in.available()));
super.close();
}
}

34
src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java
View File

@ -1,20 +1,20 @@
/*
A simple backup mod for Fabric
Copyright (C) 2020 Szum123321
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
* A simple backup mod for Fabric
* Copyright (C) 2022 Szum123321
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package net.szum123321.textile_backup.core.create;
@ -104,7 +104,7 @@ public class MakeBackupRunnable implements Runnable {
case TAR -> new AbstractTarArchiver().createArchive(world, outFile, context, coreCount);
}
new Cleanup(context.commandSource(), Utilities.getLevelName(context.server())).call();
Globals.INSTANCE.getQueueExecutor().submit(new Cleanup(context.commandSource(), Utilities.getLevelName(context.server())));
if(config.get().broadcastBackupDone) {
Utilities.notifyPlayers(

82
src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractCompressor.java
View File

@ -1,6 +1,6 @@
/*
* A simple backup mod for Fabric
* Copyright (C) 2020 Szum123321
* Copyright (C) 2022 Szum123321
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -20,11 +20,11 @@ package net.szum123321.textile_backup.core.create.compressors;
import net.szum123321.textile_backup.TextileBackup;
import net.szum123321.textile_backup.TextileLogger;
import net.szum123321.textile_backup.core.ActionInitiator;
import net.szum123321.textile_backup.core.CompressionStatus;
import net.szum123321.textile_backup.core.NoSpaceLeftOnDeviceException;
import net.szum123321.textile_backup.core.Utilities;
import net.szum123321.textile_backup.config.ConfigHelper;
import net.szum123321.textile_backup.config.ConfigPOJO;
import net.szum123321.textile_backup.core.*;
import net.szum123321.textile_backup.core.create.BackupContext;
import net.szum123321.textile_backup.core.create.BrokenFileHandler;
import net.szum123321.textile_backup.core.create.FileInputStreamSupplier;
import net.szum123321.textile_backup.core.create.InputSupplier;
@ -45,32 +45,53 @@ public abstract class AbstractCompressor {
public void createArchive(Path inputFile, Path outputFile, BackupContext ctx, int coreLimit) {
Instant start = Instant.now();
FileTreeHashBuilder fileHashBuilder = new FileTreeHashBuilder(() -> null); //TODO: select hashing algorithm
BrokenFileHandler brokenFileHandler = new BrokenFileHandler();
boolean keep = true;
try (OutputStream outStream = Files.newOutputStream(outputFile);
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream);
OutputStream arc = createArchiveOutputStream(bufferedOutputStream, ctx, coreLimit);
Stream<Path> fileStream = Files.walk(inputFile)) {
CompressionStatus.Builder statusBuilder = new CompressionStatus.Builder();
fileStream
var it = fileStream
.filter(path -> !Utilities.isBlacklisted(inputFile.relativize(path)))
.filter(Files::isRegularFile).forEach(file -> {
try {
addEntry(new FileInputStreamSupplier(file, inputFile.relativize(file).toString(), statusBuilder), arc);
} catch (IOException e) {
log.error("An exception occurred while trying to compress: {}", inputFile.relativize(file).toString(), e);
.filter(Files::isRegularFile).iterator();
if (ctx.initiator() == ActionInitiator.Player)
log.sendError(ctx, "Something went wrong while compressing files!");
}
});
while(it.hasNext()) {
Path file = it.next();
try {
addEntry(new FileInputStreamSupplier(file, inputFile.relativize(file).toString(), fileHashBuilder, brokenFileHandler), arc);
} catch (Exception e) {
if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode == ConfigPOJO.ErrorHandlingMode.STRICT) {
keep = false;
break;
}
brokenFileHandler.handle(file, e);
}
}
//If there are still files left in the stream, we should handle them
while(it.hasNext()) {
Path file = it.next();
brokenFileHandler.handle(file, new DataLeftException(Files.size(file)));
}
Instant now = Instant.now();
CompressionStatus status = new CompressionStatus (
fileHashBuilder.getValue(),
null, start.toEpochMilli(), now.toEpochMilli(),
brokenFileHandler.get()
);
//Serialize using gson?
ByteArrayOutputStream bo = new ByteArrayOutputStream();
ObjectOutputStream o = new ObjectOutputStream(bo);
o.writeObject(statusBuilder.build());
addEntry(new StatusFileInputSupplier(bo.toByteArray(), bo.size()), arc);
try (ByteArrayOutputStream bo = new ByteArrayOutputStream();
ObjectOutputStream o = new ObjectOutputStream(bo)) {
o.writeObject(status);
addEntry(new StatusFileInputSupplier(bo.toByteArray()), arc);
}
finish(arc);
} catch(NoSpaceLeftOnDeviceException e) {
@ -85,16 +106,17 @@ public abstract class AbstractCompressor {
log.sendError(ctx, "Backup failed. The file is corrupt.");
log.error("For help see: https://github.com/Szum123321/textile_backup/wiki/ZIP-Problems");
}
if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode == ConfigPOJO.ErrorHandlingMode.STRICT) keep = false;
} catch (IOException | InterruptedException | ExecutionException e) {
log.error("An exception occurred!", e);
if(ctx.initiator() == ActionInitiator.Player)
log.sendError(ctx, "Something went wrong while compressing files!");
if(ConfigHelper.INSTANCE.get().errorErrorHandlingMode == ConfigPOJO.ErrorHandlingMode.STRICT) keep = false;
} finally {
close();
}
// close();
log.sendInfoAL(ctx, "Compression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now())));
}
@ -109,17 +131,13 @@ public abstract class AbstractCompressor {
//Same as above, just for ParallelGzipCompressor to shut down ExecutorService
}
private record StatusFileInputSupplier(byte[] data, int len) implements InputSupplier {
@Override
public InputStream getInputStream() { return new ByteArrayInputStream(data, 0, len); }
private record StatusFileInputSupplier(byte[] data) implements InputSupplier {
public InputStream getInputStream() { return new ByteArrayInputStream(data); }
@Override
public Path getPath() { return Path.of(CompressionStatus.DATA_FILENAME); }
@Override
public String getName() { return CompressionStatus.DATA_FILENAME; }
@Override
public InputStream get() { return new ByteArrayInputStream(data, 0, len); }
public InputStream get() { return getInputStream(); }
}
}

4
src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelZipCompressor.java
View File

@ -1,6 +1,6 @@
/*
* A simple backup mod for Fabric
* Copyright (C) 2020 Szum123321
* Copyright (C) 2022 Szum123321
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
@ -99,7 +99,7 @@ public class ParallelZipCompressor extends ZipCompressor {
if(!STACKTRACE_NO_SPACE_ON_LEFT_ON_DEVICE[i].equals(cause.getStackTrace()[i])) match = false;
//For clarity' sake let's not throw the ExecutionException itself rather only the cause, as the EE is just the wrapper
//For clarity's sake let's not throw the ExecutionException itself rather only the cause, as the EE is just the wrapper
if(match) throw new NoSpaceLeftOnDeviceException(cause);
}
}

3
src/main/resources/assets/textile_backup/lang/en_us.json
View File

@ -47,6 +47,9 @@
"text.autoconfig.textile_backup.option.format": "Archive and compression format",
"text.autoconfig.textile_backup.option.format.@Tooltip": "See: https://github.com/Szum123321/textile_backup/wiki/Configuration#format",
"text.autoconfig.textile_backup.option.errorErrorHandlingMode": "Compression error handling mode",
"text.autoconfig.textile_backup.option.errorErrorHandlingMode.@Tooltip": "DO NOT ALTER unless fully certain",
"text.autoconfig.textile_backup.option.permissionLevel": "Min permission level",
"text.autoconfig.textile_backup.option.alwaysSingleplayerAllowed": "Always allow on single-player",

2
src/main/resources/fabric.mod.json
View File

@ -38,7 +38,7 @@
],
"depends": {
"fabricloader": ">=0.14.6",
"fabricloader": ">=0.14.0",
"fabric": "*",
"minecraft": ">=1.19.1",
"cloth-config2": "*",