Alex/textile_backup
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

zip ang zip are now multi threaded

Browse Source
This commit is contained in:
Szum123321 2020-04-18 18:23:24 +02:00
parent c74fc6c999
commit e8d9c15625
9 changed files with 127 additions and 102 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

10
build.gradle
View File

@ -15,6 +15,7 @@ minecraft {
repositories{
maven { url 'http://server.bbkr.space:8081/artifactory/libs-release' }
maven { url 'https://jitpack.io' }
}
@ -33,8 +34,13 @@ dependencies {
include "io.github.cottonmc.cotton:cotton-logging:1.0.0-rc.4"
include "io.github.cottonmc.cotton:cotton-config:1.0.0-rc.7"
modCompile "org.apache.commons:commons-compress:1.20"
include "org.apache.commons:commons-compress:1.20"
modCompile "org.apache.commons:commons-compress:1.13"
include "org.apache.commons:commons-compress:1.13"
modCompile "org.tukaani:xz:1.8"
include "org.tukaani:xz:1.8"
modCompile 'com.github.shevek:parallelgzip:master-SNAPSHOT'
}
processResources {

6
gradle.properties
View File

@ -9,6 +9,6 @@ loader_version=0.8.2+build.194
fabric_version=0.5.1+build.294-1.15
# Mod Properties
mod_version = 1.2.0B-1.15
maven_group = net.szum123321
archives_base_name = textile_backup
mod_version = 1.2.0S-1.15
maven_group = net.szum123321
archives_base_name = textile_backup

8
src/Copyright_Notice.txt Normal file
View File

@ -0,0 +1,8 @@
This project uses third party libraries as its dependencies and includes them in jar. Those are :
Apache Commons Compress version 1.20 licensed under Apache License Version 2.0 which can be found at http://www.apache.org/licenses/
Cotton config, Cotton logging, and Jankson-Fabric all by Cotton team licensed under MIT license which can be found at https://github.com/CottonMC/Cotton
XZ for Java by Tukaani released as public domain. https://tukaani.org/xz/java.html
Some code was partially or fully inspired by:
Parallel zip compression: https://stackoverflow.com/questions/54624695/how-to-implement-parallel-zip-creation-with-scatterzipoutputstream-with-zip64-su
answer by: https://stackoverflow.com/users/2987755/dkb

8
src/main/java/net/szum123321/textile_backup/ConfigHandler.java
View File

@ -59,9 +59,9 @@ public class ConfigHandler {
@Comment(value = "\nAvailable formats are:\n" +
"ZIP - normal zip archive using standard deflate compression\n" +
"BZIP2 - tar.bz2 archive using bzip2 compression\n" +
"GIZP - tar.gz using gzip compression\n" +
"LZ4 - tar.lz using lz4 compression\n")
"BZIP2 - tar.bz2 archive using bzip2 compression\n" +
"LZMA - tar.xz using lzma compression\n")
public final ArchiveFormat format = ArchiveFormat.ZIP;
@Comment("\nPrint info to game out\n")
@ -84,9 +84,9 @@ public class ConfigHandler {
public enum ArchiveFormat {
ZIP(".zip"),
BZIP2(".tar.bz2"),
GZIP(".tar.gz"),
LZ4(".tar.lz4");
BZIP2(".tar.bz2"),
LZMA(".tar.xz");
private final String extension;

8
src/main/java/net/szum123321/textile_backup/core/BackupHelper.java
View File

@ -128,9 +128,7 @@ public class BackupHelper {
}
private static String getFileExtension(File f) {
System.out.println(f.getName());
String[] parts = f.getName().split("\\.");
System.out.println(parts.length);
switch (parts[parts.length - 1]) {
case "zip":
@ -139,8 +137,8 @@ public class BackupHelper {
return ConfigHandler.ArchiveFormat.BZIP2.getExtension();
case "gz":
return ConfigHandler.ArchiveFormat.GZIP.getExtension();
case "lz4":
return ConfigHandler.ArchiveFormat.LZ4.getExtension();
case "xz":
return ConfigHandler.ArchiveFormat.LZMA.getExtension();
default:
return null;
@ -149,7 +147,7 @@ public class BackupHelper {
public static File getBackupRootPath(String worldName) {
File path = new File(TextileBackup.config.path);
File path = new File(TextileBackup.config.path).getAbsoluteFile();
if (TextileBackup.config.perWorldBackup)
path = path.toPath().resolve(worldName).toFile();

15
src/main/java/net/szum123321/textile_backup/core/MakeBackupThread.java
View File

@ -23,10 +23,11 @@ import net.minecraft.server.command.ServerCommandSource;
import net.minecraft.world.dimension.DimensionType;
import net.szum123321.textile_backup.TextileBackup;
import net.szum123321.textile_backup.core.compressors.GenericTarCompressor;
import net.szum123321.textile_backup.core.compressors.ZipCompressor;
import net.szum123321.textile_backup.core.compressors.ParallelZipCompressor;
import org.anarres.parallelgzip.ParallelGZIPOutputStream;
import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream;
import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream;
import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorOutputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
import java.io.File;
import java.io.IOException;
@ -67,7 +68,7 @@ public class MakeBackupThread implements Runnable {
switch (TextileBackup.config.format) {
case ZIP:
ZipCompressor.createArchive(world, outFile, ctx);
ParallelZipCompressor.createArchive(world, outFile, ctx);
break;
case BZIP2:
@ -75,16 +76,16 @@ public class MakeBackupThread implements Runnable {
break;
case GZIP:
GenericTarCompressor.createArchive(world, outFile, GzipCompressorOutputStream.class, ctx);
GenericTarCompressor.createArchive(world, outFile, ParallelGZIPOutputStream.class, ctx);
break;
case LZ4:
GenericTarCompressor.createArchive(world, outFile, FramedLZ4CompressorOutputStream.class, ctx);
case LZMA:
GenericTarCompressor.createArchive(world, outFile, XZCompressorOutputStream.class, ctx);
break;
default:
Utilities.log("Error! No correct compression format specified! using default compressor!", ctx);
ZipCompressor.createArchive(world, outFile, ctx);
ParallelZipCompressor.createArchive(world, outFile, ctx);
break;
}

15
src/main/java/net/szum123321/textile_backup/core/compressors/GenericTarCompressor.java
View File

@ -8,24 +8,26 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.CompressorOutputStream;
import org.apache.commons.compress.utils.IOUtils;
import java.io.*;
import java.lang.reflect.InvocationTargetException;
import java.nio.file.Files;
public class GenericTarCompressor {
public static void createArchive(File in, File out, Class<? extends CompressorOutputStream> CompressorStreamClass, ServerCommandSource ctx) {
public static void createArchive(File in, File out, Class<? extends OutputStream> CompressorStreamClass, ServerCommandSource ctx) {
Utilities.log("Starting compression...", ctx);
long start = System.nanoTime();
try (FileOutputStream outStream = new FileOutputStream(out);
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream);
CompressorOutputStream compressorStream = CompressorStreamClass.getDeclaredConstructor(OutputStream.class).newInstance(bufferedOutputStream);// CompressorStreamClass.getConstructor().newInstance(bufferedOutputStream);
OutputStream compressorStream = CompressorStreamClass.getDeclaredConstructor(OutputStream.class).newInstance(bufferedOutputStream);// CompressorStreamClass.getConstructor().newInstance(bufferedOutputStream);
TarArchiveOutputStream arc = new TarArchiveOutputStream(compressorStream)) {
arc.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);
File input = in.getCanonicalFile();
int rootPathLength = input.toString().length() + 1;
Files.walk(input.toPath()).filter(
path -> !path.equals(input.toPath()) &&
@ -36,7 +38,7 @@ public class GenericTarCompressor {
try (FileInputStream fin = new FileInputStream(file);
BufferedInputStream bfin = new BufferedInputStream(fin)){
ArchiveEntry entry = arc.createArchiveEntry(file, file.getAbsolutePath().substring(rootPathLength));
ArchiveEntry entry = arc.createArchiveEntry(file, input.toPath().relativize(path).toString());
arc.putArchiveEntry(entry);
IOUtils.copy(bfin, arc);
@ -46,11 +48,14 @@ public class GenericTarCompressor {
TextileBackup.logger.error(e.getMessage());
}
});
arc.finish();
} catch (IOException | IllegalAccessException | NoSuchMethodException | InstantiationException | InvocationTargetException e) {
TextileBackup.logger.error(e.toString());
}
Utilities.log("Compression finished", ctx);
long end = System.nanoTime();
Utilities.log("Compression took: " + ((end - start) / 1000000000.0) + "s", ctx);
}
}

83
src/main/java/net/szum123321/textile_backup/core/compressors/ParallelZipCompressor.java Normal file
View File

@ -0,0 +1,83 @@
package net.szum123321.textile_backup.core.compressors;
import net.minecraft.server.command.ServerCommandSource;
import net.szum123321.textile_backup.TextileBackup;
import net.szum123321.textile_backup.core.Utilities;
import org.apache.commons.compress.archivers.zip.*;
import org.apache.commons.compress.parallel.InputStreamSupplier;
import java.io.*;
import java.nio.file.Files;
import java.nio.file.Path;
import java.time.LocalDateTime;
import java.util.concurrent.*;
import java.util.zip.ZipEntry;
/*
This part of code is based on:
https://stackoverflow.com/questions/54624695/how-to-implement-parallel-zip-creation-with-scatterzipoutputstream-with-zip64-su
answer by:
https://stackoverflow.com/users/2987755/dkb
*/
public class ParallelZipCompressor {
public static void createArchive(File in, File out, ServerCommandSource ctx) {
Utilities.log("Starting compression...", ctx);
long start = System.nanoTime();;
try (FileOutputStream fileOutputStream = new FileOutputStream(out);
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream);
ZipArchiveOutputStream arc = new ZipArchiveOutputStream(bufferedOutputStream)) {
ParallelScatterZipCreator scatterZipCreator = new ParallelScatterZipCreator();
arc.setMethod(ZipArchiveOutputStream.DEFLATED);
arc.setUseZip64(Zip64Mode.AsNeeded);
arc.setLevel(TextileBackup.config.compression);
arc.setComment("Created on: " + Utilities.getDateTimeFormatter().format(LocalDateTime.now()));
File input = in.getCanonicalFile();
Files.walk(input.toPath()).filter(
path -> !path.equals(input.toPath()) &&
path.toFile().isFile() &&
!Utilities.isBlacklisted(input.toPath().relativize(path))
).forEach(p -> {
ZipArchiveEntry entry = new ZipArchiveEntry(input.toPath().relativize(p).toString());
entry.setMethod(ZipEntry.DEFLATED);
FileInputStreamSupplier supplier = new FileInputStreamSupplier(p);
scatterZipCreator.addArchiveEntry(entry, supplier);
});
scatterZipCreator.writeTo(arc);
arc.finish();
} catch (IOException | InterruptedException | ExecutionException e) {
TextileBackup.logger.error(e.getMessage());
}
long end = System.nanoTime();
Utilities.log("Compression took: " + ((end - start) / 1000000000.0) + "s", ctx);
}
static class FileInputStreamSupplier implements InputStreamSupplier {
private final Path sourceFile;
private InputStream stream;
FileInputStreamSupplier(Path sourceFile) {
this.sourceFile = sourceFile;
}
public InputStream get() {
try {
System.out.println("Creating: " + sourceFile);
stream = Files.newInputStream(sourceFile);
} catch (IOException e) {
e.printStackTrace();
}
return stream;
}
}
}

76
src/main/java/net/szum123321/textile_backup/core/compressors/ZipCompressor.java
View File

@ -1,76 +0,0 @@
/*
A simple backup mod for Fabric
Copyright (C) 2020 Szum123321
This program is free software: you can redistribute it and/or modify
it under the terms of the GNU General Public License as published by
the Free Software Foundation, either version 3 of the License, or
(at your option) any later version.
This program is distributed in the hope that it will be useful,
but WITHOUT ANY WARRANTY; without even the implied warranty of
MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
GNU General Public License for more details.
You should have received a copy of the GNU General Public License
along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package net.szum123321.textile_backup.core.compressors;
import net.minecraft.server.command.ServerCommandSource;
import net.szum123321.textile_backup.TextileBackup;
import net.szum123321.textile_backup.core.Utilities;
import org.apache.commons.compress.archivers.zip.Zip64Mode;
import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
import org.apache.commons.compress.utils.IOUtils;
import java.io.*;
import java.nio.file.Files;
import java.time.LocalDateTime;
public class ZipCompressor {
public static void createArchive(File in, File out, ServerCommandSource ctx) {
Utilities.log("Starting compression...", ctx);
try (FileOutputStream fileOutputStream = new FileOutputStream(out);
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream);
ZipArchiveOutputStream arc = new ZipArchiveOutputStream(bufferedOutputStream)) {
arc.setMethod(ZipArchiveOutputStream.DEFLATED);
arc.setUseZip64(Zip64Mode.AsNeeded);
arc.setLevel(TextileBackup.config.compression);
arc.setComment("Created on: " + Utilities.getDateTimeFormatter().format(LocalDateTime.now()));
File input = in.getCanonicalFile();
int rootPathLength = input.toString().length() + 1;
Files.walk(input.toPath()).filter(
path -> !path.equals(input.toPath()) &&
path.toFile().isFile() &&
!Utilities.isBlacklisted(input.toPath().relativize(path))
).forEach(path -> {
File file = path.toAbsolutePath().toFile();
try (FileInputStream fin = new FileInputStream(file);
BufferedInputStream bfin = new BufferedInputStream(fin)) {
ZipArchiveEntry entry = new ZipArchiveEntry(file, file.getAbsolutePath().substring(rootPathLength));
arc.putArchiveEntry(entry);
IOUtils.copy(bfin, arc);
arc.closeArchiveEntry();
} catch (IOException e) {
TextileBackup.logger.error(e.getMessage());
}
});
arc.finish();
} catch (IOException e) {
TextileBackup.logger.error(e.getMessage());
}
Utilities.log("Compression finished", ctx);
}
}