From e8d9c15625bc6331ffabdedb9fbde66d06ce9002 Mon Sep 17 00:00:00 2001 From: Szum123321 Date: Sat, 18 Apr 2020 18:23:24 +0200 Subject: [PATCH] zip ang zip are now multi threaded --- build.gradle | 10 ++- gradle.properties | 6 +- src/Copyright_Notice.txt | 8 ++ .../textile_backup/ConfigHandler.java | 8 +- .../textile_backup/core/BackupHelper.java | 8 +- .../textile_backup/core/MakeBackupThread.java | 15 ++-- .../compressors/GenericTarCompressor.java | 15 ++-- .../compressors/ParallelZipCompressor.java | 83 +++++++++++++++++++ .../core/compressors/ZipCompressor.java | 76 ----------------- 9 files changed, 127 insertions(+), 102 deletions(-) create mode 100644 src/Copyright_Notice.txt create mode 100644 src/main/java/net/szum123321/textile_backup/core/compressors/ParallelZipCompressor.java delete mode 100644 src/main/java/net/szum123321/textile_backup/core/compressors/ZipCompressor.java diff --git a/build.gradle b/build.gradle index 10eb4c7..67c0f6d 100644 --- a/build.gradle +++ b/build.gradle @@ -15,6 +15,7 @@ minecraft { repositories{ maven { url 'http://server.bbkr.space:8081/artifactory/libs-release' } + maven { url 'https://jitpack.io' } } @@ -33,8 +34,13 @@ dependencies { include "io.github.cottonmc.cotton:cotton-logging:1.0.0-rc.4" include "io.github.cottonmc.cotton:cotton-config:1.0.0-rc.7" - modCompile "org.apache.commons:commons-compress:1.20" - include "org.apache.commons:commons-compress:1.20" + modCompile "org.apache.commons:commons-compress:1.13" + include "org.apache.commons:commons-compress:1.13" + + modCompile "org.tukaani:xz:1.8" + include "org.tukaani:xz:1.8" + + modCompile 'com.github.shevek:parallelgzip:master-SNAPSHOT' } processResources { diff --git a/gradle.properties b/gradle.properties index eb2fd34..3d44dd5 100644 --- a/gradle.properties +++ b/gradle.properties @@ -9,6 +9,6 @@ loader_version=0.8.2+build.194 fabric_version=0.5.1+build.294-1.15 # Mod Properties - mod_version = 1.2.0B-1.15 - maven_group = net.szum123321 - archives_base_name = textile_backup \ No newline at end of file +mod_version = 1.2.0S-1.15 +maven_group = net.szum123321 +archives_base_name = textile_backup \ No newline at end of file diff --git a/src/Copyright_Notice.txt b/src/Copyright_Notice.txt new file mode 100644 index 0000000..b755408 --- /dev/null +++ b/src/Copyright_Notice.txt @@ -0,0 +1,8 @@ +This project uses third party libraries as its dependencies and includes them in jar. Those are : + Apache Commons Compress version 1.20 licensed under Apache License Version 2.0 which can be found at http://www.apache.org/licenses/ + Cotton config, Cotton logging, and Jankson-Fabric all by Cotton team licensed under MIT license which can be found at https://github.com/CottonMC/Cotton + XZ for Java by Tukaani released as public domain. https://tukaani.org/xz/java.html + +Some code was partially or fully inspired by: + Parallel zip compression: https://stackoverflow.com/questions/54624695/how-to-implement-parallel-zip-creation-with-scatterzipoutputstream-with-zip64-su + answer by: https://stackoverflow.com/users/2987755/dkb \ No newline at end of file diff --git a/src/main/java/net/szum123321/textile_backup/ConfigHandler.java b/src/main/java/net/szum123321/textile_backup/ConfigHandler.java index dba67b4..91a39e1 100644 --- a/src/main/java/net/szum123321/textile_backup/ConfigHandler.java +++ b/src/main/java/net/szum123321/textile_backup/ConfigHandler.java @@ -59,9 +59,9 @@ public class ConfigHandler { @Comment(value = "\nAvailable formats are:\n" + "ZIP - normal zip archive using standard deflate compression\n" + - "BZIP2 - tar.bz2 archive using bzip2 compression\n" + "GIZP - tar.gz using gzip compression\n" + - "LZ4 - tar.lz using lz4 compression\n") + "BZIP2 - tar.bz2 archive using bzip2 compression\n" + + "LZMA - tar.xz using lzma compression\n") public final ArchiveFormat format = ArchiveFormat.ZIP; @Comment("\nPrint info to game out\n") @@ -84,9 +84,9 @@ public class ConfigHandler { public enum ArchiveFormat { ZIP(".zip"), - BZIP2(".tar.bz2"), GZIP(".tar.gz"), - LZ4(".tar.lz4"); + BZIP2(".tar.bz2"), + LZMA(".tar.xz"); private final String extension; diff --git a/src/main/java/net/szum123321/textile_backup/core/BackupHelper.java b/src/main/java/net/szum123321/textile_backup/core/BackupHelper.java index 259604d..626354b 100644 --- a/src/main/java/net/szum123321/textile_backup/core/BackupHelper.java +++ b/src/main/java/net/szum123321/textile_backup/core/BackupHelper.java @@ -128,9 +128,7 @@ public class BackupHelper { } private static String getFileExtension(File f) { - System.out.println(f.getName()); String[] parts = f.getName().split("\\."); - System.out.println(parts.length); switch (parts[parts.length - 1]) { case "zip": @@ -139,8 +137,8 @@ public class BackupHelper { return ConfigHandler.ArchiveFormat.BZIP2.getExtension(); case "gz": return ConfigHandler.ArchiveFormat.GZIP.getExtension(); - case "lz4": - return ConfigHandler.ArchiveFormat.LZ4.getExtension(); + case "xz": + return ConfigHandler.ArchiveFormat.LZMA.getExtension(); default: return null; @@ -149,7 +147,7 @@ public class BackupHelper { public static File getBackupRootPath(String worldName) { - File path = new File(TextileBackup.config.path); + File path = new File(TextileBackup.config.path).getAbsoluteFile(); if (TextileBackup.config.perWorldBackup) path = path.toPath().resolve(worldName).toFile(); diff --git a/src/main/java/net/szum123321/textile_backup/core/MakeBackupThread.java b/src/main/java/net/szum123321/textile_backup/core/MakeBackupThread.java index 09780d3..4cb4677 100644 --- a/src/main/java/net/szum123321/textile_backup/core/MakeBackupThread.java +++ b/src/main/java/net/szum123321/textile_backup/core/MakeBackupThread.java @@ -23,10 +23,11 @@ import net.minecraft.server.command.ServerCommandSource; import net.minecraft.world.dimension.DimensionType; import net.szum123321.textile_backup.TextileBackup; import net.szum123321.textile_backup.core.compressors.GenericTarCompressor; -import net.szum123321.textile_backup.core.compressors.ZipCompressor; +import net.szum123321.textile_backup.core.compressors.ParallelZipCompressor; +import org.anarres.parallelgzip.ParallelGZIPOutputStream; import org.apache.commons.compress.compressors.bzip2.BZip2CompressorOutputStream; import org.apache.commons.compress.compressors.gzip.GzipCompressorOutputStream; -import org.apache.commons.compress.compressors.lz4.FramedLZ4CompressorOutputStream; +import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; import java.io.File; import java.io.IOException; @@ -67,7 +68,7 @@ public class MakeBackupThread implements Runnable { switch (TextileBackup.config.format) { case ZIP: - ZipCompressor.createArchive(world, outFile, ctx); + ParallelZipCompressor.createArchive(world, outFile, ctx); break; case BZIP2: @@ -75,16 +76,16 @@ public class MakeBackupThread implements Runnable { break; case GZIP: - GenericTarCompressor.createArchive(world, outFile, GzipCompressorOutputStream.class, ctx); + GenericTarCompressor.createArchive(world, outFile, ParallelGZIPOutputStream.class, ctx); break; - case LZ4: - GenericTarCompressor.createArchive(world, outFile, FramedLZ4CompressorOutputStream.class, ctx); + case LZMA: + GenericTarCompressor.createArchive(world, outFile, XZCompressorOutputStream.class, ctx); break; default: Utilities.log("Error! No correct compression format specified! using default compressor!", ctx); - ZipCompressor.createArchive(world, outFile, ctx); + ParallelZipCompressor.createArchive(world, outFile, ctx); break; } diff --git a/src/main/java/net/szum123321/textile_backup/core/compressors/GenericTarCompressor.java b/src/main/java/net/szum123321/textile_backup/core/compressors/GenericTarCompressor.java index d59881b..5c07d06 100644 --- a/src/main/java/net/szum123321/textile_backup/core/compressors/GenericTarCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/compressors/GenericTarCompressor.java @@ -8,24 +8,26 @@ import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.compressors.CompressorOutputStream; import org.apache.commons.compress.utils.IOUtils; + import java.io.*; import java.lang.reflect.InvocationTargetException; import java.nio.file.Files; public class GenericTarCompressor { - public static void createArchive(File in, File out, Class CompressorStreamClass, ServerCommandSource ctx) { + public static void createArchive(File in, File out, Class CompressorStreamClass, ServerCommandSource ctx) { Utilities.log("Starting compression...", ctx); + long start = System.nanoTime(); + try (FileOutputStream outStream = new FileOutputStream(out); BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream); - CompressorOutputStream compressorStream = CompressorStreamClass.getDeclaredConstructor(OutputStream.class).newInstance(bufferedOutputStream);// CompressorStreamClass.getConstructor().newInstance(bufferedOutputStream); + OutputStream compressorStream = CompressorStreamClass.getDeclaredConstructor(OutputStream.class).newInstance(bufferedOutputStream);// CompressorStreamClass.getConstructor().newInstance(bufferedOutputStream); TarArchiveOutputStream arc = new TarArchiveOutputStream(compressorStream)) { arc.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); File input = in.getCanonicalFile(); - int rootPathLength = input.toString().length() + 1; Files.walk(input.toPath()).filter( path -> !path.equals(input.toPath()) && @@ -36,7 +38,7 @@ public class GenericTarCompressor { try (FileInputStream fin = new FileInputStream(file); BufferedInputStream bfin = new BufferedInputStream(fin)){ - ArchiveEntry entry = arc.createArchiveEntry(file, file.getAbsolutePath().substring(rootPathLength)); + ArchiveEntry entry = arc.createArchiveEntry(file, input.toPath().relativize(path).toString()); arc.putArchiveEntry(entry); IOUtils.copy(bfin, arc); @@ -46,11 +48,14 @@ public class GenericTarCompressor { TextileBackup.logger.error(e.getMessage()); } }); + arc.finish(); } catch (IOException | IllegalAccessException | NoSuchMethodException | InstantiationException | InvocationTargetException e) { TextileBackup.logger.error(e.toString()); } - Utilities.log("Compression finished", ctx); + long end = System.nanoTime(); + + Utilities.log("Compression took: " + ((end - start) / 1000000000.0) + "s", ctx); } } \ No newline at end of file diff --git a/src/main/java/net/szum123321/textile_backup/core/compressors/ParallelZipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/compressors/ParallelZipCompressor.java new file mode 100644 index 0000000..867c175 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/compressors/ParallelZipCompressor.java @@ -0,0 +1,83 @@ +package net.szum123321.textile_backup.core.compressors; + +import net.minecraft.server.command.ServerCommandSource; +import net.szum123321.textile_backup.TextileBackup; +import net.szum123321.textile_backup.core.Utilities; +import org.apache.commons.compress.archivers.zip.*; +import org.apache.commons.compress.parallel.InputStreamSupplier; + +import java.io.*; +import java.nio.file.Files; +import java.nio.file.Path; +import java.time.LocalDateTime; +import java.util.concurrent.*; +import java.util.zip.ZipEntry; + +/* + This part of code is based on: + https://stackoverflow.com/questions/54624695/how-to-implement-parallel-zip-creation-with-scatterzipoutputstream-with-zip64-su + answer by: + https://stackoverflow.com/users/2987755/dkb +*/ + +public class ParallelZipCompressor { + public static void createArchive(File in, File out, ServerCommandSource ctx) { + Utilities.log("Starting compression...", ctx); + + long start = System.nanoTime();; + + try (FileOutputStream fileOutputStream = new FileOutputStream(out); + BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream); + ZipArchiveOutputStream arc = new ZipArchiveOutputStream(bufferedOutputStream)) { + + ParallelScatterZipCreator scatterZipCreator = new ParallelScatterZipCreator(); + + arc.setMethod(ZipArchiveOutputStream.DEFLATED); + arc.setUseZip64(Zip64Mode.AsNeeded); + arc.setLevel(TextileBackup.config.compression); + arc.setComment("Created on: " + Utilities.getDateTimeFormatter().format(LocalDateTime.now())); + + File input = in.getCanonicalFile(); + + Files.walk(input.toPath()).filter( + path -> !path.equals(input.toPath()) && + path.toFile().isFile() && + !Utilities.isBlacklisted(input.toPath().relativize(path)) + ).forEach(p -> { + ZipArchiveEntry entry = new ZipArchiveEntry(input.toPath().relativize(p).toString()); + entry.setMethod(ZipEntry.DEFLATED); + FileInputStreamSupplier supplier = new FileInputStreamSupplier(p); + scatterZipCreator.addArchiveEntry(entry, supplier); + }); + + scatterZipCreator.writeTo(arc); + + arc.finish(); + } catch (IOException | InterruptedException | ExecutionException e) { + TextileBackup.logger.error(e.getMessage()); + } + + long end = System.nanoTime(); + + Utilities.log("Compression took: " + ((end - start) / 1000000000.0) + "s", ctx); + } + + static class FileInputStreamSupplier implements InputStreamSupplier { + private final Path sourceFile; + private InputStream stream; + + FileInputStreamSupplier(Path sourceFile) { + this.sourceFile = sourceFile; + } + + public InputStream get() { + try { + System.out.println("Creating: " + sourceFile); + stream = Files.newInputStream(sourceFile); + } catch (IOException e) { + e.printStackTrace(); + } + return stream; + } + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/compressors/ZipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/compressors/ZipCompressor.java deleted file mode 100644 index a69b6f7..0000000 --- a/src/main/java/net/szum123321/textile_backup/core/compressors/ZipCompressor.java +++ /dev/null @@ -1,76 +0,0 @@ -/* - A simple backup mod for Fabric - Copyright (C) 2020 Szum123321 - - This program is free software: you can redistribute it and/or modify - it under the terms of the GNU General Public License as published by - the Free Software Foundation, either version 3 of the License, or - (at your option) any later version. - - This program is distributed in the hope that it will be useful, - but WITHOUT ANY WARRANTY; without even the implied warranty of - MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - GNU General Public License for more details. - - You should have received a copy of the GNU General Public License - along with this program. If not, see . -*/ - -package net.szum123321.textile_backup.core.compressors; - -import net.minecraft.server.command.ServerCommandSource; -import net.szum123321.textile_backup.TextileBackup; -import net.szum123321.textile_backup.core.Utilities; -import org.apache.commons.compress.archivers.zip.Zip64Mode; -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; -import org.apache.commons.compress.utils.IOUtils; - -import java.io.*; -import java.nio.file.Files; -import java.time.LocalDateTime; - -public class ZipCompressor { - public static void createArchive(File in, File out, ServerCommandSource ctx) { - Utilities.log("Starting compression...", ctx); - - try (FileOutputStream fileOutputStream = new FileOutputStream(out); - BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream); - ZipArchiveOutputStream arc = new ZipArchiveOutputStream(bufferedOutputStream)) { - - arc.setMethod(ZipArchiveOutputStream.DEFLATED); - arc.setUseZip64(Zip64Mode.AsNeeded); - arc.setLevel(TextileBackup.config.compression); - arc.setComment("Created on: " + Utilities.getDateTimeFormatter().format(LocalDateTime.now())); - - File input = in.getCanonicalFile(); - int rootPathLength = input.toString().length() + 1; - - Files.walk(input.toPath()).filter( - path -> !path.equals(input.toPath()) && - path.toFile().isFile() && - !Utilities.isBlacklisted(input.toPath().relativize(path)) - ).forEach(path -> { - File file = path.toAbsolutePath().toFile(); - - try (FileInputStream fin = new FileInputStream(file); - BufferedInputStream bfin = new BufferedInputStream(fin)) { - ZipArchiveEntry entry = new ZipArchiveEntry(file, file.getAbsolutePath().substring(rootPathLength)); - - arc.putArchiveEntry(entry); - IOUtils.copy(bfin, arc); - - arc.closeArchiveEntry(); - } catch (IOException e) { - TextileBackup.logger.error(e.getMessage()); - } - }); - - arc.finish(); - } catch (IOException e) { - TextileBackup.logger.error(e.getMessage()); - } - - Utilities.log("Compression finished", ctx); - } -} \ No newline at end of file