From a93e87729b0009744e81fab5a80bb08884671eb4 Mon Sep 17 00:00:00 2001 From: szymon Date: Sat, 8 Aug 2020 23:36:32 +0200 Subject: [PATCH] Moved most of the compression code to separate class (AbstractTarCompressor) --- .../core/create/MakeBackupRunnable.java | 11 +-- .../compressors/AbstractTarCompressor.java | 77 +++++++++++++++++++ .../create/compressors/LZMACompressor.java | 60 ++------------- .../compressors/ParallelBZip2Compressor.java | 61 ++------------- .../compressors/ParallelGzipCompressor.java | 60 +++------------ 5 files changed, 106 insertions(+), 163 deletions(-) create mode 100644 src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractTarCompressor.java diff --git a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java index 7a0f740..b2a3570 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java @@ -21,10 +21,7 @@ package net.szum123321.textile_backup.core.create; import net.minecraft.server.MinecraftServer; import net.minecraft.server.command.ServerCommandSource; import net.szum123321.textile_backup.Statics; -import net.szum123321.textile_backup.core.create.compressors.LZMACompressor; -import net.szum123321.textile_backup.core.create.compressors.ParallelBZip2Compressor; -import net.szum123321.textile_backup.core.create.compressors.ParallelGzipCompressor; -import net.szum123321.textile_backup.core.create.compressors.ParallelZipCompressor; +import net.szum123321.textile_backup.core.create.compressors.*; import net.szum123321.textile_backup.core.Utilities; import java.io.File; @@ -85,15 +82,15 @@ public class MakeBackupRunnable implements Runnable { break; case BZIP2: - ParallelBZip2Compressor.createArchive(world, outFile, commandSource, coreCount); + ParallelBZip2Compressor.getInstance().createArchive(world, outFile, commandSource, coreCount); break; case GZIP: - ParallelGzipCompressor.createArchive(world, outFile, commandSource, coreCount); + ParallelGzipCompressor.getInstance().createArchive(world, outFile, commandSource, coreCount); break; case LZMA: - LZMACompressor.createArchive(world, outFile, commandSource); // Always single-threaded ): + LZMACompressor.getInstance().createArchive(world, outFile, commandSource, coreCount); break; default: diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractTarCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractTarCompressor.java new file mode 100644 index 0000000..1be2726 --- /dev/null +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractTarCompressor.java @@ -0,0 +1,77 @@ +/* + * A simple backup mod for Fabric + * Copyright (C) 2020 Szum123321 + * + * This program is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * This program is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with this program. If not, see . + */ + +package net.szum123321.textile_backup.core.create.compressors; + +import net.minecraft.server.command.ServerCommandSource; +import net.szum123321.textile_backup.Statics; +import net.szum123321.textile_backup.core.Utilities; +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; +import org.apache.commons.compress.utils.IOUtils; + +import java.io.*; +import java.nio.file.Files; +import java.time.Duration; +import java.time.Instant; + +public abstract class AbstractTarCompressor { + protected abstract OutputStream openCompressorStream(OutputStream outputStream, int coreCountLimit) throws IOException; + + public void createArchive(File inputFile, File out, ServerCommandSource ctx, int coreLimit) { + Statics.LOGGER.sendInfo(ctx, "Starting compression..."); + + Instant start = Instant.now(); + + try (FileOutputStream outStream = new FileOutputStream(out); + BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream); + OutputStream compressorOutputStream = openCompressorStream(bufferedOutputStream, coreLimit); + TarArchiveOutputStream arc = new TarArchiveOutputStream(compressorOutputStream)) { + arc.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); + arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); + + Files.walk(inputFile.toPath()) + .filter(path -> !path.equals(inputFile.toPath())) + .filter(path -> path.toFile().isFile()) + .filter(path -> !Utilities.isBlacklisted(inputFile.toPath().relativize(path))) + .forEach(path -> { + File file = path.toAbsolutePath().toFile(); + + try (FileInputStream fileInputStream = new FileInputStream(file); + BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) { + ArchiveEntry entry = arc.createArchiveEntry(file, inputFile.toPath().relativize(path).toString()); + + arc.putArchiveEntry(entry); + IOUtils.copy(bufferedInputStream, arc); + + arc.closeArchiveEntry(); + } catch (IOException e) { + Statics.LOGGER.error("An exception occurred while trying to compress: {}", path.getFileName(), e); + Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!"); + } + }); + + arc.finish(); + } catch (IOException e) { + Statics.LOGGER.error("An exception occurred!", e); + Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!"); + } finally { + Statics.LOGGER.sendInfo(ctx, "Compression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now()))); + } + } +} diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/LZMACompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/LZMACompressor.java index 9af905f..60e535b 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/LZMACompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/LZMACompressor.java @@ -18,63 +18,19 @@ package net.szum123321.textile_backup.core.create.compressors; -import net.minecraft.server.command.ServerCommandSource; -import net.szum123321.textile_backup.Statics; -import net.szum123321.textile_backup.core.Utilities; -import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream; -import org.apache.commons.compress.utils.IOUtils; - import java.io.*; -import java.nio.file.Files; -import java.time.Duration; -import java.time.Instant; -public class LZMACompressor { - public static void createArchive(File in, File out, ServerCommandSource ctx) { - Statics.LOGGER.sendInfo(ctx, "Starting compression..."); +public class LZMACompressor extends AbstractTarCompressor { + private static final LZMACompressor INSTANCE = new LZMACompressor(); - Instant start = Instant.now(); + public static LZMACompressor getInstance() { + return INSTANCE; + } - try (FileOutputStream outStream = new FileOutputStream(out); - BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream); - XZCompressorOutputStream compressorStream = new XZCompressorOutputStream(bufferedOutputStream);// CompressorStreamClass.getConstructor().newInstance(bufferedOutputStream); - TarArchiveOutputStream arc = new TarArchiveOutputStream(compressorStream)) { - - arc.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); - arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); - - File input = in.getCanonicalFile(); - - Files.walk(input.toPath() - ).filter(path -> !path.equals(input.toPath()) && - path.toFile().isFile() && - !Utilities.isBlacklisted(input.toPath().relativize(path)) - ).forEach(path -> { - File file = path.toAbsolutePath().toFile(); - - try (FileInputStream fin = new FileInputStream(file); - BufferedInputStream bfin = new BufferedInputStream(fin)) { - ArchiveEntry entry = arc.createArchiveEntry(file, input.toPath().relativize(path).toString()); - - arc.putArchiveEntry(entry); - IOUtils.copy(bfin, arc); - - arc.closeArchiveEntry(); - } catch (IOException e) { - Statics.LOGGER.error("An exception occurred while trying to compress: {}", path.getFileName(), e); - Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!"); - } - }); - - arc.finish(); - } catch (IOException e) { - Statics.LOGGER.error("An exception occurred!", e); - Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!"); - } - - Statics.LOGGER.sendInfo(ctx, "Compression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now()))); + @Override + protected OutputStream openCompressorStream(OutputStream outputStream, int coreCountLimit) throws IOException { + return new XZCompressorOutputStream(outputStream); } } \ No newline at end of file diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelBZip2Compressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelBZip2Compressor.java index 72eb366..b938f3f 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelBZip2Compressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelBZip2Compressor.java @@ -18,65 +18,20 @@ package net.szum123321.textile_backup.core.create.compressors; -import net.minecraft.server.command.ServerCommandSource; -import net.szum123321.textile_backup.Statics; -import net.szum123321.textile_backup.core.Utilities; -import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.compress.utils.IOUtils; import org.at4j.comp.bzip2.BZip2OutputStream; import org.at4j.comp.bzip2.BZip2OutputStreamSettings; import java.io.*; -import java.nio.file.Files; -import java.time.Duration; -import java.time.Instant; -public class ParallelBZip2Compressor { - public static void createArchive(File in, File out, ServerCommandSource ctx, int coreLimit) { - Statics.LOGGER.sendInfo(ctx, "Starting compression..."); +public class ParallelBZip2Compressor extends AbstractTarCompressor { + private static final ParallelBZip2Compressor INSTANCE = new ParallelBZip2Compressor(); - BZip2OutputStreamSettings settings = new BZip2OutputStreamSettings().setNumberOfEncoderThreads(coreLimit); + public static ParallelBZip2Compressor getInstance() { + return INSTANCE; + } - Instant start = Instant.now(); - - try (FileOutputStream fileOutputStream = new FileOutputStream(out); - BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream); - BZip2OutputStream bZip2OutputStream = new BZip2OutputStream(bufferedOutputStream, settings); - TarArchiveOutputStream arc = new TarArchiveOutputStream(bZip2OutputStream)) { - - arc.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); - arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); - - File input = in.getCanonicalFile(); - - Files.walk(input.toPath() - ).filter(path -> !path.equals(input.toPath()) && - path.toFile().isFile() && - !Utilities.isBlacklisted(input.toPath().relativize(path)) - ).forEach(path -> { - File file = path.toAbsolutePath().toFile(); - - try (FileInputStream fin = new FileInputStream(file); - BufferedInputStream bfin = new BufferedInputStream(fin)) { - ArchiveEntry entry = arc.createArchiveEntry(file, input.toPath().relativize(path).toString()); - - arc.putArchiveEntry(entry); - IOUtils.copy(bfin, arc); - - arc.closeArchiveEntry(); - } catch (IOException e) { - Statics.LOGGER.error("An exception occurred while trying to compress: {}", path.getFileName(), e); - Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!"); - } - }); - - arc.finish(); - } catch (IOException e) { - Statics.LOGGER.error("An exception occurred!", e); - Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!"); - } - - Statics.LOGGER.sendInfo(ctx, "Compression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now()))); + @Override + protected OutputStream openCompressorStream(OutputStream outputStream, int coreCountLimit) throws IOException { + return new BZip2OutputStream(outputStream, new BZip2OutputStreamSettings().setNumberOfEncoderThreads(coreCountLimit)); } } \ No newline at end of file diff --git a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelGzipCompressor.java b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelGzipCompressor.java index 32942a1..5ba0f3a 100644 --- a/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelGzipCompressor.java +++ b/src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelGzipCompressor.java @@ -18,62 +18,20 @@ package net.szum123321.textile_backup.core.create.compressors; -import net.minecraft.server.command.ServerCommandSource; -import net.szum123321.textile_backup.Statics; -import net.szum123321.textile_backup.core.Utilities; import org.anarres.parallelgzip.ParallelGZIPOutputStream; -import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream; -import org.apache.commons.compress.utils.IOUtils; import java.io.*; -import java.nio.file.Files; -import java.time.Duration; -import java.time.Instant; +import java.util.concurrent.Executors; -public class ParallelGzipCompressor { - public static void createArchive(File in, File out, ServerCommandSource ctx, int coreLimit) { - Statics.LOGGER.sendInfo(ctx, "Starting compression..."); +public class ParallelGzipCompressor extends AbstractTarCompressor { + private static final ParallelGzipCompressor INSTANCE = new ParallelGzipCompressor(); - Instant start = Instant.now(); + public static ParallelGzipCompressor getInstance() { + return INSTANCE; + } - try (FileOutputStream outStream = new FileOutputStream(out); - BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream); - ParallelGZIPOutputStream gzipOutputStream = new ParallelGZIPOutputStream(bufferedOutputStream, coreLimit); - TarArchiveOutputStream arc = new TarArchiveOutputStream(gzipOutputStream)) { - - arc.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX); - arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX); - - File input = in.getCanonicalFile(); - - Files.walk(input.toPath()) - .filter(path -> !path.equals(input.toPath())) - .filter(path -> path.toFile().isFile()) - .filter(path -> !Utilities.isBlacklisted(input.toPath().relativize(path))) - .forEach(path -> { - File file = path.toAbsolutePath().toFile(); - - try (FileInputStream fin = new FileInputStream(file); - BufferedInputStream bfin = new BufferedInputStream(fin)) { - ArchiveEntry entry = arc.createArchiveEntry(file, input.toPath().relativize(path).toString()); - - arc.putArchiveEntry(entry); - IOUtils.copy(bfin, arc); - - arc.closeArchiveEntry(); - } catch (IOException e) { - Statics.LOGGER.error("An exception occurred while trying to compress: {}", path.getFileName(), e); - Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!"); - } - }); - - arc.finish(); - } catch (IOException e) { - Statics.LOGGER.error("An exception occurred!", e); - Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!"); - } - - Statics.LOGGER.sendInfo(ctx, "Compression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now()))); + @Override + protected OutputStream openCompressorStream(OutputStream outputStream, int coreCountLimit) throws IOException { + return new ParallelGZIPOutputStream(outputStream, Executors.newFixedThreadPool(coreCountLimit)); } }