Alex/textile_backup
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Moved most of the compression code to separate class (AbstractTarCompressor)

Browse Source
This commit is contained in:
szymon 2020-08-08 23:36:32 +02:00
parent 2230f28e97
commit a93e87729b
5 changed files with 106 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
src/main/java/net/szum123321/textile_backup/core/create/MakeBackupRunnable.java
View File

@ -21,10 +21,7 @@ package net.szum123321.textile_backup.core.create;
import net.minecraft.server.MinecraftServer;
import net.minecraft.server.command.ServerCommandSource;
import net.szum123321.textile_backup.Statics;
import net.szum123321.textile_backup.core.create.compressors.LZMACompressor;
import net.szum123321.textile_backup.core.create.compressors.ParallelBZip2Compressor;
import net.szum123321.textile_backup.core.create.compressors.ParallelGzipCompressor;
import net.szum123321.textile_backup.core.create.compressors.ParallelZipCompressor;
import net.szum123321.textile_backup.core.create.compressors.*;
import net.szum123321.textile_backup.core.Utilities;
import java.io.File;
@ -85,15 +82,15 @@ public class MakeBackupRunnable implements Runnable {
break;
case BZIP2:
ParallelBZip2Compressor.createArchive(world, outFile, commandSource, coreCount);
ParallelBZip2Compressor.getInstance().createArchive(world, outFile, commandSource, coreCount);
break;
case GZIP:
ParallelGzipCompressor.createArchive(world, outFile, commandSource, coreCount);
ParallelGzipCompressor.getInstance().createArchive(world, outFile, commandSource, coreCount);
break;
case LZMA:
LZMACompressor.createArchive(world, outFile, commandSource); // Always single-threaded ):
LZMACompressor.getInstance().createArchive(world, outFile, commandSource, coreCount);
break;
default:

77
src/main/java/net/szum123321/textile_backup/core/create/compressors/AbstractTarCompressor.java Normal file
View File

@ -0,0 +1,77 @@
/*
* A simple backup mod for Fabric
* Copyright (C) 2020 Szum123321
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU General Public License for more details.
*
* You should have received a copy of the GNU General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package net.szum123321.textile_backup.core.create.compressors;
import net.minecraft.server.command.ServerCommandSource;
import net.szum123321.textile_backup.Statics;
import net.szum123321.textile_backup.core.Utilities;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.utils.IOUtils;
import java.io.*;
import java.nio.file.Files;
import java.time.Duration;
import java.time.Instant;
public abstract class AbstractTarCompressor {
protected abstract OutputStream openCompressorStream(OutputStream outputStream, int coreCountLimit) throws IOException;
public void createArchive(File inputFile, File out, ServerCommandSource ctx, int coreLimit) {
Statics.LOGGER.sendInfo(ctx, "Starting compression...");
Instant start = Instant.now();
try (FileOutputStream outStream = new FileOutputStream(out);
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream);
OutputStream compressorOutputStream = openCompressorStream(bufferedOutputStream, coreLimit);
TarArchiveOutputStream arc = new TarArchiveOutputStream(compressorOutputStream)) {
arc.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);
Files.walk(inputFile.toPath())
.filter(path -> !path.equals(inputFile.toPath()))
.filter(path -> path.toFile().isFile())
.filter(path -> !Utilities.isBlacklisted(inputFile.toPath().relativize(path)))
.forEach(path -> {
File file = path.toAbsolutePath().toFile();
try (FileInputStream fileInputStream = new FileInputStream(file);
BufferedInputStream bufferedInputStream = new BufferedInputStream(fileInputStream)) {
ArchiveEntry entry = arc.createArchiveEntry(file, inputFile.toPath().relativize(path).toString());
arc.putArchiveEntry(entry);
IOUtils.copy(bufferedInputStream, arc);
arc.closeArchiveEntry();
} catch (IOException e) {
Statics.LOGGER.error("An exception occurred while trying to compress: {}", path.getFileName(), e);
Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!");
}
});
arc.finish();
} catch (IOException e) {
Statics.LOGGER.error("An exception occurred!", e);
Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!");
} finally {
Statics.LOGGER.sendInfo(ctx, "Compression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now())));
}
}
}

58
src/main/java/net/szum123321/textile_backup/core/create/compressors/LZMACompressor.java
View File

@ -18,63 +18,19 @@
package net.szum123321.textile_backup.core.create.compressors;
import net.minecraft.server.command.ServerCommandSource;
import net.szum123321.textile_backup.Statics;
import net.szum123321.textile_backup.core.Utilities;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.compressors.xz.XZCompressorOutputStream;
import org.apache.commons.compress.utils.IOUtils;
import java.io.*;
import java.nio.file.Files;
import java.time.Duration;
import java.time.Instant;
public class LZMACompressor {
public static void createArchive(File in, File out, ServerCommandSource ctx) {
Statics.LOGGER.sendInfo(ctx, "Starting compression...");
public class LZMACompressor extends AbstractTarCompressor {
private static final LZMACompressor INSTANCE = new LZMACompressor();
Instant start = Instant.now();
try (FileOutputStream outStream = new FileOutputStream(out);
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream);
XZCompressorOutputStream compressorStream = new XZCompressorOutputStream(bufferedOutputStream);// CompressorStreamClass.getConstructor().newInstance(bufferedOutputStream);
TarArchiveOutputStream arc = new TarArchiveOutputStream(compressorStream)) {
arc.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);
File input = in.getCanonicalFile();
Files.walk(input.toPath()
).filter(path -> !path.equals(input.toPath()) &&
path.toFile().isFile() &&
!Utilities.isBlacklisted(input.toPath().relativize(path))
).forEach(path -> {
File file = path.toAbsolutePath().toFile();
try (FileInputStream fin = new FileInputStream(file);
BufferedInputStream bfin = new BufferedInputStream(fin)) {
ArchiveEntry entry = arc.createArchiveEntry(file, input.toPath().relativize(path).toString());
arc.putArchiveEntry(entry);
IOUtils.copy(bfin, arc);
arc.closeArchiveEntry();
} catch (IOException e) {
Statics.LOGGER.error("An exception occurred while trying to compress: {}", path.getFileName(), e);
Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!");
}
});
arc.finish();
} catch (IOException e) {
Statics.LOGGER.error("An exception occurred!", e);
Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!");
public static LZMACompressor getInstance() {
return INSTANCE;
}
Statics.LOGGER.sendInfo(ctx, "Compression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now())));
@Override
protected OutputStream openCompressorStream(OutputStream outputStream, int coreCountLimit) throws IOException {
return new XZCompressorOutputStream(outputStream);
}
}

59
src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelBZip2Compressor.java
View File

@ -18,65 +18,20 @@
package net.szum123321.textile_backup.core.create.compressors;
import net.minecraft.server.command.ServerCommandSource;
import net.szum123321.textile_backup.Statics;
import net.szum123321.textile_backup.core.Utilities;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.utils.IOUtils;
import org.at4j.comp.bzip2.BZip2OutputStream;
import org.at4j.comp.bzip2.BZip2OutputStreamSettings;
import java.io.*;
import java.nio.file.Files;
import java.time.Duration;
import java.time.Instant;
public class ParallelBZip2Compressor {
public static void createArchive(File in, File out, ServerCommandSource ctx, int coreLimit) {
Statics.LOGGER.sendInfo(ctx, "Starting compression...");
public class ParallelBZip2Compressor extends AbstractTarCompressor {
private static final ParallelBZip2Compressor INSTANCE = new ParallelBZip2Compressor();
BZip2OutputStreamSettings settings = new BZip2OutputStreamSettings().setNumberOfEncoderThreads(coreLimit);
Instant start = Instant.now();
try (FileOutputStream fileOutputStream = new FileOutputStream(out);
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(fileOutputStream);
BZip2OutputStream bZip2OutputStream = new BZip2OutputStream(bufferedOutputStream, settings);
TarArchiveOutputStream arc = new TarArchiveOutputStream(bZip2OutputStream)) {
arc.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);
File input = in.getCanonicalFile();
Files.walk(input.toPath()
).filter(path -> !path.equals(input.toPath()) &&
path.toFile().isFile() &&
!Utilities.isBlacklisted(input.toPath().relativize(path))
).forEach(path -> {
File file = path.toAbsolutePath().toFile();
try (FileInputStream fin = new FileInputStream(file);
BufferedInputStream bfin = new BufferedInputStream(fin)) {
ArchiveEntry entry = arc.createArchiveEntry(file, input.toPath().relativize(path).toString());
arc.putArchiveEntry(entry);
IOUtils.copy(bfin, arc);
arc.closeArchiveEntry();
} catch (IOException e) {
Statics.LOGGER.error("An exception occurred while trying to compress: {}", path.getFileName(), e);
Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!");
}
});
arc.finish();
} catch (IOException e) {
Statics.LOGGER.error("An exception occurred!", e);
Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!");
public static ParallelBZip2Compressor getInstance() {
return INSTANCE;
}
Statics.LOGGER.sendInfo(ctx, "Compression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now())));
@Override
protected OutputStream openCompressorStream(OutputStream outputStream, int coreCountLimit) throws IOException {
return new BZip2OutputStream(outputStream, new BZip2OutputStreamSettings().setNumberOfEncoderThreads(coreCountLimit));
}
}

58
src/main/java/net/szum123321/textile_backup/core/create/compressors/ParallelGzipCompressor.java
View File

@ -18,62 +18,20 @@
package net.szum123321.textile_backup.core.create.compressors;
import net.minecraft.server.command.ServerCommandSource;
import net.szum123321.textile_backup.Statics;
import net.szum123321.textile_backup.core.Utilities;
import org.anarres.parallelgzip.ParallelGZIPOutputStream;
import org.apache.commons.compress.archivers.ArchiveEntry;
import org.apache.commons.compress.archivers.tar.TarArchiveOutputStream;
import org.apache.commons.compress.utils.IOUtils;
import java.io.*;
import java.nio.file.Files;
import java.time.Duration;
import java.time.Instant;
import java.util.concurrent.Executors;
public class ParallelGzipCompressor {
public static void createArchive(File in, File out, ServerCommandSource ctx, int coreLimit) {
Statics.LOGGER.sendInfo(ctx, "Starting compression...");
public class ParallelGzipCompressor extends AbstractTarCompressor {
private static final ParallelGzipCompressor INSTANCE = new ParallelGzipCompressor();
Instant start = Instant.now();
try (FileOutputStream outStream = new FileOutputStream(out);
BufferedOutputStream bufferedOutputStream = new BufferedOutputStream(outStream);
ParallelGZIPOutputStream gzipOutputStream = new ParallelGZIPOutputStream(bufferedOutputStream, coreLimit);
TarArchiveOutputStream arc = new TarArchiveOutputStream(gzipOutputStream)) {
arc.setLongFileMode(TarArchiveOutputStream.LONGFILE_POSIX);
arc.setBigNumberMode(TarArchiveOutputStream.BIGNUMBER_POSIX);
File input = in.getCanonicalFile();
Files.walk(input.toPath())
.filter(path -> !path.equals(input.toPath()))
.filter(path -> path.toFile().isFile())
.filter(path -> !Utilities.isBlacklisted(input.toPath().relativize(path)))
.forEach(path -> {
File file = path.toAbsolutePath().toFile();
try (FileInputStream fin = new FileInputStream(file);
BufferedInputStream bfin = new BufferedInputStream(fin)) {
ArchiveEntry entry = arc.createArchiveEntry(file, input.toPath().relativize(path).toString());
arc.putArchiveEntry(entry);
IOUtils.copy(bfin, arc);
arc.closeArchiveEntry();
} catch (IOException e) {
Statics.LOGGER.error("An exception occurred while trying to compress: {}", path.getFileName(), e);
Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!");
}
});
arc.finish();
} catch (IOException e) {
Statics.LOGGER.error("An exception occurred!", e);
Statics.LOGGER.sendError(ctx, "Something went wrong while compressing files!");
public static ParallelGzipCompressor getInstance() {
return INSTANCE;
}
Statics.LOGGER.sendInfo(ctx, "Compression took: {} seconds.", Utilities.formatDuration(Duration.between(start, Instant.now())));
@Override
protected OutputStream openCompressorStream(OutputStream outputStream, int coreCountLimit) throws IOException {
return new ParallelGZIPOutputStream(outputStream, Executors.newFixedThreadPool(coreCountLimit));
}
}