From c937ef14100543b9e490a22928557182afb398e0 Mon Sep 17 00:00:00 2001 From: Lyfts <127234178+Lyfts@users.noreply.github.com> Date: Wed, 31 Jul 2024 16:52:13 +0200 Subject: [PATCH 01/12] switch backups to common compress lib & add max folder size config --- .../serverutils/ServerUtilitiesConfig.java | 16 +++ .../java/serverutils/lib/util/FileUtils.java | 72 ++++++++---- .../serverutils/task/backup/BackupTask.java | 57 +++++++-- .../serverutils/task/backup/ThreadBackup.java | 110 ++++++++---------- 4 files changed, 162 insertions(+), 93 deletions(-) diff --git a/src/main/java/serverutils/ServerUtilitiesConfig.java b/src/main/java/serverutils/ServerUtilitiesConfig.java index cf989b27..8ecb2a0a 100644 --- a/src/main/java/serverutils/ServerUtilitiesConfig.java +++ b/src/main/java/serverutils/ServerUtilitiesConfig.java @@ -174,6 +174,20 @@ public static boolean sync() { .getBoolean(); backups.need_online_players = config .get(BACKUPS, "need_online_players", true, "Backups won't run if no players are online.").getBoolean(); + backups.max_folder_size = config + .get( + BACKUPS, + "max_folder_size", + 1, + """ + Max size of backup folder in GB. If total folder size exceeds this value it will delete old backups until the size is under. + 0 = Disabled and backups_to_keep will be used instead.""") + .getInt(); + backups.delete_custom_name_backups = config.get( + BACKUPS, + "delete_custom_name_backups", + true, + "Delete backups that have a custom name set through /backup start ").getBoolean(); chat.add_nickname_tilde = config.get( CHAT, @@ -472,6 +486,8 @@ public static class Backups { public boolean display_file_size; public boolean need_online_players; public boolean silent_backup; + public int max_folder_size; + public boolean delete_custom_name_backups; } public static class Login { diff --git a/src/main/java/serverutils/lib/util/FileUtils.java b/src/main/java/serverutils/lib/util/FileUtils.java index 51f0e1c9..48e9d057 100644 --- a/src/main/java/serverutils/lib/util/FileUtils.java +++ b/src/main/java/serverutils/lib/util/FileUtils.java @@ -1,5 +1,9 @@ package serverutils.lib.util; +import static serverutils.lib.util.FileUtils.SizeUnit.GB; +import static serverutils.lib.util.FileUtils.SizeUnit.KB; +import static serverutils.lib.util.FileUtils.SizeUnit.MB; + import java.io.BufferedWriter; import java.io.File; import java.io.FileInputStream; @@ -14,13 +18,23 @@ public class FileUtils { - public static final int KB = 1024; - public static final int MB = KB * 1024; - public static final int GB = MB * 1024; + public enum SizeUnit { + + B(1), + KB(B.size * 1024), + MB(KB.size * 1024), + GB(MB.size * 1024); + + private final long size; + + SizeUnit(long size) { + this.size = size; + } - public static final double KB_D = 1024D; - public static final double MB_D = KB_D * 1024D; - public static final double GB_D = MB_D * 1024D; + public long getSize() { + return size; + } + } public static File newFile(File file) { if (!file.exists()) { @@ -104,33 +118,49 @@ public static void listTree0(List list, File file) { } public static long getSize(File file) { - if (!file.exists()) { - return 0L; - } else if (file.isFile()) { - return file.length(); + return getSize(file, SizeUnit.B); + } + + public static long getSize(File file, SizeUnit sizeUnit) { + long size = getSize0(file); + if (size == 0L) return 0L; + + return switch (sizeUnit) { + case KB -> size / KB.getSize(); + case MB -> size / MB.getSize(); + case GB -> size / GB.getSize(); + default -> size; + }; + } + + private static long getSize0(File file) { + if (!file.exists()) return 0L; + long size = 0; + + if (file.isFile()) { + size += file.length(); } else if (file.isDirectory()) { - long length = 0L; File[] f1 = file.listFiles(); if (f1 != null && f1.length > 0) { for (File aF1 : f1) { - length += getSize(aF1); + size += getSize0(aF1); } } - return length; } - return 0L; + + return size; } public static String getSizeString(double b) { - if (b >= GB_D) { - return String.format("%.1fGB", b / GB_D); - } else if (b >= MB_D) { - return String.format("%.1fMB", b / MB_D); - } else if (b >= KB_D) { - return String.format("%.1fKB", b / KB_D); + if (b >= GB.getSize()) { + return String.format("%.1fGB", b / (double) GB.getSize()); + } else if (b >= MB.getSize()) { + return String.format("%.1fMB", b / (double) MB.getSize()); + } else if (b >= KB.getSize()) { + return String.format("%.1fKB", b / (double) KB.getSize()); } - return ((long) b) + "B"; + return b + "B"; } public static String getSizeString(File file) { diff --git a/src/main/java/serverutils/task/backup/BackupTask.java b/src/main/java/serverutils/task/backup/BackupTask.java index 699b97ba..61adbfa4 100644 --- a/src/main/java/serverutils/task/backup/BackupTask.java +++ b/src/main/java/serverutils/task/backup/BackupTask.java @@ -2,9 +2,15 @@ import static serverutils.ServerUtilitiesConfig.backups; import static serverutils.ServerUtilitiesNotifications.BACKUP_START; +import static serverutils.lib.util.FileUtils.SizeUnit; import java.io.File; import java.util.Arrays; +import java.util.Comparator; +import java.util.Iterator; +import java.util.List; +import java.util.regex.Pattern; +import java.util.stream.Collectors; import javax.annotation.Nullable; @@ -23,6 +29,7 @@ public class BackupTask extends Task { + public static final Pattern BACKUP_NAME_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}(.*)"); public static File backupsFolder; public static ThreadBackup thread; public static boolean hadPlayer = false; @@ -84,7 +91,7 @@ public void execute(Universe universe) { } } } catch (Exception ex) { - ServerUtilities.LOGGER.info("Error while saving world: {}", ex.getMessage()); + ServerUtilities.LOGGER.info("An error occurred while turning off auto-save.", ex); } File wd = server.getEntityWorld().getSaveHandler().getWorldDirectory(); @@ -99,19 +106,45 @@ public void execute(Universe universe) { } public static void clearOldBackups() { - String[] backups = backupsFolder.list(); + File[] files = backupsFolder.listFiles(); + if (files == null || files.length == 0) return; - if (backups != null && backups.length > ServerUtilitiesConfig.backups.backups_to_keep) { - Arrays.sort(backups); + List backupFiles = Arrays.stream(files).filter( + file -> backups.delete_custom_name_backups || BACKUP_NAME_PATTERN.matcher(file.getName()).matches()) + .sorted(Comparator.comparingLong(File::lastModified)).collect(Collectors.toList()); - int toDelete = backups.length - ServerUtilitiesConfig.backups.backups_to_keep; - ServerUtilities.LOGGER.info("Deleting {} old backups", toDelete); + int maxGb = backups.max_folder_size; + if (maxGb > 0) { + long currentSize = backupFiles.stream().mapToLong(file -> FileUtils.getSize(file, SizeUnit.GB)).sum(); + if (currentSize <= maxGb) return; + deleteOldBackups(backupFiles, currentSize, maxGb); - for (int i = 0; i < toDelete; i++) { - File f = new File(backupsFolder, backups[i]); - ServerUtilities.LOGGER.info("Deleted old backup: {}", f.getPath()); - FileUtils.delete(f); - } + } else if (backupFiles.size() > backups.backups_to_keep) { + deleteExcessBackups(backupFiles); + } + } + + private static void deleteOldBackups(List backupFiles, long currentSize, int maxGb) { + int deleted = 0; + Iterator it = backupFiles.iterator(); + while (it.hasNext() && currentSize > maxGb) { + File file = it.next(); + currentSize -= FileUtils.getSize(file, SizeUnit.GB); + ServerUtilities.LOGGER.info("Deleting old backup: {}", file.getPath()); + FileUtils.delete(file); + it.remove(); + deleted++; + } + ServerUtilities.LOGGER.info("Deleted {} old backups", deleted); + } + + private static void deleteExcessBackups(List backupFiles) { + int toDelete = backupFiles.size() - ServerUtilitiesConfig.backups.backups_to_keep; + ServerUtilities.LOGGER.info("Deleting {} old backups", toDelete); + for (int i = 0; i < toDelete; i++) { + File file = backupFiles.get(i); + ServerUtilities.LOGGER.info("Deleted old backup: {}", file.getPath()); + FileUtils.delete(file); } } @@ -140,7 +173,7 @@ private void postBackup(Universe universe) { } } } catch (Exception ex) { - ex.printStackTrace(); + ServerUtilities.LOGGER.info("An error occurred while turning on auto-save.", ex); } } } diff --git a/src/main/java/serverutils/task/backup/ThreadBackup.java b/src/main/java/serverutils/task/backup/ThreadBackup.java index d868ec00..eb5c2cbf 100644 --- a/src/main/java/serverutils/task/backup/ThreadBackup.java +++ b/src/main/java/serverutils/task/backup/ThreadBackup.java @@ -1,23 +1,25 @@ package serverutils.task.backup; +import static serverutils.ServerUtilitiesConfig.backups; import static serverutils.ServerUtilitiesNotifications.BACKUP_END1; import static serverutils.ServerUtilitiesNotifications.BACKUP_END2; import java.io.File; import java.io.FileInputStream; -import java.io.FileOutputStream; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.List; -import java.util.zip.ZipEntry; -import java.util.zip.ZipOutputStream; import net.minecraft.util.EnumChatFormatting; import net.minecraft.util.IChatComponent; +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.io.IOUtils; + import serverutils.ServerUtilities; -import serverutils.ServerUtilitiesConfig; import serverutils.ServerUtilitiesNotifications; +import serverutils.lib.math.Ticks; import serverutils.lib.util.FileUtils; import serverutils.lib.util.ServerUtils; import serverutils.lib.util.StringUtils; @@ -45,12 +47,7 @@ public static void doBackup(File src, String customName) { String time = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(Calendar.getInstance().getTime()); File dstFile = null; StringBuilder out = new StringBuilder(); - - if (customName.isEmpty()) { - out.append(time); - } else { - out.append(customName); - } + out.append(customName.isEmpty() ? time : customName); try { List files = FileUtils.listTree(src); @@ -58,64 +55,57 @@ public static void doBackup(File src, String customName) { ServerUtilities.LOGGER.info("Backing up {} files...", files.size()); long start = System.currentTimeMillis(); - if (ServerUtilitiesConfig.backups.compression_level > 0) { + if (backups.compression_level > 0) { out.append(".zip"); dstFile = FileUtils.newFile(new File(BackupTask.backupsFolder, out.toString())); - ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(dstFile)); - zos.setLevel(ServerUtilitiesConfig.backups.compression_level); - - long logMillis = System.currentTimeMillis() + 5000L; - - byte[] buffer = new byte[4096]; - - ServerUtilities.LOGGER.info("Compressing {} files!", allFiles); - - for (int i = 0; i < allFiles; i++) { - File file = files.get(i); - String filePath = file.getAbsolutePath(); - ZipEntry ze = new ZipEntry( - src.getName() + File.separator + filePath.substring(src.getAbsolutePath().length() + 1)); - - long millis = System.currentTimeMillis(); - - if (i == 0 || millis > logMillis || i == allFiles - 1) { - logMillis = millis + 5000L; - - StringBuilder log = new StringBuilder(); - log.append('['); - log.append(i); - log.append(" | "); - log.append(StringUtils.formatDouble00((i / (double) allFiles) * 100D)); - log.append("%]: "); - log.append(ze.getName()); - ServerUtilities.LOGGER.info(log.toString()); + try (ZipArchiveOutputStream zaos = new ZipArchiveOutputStream(dstFile)) { + zaos.setLevel(backups.compression_level); + + long logMillis = System.currentTimeMillis() + Ticks.SECOND.x(5).millis(); + ServerUtilities.LOGGER.info("Compressing {} files!", allFiles); + + for (int i = 0; i < allFiles; i++) { + File file = files.get(i); + String filePath = file.getAbsolutePath(); + ZipArchiveEntry ze = new ZipArchiveEntry( + src.getName() + File.separator + + filePath.substring(src.getAbsolutePath().length() + 1)); + + long millis = System.currentTimeMillis(); + + if (i == 0 || millis > logMillis || i == allFiles - 1) { + logMillis = millis + Ticks.SECOND.x(5).millis(); + + StringBuilder log = new StringBuilder(); + log.append('['); + log.append(i); + log.append(" | "); + log.append(StringUtils.formatDouble00((i / (double) allFiles) * 100D)); + log.append("%]: "); + log.append(ze.getName()); + ServerUtilities.LOGGER.info(log.toString()); + } + + zaos.putArchiveEntry(ze); + + try (FileInputStream fis = new FileInputStream(file)) { + IOUtils.copy(fis, zaos); + } + zaos.closeArchiveEntry(); } - - zos.putNextEntry(ze); - FileInputStream fis = new FileInputStream(file); - - int len; - while ((len = fis.read(buffer)) > 0) zos.write(buffer, 0, len); - zos.closeEntry(); - fis.close(); } - - zos.close(); - ServerUtilities.LOGGER.info( - "Done compressing in " + getDoneTime(start) - + " seconds (" - + FileUtils.getSizeString(dstFile) - + ")!"); + "Done compressing in {} seconds ({})!", + getDoneTime(start), + FileUtils.getSizeString(dstFile)); } else { out.append(File.separatorChar).append(src.getName()); - dstFile = new File(BackupTask.backupsFolder, out.toString()); - dstFile.mkdirs(); + dstFile = FileUtils.newFile(new File(BackupTask.backupsFolder, out.toString()));; String dstPath = dstFile.getAbsolutePath() + File.separator; String srcPath = src.getAbsolutePath(); - long logMillis = System.currentTimeMillis() + 2000L; + long logMillis = System.currentTimeMillis() + Ticks.SECOND.x(2).millis(); for (int i = 0; i < allFiles; i++) { File file = files.get(i); @@ -123,7 +113,7 @@ public static void doBackup(File src, String customName) { long millis = System.currentTimeMillis(); if (i == 0 || millis > logMillis || i == allFiles - 1) { - logMillis = millis + 2000L; + logMillis = millis + Ticks.SECOND.x(2).millis(); StringBuilder log = new StringBuilder(); log.append('['); @@ -144,7 +134,7 @@ public static void doBackup(File src, String customName) { BackupTask.clearOldBackups(); - if (ServerUtilitiesConfig.backups.display_file_size) { + if (backups.display_file_size) { String sizeB = FileUtils.getSizeString(dstFile); String sizeT = FileUtils.getSizeString(BackupTask.backupsFolder); ServerUtilitiesNotifications.backupNotification( @@ -160,8 +150,8 @@ public static void doBackup(File src, String customName) { ServerUtilities.lang(null, "cmd.backup_fail", e.getClass().getName()), EnumChatFormatting.RED); ServerUtils.notifyChat(ServerUtils.getServer(), null, c); + ServerUtilities.LOGGER.error("Error while backing up: {}", e.getMessage()); - e.printStackTrace(); if (dstFile != null) FileUtils.delete(dstFile); } } From d4aaf7e944a134f43f8cd48dcca3d27c232dfc62 Mon Sep 17 00:00:00 2001 From: Lyfts <127234178+Lyfts@users.noreply.github.com> Date: Sat, 3 Aug 2024 00:21:21 +0200 Subject: [PATCH 02/12] Change uncompressed backup to a STORED zip instead of folder --- .../serverutils/task/backup/BackupTask.java | 7 +- .../serverutils/task/backup/ThreadBackup.java | 106 ++++++------------ 2 files changed, 40 insertions(+), 73 deletions(-) diff --git a/src/main/java/serverutils/task/backup/BackupTask.java b/src/main/java/serverutils/task/backup/BackupTask.java index 61adbfa4..619d07dc 100644 --- a/src/main/java/serverutils/task/backup/BackupTask.java +++ b/src/main/java/serverutils/task/backup/BackupTask.java @@ -94,13 +94,13 @@ public void execute(Universe universe) { ServerUtilities.LOGGER.info("An error occurred while turning off auto-save.", ex); } - File wd = server.getEntityWorld().getSaveHandler().getWorldDirectory(); + File worldDir = server.getEntityWorld().getSaveHandler().getWorldDirectory(); if (backups.use_separate_thread) { - thread = new ThreadBackup(wd, customName); + thread = new ThreadBackup(worldDir, customName); thread.start(); } else { - ThreadBackup.doBackup(wd, customName); + ThreadBackup.doBackup(worldDir, customName); } universe.scheduleTask(new BackupTask(true)); } @@ -159,6 +159,7 @@ private void postBackup(Universe universe) { return; } + clearOldBackups(); thread = null; try { MinecraftServer server = ServerUtils.getServer(); diff --git a/src/main/java/serverutils/task/backup/ThreadBackup.java b/src/main/java/serverutils/task/backup/ThreadBackup.java index eb5c2cbf..2beb8a4e 100644 --- a/src/main/java/serverutils/task/backup/ThreadBackup.java +++ b/src/main/java/serverutils/task/backup/ThreadBackup.java @@ -6,9 +6,11 @@ import java.io.File; import java.io.FileInputStream; +import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.List; +import java.util.zip.ZipEntry; import net.minecraft.util.EnumChatFormatting; import net.minecraft.util.IChatComponent; @@ -26,6 +28,9 @@ public class ThreadBackup extends Thread { + private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); + + private static long logMillis; private final File src0; private final String customName; public boolean isDone = false; @@ -44,96 +49,48 @@ public void run() { public static void doBackup(File src, String customName) { - String time = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss").format(Calendar.getInstance().getTime()); + String time = DATE_FORMAT.format(Calendar.getInstance().getTime()); File dstFile = null; StringBuilder out = new StringBuilder(); - out.append(customName.isEmpty() ? time : customName); + out.append(customName.isEmpty() ? time : customName).append(".zip"); try { List files = FileUtils.listTree(src); int allFiles = files.size(); - ServerUtilities.LOGGER.info("Backing up {} files...", files.size()); long start = System.currentTimeMillis(); - if (backups.compression_level > 0) { - out.append(".zip"); - dstFile = FileUtils.newFile(new File(BackupTask.backupsFolder, out.toString())); - try (ZipArchiveOutputStream zaos = new ZipArchiveOutputStream(dstFile)) { - zaos.setLevel(backups.compression_level); - long logMillis = System.currentTimeMillis() + Ticks.SECOND.x(5).millis(); - ServerUtilities.LOGGER.info("Compressing {} files!", allFiles); - - for (int i = 0; i < allFiles; i++) { - File file = files.get(i); - String filePath = file.getAbsolutePath(); - ZipArchiveEntry ze = new ZipArchiveEntry( - src.getName() + File.separator - + filePath.substring(src.getAbsolutePath().length() + 1)); - - long millis = System.currentTimeMillis(); - - if (i == 0 || millis > logMillis || i == allFiles - 1) { - logMillis = millis + Ticks.SECOND.x(5).millis(); - - StringBuilder log = new StringBuilder(); - log.append('['); - log.append(i); - log.append(" | "); - log.append(StringUtils.formatDouble00((i / (double) allFiles) * 100D)); - log.append("%]: "); - log.append(ze.getName()); - ServerUtilities.LOGGER.info(log.toString()); - } - - zaos.putArchiveEntry(ze); - - try (FileInputStream fis = new FileInputStream(file)) { - IOUtils.copy(fis, zaos); - } - zaos.closeArchiveEntry(); - } - } - ServerUtilities.LOGGER.info( - "Done compressing in {} seconds ({})!", - getDoneTime(start), - FileUtils.getSizeString(dstFile)); - } else { - out.append(File.separatorChar).append(src.getName()); - dstFile = FileUtils.newFile(new File(BackupTask.backupsFolder, out.toString()));; + dstFile = FileUtils.newFile(new File(BackupTask.backupsFolder, out.toString())); + try (ZipArchiveOutputStream zaos = new ZipArchiveOutputStream(dstFile)) { - String dstPath = dstFile.getAbsolutePath() + File.separator; - String srcPath = src.getAbsolutePath(); + if (backups.compression_level == 0) { + zaos.setMethod(ZipEntry.STORED); + } else { + zaos.setLevel(backups.compression_level); + } - long logMillis = System.currentTimeMillis() + Ticks.SECOND.x(2).millis(); + logMillis = System.currentTimeMillis() + Ticks.SECOND.x(5).millis(); + ServerUtilities.LOGGER.info("Backing up {} files!", allFiles); for (int i = 0; i < allFiles; i++) { File file = files.get(i); - - long millis = System.currentTimeMillis(); - - if (i == 0 || millis > logMillis || i == allFiles - 1) { - logMillis = millis + Ticks.SECOND.x(2).millis(); - - StringBuilder log = new StringBuilder(); - log.append('['); - log.append(i); - log.append(" | "); - log.append(StringUtils.formatDouble00((i / (double) allFiles) * 100D)); - log.append("%]: "); - log.append(file.getName()); - ServerUtilities.LOGGER.info(log.toString()); + String filePath = file.getAbsolutePath(); + ZipArchiveEntry ze = new ZipArchiveEntry( + src.getName() + File.separator + filePath.substring(src.getAbsolutePath().length() + 1)); + + logProgress(i, allFiles, filePath); + zaos.putArchiveEntry(ze); + try (FileInputStream fis = new FileInputStream(file)) { + IOUtils.copy(fis, zaos); } - - File dst1 = new File(dstPath + (file.getAbsolutePath().replace(srcPath, ""))); - FileUtils.copyFile(file, dst1); + zaos.closeArchiveEntry(); } } + ServerUtilities.LOGGER + .info("Backup done in {} seconds ({})!", getDoneTime(start), FileUtils.getSizeString(dstFile)); ServerUtilities.LOGGER.info("Created {} from {}", dstFile.getAbsolutePath(), src.getAbsolutePath()); - BackupTask.clearOldBackups(); - if (backups.display_file_size) { String sizeB = FileUtils.getSizeString(dstFile); String sizeT = FileUtils.getSizeString(BackupTask.backupsFolder); @@ -156,6 +113,15 @@ public static void doBackup(File src, String customName) { } } + private static void logProgress(int i, int allFiles, String name) { + long millis = System.currentTimeMillis(); + if (i == 0 || millis > logMillis || i == allFiles - 1) { + logMillis = millis + Ticks.SECOND.x(5).millis(); + ServerUtilities.LOGGER + .info("[{} | {}%]: {}", i, StringUtils.formatDouble00((i / (double) allFiles) * 100D), name); + } + } + private static String getDoneTime(long l) { return StringUtils.getTimeString(System.currentTimeMillis() - l); } From 84494021b83ed77206e1f807fb95d75e29590c32 Mon Sep 17 00:00:00 2001 From: Lyfts <127234178+Lyfts@users.noreply.github.com> Date: Sat, 3 Aug 2024 00:48:30 +0200 Subject: [PATCH 03/12] disabled by default --- src/main/java/serverutils/ServerUtilitiesConfig.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/serverutils/ServerUtilitiesConfig.java b/src/main/java/serverutils/ServerUtilitiesConfig.java index 8ecb2a0a..d006d19e 100644 --- a/src/main/java/serverutils/ServerUtilitiesConfig.java +++ b/src/main/java/serverutils/ServerUtilitiesConfig.java @@ -178,7 +178,7 @@ public static boolean sync() { .get( BACKUPS, "max_folder_size", - 1, + 0, """ Max size of backup folder in GB. If total folder size exceeds this value it will delete old backups until the size is under. 0 = Disabled and backups_to_keep will be used instead.""") @@ -186,7 +186,7 @@ public static boolean sync() { backups.delete_custom_name_backups = config.get( BACKUPS, "delete_custom_name_backups", - true, + false, "Delete backups that have a custom name set through /backup start ").getBoolean(); chat.add_nickname_tilde = config.get( From 3cfa172f1b0935c5cb183269f8f91149e0a1b872 Mon Sep 17 00:00:00 2001 From: Lyfts <127234178+Lyfts@users.noreply.github.com> Date: Sat, 3 Aug 2024 01:09:39 +0200 Subject: [PATCH 04/12] stringbuilder be gone --- .../serverutils/task/backup/ThreadBackup.java | 23 ++++++++----------- 1 file changed, 9 insertions(+), 14 deletions(-) diff --git a/src/main/java/serverutils/task/backup/ThreadBackup.java b/src/main/java/serverutils/task/backup/ThreadBackup.java index 2beb8a4e..c2362a9f 100644 --- a/src/main/java/serverutils/task/backup/ThreadBackup.java +++ b/src/main/java/serverutils/task/backup/ThreadBackup.java @@ -48,19 +48,16 @@ public void run() { } public static void doBackup(File src, String customName) { - - String time = DATE_FORMAT.format(Calendar.getInstance().getTime()); + String outName = (customName.isEmpty() ? DATE_FORMAT.format(Calendar.getInstance().getTime()) : customName) + + ".zip"; File dstFile = null; - StringBuilder out = new StringBuilder(); - out.append(customName.isEmpty() ? time : customName).append(".zip"); - try { List files = FileUtils.listTree(src); int allFiles = files.size(); ServerUtilities.LOGGER.info("Backing up {} files...", files.size()); long start = System.currentTimeMillis(); - dstFile = FileUtils.newFile(new File(BackupTask.backupsFolder, out.toString())); + dstFile = FileUtils.newFile(new File(BackupTask.backupsFolder, outName)); try (ZipArchiveOutputStream zaos = new ZipArchiveOutputStream(dstFile)) { if (backups.compression_level == 0) { @@ -75,30 +72,28 @@ public static void doBackup(File src, String customName) { for (int i = 0; i < allFiles; i++) { File file = files.get(i); String filePath = file.getAbsolutePath(); - ZipArchiveEntry ze = new ZipArchiveEntry( + ZipArchiveEntry entry = new ZipArchiveEntry( src.getName() + File.separator + filePath.substring(src.getAbsolutePath().length() + 1)); logProgress(i, allFiles, filePath); - zaos.putArchiveEntry(ze); + zaos.putArchiveEntry(entry); try (FileInputStream fis = new FileInputStream(file)) { IOUtils.copy(fis, zaos); } zaos.closeArchiveEntry(); } } - - ServerUtilities.LOGGER - .info("Backup done in {} seconds ({})!", getDoneTime(start), FileUtils.getSizeString(dstFile)); + String backupSize = FileUtils.getSizeString(dstFile); + ServerUtilities.LOGGER.info("Backup done in {} seconds ({})!", getDoneTime(start), backupSize); ServerUtilities.LOGGER.info("Created {} from {}", dstFile.getAbsolutePath(), src.getAbsolutePath()); if (backups.display_file_size) { - String sizeB = FileUtils.getSizeString(dstFile); String sizeT = FileUtils.getSizeString(BackupTask.backupsFolder); ServerUtilitiesNotifications.backupNotification( BACKUP_END2, "cmd.backup_end_2", getDoneTime(start), - (sizeB.equals(sizeT) ? sizeB : (sizeB + " | " + sizeT))); + (backupSize.equals(sizeT) ? backupSize : (backupSize + " | " + sizeT))); } else { ServerUtilitiesNotifications.backupNotification(BACKUP_END1, "cmd.backup_end_1", getDoneTime(start)); } @@ -107,7 +102,7 @@ public static void doBackup(File src, String customName) { ServerUtilities.lang(null, "cmd.backup_fail", e.getClass().getName()), EnumChatFormatting.RED); ServerUtils.notifyChat(ServerUtils.getServer(), null, c); - ServerUtilities.LOGGER.error("Error while backing up: {}", e.getMessage()); + ServerUtilities.LOGGER.error("Error while backing up", e); if (dstFile != null) FileUtils.delete(dstFile); } From 6c5e8127934a1383cdd2180bdd9255e0dcb271c0 Mon Sep 17 00:00:00 2001 From: Lyfts <127234178+Lyfts@users.noreply.github.com> Date: Sat, 3 Aug 2024 10:56:22 +0200 Subject: [PATCH 05/12] custom-named backups should be deleted by default --- src/main/java/serverutils/ServerUtilitiesConfig.java | 7 ++++--- src/main/java/serverutils/task/backup/BackupTask.java | 3 ++- 2 files changed, 6 insertions(+), 4 deletions(-) diff --git a/src/main/java/serverutils/ServerUtilitiesConfig.java b/src/main/java/serverutils/ServerUtilitiesConfig.java index d006d19e..b15af98e 100644 --- a/src/main/java/serverutils/ServerUtilitiesConfig.java +++ b/src/main/java/serverutils/ServerUtilitiesConfig.java @@ -180,14 +180,15 @@ public static boolean sync() { "max_folder_size", 0, """ - Max size of backup folder in GB. If total folder size exceeds this value it will delete old backups until the size is under. + Max size of backup folder in GB. If total folder size exceeds this value old backups will be deleted until the size is under. 0 = Disabled and backups_to_keep will be used instead.""") .getInt(); backups.delete_custom_name_backups = config.get( BACKUPS, "delete_custom_name_backups", - false, - "Delete backups that have a custom name set through /backup start ").getBoolean(); + true, + "Include backups that have a custom name set through /backup start when deleting old backups") + .getBoolean(); chat.add_nickname_tilde = config.get( CHAT, diff --git a/src/main/java/serverutils/task/backup/BackupTask.java b/src/main/java/serverutils/task/backup/BackupTask.java index 619d07dc..ed117a8e 100644 --- a/src/main/java/serverutils/task/backup/BackupTask.java +++ b/src/main/java/serverutils/task/backup/BackupTask.java @@ -17,6 +17,7 @@ import net.minecraft.command.ICommandSender; import net.minecraft.server.MinecraftServer; import net.minecraft.world.WorldServer; +import net.minecraftforge.common.DimensionManager; import serverutils.ServerUtilities; import serverutils.ServerUtilitiesConfig; @@ -94,7 +95,7 @@ public void execute(Universe universe) { ServerUtilities.LOGGER.info("An error occurred while turning off auto-save.", ex); } - File worldDir = server.getEntityWorld().getSaveHandler().getWorldDirectory(); + File worldDir = DimensionManager.getCurrentSaveRootDirectory(); if (backups.use_separate_thread) { thread = new ThreadBackup(worldDir, customName); From 91ca07f2499a7e26053d0a9e99de07821abc7d9f Mon Sep 17 00:00:00 2001 From: Lyfts <127234178+Lyfts@users.noreply.github.com> Date: Fri, 23 Aug 2024 21:37:16 +0200 Subject: [PATCH 06/12] iterator isn't necessary anymore --- src/main/java/serverutils/task/backup/BackupTask.java | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/src/main/java/serverutils/task/backup/BackupTask.java b/src/main/java/serverutils/task/backup/BackupTask.java index ed117a8e..74d9eecd 100644 --- a/src/main/java/serverutils/task/backup/BackupTask.java +++ b/src/main/java/serverutils/task/backup/BackupTask.java @@ -7,7 +7,6 @@ import java.io.File; import java.util.Arrays; import java.util.Comparator; -import java.util.Iterator; import java.util.List; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -127,13 +126,11 @@ public static void clearOldBackups() { private static void deleteOldBackups(List backupFiles, long currentSize, int maxGb) { int deleted = 0; - Iterator it = backupFiles.iterator(); - while (it.hasNext() && currentSize > maxGb) { - File file = it.next(); + for (File file : backupFiles) { + if (currentSize <= maxGb) break; currentSize -= FileUtils.getSize(file, SizeUnit.GB); ServerUtilities.LOGGER.info("Deleting old backup: {}", file.getPath()); FileUtils.delete(file); - it.remove(); deleted++; } ServerUtilities.LOGGER.info("Deleted {} old backups", deleted); From 971d7e1fdb8db5ee9dc0d397e07576093a2094c3 Mon Sep 17 00:00:00 2001 From: Lyfts <127234178+Lyfts@users.noreply.github.com> Date: Tue, 6 Aug 2024 17:49:17 +0200 Subject: [PATCH 07/12] add option to only backup claimed chunks --- .../serverutils/ServerUtilitiesConfig.java | 5 + .../java/serverutils/data/ClaimedChunks.java | 4 + .../serverutils/lib/math/ChunkDimPos.java | 17 +- .../java/serverutils/lib/util/FileUtils.java | 4 + .../serverutils/task/backup/BackupTask.java | 17 +- .../serverutils/task/backup/ThreadBackup.java | 229 ++++++++++++++---- 6 files changed, 231 insertions(+), 45 deletions(-) diff --git a/src/main/java/serverutils/ServerUtilitiesConfig.java b/src/main/java/serverutils/ServerUtilitiesConfig.java index b15af98e..c398cac6 100644 --- a/src/main/java/serverutils/ServerUtilitiesConfig.java +++ b/src/main/java/serverutils/ServerUtilitiesConfig.java @@ -189,6 +189,10 @@ public static boolean sync() { true, "Include backups that have a custom name set through /backup start when deleting old backups") .getBoolean(); + backups.only_backup_claimed_chunks = config.get(BACKUPS, "only_backup_claimed_chunks", false, """ + Only claimed chunks will be included in backups. + Backups will be much faster and smaller, but any unclaimed chunk will be unrecoverable.""") + .getBoolean(); chat.add_nickname_tilde = config.get( CHAT, @@ -489,6 +493,7 @@ public static class Backups { public boolean silent_backup; public int max_folder_size; public boolean delete_custom_name_backups; + public boolean only_backup_claimed_chunks; } public static class Login { diff --git a/src/main/java/serverutils/data/ClaimedChunks.java b/src/main/java/serverutils/data/ClaimedChunks.java index 86a03767..766f6d95 100644 --- a/src/main/java/serverutils/data/ClaimedChunks.java +++ b/src/main/java/serverutils/data/ClaimedChunks.java @@ -172,6 +172,10 @@ public Collection getAllChunks() { return map.isEmpty() ? Collections.emptyList() : map.values(); } + public Set getAllClaimedPositions() { + return map.isEmpty() ? Collections.emptySet() : map.keySet(); + } + public Set getTeamChunks(@Nullable ForgeTeam team, OptionalInt dimension, boolean includePending) { if (team == null) { return Collections.emptySet(); diff --git a/src/main/java/serverutils/lib/math/ChunkDimPos.java b/src/main/java/serverutils/lib/math/ChunkDimPos.java index acbd580e..b5f03b0d 100644 --- a/src/main/java/serverutils/lib/math/ChunkDimPos.java +++ b/src/main/java/serverutils/lib/math/ChunkDimPos.java @@ -3,9 +3,13 @@ import net.minecraft.entity.Entity; import net.minecraft.world.ChunkCoordIntPair; +import com.gtnewhorizon.gtnhlib.util.CoordinatePacker; + public final class ChunkDimPos { - public final int posX, posZ, dim; + public int posX, posZ, dim; + + public ChunkDimPos() {} public ChunkDimPos(int x, int z, int d) { posX = x; @@ -25,6 +29,17 @@ public ChunkDimPos(Entity entity) { this(MathUtils.chunk(entity.posX), MathUtils.chunk(entity.posZ), entity.worldObj.provider.dimensionId); } + public ChunkDimPos set(int x, int z, int dim) { + this.posX = x; + this.posZ = z; + this.dim = dim; + return this; + } + + public ChunkDimPos set(long packed, int dim) { + return set(CoordinatePacker.unpackX(packed), CoordinatePacker.unpackZ(packed), dim); + } + public boolean equals(Object o) { if (o == null) { return false; diff --git a/src/main/java/serverutils/lib/util/FileUtils.java b/src/main/java/serverutils/lib/util/FileUtils.java index 48e9d057..5b12f68c 100644 --- a/src/main/java/serverutils/lib/util/FileUtils.java +++ b/src/main/java/serverutils/lib/util/FileUtils.java @@ -230,4 +230,8 @@ public static String getBaseName(File file) { return index == -1 ? name : name.substring(0, index); } } + + public static String getRelativePath(File dir, File file) { + return dir.getName() + File.separator + file.getAbsolutePath().substring(dir.getAbsolutePath().length() + 1); + } } diff --git a/src/main/java/serverutils/task/backup/BackupTask.java b/src/main/java/serverutils/task/backup/BackupTask.java index ed117a8e..49816684 100644 --- a/src/main/java/serverutils/task/backup/BackupTask.java +++ b/src/main/java/serverutils/task/backup/BackupTask.java @@ -7,8 +7,10 @@ import java.io.File; import java.util.Arrays; import java.util.Comparator; +import java.util.HashSet; import java.util.Iterator; import java.util.List; +import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -22,7 +24,9 @@ import serverutils.ServerUtilities; import serverutils.ServerUtilitiesConfig; import serverutils.ServerUtilitiesNotifications; +import serverutils.data.ClaimedChunks; import serverutils.lib.data.Universe; +import serverutils.lib.math.ChunkDimPos; import serverutils.lib.math.Ticks; import serverutils.lib.util.FileUtils; import serverutils.lib.util.ServerUtils; @@ -31,6 +35,7 @@ public class BackupTask extends Task { public static final Pattern BACKUP_NAME_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}(.*)"); + public static final File BACKUP_TEMP_FOLDER = new File("serverutilities/temp/"); public static File backupsFolder; public static ThreadBackup thread; public static boolean hadPlayer = false; @@ -97,11 +102,17 @@ public void execute(Universe universe) { File worldDir = DimensionManager.getCurrentSaveRootDirectory(); + Set backupChunks = new HashSet<>(); + if (backups.only_backup_claimed_chunks && ClaimedChunks.isActive()) { + backupChunks.addAll(ClaimedChunks.instance.getAllClaimedPositions()); + BACKUP_TEMP_FOLDER.mkdirs(); + } + if (backups.use_separate_thread) { - thread = new ThreadBackup(worldDir, customName); + thread = new ThreadBackup(worldDir, customName, backupChunks); thread.start(); } else { - ThreadBackup.doBackup(worldDir, customName); + ThreadBackup.doBackup(worldDir, customName, backupChunks); } universe.scheduleTask(new BackupTask(true)); } @@ -161,6 +172,8 @@ private void postBackup(Universe universe) { } clearOldBackups(); + FileUtils.delete(BACKUP_TEMP_FOLDER); + thread = null; try { MinecraftServer server = ServerUtils.getServer(); diff --git a/src/main/java/serverutils/task/backup/ThreadBackup.java b/src/main/java/serverutils/task/backup/ThreadBackup.java index c2362a9f..7d1ab236 100644 --- a/src/main/java/serverutils/task/backup/ThreadBackup.java +++ b/src/main/java/serverutils/task/backup/ThreadBackup.java @@ -3,24 +3,45 @@ import static serverutils.ServerUtilitiesConfig.backups; import static serverutils.ServerUtilitiesNotifications.BACKUP_END1; import static serverutils.ServerUtilitiesNotifications.BACKUP_END2; +import static serverutils.task.backup.BackupTask.BACKUP_TEMP_FOLDER; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.File; import java.io.FileInputStream; +import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.List; +import java.util.Set; import java.util.zip.ZipEntry; +import net.minecraft.nbt.CompressedStreamTools; +import net.minecraft.nbt.NBTTagCompound; +import net.minecraft.server.MinecraftServer; import net.minecraft.util.EnumChatFormatting; import net.minecraft.util.IChatComponent; +import net.minecraft.world.WorldServer; +import net.minecraft.world.chunk.storage.RegionFile; +import net.minecraft.world.chunk.storage.RegionFileCache; +import org.apache.commons.compress.archivers.ArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; import org.apache.commons.io.IOUtils; +import com.gtnewhorizon.gtnhlib.util.CoordinatePacker; + +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.longs.LongOpenHashSet; +import it.unimi.dsi.fastutil.longs.LongSet; import serverutils.ServerUtilities; import serverutils.ServerUtilitiesNotifications; +import serverutils.lib.math.ChunkDimPos; import serverutils.lib.math.Ticks; import serverutils.lib.util.FileUtils; import serverutils.lib.util.ServerUtils; @@ -29,78 +50,67 @@ public class ThreadBackup extends Thread { private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); - private static long logMillis; private final File src0; private final String customName; + private final Set chunksToBackup; public boolean isDone = false; - public ThreadBackup(File w, String s) { - src0 = w; - customName = s; + public ThreadBackup(File sourceFile, String backupName, Set backupChunks) { + src0 = sourceFile; + customName = backupName; + chunksToBackup = backupChunks; setPriority(7); } public void run() { isDone = false; - doBackup(src0, customName); + doBackup(src0, customName, chunksToBackup); isDone = true; } - public static void doBackup(File src, String customName) { + public static void doBackup(File src, String customName, Set chunks) { String outName = (customName.isEmpty() ? DATE_FORMAT.format(Calendar.getInstance().getTime()) : customName) + ".zip"; File dstFile = null; try { List files = FileUtils.listTree(src); - int allFiles = files.size(); - ServerUtilities.LOGGER.info("Backing up {} files...", files.size()); long start = System.currentTimeMillis(); + logMillis = start + Ticks.SECOND.x(5).millis(); dstFile = FileUtils.newFile(new File(BackupTask.backupsFolder, outName)); try (ZipArchiveOutputStream zaos = new ZipArchiveOutputStream(dstFile)) { - if (backups.compression_level == 0) { zaos.setMethod(ZipEntry.STORED); } else { zaos.setLevel(backups.compression_level); } - logMillis = System.currentTimeMillis() + Ticks.SECOND.x(5).millis(); - ServerUtilities.LOGGER.info("Backing up {} files!", allFiles); - - for (int i = 0; i < allFiles; i++) { - File file = files.get(i); - String filePath = file.getAbsolutePath(); - ZipArchiveEntry entry = new ZipArchiveEntry( - src.getName() + File.separator + filePath.substring(src.getAbsolutePath().length() + 1)); - - logProgress(i, allFiles, filePath); - zaos.putArchiveEntry(entry); - try (FileInputStream fis = new FileInputStream(file)) { - IOUtils.copy(fis, zaos); - } - zaos.closeArchiveEntry(); + if (!chunks.isEmpty() && backups.only_backup_claimed_chunks) { + backupRegions(src, files, chunks, zaos); + } else { + compressFiles(src, files, zaos); + } + + String backupSize = FileUtils.getSizeString(dstFile); + ServerUtilities.LOGGER.info("Backup done in {} seconds ({})!", getDoneTime(start), backupSize); + ServerUtilities.LOGGER.info("Created {} from {}", dstFile.getAbsolutePath(), src.getAbsolutePath()); + + if (backups.display_file_size) { + String sizeT = FileUtils.getSizeString(BackupTask.backupsFolder); + ServerUtilitiesNotifications.backupNotification( + BACKUP_END2, + "cmd.backup_end_2", + getDoneTime(start), + (backupSize.equals(sizeT) ? backupSize : (backupSize + " | " + sizeT))); + } else { + ServerUtilitiesNotifications + .backupNotification(BACKUP_END1, "cmd.backup_end_1", getDoneTime(start)); } - } - String backupSize = FileUtils.getSizeString(dstFile); - ServerUtilities.LOGGER.info("Backup done in {} seconds ({})!", getDoneTime(start), backupSize); - ServerUtilities.LOGGER.info("Created {} from {}", dstFile.getAbsolutePath(), src.getAbsolutePath()); - - if (backups.display_file_size) { - String sizeT = FileUtils.getSizeString(BackupTask.backupsFolder); - ServerUtilitiesNotifications.backupNotification( - BACKUP_END2, - "cmd.backup_end_2", - getDoneTime(start), - (backupSize.equals(sizeT) ? backupSize : (backupSize + " | " + sizeT))); - } else { - ServerUtilitiesNotifications.backupNotification(BACKUP_END1, "cmd.backup_end_1", getDoneTime(start)); } } catch (Exception e) { - IChatComponent c = StringUtils.color( - ServerUtilities.lang(null, "cmd.backup_fail", e.getClass().getName()), - EnumChatFormatting.RED); + IChatComponent c = StringUtils + .color(ServerUtilities.lang(null, "cmd.backup_fail", e), EnumChatFormatting.RED); ServerUtils.notifyChat(ServerUtils.getServer(), null, c); ServerUtilities.LOGGER.error("Error while backing up", e); @@ -110,14 +120,149 @@ public static void doBackup(File src, String customName) { private static void logProgress(int i, int allFiles, String name) { long millis = System.currentTimeMillis(); - if (i == 0 || millis > logMillis || i == allFiles - 1) { + boolean first = i == 0; + if (first) { + ServerUtilities.LOGGER.info("Backing up {} files...", allFiles); + } + + if (first || millis > logMillis || i == allFiles - 1) { logMillis = millis + Ticks.SECOND.x(5).millis(); ServerUtilities.LOGGER .info("[{} | {}%]: {}", i, StringUtils.formatDouble00((i / (double) allFiles) * 100D), name); } } + private static void compressFiles(File sourceDir, List files, ZipArchiveOutputStream zaos) + throws IOException { + int allFiles = files.size(); + for (int i = 0; i < allFiles; i++) { + File file = files.get(i); + compressFile(FileUtils.getRelativePath(sourceDir, file), file, zaos, i, allFiles); + } + } + + private static void compressFile(String entryName, File file, ZipArchiveOutputStream out, int index, int totalFiles) + throws IOException { + ArchiveEntry entry = new ZipArchiveEntry(file, entryName); + logProgress(index, totalFiles, file.getAbsolutePath()); + out.putArchiveEntry(entry); + try (FileInputStream fis = new FileInputStream(file)) { + IOUtils.copy(fis, out); + } + out.closeArchiveEntry(); + + } + + private static void backupRegions(File sourceFolder, List files, Set chunksToBackup, + ZipArchiveOutputStream out) throws IOException { + Int2ObjectMap> claimedChunks = mapClaimsToRegion(chunksToBackup); + Int2ObjectMap> dim2RegionFiles = mapRegionFilesToLong(claimedChunks); + files.removeIf(f -> f.getName().endsWith(".mca")); + + int index = 0; + int savedChunks = 0; + int totalFiles = files.size(); + ChunkDimPos mutableTemp = new ChunkDimPos(); + for (Int2ObjectMap.Entry> dimEntry : dim2RegionFiles.int2ObjectEntrySet()) { + Long2ObjectMap regionFiles = dimEntry.getValue(); + Long2ObjectMap dimClaims = claimedChunks.get(dimEntry.getIntKey()); + totalFiles += regionFiles.size(); + int dim = dimEntry.getIntKey(); + for (Long2ObjectMap.Entry entry : regionFiles.long2ObjectEntrySet()) { + File file = entry.getValue(); + LongSet chunks = dimClaims.get(entry.getLongKey()); + if (chunks == null || chunks.isEmpty()) continue; + File dimensionRoot = file.getParentFile().getParentFile(); + + File tempFile = FileUtils.newFile(new File(BACKUP_TEMP_FOLDER, file.getName())); + RegionFile tempRegion = new RegionFile(tempFile); + boolean hasData = false; + for (long pos : chunks) { + mutableTemp.set(pos, dim); + DataInputStream in = RegionFileCache + .getChunkInputStream(dimensionRoot, mutableTemp.posX, mutableTemp.posZ); + + if (in == null) continue; + savedChunks++; + hasData = true; + NBTTagCompound tag = CompressedStreamTools.read(in); + DataOutputStream tempOut = tempRegion + .getChunkDataOutputStream(mutableTemp.posX & 31, mutableTemp.posZ & 31); + CompressedStreamTools.write(tag, tempOut); + tempOut.close(); + } + + tempRegion.close(); + if (hasData) { + compressFile(FileUtils.getRelativePath(sourceFolder, file), tempFile, out, index++, totalFiles); + } + + FileUtils.delete(tempFile); + } + + for (File file : files) { + compressFile(FileUtils.getRelativePath(sourceFolder, file), file, out, index++, totalFiles); + } + + ServerUtilities.LOGGER + .info("Backed up {} regions containing {} claimed chunks", regionFiles.size(), savedChunks); + } + } + + private static Int2ObjectMap> mapRegionFilesToLong( + Int2ObjectMap> dimClaims) { + Int2ObjectMap> regionFiles = new Int2ObjectOpenHashMap<>(); + MinecraftServer server = ServerUtils.getServer(); + for (WorldServer worldserver : server.worldServers) { + if (worldserver == null) continue; + + int dim = worldserver.provider.dimensionId; + File regionFolder = new File(worldserver.getChunkSaveLocation(), "region"); + if (!regionFolder.exists()) continue; + + File[] regions = regionFolder.listFiles(); + if (regions == null) continue; + + Long2ObjectMap regionClaims = dimClaims.get(dim); + for (File file : regions) { + int[] coords = getRegionCoords(file); + long key = CoordinatePacker.pack(coords[0], 0, coords[1]); + if (!regionClaims.containsKey(key)) { + ServerUtilities.LOGGER.info("Skipping region file: {}", file.getName()); + continue; + } + regionFiles.computeIfAbsent(dim, k -> new Long2ObjectOpenHashMap<>()).put(key, file); + } + } + return regionFiles; + } + + private static Int2ObjectMap> mapClaimsToRegion(Set chunksToBackup) { + Int2ObjectMap> dimClaims = new Int2ObjectOpenHashMap<>(); + chunksToBackup.forEach(pos -> { + long region = getRegionFromChunk(pos.posX, pos.posZ); + dimClaims.computeIfAbsent(pos.dim, k -> new Long2ObjectOpenHashMap<>()) + .computeIfAbsent(region, k -> new LongOpenHashSet()) + .add(CoordinatePacker.pack(pos.posX, 0, pos.posZ)); + }); + return dimClaims; + } + + private static int[] getRegionCoords(File f) { + String fileName = f.getName(); + int firstDot = fileName.indexOf('.'); + int secondDot = fileName.indexOf('.', firstDot + 1); + + int x = Integer.parseInt(fileName.substring(firstDot + 1, secondDot)); + int z = Integer.parseInt(fileName.substring(secondDot + 1, fileName.lastIndexOf('.'))); + return new int[] { x, z }; + } + private static String getDoneTime(long l) { return StringUtils.getTimeString(System.currentTimeMillis() - l); } + + private static long getRegionFromChunk(int chunkX, int chunkZ) { + return CoordinatePacker.pack(chunkX >> 5, 0, chunkZ >> 5); + } } From 5b83777c31e15f0a6d80232f4c63f667f5a8be36 Mon Sep 17 00:00:00 2001 From: Lyfts <127234178+Lyfts@users.noreply.github.com> Date: Thu, 5 Sep 2024 16:36:48 +0200 Subject: [PATCH 08/12] no need to remap the ChunkDimPos' to long --- .../serverutils/task/backup/ThreadBackup.java | 40 +++++++++---------- 1 file changed, 19 insertions(+), 21 deletions(-) diff --git a/src/main/java/serverutils/task/backup/ThreadBackup.java b/src/main/java/serverutils/task/backup/ThreadBackup.java index 7d1ab236..3b39126f 100644 --- a/src/main/java/serverutils/task/backup/ThreadBackup.java +++ b/src/main/java/serverutils/task/backup/ThreadBackup.java @@ -37,9 +37,10 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; -import it.unimi.dsi.fastutil.longs.LongOpenHashSet; -import it.unimi.dsi.fastutil.longs.LongSet; +import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; +import it.unimi.dsi.fastutil.objects.ObjectSet; import serverutils.ServerUtilities; +import serverutils.ServerUtilitiesConfig; import serverutils.ServerUtilitiesNotifications; import serverutils.lib.math.ChunkDimPos; import serverutils.lib.math.Ticks; @@ -155,39 +156,33 @@ private static void compressFile(String entryName, File file, ZipArchiveOutputSt private static void backupRegions(File sourceFolder, List files, Set chunksToBackup, ZipArchiveOutputStream out) throws IOException { - Int2ObjectMap> claimedChunks = mapClaimsToRegion(chunksToBackup); + Int2ObjectMap>> claimedChunks = mapClaimsToRegion(chunksToBackup); Int2ObjectMap> dim2RegionFiles = mapRegionFilesToLong(claimedChunks); files.removeIf(f -> f.getName().endsWith(".mca")); int index = 0; int savedChunks = 0; int totalFiles = files.size(); - ChunkDimPos mutableTemp = new ChunkDimPos(); for (Int2ObjectMap.Entry> dimEntry : dim2RegionFiles.int2ObjectEntrySet()) { Long2ObjectMap regionFiles = dimEntry.getValue(); - Long2ObjectMap dimClaims = claimedChunks.get(dimEntry.getIntKey()); + Long2ObjectMap> dimClaims = claimedChunks.get(dimEntry.getIntKey()); totalFiles += regionFiles.size(); - int dim = dimEntry.getIntKey(); for (Long2ObjectMap.Entry entry : regionFiles.long2ObjectEntrySet()) { File file = entry.getValue(); - LongSet chunks = dimClaims.get(entry.getLongKey()); + ObjectSet chunks = dimClaims.get(entry.getLongKey()); if (chunks == null || chunks.isEmpty()) continue; File dimensionRoot = file.getParentFile().getParentFile(); - File tempFile = FileUtils.newFile(new File(BACKUP_TEMP_FOLDER, file.getName())); RegionFile tempRegion = new RegionFile(tempFile); boolean hasData = false; - for (long pos : chunks) { - mutableTemp.set(pos, dim); - DataInputStream in = RegionFileCache - .getChunkInputStream(dimensionRoot, mutableTemp.posX, mutableTemp.posZ); + for (ChunkDimPos pos : chunks) { + DataInputStream in = RegionFileCache.getChunkInputStream(dimensionRoot, pos.posX, pos.posZ); if (in == null) continue; savedChunks++; hasData = true; NBTTagCompound tag = CompressedStreamTools.read(in); - DataOutputStream tempOut = tempRegion - .getChunkDataOutputStream(mutableTemp.posX & 31, mutableTemp.posZ & 31); + DataOutputStream tempOut = tempRegion.getChunkDataOutputStream(pos.posX & 31, pos.posZ & 31); CompressedStreamTools.write(tag, tempOut); tempOut.close(); } @@ -210,7 +205,7 @@ private static void backupRegions(File sourceFolder, List files, Set> mapRegionFilesToLong( - Int2ObjectMap> dimClaims) { + Int2ObjectMap>> dimClaims) { Int2ObjectMap> regionFiles = new Int2ObjectOpenHashMap<>(); MinecraftServer server = ServerUtils.getServer(); for (WorldServer worldserver : server.worldServers) { @@ -223,12 +218,15 @@ private static Int2ObjectMap> mapRegionFilesToLong( File[] regions = regionFolder.listFiles(); if (regions == null) continue; - Long2ObjectMap regionClaims = dimClaims.get(dim); + Long2ObjectMap> regionClaims = dimClaims.get(dim); + if (regionClaims == null) continue; for (File file : regions) { int[] coords = getRegionCoords(file); long key = CoordinatePacker.pack(coords[0], 0, coords[1]); if (!regionClaims.containsKey(key)) { - ServerUtilities.LOGGER.info("Skipping region file: {}", file.getName()); + if (ServerUtilitiesConfig.debugging.print_more_info) { + ServerUtilities.LOGGER.info("Skipping region file {} from dimension {}", file.getName(), dim); + } continue; } regionFiles.computeIfAbsent(dim, k -> new Long2ObjectOpenHashMap<>()).put(key, file); @@ -237,13 +235,13 @@ private static Int2ObjectMap> mapRegionFilesToLong( return regionFiles; } - private static Int2ObjectMap> mapClaimsToRegion(Set chunksToBackup) { - Int2ObjectMap> dimClaims = new Int2ObjectOpenHashMap<>(); + private static Int2ObjectMap>> mapClaimsToRegion( + Set chunksToBackup) { + Int2ObjectMap>> dimClaims = new Int2ObjectOpenHashMap<>(); chunksToBackup.forEach(pos -> { long region = getRegionFromChunk(pos.posX, pos.posZ); dimClaims.computeIfAbsent(pos.dim, k -> new Long2ObjectOpenHashMap<>()) - .computeIfAbsent(region, k -> new LongOpenHashSet()) - .add(CoordinatePacker.pack(pos.posX, 0, pos.posZ)); + .computeIfAbsent(region, k -> new ObjectOpenHashSet<>()).add(pos); }); return dimClaims; } From a6ee988a3e2ef070b740ad8213840140776b52e5 Mon Sep 17 00:00:00 2001 From: Lyfts <127234178+Lyfts@users.noreply.github.com> Date: Thu, 5 Sep 2024 17:45:07 +0200 Subject: [PATCH 09/12] much more optimized solution for mapping claims to their region file --- .../serverutils/task/backup/ThreadBackup.java | 107 ++++++++---------- 1 file changed, 47 insertions(+), 60 deletions(-) diff --git a/src/main/java/serverutils/task/backup/ThreadBackup.java b/src/main/java/serverutils/task/backup/ThreadBackup.java index 3b39126f..3bd8edc7 100644 --- a/src/main/java/serverutils/task/backup/ThreadBackup.java +++ b/src/main/java/serverutils/task/backup/ThreadBackup.java @@ -19,7 +19,6 @@ import net.minecraft.nbt.CompressedStreamTools; import net.minecraft.nbt.NBTTagCompound; -import net.minecraft.server.MinecraftServer; import net.minecraft.util.EnumChatFormatting; import net.minecraft.util.IChatComponent; import net.minecraft.world.WorldServer; @@ -37,6 +36,8 @@ import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; import it.unimi.dsi.fastutil.longs.Long2ObjectMap; import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.objects.Object2ObjectMap; +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; import it.unimi.dsi.fastutil.objects.ObjectSet; import serverutils.ServerUtilities; @@ -156,94 +157,80 @@ private static void compressFile(String entryName, File file, ZipArchiveOutputSt private static void backupRegions(File sourceFolder, List files, Set chunksToBackup, ZipArchiveOutputStream out) throws IOException { - Int2ObjectMap>> claimedChunks = mapClaimsToRegion(chunksToBackup); - Int2ObjectMap> dim2RegionFiles = mapRegionFilesToLong(claimedChunks); + Object2ObjectMap> dimRegionClaims = mapClaimsToRegionFile(chunksToBackup); files.removeIf(f -> f.getName().endsWith(".mca")); int index = 0; int savedChunks = 0; - int totalFiles = files.size(); - for (Int2ObjectMap.Entry> dimEntry : dim2RegionFiles.int2ObjectEntrySet()) { - Long2ObjectMap regionFiles = dimEntry.getValue(); - Long2ObjectMap> dimClaims = claimedChunks.get(dimEntry.getIntKey()); - totalFiles += regionFiles.size(); - for (Long2ObjectMap.Entry entry : regionFiles.long2ObjectEntrySet()) { - File file = entry.getValue(); - ObjectSet chunks = dimClaims.get(entry.getLongKey()); - if (chunks == null || chunks.isEmpty()) continue; - File dimensionRoot = file.getParentFile().getParentFile(); - File tempFile = FileUtils.newFile(new File(BACKUP_TEMP_FOLDER, file.getName())); - RegionFile tempRegion = new RegionFile(tempFile); - boolean hasData = false; - - for (ChunkDimPos pos : chunks) { - DataInputStream in = RegionFileCache.getChunkInputStream(dimensionRoot, pos.posX, pos.posZ); - if (in == null) continue; - savedChunks++; - hasData = true; - NBTTagCompound tag = CompressedStreamTools.read(in); - DataOutputStream tempOut = tempRegion.getChunkDataOutputStream(pos.posX & 31, pos.posZ & 31); - CompressedStreamTools.write(tag, tempOut); - tempOut.close(); - } - - tempRegion.close(); - if (hasData) { - compressFile(FileUtils.getRelativePath(sourceFolder, file), tempFile, out, index++, totalFiles); - } - - FileUtils.delete(tempFile); + int regionFiles = dimRegionClaims.size(); + int totalFiles = files.size() + regionFiles; + for (Object2ObjectMap.Entry> entry : dimRegionClaims.object2ObjectEntrySet()) { + File file = entry.getKey(); + File dimensionRoot = file.getParentFile().getParentFile(); + File tempFile = FileUtils.newFile(new File(BACKUP_TEMP_FOLDER, file.getName())); + RegionFile tempRegion = new RegionFile(tempFile); + boolean hasData = false; + + for (ChunkDimPos pos : entry.getValue()) { + DataInputStream in = RegionFileCache.getChunkInputStream(dimensionRoot, pos.posX, pos.posZ); + if (in == null) continue; + savedChunks++; + hasData = true; + NBTTagCompound tag = CompressedStreamTools.read(in); + DataOutputStream tempOut = tempRegion.getChunkDataOutputStream(pos.posX & 31, pos.posZ & 31); + CompressedStreamTools.write(tag, tempOut); + tempOut.close(); } - for (File file : files) { - compressFile(FileUtils.getRelativePath(sourceFolder, file), file, out, index++, totalFiles); + tempRegion.close(); + if (hasData) { + compressFile(FileUtils.getRelativePath(sourceFolder, file), tempFile, out, index++, totalFiles); } - ServerUtilities.LOGGER - .info("Backed up {} regions containing {} claimed chunks", regionFiles.size(), savedChunks); + FileUtils.delete(tempFile); + } + + for (File file : files) { + compressFile(FileUtils.getRelativePath(sourceFolder, file), file, out, index++, totalFiles); } + + ServerUtilities.LOGGER.info("Backed up {} regions containing {} claimed chunks", regionFiles, savedChunks); } - private static Int2ObjectMap> mapRegionFilesToLong( - Int2ObjectMap>> dimClaims) { - Int2ObjectMap> regionFiles = new Int2ObjectOpenHashMap<>(); - MinecraftServer server = ServerUtils.getServer(); - for (WorldServer worldserver : server.worldServers) { + private static Object2ObjectMap> mapClaimsToRegionFile( + Set chunksToBackup) { + Int2ObjectMap>> regionClaimsByDim = new Int2ObjectOpenHashMap<>(); + chunksToBackup.forEach( + pos -> regionClaimsByDim.computeIfAbsent(pos.dim, k -> new Long2ObjectOpenHashMap<>()) + .computeIfAbsent(getRegionFromChunk(pos.posX, pos.posZ), k -> new ObjectOpenHashSet<>()) + .add(pos)); + + Object2ObjectMap> regionFilesToBackup = new Object2ObjectOpenHashMap<>(); + for (WorldServer worldserver : ServerUtils.getServer().worldServers) { if (worldserver == null) continue; int dim = worldserver.provider.dimensionId; File regionFolder = new File(worldserver.getChunkSaveLocation(), "region"); - if (!regionFolder.exists()) continue; + Long2ObjectMap> regionClaims = regionClaimsByDim.get(dim); + if (!regionFolder.exists() || regionClaims == null) continue; File[] regions = regionFolder.listFiles(); if (regions == null) continue; - Long2ObjectMap> regionClaims = dimClaims.get(dim); - if (regionClaims == null) continue; for (File file : regions) { int[] coords = getRegionCoords(file); long key = CoordinatePacker.pack(coords[0], 0, coords[1]); - if (!regionClaims.containsKey(key)) { + ObjectSet claims = regionClaims.get(key); + if (claims == null) { if (ServerUtilitiesConfig.debugging.print_more_info) { ServerUtilities.LOGGER.info("Skipping region file {} from dimension {}", file.getName(), dim); } continue; } - regionFiles.computeIfAbsent(dim, k -> new Long2ObjectOpenHashMap<>()).put(key, file); + regionFilesToBackup.put(file, claims); } } - return regionFiles; - } - - private static Int2ObjectMap>> mapClaimsToRegion( - Set chunksToBackup) { - Int2ObjectMap>> dimClaims = new Int2ObjectOpenHashMap<>(); - chunksToBackup.forEach(pos -> { - long region = getRegionFromChunk(pos.posX, pos.posZ); - dimClaims.computeIfAbsent(pos.dim, k -> new Long2ObjectOpenHashMap<>()) - .computeIfAbsent(region, k -> new ObjectOpenHashSet<>()).add(pos); - }); - return dimClaims; + return regionFilesToBackup; } private static int[] getRegionCoords(File f) { From 8182c277c2bd5376cdb7372ff04c954162b71c68 Mon Sep 17 00:00:00 2001 From: Lyfts <127234178+Lyfts@users.noreply.github.com> Date: Fri, 6 Sep 2024 18:08:21 +0200 Subject: [PATCH 10/12] update configs --- src/main/java/serverutils/ServerUtilitiesConfig.java | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/src/main/java/serverutils/ServerUtilitiesConfig.java b/src/main/java/serverutils/ServerUtilitiesConfig.java index 9c270af5..277190bc 100644 --- a/src/main/java/serverutils/ServerUtilitiesConfig.java +++ b/src/main/java/serverutils/ServerUtilitiesConfig.java @@ -312,6 +312,16 @@ public static class Backups { @Config.Comment("Silence backup notifications.") @Config.DefaultBoolean(false) public boolean silent_backup; + + @Config.Comment(""" + Max size of backup folder in GB. If total folder size exceeds this value it will delete old backups until the size is under. + 0 = Disabled and backups_to_keep will be used instead.""") + @Config.DefaultInt(0) + public int max_folder_size; + + @Config.Comment("Delete backups that have a custom name set through /backup start ") + @Config.DefaultBoolean(true) + public boolean delete_custom_name_backups; } public static class Login { From 8377ee18dc0b85d3e6f47b3087ae6fd42d392a03 Mon Sep 17 00:00:00 2001 From: Lyfts <127234178+Lyfts@users.noreply.github.com> Date: Fri, 6 Sep 2024 18:13:08 +0200 Subject: [PATCH 11/12] update configs --- src/main/java/serverutils/ServerUtilitiesConfig.java | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/src/main/java/serverutils/ServerUtilitiesConfig.java b/src/main/java/serverutils/ServerUtilitiesConfig.java index 277190bc..8d8c9b66 100644 --- a/src/main/java/serverutils/ServerUtilitiesConfig.java +++ b/src/main/java/serverutils/ServerUtilitiesConfig.java @@ -322,6 +322,12 @@ public static class Backups { @Config.Comment("Delete backups that have a custom name set through /backup start ") @Config.DefaultBoolean(true) public boolean delete_custom_name_backups; + + @Config.Comment(""" + Only include claimed chunks in backup. + Backups will be much faster and smaller, but any unclaimed chunk will be unrecoverable.""") + @Config.DefaultBoolean(false) + public boolean only_backup_claimed_chunks; } public static class Login { From cc18a9e7b90bf16f3185e9252cc7e635cd709987 Mon Sep 17 00:00:00 2001 From: Lyfts <127234178+Lyfts@users.noreply.github.com> Date: Mon, 9 Sep 2024 19:46:57 +0200 Subject: [PATCH 12/12] make backups fall back to using native java zip if commons compress is somehow missing --- .../util/compression/CommonsCompressor.java | 46 +++++++++++++++++ .../lib/util/compression/ICompress.java | 11 ++++ .../util/compression/LegacyCompressor.java | 46 +++++++++++++++++ .../serverutils/task/backup/BackupTask.java | 17 ++++++- .../serverutils/task/backup/ThreadBackup.java | 50 +++++++------------ 5 files changed, 135 insertions(+), 35 deletions(-) create mode 100644 src/main/java/serverutils/lib/util/compression/CommonsCompressor.java create mode 100644 src/main/java/serverutils/lib/util/compression/ICompress.java create mode 100644 src/main/java/serverutils/lib/util/compression/LegacyCompressor.java diff --git a/src/main/java/serverutils/lib/util/compression/CommonsCompressor.java b/src/main/java/serverutils/lib/util/compression/CommonsCompressor.java new file mode 100644 index 00000000..176cb461 --- /dev/null +++ b/src/main/java/serverutils/lib/util/compression/CommonsCompressor.java @@ -0,0 +1,46 @@ +package serverutils.lib.util.compression; + +import static serverutils.ServerUtilitiesConfig.backups; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.zip.ZipEntry; + +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.io.IOUtils; + +public class CommonsCompressor implements ICompress { + + private ArchiveOutputStream output; + + @Override + public void createOutputStream(File file) throws IOException { + ZipArchiveOutputStream zaos = new ZipArchiveOutputStream(file); + if (backups.compression_level == 0) { + zaos.setMethod(ZipEntry.STORED); + } else { + zaos.setLevel(backups.compression_level); + } + output = zaos; + } + + @Override + public void addFileToArchive(File file, String name) throws IOException { + ArchiveEntry entry = output.createArchiveEntry(file, name); + output.putArchiveEntry(entry); + try (FileInputStream fis = new FileInputStream(file)) { + IOUtils.copy(fis, output); + } + output.closeArchiveEntry(); + } + + @Override + public void close() throws Exception { + if (output != null) { + output.close(); + } + } +} diff --git a/src/main/java/serverutils/lib/util/compression/ICompress.java b/src/main/java/serverutils/lib/util/compression/ICompress.java new file mode 100644 index 00000000..426a2336 --- /dev/null +++ b/src/main/java/serverutils/lib/util/compression/ICompress.java @@ -0,0 +1,11 @@ +package serverutils.lib.util.compression; + +import java.io.File; +import java.io.IOException; + +public interface ICompress extends AutoCloseable { + + void createOutputStream(File file) throws IOException; + + void addFileToArchive(File file, String name) throws IOException; +} diff --git a/src/main/java/serverutils/lib/util/compression/LegacyCompressor.java b/src/main/java/serverutils/lib/util/compression/LegacyCompressor.java new file mode 100644 index 00000000..7e3d3803 --- /dev/null +++ b/src/main/java/serverutils/lib/util/compression/LegacyCompressor.java @@ -0,0 +1,46 @@ +package serverutils.lib.util.compression; + +import static serverutils.ServerUtilitiesConfig.backups; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.apache.commons.io.IOUtils; + +public class LegacyCompressor implements ICompress { + + private ZipOutputStream output; + + @Override + public void createOutputStream(File file) throws IOException { + ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(file)); + if (backups.compression_level == 0) { + zos.setMethod(ZipOutputStream.STORED); + } else { + zos.setLevel(backups.compression_level); + } + + output = zos; + } + + @Override + public void addFileToArchive(File file, String name) throws IOException { + ZipEntry entry = new ZipEntry(name); + output.putNextEntry(entry); + try (FileInputStream fis = new FileInputStream(file)) { + IOUtils.copy(fis, output); + } + output.closeEntry(); + } + + @Override + public void close() throws Exception { + if (output != null) { + output.close(); + } + } +} diff --git a/src/main/java/serverutils/task/backup/BackupTask.java b/src/main/java/serverutils/task/backup/BackupTask.java index 98a595a9..e4620b0a 100644 --- a/src/main/java/serverutils/task/backup/BackupTask.java +++ b/src/main/java/serverutils/task/backup/BackupTask.java @@ -27,14 +27,19 @@ import serverutils.lib.data.Universe; import serverutils.lib.math.ChunkDimPos; import serverutils.lib.math.Ticks; +import serverutils.lib.util.CommonUtils; import serverutils.lib.util.FileUtils; import serverutils.lib.util.ServerUtils; +import serverutils.lib.util.compression.CommonsCompressor; +import serverutils.lib.util.compression.ICompress; +import serverutils.lib.util.compression.LegacyCompressor; import serverutils.task.Task; public class BackupTask extends Task { public static final Pattern BACKUP_NAME_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}(.*)"); public static final File BACKUP_TEMP_FOLDER = new File("serverutilities/temp/"); + private static final boolean useLegacy; public static File backupsFolder; public static ThreadBackup thread; public static boolean hadPlayer = false; @@ -48,6 +53,7 @@ public class BackupTask extends Task { if (!backupsFolder.exists()) backupsFolder.mkdirs(); clearOldBackups(); ServerUtilities.LOGGER.info("Backups folder - {}", backupsFolder.getAbsolutePath()); + useLegacy = !CommonUtils.getClassExists("org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream"); } public BackupTask() { @@ -107,11 +113,18 @@ public void execute(Universe universe) { BACKUP_TEMP_FOLDER.mkdirs(); } + ICompress compressor; + if (useLegacy) { + compressor = new LegacyCompressor(); + } else { + compressor = new CommonsCompressor(); + } + if (backups.use_separate_thread) { - thread = new ThreadBackup(worldDir, customName, backupChunks); + thread = new ThreadBackup(compressor, worldDir, customName, backupChunks); thread.start(); } else { - ThreadBackup.doBackup(worldDir, customName, backupChunks); + ThreadBackup.doBackup(compressor, worldDir, customName, backupChunks); } universe.scheduleTask(new BackupTask(true)); } diff --git a/src/main/java/serverutils/task/backup/ThreadBackup.java b/src/main/java/serverutils/task/backup/ThreadBackup.java index 3bd8edc7..31cf4d6b 100644 --- a/src/main/java/serverutils/task/backup/ThreadBackup.java +++ b/src/main/java/serverutils/task/backup/ThreadBackup.java @@ -8,14 +8,12 @@ import java.io.DataInputStream; import java.io.DataOutputStream; import java.io.File; -import java.io.FileInputStream; import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.List; import java.util.Set; -import java.util.zip.ZipEntry; import net.minecraft.nbt.CompressedStreamTools; import net.minecraft.nbt.NBTTagCompound; @@ -25,11 +23,6 @@ import net.minecraft.world.chunk.storage.RegionFile; import net.minecraft.world.chunk.storage.RegionFileCache; -import org.apache.commons.compress.archivers.ArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; -import org.apache.commons.io.IOUtils; - import com.gtnewhorizon.gtnhlib.util.CoordinatePacker; import it.unimi.dsi.fastutil.ints.Int2ObjectMap; @@ -48,6 +41,7 @@ import serverutils.lib.util.FileUtils; import serverutils.lib.util.ServerUtils; import serverutils.lib.util.StringUtils; +import serverutils.lib.util.compression.ICompress; public class ThreadBackup extends Thread { @@ -57,21 +51,23 @@ public class ThreadBackup extends Thread { private final String customName; private final Set chunksToBackup; public boolean isDone = false; + private final ICompress compressor; - public ThreadBackup(File sourceFile, String backupName, Set backupChunks) { + public ThreadBackup(ICompress compress, File sourceFile, String backupName, Set backupChunks) { src0 = sourceFile; customName = backupName; chunksToBackup = backupChunks; + compressor = compress; setPriority(7); } public void run() { isDone = false; - doBackup(src0, customName, chunksToBackup); + doBackup(compressor, src0, customName, chunksToBackup); isDone = true; } - public static void doBackup(File src, String customName, Set chunks) { + public static void doBackup(ICompress compressor, File src, String customName, Set chunks) { String outName = (customName.isEmpty() ? DATE_FORMAT.format(Calendar.getInstance().getTime()) : customName) + ".zip"; File dstFile = null; @@ -81,17 +77,12 @@ public static void doBackup(File src, String customName, Set chunks logMillis = start + Ticks.SECOND.x(5).millis(); dstFile = FileUtils.newFile(new File(BackupTask.backupsFolder, outName)); - try (ZipArchiveOutputStream zaos = new ZipArchiveOutputStream(dstFile)) { - if (backups.compression_level == 0) { - zaos.setMethod(ZipEntry.STORED); - } else { - zaos.setLevel(backups.compression_level); - } - + try (compressor) { + compressor.createOutputStream(dstFile); if (!chunks.isEmpty() && backups.only_backup_claimed_chunks) { - backupRegions(src, files, chunks, zaos); + backupRegions(src, files, chunks, compressor); } else { - compressFiles(src, files, zaos); + compressFiles(src, files, compressor); } String backupSize = FileUtils.getSizeString(dstFile); @@ -134,29 +125,22 @@ private static void logProgress(int i, int allFiles, String name) { } } - private static void compressFiles(File sourceDir, List files, ZipArchiveOutputStream zaos) - throws IOException { + private static void compressFiles(File sourceDir, List files, ICompress compressor) throws IOException { int allFiles = files.size(); for (int i = 0; i < allFiles; i++) { File file = files.get(i); - compressFile(FileUtils.getRelativePath(sourceDir, file), file, zaos, i, allFiles); + compressFile(FileUtils.getRelativePath(sourceDir, file), file, compressor, i, allFiles); } } - private static void compressFile(String entryName, File file, ZipArchiveOutputStream out, int index, int totalFiles) + private static void compressFile(String entryName, File file, ICompress compressor, int index, int totalFiles) throws IOException { - ArchiveEntry entry = new ZipArchiveEntry(file, entryName); logProgress(index, totalFiles, file.getAbsolutePath()); - out.putArchiveEntry(entry); - try (FileInputStream fis = new FileInputStream(file)) { - IOUtils.copy(fis, out); - } - out.closeArchiveEntry(); - + compressor.addFileToArchive(file, entryName); } private static void backupRegions(File sourceFolder, List files, Set chunksToBackup, - ZipArchiveOutputStream out) throws IOException { + ICompress compressor) throws IOException { Object2ObjectMap> dimRegionClaims = mapClaimsToRegionFile(chunksToBackup); files.removeIf(f -> f.getName().endsWith(".mca")); @@ -184,14 +168,14 @@ private static void backupRegions(File sourceFolder, List files, Set