diff --git a/src/main/java/serverutils/ServerUtilitiesConfig.java b/src/main/java/serverutils/ServerUtilitiesConfig.java index cb3577a0..ed7ef30f 100644 --- a/src/main/java/serverutils/ServerUtilitiesConfig.java +++ b/src/main/java/serverutils/ServerUtilitiesConfig.java @@ -325,6 +325,12 @@ public static class Backups { @Config.Comment("Delete backups that have a custom name set through /backup start ") @Config.DefaultBoolean(true) public boolean delete_custom_name_backups; + + @Config.Comment(""" + Only include claimed chunks in backup. + Backups will be much faster and smaller, but any unclaimed chunk will be unrecoverable.""") + @Config.DefaultBoolean(false) + public boolean only_backup_claimed_chunks; } public static class Login { diff --git a/src/main/java/serverutils/data/ClaimedChunks.java b/src/main/java/serverutils/data/ClaimedChunks.java index fb77bc5d..0d544521 100644 --- a/src/main/java/serverutils/data/ClaimedChunks.java +++ b/src/main/java/serverutils/data/ClaimedChunks.java @@ -163,6 +163,10 @@ public Collection getAllChunks() { return map.isEmpty() ? Collections.emptyList() : map.values(); } + public Set getAllClaimedPositions() { + return map.isEmpty() ? Collections.emptySet() : map.keySet(); + } + public Set getTeamChunks(@Nullable ForgeTeam team, OptionalInt dimension, boolean includePending) { if (team == null) { return Collections.emptySet(); diff --git a/src/main/java/serverutils/lib/math/ChunkDimPos.java b/src/main/java/serverutils/lib/math/ChunkDimPos.java index 2d864b6c..b5f03b0d 100644 --- a/src/main/java/serverutils/lib/math/ChunkDimPos.java +++ b/src/main/java/serverutils/lib/math/ChunkDimPos.java @@ -36,11 +36,8 @@ public ChunkDimPos set(int x, int z, int dim) { return this; } - public ChunkDimPos set(long packedPos, int dim) { - this.posX = CoordinatePacker.unpackX(packedPos); - this.posZ = CoordinatePacker.unpackZ(packedPos); - this.dim = dim; - return this; + public ChunkDimPos set(long packed, int dim) { + return set(CoordinatePacker.unpackX(packed), CoordinatePacker.unpackZ(packed), dim); } public boolean equals(Object o) { diff --git a/src/main/java/serverutils/lib/util/FileUtils.java b/src/main/java/serverutils/lib/util/FileUtils.java index 48e9d057..5b12f68c 100644 --- a/src/main/java/serverutils/lib/util/FileUtils.java +++ b/src/main/java/serverutils/lib/util/FileUtils.java @@ -230,4 +230,8 @@ public static String getBaseName(File file) { return index == -1 ? name : name.substring(0, index); } } + + public static String getRelativePath(File dir, File file) { + return dir.getName() + File.separator + file.getAbsolutePath().substring(dir.getAbsolutePath().length() + 1); + } } diff --git a/src/main/java/serverutils/lib/util/compression/CommonsCompressor.java b/src/main/java/serverutils/lib/util/compression/CommonsCompressor.java new file mode 100644 index 00000000..176cb461 --- /dev/null +++ b/src/main/java/serverutils/lib/util/compression/CommonsCompressor.java @@ -0,0 +1,46 @@ +package serverutils.lib.util.compression; + +import static serverutils.ServerUtilitiesConfig.backups; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.util.zip.ZipEntry; + +import org.apache.commons.compress.archivers.ArchiveEntry; +import org.apache.commons.compress.archivers.ArchiveOutputStream; +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; +import org.apache.commons.io.IOUtils; + +public class CommonsCompressor implements ICompress { + + private ArchiveOutputStream output; + + @Override + public void createOutputStream(File file) throws IOException { + ZipArchiveOutputStream zaos = new ZipArchiveOutputStream(file); + if (backups.compression_level == 0) { + zaos.setMethod(ZipEntry.STORED); + } else { + zaos.setLevel(backups.compression_level); + } + output = zaos; + } + + @Override + public void addFileToArchive(File file, String name) throws IOException { + ArchiveEntry entry = output.createArchiveEntry(file, name); + output.putArchiveEntry(entry); + try (FileInputStream fis = new FileInputStream(file)) { + IOUtils.copy(fis, output); + } + output.closeArchiveEntry(); + } + + @Override + public void close() throws Exception { + if (output != null) { + output.close(); + } + } +} diff --git a/src/main/java/serverutils/lib/util/compression/ICompress.java b/src/main/java/serverutils/lib/util/compression/ICompress.java new file mode 100644 index 00000000..426a2336 --- /dev/null +++ b/src/main/java/serverutils/lib/util/compression/ICompress.java @@ -0,0 +1,11 @@ +package serverutils.lib.util.compression; + +import java.io.File; +import java.io.IOException; + +public interface ICompress extends AutoCloseable { + + void createOutputStream(File file) throws IOException; + + void addFileToArchive(File file, String name) throws IOException; +} diff --git a/src/main/java/serverutils/lib/util/compression/LegacyCompressor.java b/src/main/java/serverutils/lib/util/compression/LegacyCompressor.java new file mode 100644 index 00000000..7e3d3803 --- /dev/null +++ b/src/main/java/serverutils/lib/util/compression/LegacyCompressor.java @@ -0,0 +1,46 @@ +package serverutils.lib.util.compression; + +import static serverutils.ServerUtilitiesConfig.backups; + +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.util.zip.ZipEntry; +import java.util.zip.ZipOutputStream; + +import org.apache.commons.io.IOUtils; + +public class LegacyCompressor implements ICompress { + + private ZipOutputStream output; + + @Override + public void createOutputStream(File file) throws IOException { + ZipOutputStream zos = new ZipOutputStream(new FileOutputStream(file)); + if (backups.compression_level == 0) { + zos.setMethod(ZipOutputStream.STORED); + } else { + zos.setLevel(backups.compression_level); + } + + output = zos; + } + + @Override + public void addFileToArchive(File file, String name) throws IOException { + ZipEntry entry = new ZipEntry(name); + output.putNextEntry(entry); + try (FileInputStream fis = new FileInputStream(file)) { + IOUtils.copy(fis, output); + } + output.closeEntry(); + } + + @Override + public void close() throws Exception { + if (output != null) { + output.close(); + } + } +} diff --git a/src/main/java/serverutils/task/backup/BackupTask.java b/src/main/java/serverutils/task/backup/BackupTask.java index 74d9eecd..e4620b0a 100644 --- a/src/main/java/serverutils/task/backup/BackupTask.java +++ b/src/main/java/serverutils/task/backup/BackupTask.java @@ -7,7 +7,9 @@ import java.io.File; import java.util.Arrays; import java.util.Comparator; +import java.util.HashSet; import java.util.List; +import java.util.Set; import java.util.regex.Pattern; import java.util.stream.Collectors; @@ -21,15 +23,23 @@ import serverutils.ServerUtilities; import serverutils.ServerUtilitiesConfig; import serverutils.ServerUtilitiesNotifications; +import serverutils.data.ClaimedChunks; import serverutils.lib.data.Universe; +import serverutils.lib.math.ChunkDimPos; import serverutils.lib.math.Ticks; +import serverutils.lib.util.CommonUtils; import serverutils.lib.util.FileUtils; import serverutils.lib.util.ServerUtils; +import serverutils.lib.util.compression.CommonsCompressor; +import serverutils.lib.util.compression.ICompress; +import serverutils.lib.util.compression.LegacyCompressor; import serverutils.task.Task; public class BackupTask extends Task { public static final Pattern BACKUP_NAME_PATTERN = Pattern.compile("\\d{4}-\\d{2}-\\d{2}-\\d{2}-\\d{2}-\\d{2}(.*)"); + public static final File BACKUP_TEMP_FOLDER = new File("serverutilities/temp/"); + private static final boolean useLegacy; public static File backupsFolder; public static ThreadBackup thread; public static boolean hadPlayer = false; @@ -43,6 +53,7 @@ public class BackupTask extends Task { if (!backupsFolder.exists()) backupsFolder.mkdirs(); clearOldBackups(); ServerUtilities.LOGGER.info("Backups folder - {}", backupsFolder.getAbsolutePath()); + useLegacy = !CommonUtils.getClassExists("org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream"); } public BackupTask() { @@ -96,11 +107,24 @@ public void execute(Universe universe) { File worldDir = DimensionManager.getCurrentSaveRootDirectory(); + Set backupChunks = new HashSet<>(); + if (backups.only_backup_claimed_chunks && ClaimedChunks.isActive()) { + backupChunks.addAll(ClaimedChunks.instance.getAllClaimedPositions()); + BACKUP_TEMP_FOLDER.mkdirs(); + } + + ICompress compressor; + if (useLegacy) { + compressor = new LegacyCompressor(); + } else { + compressor = new CommonsCompressor(); + } + if (backups.use_separate_thread) { - thread = new ThreadBackup(worldDir, customName); + thread = new ThreadBackup(compressor, worldDir, customName, backupChunks); thread.start(); } else { - ThreadBackup.doBackup(worldDir, customName); + ThreadBackup.doBackup(compressor, worldDir, customName, backupChunks); } universe.scheduleTask(new BackupTask(true)); } @@ -158,6 +182,8 @@ private void postBackup(Universe universe) { } clearOldBackups(); + FileUtils.delete(BACKUP_TEMP_FOLDER); + thread = null; try { MinecraftServer server = ServerUtils.getServer(); diff --git a/src/main/java/serverutils/task/backup/ThreadBackup.java b/src/main/java/serverutils/task/backup/ThreadBackup.java index c2362a9f..31cf4d6b 100644 --- a/src/main/java/serverutils/task/backup/ThreadBackup.java +++ b/src/main/java/serverutils/task/backup/ThreadBackup.java @@ -3,104 +3,107 @@ import static serverutils.ServerUtilitiesConfig.backups; import static serverutils.ServerUtilitiesNotifications.BACKUP_END1; import static serverutils.ServerUtilitiesNotifications.BACKUP_END2; +import static serverutils.task.backup.BackupTask.BACKUP_TEMP_FOLDER; +import java.io.DataInputStream; +import java.io.DataOutputStream; import java.io.File; -import java.io.FileInputStream; +import java.io.IOException; import java.text.DateFormat; import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.List; -import java.util.zip.ZipEntry; +import java.util.Set; +import net.minecraft.nbt.CompressedStreamTools; +import net.minecraft.nbt.NBTTagCompound; import net.minecraft.util.EnumChatFormatting; import net.minecraft.util.IChatComponent; - -import org.apache.commons.compress.archivers.zip.ZipArchiveEntry; -import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream; -import org.apache.commons.io.IOUtils; - +import net.minecraft.world.WorldServer; +import net.minecraft.world.chunk.storage.RegionFile; +import net.minecraft.world.chunk.storage.RegionFileCache; + +import com.gtnewhorizon.gtnhlib.util.CoordinatePacker; + +import it.unimi.dsi.fastutil.ints.Int2ObjectMap; +import it.unimi.dsi.fastutil.ints.Int2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectMap; +import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.objects.Object2ObjectMap; +import it.unimi.dsi.fastutil.objects.Object2ObjectOpenHashMap; +import it.unimi.dsi.fastutil.objects.ObjectOpenHashSet; +import it.unimi.dsi.fastutil.objects.ObjectSet; import serverutils.ServerUtilities; +import serverutils.ServerUtilitiesConfig; import serverutils.ServerUtilitiesNotifications; +import serverutils.lib.math.ChunkDimPos; import serverutils.lib.math.Ticks; import serverutils.lib.util.FileUtils; import serverutils.lib.util.ServerUtils; import serverutils.lib.util.StringUtils; +import serverutils.lib.util.compression.ICompress; public class ThreadBackup extends Thread { private static final DateFormat DATE_FORMAT = new SimpleDateFormat("yyyy-MM-dd-HH-mm-ss"); - private static long logMillis; private final File src0; private final String customName; + private final Set chunksToBackup; public boolean isDone = false; + private final ICompress compressor; - public ThreadBackup(File w, String s) { - src0 = w; - customName = s; + public ThreadBackup(ICompress compress, File sourceFile, String backupName, Set backupChunks) { + src0 = sourceFile; + customName = backupName; + chunksToBackup = backupChunks; + compressor = compress; setPriority(7); } public void run() { isDone = false; - doBackup(src0, customName); + doBackup(compressor, src0, customName, chunksToBackup); isDone = true; } - public static void doBackup(File src, String customName) { + public static void doBackup(ICompress compressor, File src, String customName, Set chunks) { String outName = (customName.isEmpty() ? DATE_FORMAT.format(Calendar.getInstance().getTime()) : customName) + ".zip"; File dstFile = null; try { List files = FileUtils.listTree(src); - int allFiles = files.size(); - ServerUtilities.LOGGER.info("Backing up {} files...", files.size()); long start = System.currentTimeMillis(); + logMillis = start + Ticks.SECOND.x(5).millis(); dstFile = FileUtils.newFile(new File(BackupTask.backupsFolder, outName)); - try (ZipArchiveOutputStream zaos = new ZipArchiveOutputStream(dstFile)) { - - if (backups.compression_level == 0) { - zaos.setMethod(ZipEntry.STORED); + try (compressor) { + compressor.createOutputStream(dstFile); + if (!chunks.isEmpty() && backups.only_backup_claimed_chunks) { + backupRegions(src, files, chunks, compressor); } else { - zaos.setLevel(backups.compression_level); + compressFiles(src, files, compressor); } - logMillis = System.currentTimeMillis() + Ticks.SECOND.x(5).millis(); - ServerUtilities.LOGGER.info("Backing up {} files!", allFiles); - - for (int i = 0; i < allFiles; i++) { - File file = files.get(i); - String filePath = file.getAbsolutePath(); - ZipArchiveEntry entry = new ZipArchiveEntry( - src.getName() + File.separator + filePath.substring(src.getAbsolutePath().length() + 1)); - - logProgress(i, allFiles, filePath); - zaos.putArchiveEntry(entry); - try (FileInputStream fis = new FileInputStream(file)) { - IOUtils.copy(fis, zaos); - } - zaos.closeArchiveEntry(); + String backupSize = FileUtils.getSizeString(dstFile); + ServerUtilities.LOGGER.info("Backup done in {} seconds ({})!", getDoneTime(start), backupSize); + ServerUtilities.LOGGER.info("Created {} from {}", dstFile.getAbsolutePath(), src.getAbsolutePath()); + + if (backups.display_file_size) { + String sizeT = FileUtils.getSizeString(BackupTask.backupsFolder); + ServerUtilitiesNotifications.backupNotification( + BACKUP_END2, + "cmd.backup_end_2", + getDoneTime(start), + (backupSize.equals(sizeT) ? backupSize : (backupSize + " | " + sizeT))); + } else { + ServerUtilitiesNotifications + .backupNotification(BACKUP_END1, "cmd.backup_end_1", getDoneTime(start)); } } - String backupSize = FileUtils.getSizeString(dstFile); - ServerUtilities.LOGGER.info("Backup done in {} seconds ({})!", getDoneTime(start), backupSize); - ServerUtilities.LOGGER.info("Created {} from {}", dstFile.getAbsolutePath(), src.getAbsolutePath()); - - if (backups.display_file_size) { - String sizeT = FileUtils.getSizeString(BackupTask.backupsFolder); - ServerUtilitiesNotifications.backupNotification( - BACKUP_END2, - "cmd.backup_end_2", - getDoneTime(start), - (backupSize.equals(sizeT) ? backupSize : (backupSize + " | " + sizeT))); - } else { - ServerUtilitiesNotifications.backupNotification(BACKUP_END1, "cmd.backup_end_1", getDoneTime(start)); - } } catch (Exception e) { - IChatComponent c = StringUtils.color( - ServerUtilities.lang(null, "cmd.backup_fail", e.getClass().getName()), - EnumChatFormatting.RED); + IChatComponent c = StringUtils + .color(ServerUtilities.lang(null, "cmd.backup_fail", e), EnumChatFormatting.RED); ServerUtils.notifyChat(ServerUtils.getServer(), null, c); ServerUtilities.LOGGER.error("Error while backing up", e); @@ -110,14 +113,125 @@ public static void doBackup(File src, String customName) { private static void logProgress(int i, int allFiles, String name) { long millis = System.currentTimeMillis(); - if (i == 0 || millis > logMillis || i == allFiles - 1) { + boolean first = i == 0; + if (first) { + ServerUtilities.LOGGER.info("Backing up {} files...", allFiles); + } + + if (first || millis > logMillis || i == allFiles - 1) { logMillis = millis + Ticks.SECOND.x(5).millis(); ServerUtilities.LOGGER .info("[{} | {}%]: {}", i, StringUtils.formatDouble00((i / (double) allFiles) * 100D), name); } } + private static void compressFiles(File sourceDir, List files, ICompress compressor) throws IOException { + int allFiles = files.size(); + for (int i = 0; i < allFiles; i++) { + File file = files.get(i); + compressFile(FileUtils.getRelativePath(sourceDir, file), file, compressor, i, allFiles); + } + } + + private static void compressFile(String entryName, File file, ICompress compressor, int index, int totalFiles) + throws IOException { + logProgress(index, totalFiles, file.getAbsolutePath()); + compressor.addFileToArchive(file, entryName); + } + + private static void backupRegions(File sourceFolder, List files, Set chunksToBackup, + ICompress compressor) throws IOException { + Object2ObjectMap> dimRegionClaims = mapClaimsToRegionFile(chunksToBackup); + files.removeIf(f -> f.getName().endsWith(".mca")); + + int index = 0; + int savedChunks = 0; + int regionFiles = dimRegionClaims.size(); + int totalFiles = files.size() + regionFiles; + for (Object2ObjectMap.Entry> entry : dimRegionClaims.object2ObjectEntrySet()) { + File file = entry.getKey(); + File dimensionRoot = file.getParentFile().getParentFile(); + File tempFile = FileUtils.newFile(new File(BACKUP_TEMP_FOLDER, file.getName())); + RegionFile tempRegion = new RegionFile(tempFile); + boolean hasData = false; + + for (ChunkDimPos pos : entry.getValue()) { + DataInputStream in = RegionFileCache.getChunkInputStream(dimensionRoot, pos.posX, pos.posZ); + if (in == null) continue; + savedChunks++; + hasData = true; + NBTTagCompound tag = CompressedStreamTools.read(in); + DataOutputStream tempOut = tempRegion.getChunkDataOutputStream(pos.posX & 31, pos.posZ & 31); + CompressedStreamTools.write(tag, tempOut); + tempOut.close(); + } + + tempRegion.close(); + if (hasData) { + compressFile(FileUtils.getRelativePath(sourceFolder, file), tempFile, compressor, index++, totalFiles); + } + + FileUtils.delete(tempFile); + } + + for (File file : files) { + compressFile(FileUtils.getRelativePath(sourceFolder, file), file, compressor, index++, totalFiles); + } + + ServerUtilities.LOGGER.info("Backed up {} regions containing {} claimed chunks", regionFiles, savedChunks); + } + + private static Object2ObjectMap> mapClaimsToRegionFile( + Set chunksToBackup) { + Int2ObjectMap>> regionClaimsByDim = new Int2ObjectOpenHashMap<>(); + chunksToBackup.forEach( + pos -> regionClaimsByDim.computeIfAbsent(pos.dim, k -> new Long2ObjectOpenHashMap<>()) + .computeIfAbsent(getRegionFromChunk(pos.posX, pos.posZ), k -> new ObjectOpenHashSet<>()) + .add(pos)); + + Object2ObjectMap> regionFilesToBackup = new Object2ObjectOpenHashMap<>(); + for (WorldServer worldserver : ServerUtils.getServer().worldServers) { + if (worldserver == null) continue; + + int dim = worldserver.provider.dimensionId; + File regionFolder = new File(worldserver.getChunkSaveLocation(), "region"); + Long2ObjectMap> regionClaims = regionClaimsByDim.get(dim); + if (!regionFolder.exists() || regionClaims == null) continue; + + File[] regions = regionFolder.listFiles(); + if (regions == null) continue; + + for (File file : regions) { + int[] coords = getRegionCoords(file); + long key = CoordinatePacker.pack(coords[0], 0, coords[1]); + ObjectSet claims = regionClaims.get(key); + if (claims == null) { + if (ServerUtilitiesConfig.debugging.print_more_info) { + ServerUtilities.LOGGER.info("Skipping region file {} from dimension {}", file.getName(), dim); + } + continue; + } + regionFilesToBackup.put(file, claims); + } + } + return regionFilesToBackup; + } + + private static int[] getRegionCoords(File f) { + String fileName = f.getName(); + int firstDot = fileName.indexOf('.'); + int secondDot = fileName.indexOf('.', firstDot + 1); + + int x = Integer.parseInt(fileName.substring(firstDot + 1, secondDot)); + int z = Integer.parseInt(fileName.substring(secondDot + 1, fileName.lastIndexOf('.'))); + return new int[] { x, z }; + } + private static String getDoneTime(long l) { return StringUtils.getTimeString(System.currentTimeMillis() - l); } + + private static long getRegionFromChunk(int chunkX, int chunkZ) { + return CoordinatePacker.pack(chunkX >> 5, 0, chunkZ >> 5); + } }