diff --git a/patches/server/Allow-Saving-of-Oversized-Chunks.patch b/patches/server/Allow-Saving-of-Oversized-Chunks.patch new file mode 100644 index 0000000000..d84e03b3df --- /dev/null +++ b/patches/server/Allow-Saving-of-Oversized-Chunks.patch @@ -0,0 +1,253 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Aikar +Date: Fri, 15 Feb 2019 01:08:19 -0500 +Subject: [PATCH] Allow Saving of Oversized Chunks + +Note 1.17 update: With 1.17, Entities are no longer stored in chunk slices, so this needs updating!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!! + +The Minecraft World Region File format has a hard cap of 1MB per chunk. +This is due to the fact that the header of the file format only allocates +a single byte for sector count, meaning a maximum of 256 sectors, at 4k per sector. + +This limit can be reached fairly easily with books, resulting in the chunk being unable +to save to the world. Worse off, is that nothing printed when this occured, and silently +performed a chunk rollback on next load. + +This leads to security risk with duplication and is being actively exploited. + +This patch catches the too large scenario, falls back and moves any large Entity +or Tile Entity into a new compound, and this compound is saved into a different file. + +On Chunk Load, we check for oversized status, and if so, we load the extra file and +merge the Entities and Tile Entities from the oversized chunk back into the level to +then be loaded as normal. + +Once a chunk is returned back to normal size, the oversized flag will clear, and no +extra data file will exist. + +This fix maintains compatability with all existing Anvil Region Format tools as it +does not alter the save format. They will just not know about the extra entities. + +This fix also maintains compatability if someone switches server jars to one without +this fix, as the data will remain in the oversized file. Once the server returns +to a jar with this fix, the data will be restored. + +diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java +index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 +--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java ++++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java +@@ -0,0 +0,0 @@ import java.nio.file.LinkOption; + import java.nio.file.Path; + import java.nio.file.StandardCopyOption; + import java.nio.file.StandardOpenOption; ++import java.util.zip.InflaterInputStream; // Paper ++ + import javax.annotation.Nullable; + import net.minecraft.Util; ++import net.minecraft.nbt.CompoundTag; ++import net.minecraft.nbt.NbtIo; + import net.minecraft.world.level.ChunkPos; + import org.apache.logging.log4j.LogManager; + import org.apache.logging.log4j.Logger; +@@ -0,0 +0,0 @@ public class RegionFile implements AutoCloseable { + @VisibleForTesting + protected final RegionBitmap usedSectors; + public final java.util.concurrent.locks.ReentrantLock fileLock = new java.util.concurrent.locks.ReentrantLock(true); // Paper ++ public final File regionFile; // Paper + + public RegionFile(File file, File directory, boolean dsync) throws IOException { + this(file.toPath(), directory.toPath(), RegionFileVersion.VERSION_DEFLATE, dsync); +@@ -0,0 +0,0 @@ public class RegionFile implements AutoCloseable { + + public RegionFile(Path file, Path directory, RegionFileVersion outputChunkStreamVersion, boolean dsync) throws IOException { + this.header = ByteBuffer.allocateDirect(8192); ++ this.regionFile = file.toFile(); // Paper ++ initOversizedState(); // Paper + this.usedSectors = new RegionBitmap(); + this.version = outputChunkStreamVersion; + if (!Files.isDirectory(directory, new LinkOption[0])) { +@@ -0,0 +0,0 @@ public class RegionFile implements AutoCloseable { + + } + ++ // Paper start ++ private final byte[] oversized = new byte[1024]; ++ private int oversizedCount = 0; ++ ++ private synchronized void initOversizedState() throws IOException { ++ File metaFile = getOversizedMetaFile(); ++ if (metaFile.exists()) { ++ final byte[] read = java.nio.file.Files.readAllBytes(metaFile.toPath()); ++ System.arraycopy(read, 0, oversized, 0, oversized.length); ++ for (byte temp : oversized) { ++ oversizedCount += temp; ++ } ++ } ++ } ++ ++ private static int getChunkIndex(int x, int z) { ++ return (x & 31) + (z & 31) * 32; ++ } ++ synchronized boolean isOversized(int x, int z) { ++ return this.oversized[getChunkIndex(x, z)] == 1; ++ } ++ synchronized void setOversized(int x, int z, boolean oversized) throws IOException { ++ final int offset = getChunkIndex(x, z); ++ boolean previous = this.oversized[offset] == 1; ++ this.oversized[offset] = (byte) (oversized ? 1 : 0); ++ if (!previous && oversized) { ++ oversizedCount++; ++ } else if (!oversized && previous) { ++ oversizedCount--; ++ } ++ if (previous && !oversized) { ++ File oversizedFile = getOversizedFile(x, z); ++ if (oversizedFile.exists()) { ++ oversizedFile.delete(); ++ } ++ } ++ if (oversizedCount > 0) { ++ if (previous != oversized) { ++ writeOversizedMeta(); ++ } ++ } else if (previous) { ++ File oversizedMetaFile = getOversizedMetaFile(); ++ if (oversizedMetaFile.exists()) { ++ oversizedMetaFile.delete(); ++ } ++ } ++ } ++ ++ private void writeOversizedMeta() throws IOException { ++ java.nio.file.Files.write(getOversizedMetaFile().toPath(), oversized); ++ } ++ ++ private File getOversizedMetaFile() { ++ return new File(this.regionFile.getParentFile(), this.regionFile.getName().replaceAll("\\.mca$", "") + ".oversized.nbt"); ++ } ++ ++ private File getOversizedFile(int x, int z) { ++ return new File(this.regionFile.getParentFile(), this.regionFile.getName().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt"); ++ } ++ ++ synchronized CompoundTag getOversizedData(int x, int z) throws IOException { ++ File file = getOversizedFile(x, z); ++ try (DataInputStream out = new DataInputStream(new BufferedInputStream(new InflaterInputStream(new java.io.FileInputStream(file))))) { ++ return NbtIo.read((java.io.DataInput) out); ++ } ++ ++ } ++ // Paper end + private class ChunkBuffer extends ByteArrayOutputStream { + + private final ChunkPos pos; +diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 +--- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java ++++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFileStorage.java +@@ -0,0 +0,0 @@ import java.io.File; + import java.io.IOException; + import javax.annotation.Nullable; + import net.minecraft.nbt.CompoundTag; ++import net.minecraft.nbt.ListTag; + import net.minecraft.nbt.NbtIo; ++import net.minecraft.nbt.Tag; + import net.minecraft.server.MinecraftServer; + import net.minecraft.util.ExceptionCollector; + import net.minecraft.world.level.ChunkPos; +@@ -0,0 +0,0 @@ public class RegionFileStorage implements AutoCloseable { + } + } + ++ // Paper start ++ private static void printOversizedLog(String msg, File file, int x, int z) { ++ org.apache.logging.log4j.LogManager.getLogger().fatal(msg + " (" + file.toString().replaceAll(".+[\\\\/]", "") + " - " + x + "," + z + ") Go clean it up to remove this message. /minecraft:tp " + (x<<4)+" 128 "+(z<<4) + " - DO NOT REPORT THIS TO PAPER - You may ask for help on Discord, but do not file an issue. These error messages can not be removed."); ++ } ++ ++ private static final int DEFAULT_SIZE_THRESHOLD = 1024 * 8; ++ private static final int OVERZEALOUS_TOTAL_THRESHOLD = 1024 * 64; ++ private static final int OVERZEALOUS_THRESHOLD = 1024; ++ private static int SIZE_THRESHOLD = DEFAULT_SIZE_THRESHOLD; ++ private static void resetFilterThresholds() { ++ SIZE_THRESHOLD = Math.max(1024 * 4, Integer.getInteger("Paper.FilterThreshhold", DEFAULT_SIZE_THRESHOLD)); ++ } ++ static { ++ resetFilterThresholds(); ++ } ++ ++ static boolean isOverzealous() { ++ return SIZE_THRESHOLD == OVERZEALOUS_THRESHOLD; ++ } ++ ++ ++ private static CompoundTag readOversizedChunk(RegionFile regionfile, ChunkPos chunkCoordinate) throws IOException { ++ synchronized (regionfile) { ++ try (DataInputStream datainputstream = regionfile.getReadStream(chunkCoordinate)) { ++ CompoundTag oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z); ++ CompoundTag chunk = NbtIo.read((DataInput) datainputstream); ++ if (oversizedData == null) { ++ return chunk; ++ } ++ CompoundTag oversizedLevel = oversizedData.getCompound("Level"); ++ CompoundTag level = chunk.getCompound("Level"); ++ ++ mergeChunkList(level, oversizedLevel, "Entities"); ++ mergeChunkList(level, oversizedLevel, "TileEntities"); ++ ++ chunk.put("Level", level); ++ ++ return chunk; ++ } catch (Throwable throwable) { ++ throwable.printStackTrace(); ++ throw throwable; ++ } ++ } ++ } ++ ++ private static void mergeChunkList(CompoundTag level, CompoundTag oversizedLevel, String key) { ++ ListTag levelList = level.getList(key, 10); ++ ListTag oversizedList = oversizedLevel.getList(key, 10); ++ ++ if (!oversizedList.isEmpty()) { ++ levelList.addAll(oversizedList); ++ level.put(key, levelList); ++ } ++ } ++ ++ private static int getNBTSize(Tag nbtBase) { ++ DataOutputStream test = new DataOutputStream(new org.apache.commons.io.output.NullOutputStream()); ++ try { ++ nbtBase.write(test); ++ return test.size(); ++ } catch (IOException e) { ++ e.printStackTrace(); ++ return 0; ++ } ++ } ++ ++ // Paper End ++ + @Nullable + public CompoundTag read(ChunkPos pos) throws IOException { + // CraftBukkit start - SPIGOT-5680: There's no good reason to preemptively create files on read, save that for writing +@@ -0,0 +0,0 @@ public class RegionFileStorage implements AutoCloseable { + try { // Paper + DataInputStream datainputstream = regionfile.getChunkDataInputStream(pos); + ++ // Paper start ++ if (regionfile.isOversized(pos.x, pos.z)) { ++ printOversizedLog("Loading Oversized Chunk!", regionfile.regionFile, pos.x, pos.z); ++ return readOversizedChunk(regionfile, pos); ++ } ++ // Paper end + CompoundTag nbttagcompound; + label43: + { +@@ -0,0 +0,0 @@ public class RegionFileStorage implements AutoCloseable { + + try { + NbtIo.write(nbt, (DataOutput) dataoutputstream); ++ regionfile.setOversized(pos.x, pos.z, false); // Paper - We don't do this anymore, mojang stores differently, but clear old meta flag if it exists to get rid of our own meta file once last oversized is gone + } catch (Throwable throwable) { + if (dataoutputstream != null) { + try { diff --git a/patches/server/Duplicate-UUID-Resolve-Option.patch b/patches/server/Duplicate-UUID-Resolve-Option.patch new file mode 100644 index 0000000000..da978470ce --- /dev/null +++ b/patches/server/Duplicate-UUID-Resolve-Option.patch @@ -0,0 +1,228 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Aikar +Date: Sat, 21 Jul 2018 14:27:34 -0400 +Subject: [PATCH] Duplicate UUID Resolve Option + +Due to a bug in https://github.com/PaperMC/Paper/commit/2e29af3df05ec0a383f48be549d1c03200756d24 +which was added all the way back in March of 2016, it was unknown (potentially not at the time) +that an entity might actually change the seed of the random object. + +At some point, EntitySquid did start setting the seed. Due to this shared random, this caused +every entity to use a Random object with a predictable seed. + +This has caused entities to potentially generate with the same UUID.... + +Over the years, servers have had entities disappear, but no sign of trouble +because CraftBukkit removed the log lines indicating that something was wrong. + +We have fixed the root issue causing duplicate UUID's, however we now have chunk +files full of entities that have the same UUID as another entity! + +When these chunks load, the 2nd entity will not be added to the world correctly. + +If that chunk loads in a different order in the future, then it will reverse and the +missing one is now the one added to the world and not the other. This results in very +inconsistent entity behavior. + +This change allows you to recover any duplicate entity by generating a new UUID for it. +This also lets you delete them instead if you don't want to risk having new entities added to +the world that you previously did not see. + +But for those who are ok with leaving this inconsistent behavior, you may use WARN or NOTHING options. + +It is recommended you regenerate the entities, as these were legit entities, and deserve your love. + +diff --git a/src/main/java/com/destroystokyo/paper/PaperWorldConfig.java b/src/main/java/com/destroystokyo/paper/PaperWorldConfig.java +index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 +--- a/src/main/java/com/destroystokyo/paper/PaperWorldConfig.java ++++ b/src/main/java/com/destroystokyo/paper/PaperWorldConfig.java +@@ -0,0 +0,0 @@ public class PaperWorldConfig { + preventMovingIntoUnloadedChunks = getBoolean("prevent-moving-into-unloaded-chunks", false); + } + ++ public enum DuplicateUUIDMode { ++ SAFE_REGEN, DELETE, NOTHING, WARN ++ } ++ public DuplicateUUIDMode duplicateUUIDMode = DuplicateUUIDMode.SAFE_REGEN; ++ public int duplicateUUIDDeleteRange = 32; ++ private void repairDuplicateUUID() { ++ String desiredMode = getString("duplicate-uuid-resolver", "saferegen").toLowerCase().trim(); ++ duplicateUUIDDeleteRange = getInt("duplicate-uuid-saferegen-delete-range", duplicateUUIDDeleteRange); ++ switch (desiredMode.toLowerCase()) { ++ case "regen": ++ case "regenerate": ++ case "saferegen": ++ case "saferegenerate": ++ duplicateUUIDMode = DuplicateUUIDMode.SAFE_REGEN; ++ log("Duplicate UUID Resolve: Regenerate New UUID if distant (Delete likely duplicates within " + duplicateUUIDDeleteRange + " blocks)"); ++ break; ++ case "remove": ++ case "delete": ++ duplicateUUIDMode = DuplicateUUIDMode.DELETE; ++ log("Duplicate UUID Resolve: Delete Entity"); ++ break; ++ case "silent": ++ case "nothing": ++ duplicateUUIDMode = DuplicateUUIDMode.NOTHING; ++ logError("Duplicate UUID Resolve: Do Nothing (no logs) - Warning, may lose indication of bad things happening"); ++ break; ++ case "log": ++ case "warn": ++ duplicateUUIDMode = DuplicateUUIDMode.WARN; ++ log("Duplicate UUID Resolve: Warn (do nothing but log it happened, may be spammy)"); ++ break; ++ default: ++ duplicateUUIDMode = DuplicateUUIDMode.WARN; ++ logError("Warning: Invalid duplicate-uuid-resolver config " + desiredMode + " - must be one of: regen, delete, nothing, warn"); ++ log("Duplicate UUID Resolve: Warn (do nothing but log it happened, may be spammy)"); ++ break; ++ } ++ } ++ + public boolean countAllMobsForSpawning = false; + private void countAllMobsForSpawning() { + countAllMobsForSpawning = getBoolean("count-all-mobs-for-spawning", false); +diff --git a/src/main/java/net/minecraft/server/level/ChunkMap.java b/src/main/java/net/minecraft/server/level/ChunkMap.java +index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 +--- a/src/main/java/net/minecraft/server/level/ChunkMap.java ++++ b/src/main/java/net/minecraft/server/level/ChunkMap.java +@@ -0,0 +0,0 @@ + package net.minecraft.server.level; + + import co.aikar.timings.Timing; // Paper ++import com.destroystokyo.paper.PaperWorldConfig; // Paper + import com.google.common.collect.ImmutableList; + import com.google.common.collect.Iterables; + import com.google.common.collect.ComparisonChain; // Paper +@@ -0,0 +0,0 @@ import java.io.File; + import java.io.IOException; + import java.io.Writer; + import java.util.BitSet; ++import java.util.HashMap; // Paper ++import java.util.Collection; + import java.util.Iterator; + import java.util.List; ++import java.util.Map; // Paper + import java.util.Objects; + import java.util.Optional; + import java.util.Queue; + import java.util.Set; + import java.util.concurrent.CancellationException; ++import java.util.UUID; // Paper + import java.util.concurrent.CompletableFuture; + import java.util.concurrent.CompletionException; + import java.util.concurrent.CompletionStage; +@@ -0,0 +0,0 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider + entity.discard(); + needsRemoval = true; + } ++ checkDupeUUID(worldserver, entity); // Paper + return !needsRemoval; + })); + // CraftBukkit end +@@ -0,0 +0,0 @@ public class ChunkMap extends ChunkStorage implements ChunkHolder.PlayerProvider + }); + } + ++ // Paper start ++ private static void checkDupeUUID(ServerLevel level, Entity entity) { ++ PaperWorldConfig.DuplicateUUIDMode mode = level.paperConfig.duplicateUUIDMode; ++ if (mode != PaperWorldConfig.DuplicateUUIDMode.WARN ++ && mode != PaperWorldConfig.DuplicateUUIDMode.DELETE ++ && mode != PaperWorldConfig.DuplicateUUIDMode.SAFE_REGEN) { ++ return; ++ } ++ Entity other = level.getEntity(entity.getUUID()); ++ ++ if (mode == PaperWorldConfig.DuplicateUUIDMode.SAFE_REGEN && other != null && !other.isRemoved() ++ && Objects.equals(other.getEncodeId(), entity.getEncodeId()) ++ && entity.getBukkitEntity().getLocation().distance(other.getBukkitEntity().getLocation()) < level.paperConfig.duplicateUUIDDeleteRange ++ ) { ++ if (Level.DEBUG_ENTITIES) LOGGER.warn("[DUPE-UUID] Duplicate UUID found used by " + other + ", deleted entity " + entity + " because it was near the duplicate and likely an actual duplicate. See https://github.com/PaperMC/Paper/issues/1223 for discussion on what this is about."); ++ entity.discard(); ++ return; ++ } ++ if (other != null && !other.isRemoved()) { ++ switch (mode) { ++ case SAFE_REGEN: { ++ entity.setUUID(UUID.randomUUID()); ++ if (Level.DEBUG_ENTITIES) LOGGER.warn("[DUPE-UUID] Duplicate UUID found used by " + other + ", regenerated UUID for " + entity + ". See https://github.com/PaperMC/Paper/issues/1223 for discussion on what this is about."); ++ break; ++ } ++ case DELETE: { ++ if (Level.DEBUG_ENTITIES) LOGGER.warn("[DUPE-UUID] Duplicate UUID found used by " + other + ", deleted entity " + entity + ". See https://github.com/PaperMC/Paper/issues/1223 for discussion on what this is about."); ++ entity.discard(); ++ break; ++ } ++ default: ++ if (Level.DEBUG_ENTITIES) LOGGER.warn("[DUPE-UUID] Duplicate UUID found used by " + other + ", doing nothing to " + entity + ". See https://github.com/PaperMC/Paper/issues/1223 for discussion on what this is about."); ++ break; ++ } ++ } ++ } ++ // Paper end + public CompletableFuture> prepareTickingChunk(ChunkHolder holder) { + ChunkPos chunkcoordintpair = holder.getPos(); + CompletableFuture, ChunkHolder.ChunkLoadingFailure>> completablefuture = this.getChunkRangeFuture(chunkcoordintpair, 1, (i) -> { +diff --git a/src/main/java/net/minecraft/world/level/entity/PersistentEntitySectionManager.java b/src/main/java/net/minecraft/world/level/entity/PersistentEntitySectionManager.java +index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 +--- a/src/main/java/net/minecraft/world/level/entity/PersistentEntitySectionManager.java ++++ b/src/main/java/net/minecraft/world/level/entity/PersistentEntitySectionManager.java +@@ -0,0 +0,0 @@ public class PersistentEntitySectionManager implements A + + private boolean addEntityUuid(T entity) { + if (!this.knownUuids.add(entity.getUUID())) { ++ // Paper start ++ if (((Entity) entity).isRemoved()) { ++ stopTracking(entity); // remove the existing entity ++ return false; ++ } ++ // Paper end + LOGGER.warn("UUID of added entity already exists: {}", (Object)entity); ++ // Paper start ++ if (net.minecraft.world.level.Level.DEBUG_ENTITIES && ((Entity) entity).level.paperConfig.duplicateUUIDMode != com.destroystokyo.paper.PaperWorldConfig.DuplicateUUIDMode.NOTHING) { ++ if (((Entity) entity).addedToWorldStack != null) { ++ ((Entity) entity).addedToWorldStack.printStackTrace(); ++ } ++ net.minecraft.server.level.ServerLevel.getAddToWorldStackTrace((net.minecraft.world.entity.Entity) entity).printStackTrace(); ++ } ++ // Paper end + return false; + } else { + return true; +@@ -0,0 +0,0 @@ public class PersistentEntitySectionManager implements A + } + + private void processUnloads() { +- this.chunksToUnload.removeIf((pos) -> { ++ this.chunksToUnload.removeIf((java.util.function.LongPredicate) (pos) -> { // Paper - decompile fix + return this.chunkVisibility.get(pos) != Visibility.HIDDEN ? true : this.processChunkUnload(pos); + }); + } +@@ -0,0 +0,0 @@ public class PersistentEntitySectionManager implements A + } + + public void autoSave() { +- this.getAllChunksToSave().forEach((pos) -> { ++ this.getAllChunksToSave().forEach((java.util.function.LongConsumer) (pos) -> { // Paper - decompile fix + boolean bl = this.chunkVisibility.get(pos) == Visibility.HIDDEN; + if (bl) { + this.processChunkUnload(pos); +@@ -0,0 +0,0 @@ public class PersistentEntitySectionManager implements A + while(!longSet.isEmpty()) { + this.permanentStorage.flush(); + this.processPendingLoads(); +- longSet.removeIf((pos) -> { ++ longSet.removeIf((java.util.function.LongPredicate) (pos) -> { // Paper - decompile fix + boolean bl = this.chunkVisibility.get(pos) == Visibility.HIDDEN; + return bl ? this.processChunkUnload(pos) : this.storeChunkSections(pos, (entityAccess) -> { + }); +@@ -0,0 +0,0 @@ public class PersistentEntitySectionManager implements A + + public void dumpSections(Writer writer) throws IOException { + CsvOutput csvOutput = CsvOutput.builder().addColumn("x").addColumn("y").addColumn("z").addColumn("visibility").addColumn("load_status").addColumn("entity_count").build(writer); +- this.sectionStorage.getAllChunksWithExistingSections().forEach((chunkPos) -> { ++ this.sectionStorage.getAllChunksWithExistingSections().forEach((java.util.function.LongConsumer) (chunkPos) -> { // Paper - decompile fix + PersistentEntitySectionManager.ChunkLoadStatus chunkLoadStatus = this.chunkLoadStatuses.get(chunkPos); + this.sectionStorage.getExistingSectionPositionsInChunk(chunkPos).forEach((sectionPos) -> { + EntitySection entitySection = this.sectionStorage.getSection(sectionPos); diff --git a/patches/server/Fix-World-isChunkGenerated-calls.patch b/patches/server/Fix-World-isChunkGenerated-calls.patch index f3a19957df..da846155c6 100644 --- a/patches/server/Fix-World-isChunkGenerated-calls.patch +++ b/patches/server/Fix-World-isChunkGenerated-calls.patch @@ -175,17 +175,17 @@ diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.ja index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 --- a/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java +++ b/src/main/java/net/minecraft/world/level/chunk/storage/RegionFile.java -@@ -0,0 +0,0 @@ import java.nio.file.StandardOpenOption; - import javax.annotation.Nullable; - import net.minecraft.Util; +@@ -0,0 +0,0 @@ import net.minecraft.Util; + import net.minecraft.nbt.CompoundTag; + import net.minecraft.nbt.NbtIo; import net.minecraft.world.level.ChunkPos; +import net.minecraft.world.level.chunk.ChunkStatus; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -0,0 +0,0 @@ public class RegionFile implements AutoCloseable { - protected final RegionBitmap usedSectors; public final java.util.concurrent.locks.ReentrantLock fileLock = new java.util.concurrent.locks.ReentrantLock(true); // Paper + public final File regionFile; // Paper + // Paper start - Cache chunk status + private final ChunkStatus[] statuses = new ChunkStatus[32 * 32]; @@ -239,14 +239,9 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000 try { NbtIo.write(nbt, (DataOutput) dataoutputstream); + regionfile.setStatus(pos.x, pos.z, ChunkSerializer.getStatus(nbt)); // Paper - cache status on disk + regionfile.setOversized(pos.x, pos.z, false); // Paper - We don't do this anymore, mojang stores differently, but clear old meta flag if it exists to get rid of our own meta file once last oversized is gone } catch (Throwable throwable) { if (dataoutputstream != null) { - try { -@@ -0,0 +0,0 @@ public class RegionFileStorage implements AutoCloseable { - - } - } -+ diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 --- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java diff --git a/patches/server/No-Tick-view-distance-implementation.patch b/patches/server/No-Tick-view-distance-implementation.patch index 69a7d70bb0..a1ad44c0a3 100644 --- a/patches/server/No-Tick-view-distance-implementation.patch +++ b/patches/server/No-Tick-view-distance-implementation.patch @@ -401,13 +401,14 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000 public Stream getPlayers(ChunkPos chunkPos, boolean onlyOnWatchDistanceEdge) { - return this.playerMap.getPlayers(chunkPos.toLong()).filter((entityplayer) -> { - int i = ChunkMap.checkerboardDistance(chunkPos, entityplayer, true); +- +- return i > this.viewDistance ? false : !onlyOnWatchDistanceEdge || i == this.viewDistance; +- }); + // Paper start - per player view distance + // there can be potential desync with player's last mapped section and the view distance map, so use the + // view distance map here. + com.destroystokyo.paper.util.misc.PooledLinkedHashSets.PooledObjectLinkedOpenHashSet inRange = this.playerViewDistanceBroadcastMap.getObjectsInRange(chunkPos); - -- return i > this.viewDistance ? false : !onlyOnWatchDistanceEdge || i == this.viewDistance; -- }); ++ + if (inRange == null) { + return Stream.empty(); + } diff --git a/patches/server/Optimize-NibbleArray-to-use-pooled-buffers.patch b/patches/server/Optimize-NibbleArray-to-use-pooled-buffers.patch new file mode 100644 index 0000000000..cfc7914f6e --- /dev/null +++ b/patches/server/Optimize-NibbleArray-to-use-pooled-buffers.patch @@ -0,0 +1,331 @@ +From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001 +From: Aikar +Date: Wed, 6 May 2020 23:30:30 -0400 +Subject: [PATCH] Optimize NibbleArray to use pooled buffers + +Massively reduces memory allocation of 2048 byte buffers by using +an object pool for these. + +Uses lots of advanced new capabilities of the Paper codebase :) + +1.17 update note: ClientboundLightUpdatePacket has has made changes which necessitate updating this patch + +diff --git a/src/main/java/net/minecraft/network/protocol/game/ClientboundLightUpdatePacket.java b/src/main/java/net/minecraft/network/protocol/game/ClientboundLightUpdatePacket.java +index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 +--- a/src/main/java/net/minecraft/network/protocol/game/ClientboundLightUpdatePacket.java ++++ b/src/main/java/net/minecraft/network/protocol/game/ClientboundLightUpdatePacket.java +@@ -0,0 +0,0 @@ package net.minecraft.network.protocol.game; + + import com.google.common.collect.Lists; + import java.util.BitSet; ++import io.netty.channel.ChannelFuture; // Paper + import java.util.List; + import javax.annotation.Nullable; + import net.minecraft.core.SectionPos; + import net.minecraft.network.FriendlyByteBuf; + import net.minecraft.network.protocol.Packet; ++import net.minecraft.server.MCUtil; ++import net.minecraft.server.level.ServerPlayer; + import net.minecraft.world.level.ChunkPos; + import net.minecraft.world.level.LightLayer; + import net.minecraft.world.level.chunk.DataLayer; +@@ -0,0 +0,0 @@ public class ClientboundLightUpdatePacket implements Packet skyUpdates; + private final List blockUpdates; + private final boolean trustEdges; ++ // Paper start ++ java.lang.Runnable cleaner1; ++ java.lang.Runnable cleaner2; ++ java.util.concurrent.atomic.AtomicInteger remainingSends = new java.util.concurrent.atomic.AtomicInteger(0); ++ ++ @Override ++ public void onPacketDispatch(ServerPlayer player) { ++ remainingSends.incrementAndGet(); ++ } ++ ++ @Override ++ public void onPacketDispatchFinish(ServerPlayer player, ChannelFuture future) { ++ if (remainingSends.decrementAndGet() <= 0) { ++ // incase of any race conditions, schedule this delayed ++ MCUtil.scheduleTask(5, () -> { ++ if (remainingSends.get() == 0) { ++ cleaner1.run(); ++ cleaner2.run(); ++ } ++ }, "Light Packet Release"); ++ } ++ } ++ ++ @Override ++ public boolean hasFinishListener() { ++ return true; ++ } ++ ++ // Paper end + + public ClientboundLightUpdatePacket(ChunkPos chunkPos, LevelLightEngine lightProvider, @Nullable BitSet bitSet, @Nullable BitSet bitSet2, boolean nonEdge) { + this.x = chunkPos.x; +@@ -0,0 +0,0 @@ public class ClientboundLightUpdatePacket implements Packet BYTE_2048 = new com.destroystokyo.paper.util.pooled.PooledObjects<>(() -> new byte[2048], maxPoolSize); ++ public static void releaseBytes(byte[] bytes) { ++ if (bytes != null && bytes != EMPTY_NIBBLE && bytes.length == 2048) { ++ System.arraycopy(EMPTY_NIBBLE, 0, bytes, 0, 2048); ++ BYTE_2048.release(bytes); ++ } ++ } + ++ public DataLayer markPoolSafe(byte[] bytes) { ++ if (bytes != EMPTY_NIBBLE) this.data = bytes; ++ return markPoolSafe(); ++ } ++ public DataLayer markPoolSafe() { ++ poolSafe = true; ++ return this; ++ } ++ public byte[] getIfSet() { ++ return this.data != null ? this.data : EMPTY_NIBBLE; ++ } ++ public byte[] getCloneIfSet() { ++ if (data == null) { ++ return EMPTY_NIBBLE; ++ } ++ byte[] ret = BYTE_2048.acquire(); ++ System.arraycopy(getIfSet(), 0, ret, 0, 2048); ++ return ret; ++ } ++ ++ public DataLayer cloneAndSet(byte[] bytes) { ++ if (bytes != null && bytes != EMPTY_NIBBLE) { ++ this.data = BYTE_2048.acquire(); ++ System.arraycopy(bytes, 0, this.data, 0, 2048); ++ } ++ return this; ++ } ++ boolean poolSafe = false; ++ public java.lang.Runnable cleaner; ++ private void registerCleaner() { ++ if (!poolSafe) { ++ cleaner = net.minecraft.server.MCUtil.registerCleaner(this, this.data, DataLayer::releaseBytes); ++ } else { ++ cleaner = net.minecraft.server.MCUtil.once(() -> DataLayer.releaseBytes(this.data)); ++ } ++ } + public DataLayer() {} + + public DataLayer(byte[] bytes) { ++ // Paper start ++ this(bytes, false); ++ } ++ public DataLayer(byte[] bytes, boolean isSafe) { + this.data = bytes; ++ if (!isSafe) this.data = getCloneIfSet(); // Paper - clone for safety ++ registerCleaner(); ++ // Paper end + if (bytes.length != 2048) { + throw (IllegalArgumentException) Util.pauseInIde((Throwable) (new IllegalArgumentException("ChunkNibbleArrays should be 2048 bytes not: " + bytes.length))); + } +@@ -0,0 +0,0 @@ public class DataLayer { + + public void set(int index, int value) { // PAIL: private -> public + if (this.data == null) { +- this.data = new byte[2048]; ++ this.data = BYTE_2048.acquire(); // Paper ++ registerCleaner();// Paper + } + + int k = this.getPosition(index); +@@ -0,0 +0,0 @@ public class DataLayer { + public byte[] getData() { + if (this.data == null) { + this.data = new byte[2048]; ++ } else { // Paper start ++ // Accessor may need this object past garbage collection so need to clone it and return pooled value ++ // If we know its safe for pre GC access, use asBytesPoolSafe(). If you just need read, use getIfSet() ++ Runnable cleaner = this.cleaner; ++ if (cleaner != null) { ++ this.data = this.data.clone(); ++ cleaner.run(); // release the previously pooled value ++ this.cleaner = null; ++ } + } ++ // Paper end + + return this.data; + } + ++ @javax.annotation.Nonnull ++ public byte[] asBytesPoolSafe() { ++ if (this.data == null) { ++ this.data = BYTE_2048.acquire(); // Paper ++ registerCleaner(); // Paper ++ } ++ ++ return this.data; ++ } ++ // Paper end + public DataLayer copy() { +- return this.data == null ? new DataLayer() : new DataLayer((byte[]) this.data.clone()); ++ return this.data == null ? new DataLayer() : new DataLayer(this.data); // Paper - clone in ctor + } + + public String toString() { +diff --git a/src/main/java/net/minecraft/world/level/chunk/storage/ChunkSerializer.java b/src/main/java/net/minecraft/world/level/chunk/storage/ChunkSerializer.java +index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 +--- a/src/main/java/net/minecraft/world/level/chunk/storage/ChunkSerializer.java ++++ b/src/main/java/net/minecraft/world/level/chunk/storage/ChunkSerializer.java +@@ -0,0 +0,0 @@ public class ChunkSerializer { + } + + if (nibblearray != null && !nibblearray.isEmpty()) { +- nbttagcompound2.putByteArray("BlockLight", nibblearray.getData()); ++ nbttagcompound2.putByteArray("BlockLight", nibblearray.asBytesPoolSafe().clone()); // Paper + } + + if (nibblearray1 != null && !nibblearray1.isEmpty()) { +- nbttagcompound2.putByteArray("SkyLight", nibblearray1.getData()); ++ nbttagcompound2.putByteArray("SkyLight", nibblearray1.asBytesPoolSafe().clone()); // Paper + } + + nbttaglist.add(nbttagcompound2); +diff --git a/src/main/java/net/minecraft/world/level/lighting/DataLayerStorageMap.java b/src/main/java/net/minecraft/world/level/lighting/DataLayerStorageMap.java +index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 +--- a/src/main/java/net/minecraft/world/level/lighting/DataLayerStorageMap.java ++++ b/src/main/java/net/minecraft/world/level/lighting/DataLayerStorageMap.java +@@ -0,0 +0,0 @@ public abstract class DataLayerStorageMap> { + + public void copyDataLayer(long pos) { + if (this.isVisible) { throw new IllegalStateException("writing to visible data"); } // Paper - avoid copying light data +- this.data.queueUpdate(pos, ((DataLayer) this.data.getUpdating(pos)).copy()); // Paper - avoid copying light data ++ DataLayer updating = this.data.getUpdating(pos); // Paper - pool nibbles ++ this.data.queueUpdate(pos, new DataLayer().markPoolSafe(updating.getCloneIfSet())); // Paper - avoid copying light data - pool safe clone ++ if (updating.cleaner != null) net.minecraft.server.MCUtil.scheduleTask(2, updating.cleaner, "Light Engine Release"); // Paper - delay clean incase anything holding ref was still using it + this.clearCache(); + } + +diff --git a/src/main/java/net/minecraft/world/level/lighting/FlatDataLayer.java b/src/main/java/net/minecraft/world/level/lighting/FlatDataLayer.java +index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 +--- a/src/main/java/net/minecraft/world/level/lighting/FlatDataLayer.java ++++ b/src/main/java/net/minecraft/world/level/lighting/FlatDataLayer.java +@@ -0,0 +0,0 @@ public class FlatDataLayer extends DataLayer { + + public FlatDataLayer(DataLayer chunkNibbleArray, int offset) { + super(128); +- System.arraycopy(chunkNibbleArray.getData(), offset * 128, this.data, 0, 128); ++ System.arraycopy(chunkNibbleArray.getIfSet(), offset * 128, this.data, 0, 128); // Paper + } + + @Override +@@ -0,0 +0,0 @@ public class FlatDataLayer extends DataLayer { + + @Override + public byte[] getData() { +- byte[] bs = new byte[2048]; ++ byte[] bs = BYTE_2048.acquire(); // Paper + + for(int i = 0; i < 16; ++i) { + System.arraycopy(this.data, 0, bs, i * 128, 128); +diff --git a/src/main/java/net/minecraft/world/level/lighting/LayerLightSectionStorage.java b/src/main/java/net/minecraft/world/level/lighting/LayerLightSectionStorage.java +index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 +--- a/src/main/java/net/minecraft/world/level/lighting/LayerLightSectionStorage.java ++++ b/src/main/java/net/minecraft/world/level/lighting/LayerLightSectionStorage.java +@@ -0,0 +0,0 @@ public abstract class LayerLightSectionStorage> + + protected DataLayer createDataLayer(long sectionPos) { + DataLayer dataLayer = this.queuedSections.get(sectionPos); +- return dataLayer != null ? dataLayer : new DataLayer(); ++ return dataLayer != null ? dataLayer : new DataLayer().markPoolSafe(); // Paper + } + + protected void clearQueuedSectionBlocks(LayerLightEngine storage, long sectionPos) { +@@ -0,0 +0,0 @@ public abstract class LayerLightSectionStorage> + + protected void queueSectionData(long sectionPos, @Nullable DataLayer array, boolean bl) { + if (array != null) { +- this.queuedSections.put(sectionPos, array); ++ DataLayer remove = this.queuedSections.put(sectionPos, array); if (remove != null && remove.cleaner != null) remove.cleaner.run(); // Paper - clean up when removed + if (!bl) { + this.untrustedSections.add(sectionPos); + } + } else { +- this.queuedSections.remove(sectionPos); ++ DataLayer remove = this.queuedSections.remove(sectionPos); if (remove != null && remove.cleaner != null) remove.cleaner.run(); // Paper - clean up when removed + } + + } +diff --git a/src/main/java/net/minecraft/world/level/lighting/SkyLightSectionStorage.java b/src/main/java/net/minecraft/world/level/lighting/SkyLightSectionStorage.java +index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644 +--- a/src/main/java/net/minecraft/world/level/lighting/SkyLightSectionStorage.java ++++ b/src/main/java/net/minecraft/world/level/lighting/SkyLightSectionStorage.java +@@ -0,0 +0,0 @@ public class SkyLightSectionStorage extends LayerLightSectionStorage