Allow reading of oversized region files

This commit is contained in:
Shane Freeder 2019-12-13 16:22:16 +00:00
parent 7b9f9f4ab6
commit 64bc3e1f0b
13 changed files with 286 additions and 19 deletions

View file

@ -20,7 +20,7 @@ index 7d39a8fbc..9889b0543 100644
if (i >= -30000000 && j >= -30000000 && i < 30000000 && j < 30000000) {
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
index 578fe8d19..12362d560 100644
index aec657952..38f61fddf 100644
--- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
@@ -0,0 +0,0 @@ public class CraftWorld implements World {

View file

@ -0,0 +1,265 @@
From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Aikar <aikar@aikar.co>
Date: Fri, 15 Feb 2019 01:08:19 -0500
Subject: [PATCH] Allow Saving of Oversized Chunks
The Minecraft World Region File format has a hard cap of 1MB per chunk.
This is due to the fact that the header of the file format only allocates
a single byte for sector count, meaning a maximum of 256 sectors, at 4k per sector.
This limit can be reached fairly easily with books, resulting in the chunk being unable
to save to the world. Worse off, is that nothing printed when this occured, and silently
performed a chunk rollback on next load.
This leads to security risk with duplication and is being actively exploited.
This patch catches the too large scenario, falls back and moves any large Entity
or Tile Entity into a new compound, and this compound is saved into a different file.
On Chunk Load, we check for oversized status, and if so, we load the extra file and
merge the Entities and Tile Entities from the oversized chunk back into the level to
then be loaded as normal.
Once a chunk is returned back to normal size, the oversized flag will clear, and no
extra data file will exist.
This fix maintains compatability with all existing Anvil Region Format tools as it
does not alter the save format. They will just not know about the extra entities.
This fix also maintains compatability if someone switches server jars to one without
this fix, as the data will remain in the oversized file. Once the server returns
to a jar with this fix, the data will be restored.
diff --git a/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java b/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
index db66d4ac7..2322c0c8c 100644
--- a/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
+++ b/src/main/java/net/minecraft/server/NBTCompressedStreamTools.java
@@ -0,0 +0,0 @@ public class NBTCompressedStreamTools {
}
+ public static NBTTagCompound readNBT(DataInputStream datainputstream) throws IOException { return a(datainputstream); } // Paper - OBFHELPER
public static NBTTagCompound a(DataInputStream datainputstream) throws IOException {
return a((DataInput) datainputstream, NBTReadLimiter.a);
}
@@ -0,0 +0,0 @@ public class NBTCompressedStreamTools {
}
}
+ public static void writeNBT(NBTTagCompound nbttagcompound, DataOutput dataoutput) throws IOException { a(nbttagcompound, dataoutput); } // Paper - OBFHELPER
public static void a(NBTTagCompound nbttagcompound, DataOutput dataoutput) throws IOException {
a((NBTBase) nbttagcompound, dataoutput);
}
diff --git a/src/main/java/net/minecraft/server/RegionFile.java b/src/main/java/net/minecraft/server/RegionFile.java
index 877fc4be6..ebb1b6b8b 100644
--- a/src/main/java/net/minecraft/server/RegionFile.java
+++ b/src/main/java/net/minecraft/server/RegionFile.java
@@ -0,0 +0,0 @@ import java.nio.file.Files;
import java.nio.file.LinkOption;
import java.nio.file.StandardCopyOption;
import java.nio.file.StandardOpenOption;
+import java.util.zip.InflaterInputStream; // Paper
+
import javax.annotation.Nullable;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
@@ -0,0 +0,0 @@ public class RegionFile implements AutoCloseable {
private final IntBuffer g;
private final IntBuffer h;
private final RegionFileBitSet freeSectors;
+ public final File file;
public RegionFile(File file, File file1) throws IOException {
this(file.toPath(), file1.toPath(), RegionFileCompression.b);
}
public RegionFile(java.nio.file.Path java_nio_file_path, java.nio.file.Path java_nio_file_path1, RegionFileCompression regionfilecompression) throws IOException {
+ this.file = java_nio_file_path.toFile(); // Paper
+ System.out.println(file.toString());
this.f = ByteBuffer.allocateDirect(8192);
+ initOversizedState();
this.freeSectors = new RegionFileBitSet();
this.e = regionfilecompression;
if (!Files.isDirectory(java_nio_file_path1, new LinkOption[0])) {
@@ -0,0 +0,0 @@ public class RegionFile implements AutoCloseable {
return this.d.resolve(s);
}
- @Nullable
+ @Nullable public synchronized DataInputStream getReadStream(ChunkCoordIntPair chunkCoordIntPair) { return a(chunkCoordIntPair);} // Paper - OBFHELPER
public synchronized DataInputStream a(ChunkCoordIntPair chunkcoordintpair) throws IOException {
int i = this.getOffset(chunkcoordintpair);
@@ -0,0 +0,0 @@ public class RegionFile implements AutoCloseable {
void run() throws IOException;
}
+ private final byte[] oversized = new byte[1024];
+ private int oversizedCount = 0;
+
+ private synchronized void initOversizedState() throws IOException {
+ File metaFile = getOversizedMetaFile();
+ if (metaFile.exists()) {
+ final byte[] read = java.nio.file.Files.readAllBytes(metaFile.toPath());
+ System.arraycopy(read, 0, oversized, 0, oversized.length);
+ for (byte temp : oversized) {
+ oversizedCount += temp;
+ }
+ }
+ }
+
+ private static int getChunkIndex(int x, int z) {
+ return (x & 31) + (z & 31) * 32;
+ }
+ synchronized boolean isOversized(int x, int z) {
+ return this.oversized[getChunkIndex(x, z)] == 1;
+ }
+ synchronized void setOversized(int x, int z, boolean oversized) throws IOException {
+ final int offset = getChunkIndex(x, z);
+ boolean previous = this.oversized[offset] == 1;
+ this.oversized[offset] = (byte) (oversized ? 1 : 0);
+ if (!previous && oversized) {
+ oversizedCount++;
+ } else if (!oversized && previous) {
+ oversizedCount--;
+ }
+ if (previous && !oversized) {
+ File oversizedFile = getOversizedFile(x, z);
+ if (oversizedFile.exists()) {
+ oversizedFile.delete();
+ }
+ }
+ if (oversizedCount > 0) {
+ if (previous != oversized) {
+ writeOversizedMeta();
+ }
+ } else if (previous) {
+ File oversizedMetaFile = getOversizedMetaFile();
+ if (oversizedMetaFile.exists()) {
+ oversizedMetaFile.delete();
+ }
+ }
+ }
+
+ private void writeOversizedMeta() throws IOException {
+ java.nio.file.Files.write(getOversizedMetaFile().toPath(), oversized);
+ }
+
+ private File getOversizedMetaFile() {
+ return new File(this.file.getParentFile(), this.file.getName().replaceAll("\\.mca$", "") + ".oversized.nbt");
+ }
+
+ private File getOversizedFile(int x, int z) {
+ return new File(this.file.getParentFile(), this.file.getName().replaceAll("\\.mca$", "") + "_oversized_" + x + "_" + z + ".nbt");
+ }
+
+ synchronized NBTTagCompound getOversizedData(int x, int z) throws IOException {
+ File file = getOversizedFile(x, z);
+ try (DataInputStream out = new DataInputStream(new BufferedInputStream(new InflaterInputStream(new java.io.FileInputStream(file))))) {
+ return NBTCompressedStreamTools.readNBT(out);
+ }
+
+ }
+ // Paper end
+
class ChunkBuffer extends ByteArrayOutputStream {
private final ChunkCoordIntPair b;
diff --git a/src/main/java/net/minecraft/server/RegionFileCache.java b/src/main/java/net/minecraft/server/RegionFileCache.java
index 57ce53cfd..b3d1bb5fd 100644
--- a/src/main/java/net/minecraft/server/RegionFileCache.java
+++ b/src/main/java/net/minecraft/server/RegionFileCache.java
@@ -0,0 +0,0 @@ public final class RegionFileCache implements AutoCloseable {
}
}
+ // Paper start
+ private static void printOversizedLog(String msg, File file, int x, int z) {
+ org.apache.logging.log4j.LogManager.getLogger().fatal(msg + " (" + file.toString().replaceAll(".+[\\\\/]", "") + " - " + x + "," + z + ") Go clean it up to remove this message. /minecraft:tp " + (x<<4)+" 128 "+(z<<4) + " - DO NOT REPORT THIS TO PAPER - You may ask for help on Discord, but do not file an issue. These error messages can not be removed.");
+ }
+
+ private static final int DEFAULT_SIZE_THRESHOLD = 1024 * 8;
+ private static final int OVERZEALOUS_TOTAL_THRESHOLD = 1024 * 64;
+ private static final int OVERZEALOUS_THRESHOLD = 1024;
+ private static int SIZE_THRESHOLD = DEFAULT_SIZE_THRESHOLD;
+ private static void resetFilterThresholds() {
+ SIZE_THRESHOLD = Math.max(1024 * 4, Integer.getInteger("Paper.FilterThreshhold", DEFAULT_SIZE_THRESHOLD));
+ }
+ static {
+ resetFilterThresholds();
+ }
+
+ static boolean isOverzealous() {
+ return SIZE_THRESHOLD == OVERZEALOUS_THRESHOLD;
+ }
+
+
+ private static NBTTagCompound readOversizedChunk(RegionFile regionfile, ChunkCoordIntPair chunkCoordinate) throws IOException {
+ synchronized (regionfile) {
+ try (DataInputStream datainputstream = regionfile.getReadStream(chunkCoordinate)) {
+ NBTTagCompound oversizedData = regionfile.getOversizedData(chunkCoordinate.x, chunkCoordinate.z);
+ NBTTagCompound chunk = NBTCompressedStreamTools.readNBT(datainputstream);
+ if (oversizedData == null) {
+ return chunk;
+ }
+ NBTTagCompound oversizedLevel = oversizedData.getCompound("Level");
+ NBTTagCompound level = chunk.getCompound("Level");
+
+ mergeChunkList(level, oversizedLevel, "Entities");
+ mergeChunkList(level, oversizedLevel, "TileEntities");
+
+ chunk.set("Level", level);
+
+ return chunk;
+ } catch (Throwable throwable) {
+ throwable.printStackTrace();
+ throw throwable;
+ }
+ }
+ }
+
+ private static void mergeChunkList(NBTTagCompound level, NBTTagCompound oversizedLevel, String key) {
+ NBTTagList levelList = level.getList(key, 10);
+ NBTTagList oversizedList = oversizedLevel.getList(key, 10);
+
+ if (!oversizedList.isEmpty()) {
+ levelList.addAll(oversizedList);
+ level.set(key, levelList);
+ }
+ }
+
+ private static int getNBTSize(NBTBase nbtBase) {
+ DataOutputStream test = new DataOutputStream(new org.apache.commons.io.output.NullOutputStream());
+ try {
+ nbtBase.write(test);
+ return test.size();
+ } catch (IOException e) {
+ e.printStackTrace();
+ return 0;
+ }
+ }
+
+ // Paper End
+
@Nullable
public NBTTagCompound read(ChunkCoordIntPair chunkcoordintpair) throws IOException {
RegionFile regionfile = this.getFile(chunkcoordintpair, false); // CraftBukkit
DataInputStream datainputstream = regionfile.a(chunkcoordintpair);
+ // Paper start
+ if (regionfile.isOversized(chunkcoordintpair.x, chunkcoordintpair.z)) {
+ printOversizedLog("Loading Oversized Chunk!", regionfile.file, chunkcoordintpair.x, chunkcoordintpair.z);
+ return readOversizedChunk(regionfile, chunkcoordintpair);
+ }
+ // Paper end
Throwable throwable = null;
NBTTagCompound nbttagcompound;
@@ -0,0 +0,0 @@ public final class RegionFileCache implements AutoCloseable {
try {
NBTCompressedStreamTools.a(nbttagcompound, (DataOutput) dataoutputstream);
+ regionfile.setOversized(chunkcoordintpair.x, chunkcoordintpair.z, false); // We don't do this anymore
} catch (Throwable throwable1) {
throwable = throwable1;
throw throwable1;
--

View file

@ -3542,7 +3542,7 @@ index 6a54ccb86..fce37d0d6 100644
return this.m;
}
diff --git a/src/main/java/net/minecraft/server/RegionFile.java b/src/main/java/net/minecraft/server/RegionFile.java
index 5cb19b105..be3726a83 100644
index e28fea44e..1d74a59b5 100644
--- a/src/main/java/net/minecraft/server/RegionFile.java
+++ b/src/main/java/net/minecraft/server/RegionFile.java
@@ -0,0 +0,0 @@ public class RegionFile implements AutoCloseable {
@ -3564,7 +3564,7 @@ index 5cb19b105..be3726a83 100644
try {
this.c();
diff --git a/src/main/java/net/minecraft/server/RegionFileCache.java b/src/main/java/net/minecraft/server/RegionFileCache.java
index 1a6be7c6d..386f47dc8 100644
index e07ae9854..d927f9321 100644
--- a/src/main/java/net/minecraft/server/RegionFileCache.java
+++ b/src/main/java/net/minecraft/server/RegionFileCache.java
@@ -0,0 +0,0 @@ import java.io.File;

View file

@ -5,7 +5,7 @@ Subject: [PATCH] Catch exceptions from dispenser entity spawns
diff --git a/src/main/java/net/minecraft/server/IDispenseBehavior.java b/src/main/java/net/minecraft/server/IDispenseBehavior.java
index 8c4a8b4ba..83877d672 100644
index abc0cd346..9a8be1474 100644
--- a/src/main/java/net/minecraft/server/IDispenseBehavior.java
+++ b/src/main/java/net/minecraft/server/IDispenseBehavior.java
@@ -0,0 +0,0 @@ public interface IDispenseBehavior {

View file

@ -7,7 +7,7 @@ If the Bukkit generator already has a spawn, use it immediately instead
of spending time generating one that we won't use
diff --git a/src/main/java/net/minecraft/server/WorldServer.java b/src/main/java/net/minecraft/server/WorldServer.java
index 657f0e024..1d750eba5 100644
index 59b2fc629..3283fb49e 100644
--- a/src/main/java/net/minecraft/server/WorldServer.java
+++ b/src/main/java/net/minecraft/server/WorldServer.java
@@ -0,0 +0,0 @@ public class WorldServer extends World {

View file

@ -200,12 +200,12 @@ index 4379434f6..8e2208422 100644
boolean isOutsideOfRange(ChunkCoordIntPair chunkcoordintpair) {
// Spigot start
diff --git a/src/main/java/net/minecraft/server/RegionFile.java b/src/main/java/net/minecraft/server/RegionFile.java
index 5d2cbbad2..7eb87c517 100644
index ebb1b6b8b..e28fea44e 100644
--- a/src/main/java/net/minecraft/server/RegionFile.java
+++ b/src/main/java/net/minecraft/server/RegionFile.java
@@ -0,0 +0,0 @@ public class RegionFile implements AutoCloseable {
private final IntBuffer h;
private final RegionFileBitSet freeSectors;
public final File file;
+ // Paper start - Cache chunk status
+ private final ChunkStatus[] statuses = new ChunkStatus[32 * 32];
@ -249,7 +249,7 @@ index 5d2cbbad2..7eb87c517 100644
this.c();
} finally {
diff --git a/src/main/java/net/minecraft/server/RegionFileCache.java b/src/main/java/net/minecraft/server/RegionFileCache.java
index 57ce53cfd..1a6be7c6d 100644
index b3d1bb5fd..e07ae9854 100644
--- a/src/main/java/net/minecraft/server/RegionFileCache.java
+++ b/src/main/java/net/minecraft/server/RegionFileCache.java
@@ -0,0 +0,0 @@ public final class RegionFileCache implements AutoCloseable {
@ -272,12 +272,14 @@ index 57ce53cfd..1a6be7c6d 100644
try {
NBTCompressedStreamTools.a(nbttagcompound, (DataOutput) dataoutputstream);
- regionfile.setOversized(chunkcoordintpair.x, chunkcoordintpair.z, false); // We don't do this anymore
+ regionfile.setStatus(chunkcoordintpair.x, chunkcoordintpair.z, ChunkRegionLoader.getStatus(nbttagcompound)); // Paper - cache status on disk
+ regionfile.setOversized(chunkcoordintpair.x, chunkcoordintpair.z, false);
} catch (Throwable throwable1) {
throwable = throwable1;
throw throwable1;
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
index b1ae19be7..7d509856b 100644
index ceb638c98..b824518a2 100644
--- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
@@ -0,0 +0,0 @@ import java.util.Objects;

View file

@ -5,7 +5,7 @@ Subject: [PATCH] Fix some generation concurrency issues
diff --git a/src/main/java/net/minecraft/server/World.java b/src/main/java/net/minecraft/server/World.java
index 10bb06489..9ad76ab32 100644
index 9889b0543..e80cb3b97 100644
--- a/src/main/java/net/minecraft/server/World.java
+++ b/src/main/java/net/minecraft/server/World.java
@@ -0,0 +0,0 @@ public abstract class World implements GeneratorAccess, AutoCloseable {

View file

@ -5,7 +5,7 @@ Subject: [PATCH] Fix stuck in sneak when changing worlds (MC-10657)
diff --git a/src/main/java/net/minecraft/server/EntityPlayer.java b/src/main/java/net/minecraft/server/EntityPlayer.java
index c9492ed37..95a5643a6 100644
index 1b0ef1442..30380c8db 100644
--- a/src/main/java/net/minecraft/server/EntityPlayer.java
+++ b/src/main/java/net/minecraft/server/EntityPlayer.java
@@ -0,0 +0,0 @@ public class EntityPlayer extends EntityHuman implements ICrafting {

View file

@ -81,7 +81,7 @@ index 55b07444e..3bc3c5aa2 100644
for (i = 1; worldserver.getType(blockposition.down(i)).getBlock() == this; ++i) {
diff --git a/src/main/java/net/minecraft/server/WorldServer.java b/src/main/java/net/minecraft/server/WorldServer.java
index 4da5c8982..b15fb2b63 100644
index 4da34b6dd..722384a91 100644
--- a/src/main/java/net/minecraft/server/WorldServer.java
+++ b/src/main/java/net/minecraft/server/WorldServer.java
@@ -0,0 +0,0 @@ public class WorldServer extends World {

View file

@ -39,7 +39,7 @@ index 2e95069c1..8977c3516 100644
double deltaX = this.locX() - player.locX();
double deltaZ = this.locZ() - player.locZ();
diff --git a/src/main/java/org/bukkit/craftbukkit/entity/CraftPlayer.java b/src/main/java/org/bukkit/craftbukkit/entity/CraftPlayer.java
index b176b2346..5603ed126 100644
index 10c653e8e..712056cad 100644
--- a/src/main/java/org/bukkit/craftbukkit/entity/CraftPlayer.java
+++ b/src/main/java/org/bukkit/craftbukkit/entity/CraftPlayer.java
@@ -0,0 +0,0 @@ public class CraftPlayer extends CraftHumanEntity implements Player {

View file

@ -8,7 +8,7 @@ Exposes a mutable array on items a player should keep on death
Example Usage: https://gist.github.com/aikar/5bb202de6057a051a950ce1f29feb0b4
diff --git a/src/main/java/net/minecraft/server/EntityPlayer.java b/src/main/java/net/minecraft/server/EntityPlayer.java
index cc9e69651..ca2398386 100644
index 65b6051cf..72938a2e2 100644
--- a/src/main/java/net/minecraft/server/EntityPlayer.java
+++ b/src/main/java/net/minecraft/server/EntityPlayer.java
@@ -0,0 +0,0 @@ public class EntityPlayer extends EntityHuman implements ICrafting {

View file

@ -599,7 +599,7 @@ index 1f6b1c4f1..cbe4b23e1 100644
}
}
diff --git a/src/main/java/net/minecraft/server/EntityPlayer.java b/src/main/java/net/minecraft/server/EntityPlayer.java
index fc3cb17e6..c9492ed37 100644
index 72938a2e2..1b0ef1442 100644
--- a/src/main/java/net/minecraft/server/EntityPlayer.java
+++ b/src/main/java/net/minecraft/server/EntityPlayer.java
@@ -0,0 +0,0 @@ public class EntityPlayer extends EntityHuman implements ICrafting {
@ -643,7 +643,7 @@ index d49ad0308..2fb04e3e9 100644
return this.bb;
}
diff --git a/src/main/java/net/minecraft/server/PlayerChunkMap.java b/src/main/java/net/minecraft/server/PlayerChunkMap.java
index 66bd402e9..041fedcfa 100644
index fce37d0d6..b37c1161d 100644
--- a/src/main/java/net/minecraft/server/PlayerChunkMap.java
+++ b/src/main/java/net/minecraft/server/PlayerChunkMap.java
@@ -0,0 +0,0 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
@ -682,7 +682,7 @@ index 66bd402e9..041fedcfa 100644
private static double a(ChunkCoordIntPair chunkcoordintpair, Entity entity) {
diff --git a/src/main/java/net/minecraft/server/SpawnerCreature.java b/src/main/java/net/minecraft/server/SpawnerCreature.java
index e168c528c..56dabbc15 100644
index fdac5bb3a..58bbf2f9d 100644
--- a/src/main/java/net/minecraft/server/SpawnerCreature.java
+++ b/src/main/java/net/minecraft/server/SpawnerCreature.java
@@ -0,0 +0,0 @@ package net.minecraft.server;
@ -755,7 +755,7 @@ index e168c528c..56dabbc15 100644
@Nullable
diff --git a/src/main/java/net/minecraft/server/WorldServer.java b/src/main/java/net/minecraft/server/WorldServer.java
index b9e37d630..25a1ef134 100644
index 3283fb49e..4da34b6dd 100644
--- a/src/main/java/net/minecraft/server/WorldServer.java
+++ b/src/main/java/net/minecraft/server/WorldServer.java
@@ -0,0 +0,0 @@ public class WorldServer extends World {

View file

@ -9,7 +9,7 @@ waiting for the execution queue to get to our request; We can just query
the chunk status and get a response now, vs having to wait
diff --git a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
index 12362d560..29c14c707 100644
index 38f61fddf..f773534d9 100644
--- a/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
+++ b/src/main/java/org/bukkit/craftbukkit/CraftWorld.java
@@ -0,0 +0,0 @@ public class CraftWorld implements World {