Faster chunk conversion (#1400)

* BlockStorage is never used concurrently, no need to synchronize

* initial, semi-functional, faster chunk conversion

* faster chunk conversion works well for every situation except spigot

* delete unused ChunkPosition class

* preallocate and pool chunk encoding buffers

* make it work correctly on spigot

* make field naming more consistent

* attempt to upgrade to latest MCProtocolLib

* remove debug code

* compile against my MCProtocolLib fork while i wait for my upstream PR to be accepted

* return to Steveice10 MCProtocolLib

bootstrap/spigot/src/main/java/org/geysermc/platform/spigot/world/GeyserSpigotWorldManager.java

@@ -26,12 +26,14 @@
 package org.geysermc.platform.spigot.world;
 
 import com.fasterxml.jackson.databind.JsonNode;
+import com.github.steveice10.mc.protocol.data.game.chunk.Chunk;
 import it.unimi.dsi.fastutil.ints.Int2IntMap;
 import it.unimi.dsi.fastutil.ints.Int2IntOpenHashMap;
 import org.bukkit.Bukkit;
 import org.bukkit.World;
 import org.bukkit.block.Biome;
 import org.bukkit.block.Block;
+import org.bukkit.entity.Player;
 import org.geysermc.connector.GeyserConnector;
 import org.geysermc.connector.network.session.GeyserSession;
 import org.geysermc.connector.network.translators.world.GeyserWorldManager;
@@ -93,23 +95,32 @@ public class GeyserSpigotWorldManager extends GeyserWorldManager {
 
     @Override
     public int getBlockAt(GeyserSession session, int x, int y, int z) {
-        if (session.getPlayerEntity() == null) {
+        Player bukkitPlayer;
-            return BlockTranslator.AIR;
+        if ((this.isLegacy && !this.isViaVersion)
-        }
+            || session.getPlayerEntity() == null
-        if (Bukkit.getPlayer(session.getPlayerEntity().getUsername()) == null) {
+            || (bukkitPlayer = Bukkit.getPlayer(session.getPlayerEntity().getUsername())) == null) {
             return BlockTranslator.AIR;
         }
+        World world = bukkitPlayer.getWorld();
         if (isLegacy) {
-            return getLegacyBlock(session, x, y, z, isViaVersion);
+            return getLegacyBlock(session, x, y, z, true);
         }
         //TODO possibly: detect server version for all versions and use ViaVersion for block state mappings like below
-        return BlockTranslator.getJavaIdBlockMap().getOrDefault(Bukkit.getPlayer(session.getPlayerEntity().getUsername()).getWorld().getBlockAt(x, y, z).getBlockData().getAsString(), 0);
+        return BlockTranslator.getJavaIdBlockMap().getOrDefault(world.getBlockAt(x, y, z).getBlockData().getAsString(), 0);
+    }
+
+    public static int getLegacyBlock(GeyserSession session, int x, int y, int z, boolean isViaVersion) {
+        if (isViaVersion) {
+            return getLegacyBlock(Bukkit.getPlayer(session.getPlayerEntity().getUsername()).getWorld(), x, y, z, true);
+        } else {
+            return BlockTranslator.AIR;
+        }
     }
 
     @SuppressWarnings("deprecation")
-    public static int getLegacyBlock(GeyserSession session, int x, int y, int z, boolean isViaVersion) {
+    public static int getLegacyBlock(World world, int x, int y, int z, boolean isViaVersion) {
         if (isViaVersion) {
-            Block block = Bukkit.getPlayer(session.getPlayerEntity().getUsername()).getWorld().getBlockAt(x, y, z);
+            Block block = world.getBlockAt(x, y, z);
             // Black magic that gets the old block state ID
             int oldBlockId = (block.getType().getId() << 4) | (block.getData() & 0xF);
             // Convert block state from old version -> 1.13 -> 1.13.1 -> 1.14 -> 1.15 -> 1.16 -> 1.16.2
@@ -124,6 +135,42 @@ public class GeyserSpigotWorldManager extends GeyserWorldManager {
         }
     }
 
+    @Override
+    public void getBlocksInSection(GeyserSession session, int x, int y, int z, Chunk chunk) {
+        Player bukkitPlayer;
+        if ((this.isLegacy && !this.isViaVersion)
+            || session.getPlayerEntity() == null
+            || (bukkitPlayer = Bukkit.getPlayer(session.getPlayerEntity().getUsername())) == null) {
+            return;
+        }
+        World world = bukkitPlayer.getWorld();
+        if (this.isLegacy)  {
+            for (int blockY = 0; blockY < 16; blockY++) { // Cache-friendly iteration order
+                for (int blockZ = 0; blockZ < 16; blockZ++) {
+                    for (int blockX = 0; blockX < 16; blockX++) {
+                        chunk.set(blockX, blockY, blockZ, getLegacyBlock(world, (x << 4) + blockX, (y << 4) + blockY, (z << 4) + blockZ, true));
+                    }
+                }
+            }
+        } else {
+            //TODO: see above TODO in getBlockAt
+            for (int blockY = 0; blockY < 16; blockY++) { // Cache-friendly iteration order
+                for (int blockZ = 0; blockZ < 16; blockZ++) {
+                    for (int blockX = 0; blockX < 16; blockX++) {
+                        Block block = world.getBlockAt((x << 4) + blockX, (y << 4) + blockY, (z << 4) + blockZ);
+                        int id = BlockTranslator.getJavaIdBlockMap().getOrDefault(block.getBlockData().getAsString(), 0);
+                        chunk.set(blockX, blockY, blockZ, id);
+                    }
+                }
+            }
+        }
+    }
+
+    @Override
+    public boolean hasMoreBlockDataThanChunkCache() {
+        return true;
+    }
+
     @Override
     @SuppressWarnings("deprecation")
     public int[] getBiomeDataAt(GeyserSession session, int x, int z) {

connector/pom.xml

@@ -111,7 +111,7 @@
         <dependency>
             <groupId>com.github.steveice10</groupId>
             <artifactId>mcprotocollib</artifactId>
-            <version>976c2d0f89</version>
+            <version>3a69a0614c</version>
             <scope>compile</scope>
             <exclusions>
                 <exclusion>

connector/src/main/java/org/geysermc/connector/network/session/cache/ChunkCache.java

@@ -32,7 +32,7 @@ import it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap;
 import org.geysermc.connector.bootstrap.GeyserBootstrap;
 import org.geysermc.connector.network.session.GeyserSession;
 import org.geysermc.connector.network.translators.world.block.BlockTranslator;
-import org.geysermc.connector.network.translators.world.chunk.ChunkPosition;
+import org.geysermc.connector.utils.MathUtils;
 
 public class ChunkCache {
 
@@ -48,27 +48,31 @@ public class ChunkCache {
         }
     }
 
-    public void addToCache(Column chunk) {
+    public Column addToCache(Column chunk) {
         if (!cache) {
-            return;
+            return chunk;
         }
 
-        long chunkPosition = ChunkPosition.toLong(chunk.getX(), chunk.getZ());
+        long chunkPosition = MathUtils.chunkPositionToLong(chunk.getX(), chunk.getZ());
         Column existingChunk;
-        if (chunk.getBiomeData() != null // Only consider merging columns if the new chunk isn't a full chunk
+        if (chunk.getBiomeData() == null // Only consider merging columns if the new chunk isn't a full chunk
             && (existingChunk = chunks.getOrDefault(chunkPosition, null)) != null) { // Column is already present in cache, we can merge with existing
+            boolean changed = false;
             for (int i = 0; i < chunk.getChunks().length; i++) { // The chunks member is final, so chunk.getChunks() will probably be inlined and then completely optimized away
                 if (chunk.getChunks()[i] != null) {
                     existingChunk.getChunks()[i] = chunk.getChunks()[i];
+                    changed = true;
                 }
             }
+            return changed ? existingChunk : null;
         } else {
             chunks.put(chunkPosition, chunk);
+            return chunk;
         }
     }
 
     public Column getChunk(int chunkX, int chunkZ)  {
-        long chunkPosition = ChunkPosition.toLong(chunkX, chunkZ);
+        long chunkPosition = MathUtils.chunkPositionToLong(chunkX, chunkZ);
         return chunks.getOrDefault(chunkPosition, null);
     }
 
@@ -111,7 +115,7 @@ public class ChunkCache {
             return;
         }
 
-        long chunkPosition = ChunkPosition.toLong(chunkX, chunkZ);
+        long chunkPosition = MathUtils.chunkPositionToLong(chunkX, chunkZ);
         chunks.remove(chunkPosition);
     }
 }

connector/src/main/java/org/geysermc/connector/network/translators/java/world/JavaChunkDataTranslator.java

@@ -25,6 +25,7 @@
 
 package org.geysermc.connector.network.translators.java.world;
 
+import com.github.steveice10.mc.protocol.data.game.chunk.Column;
 import com.github.steveice10.mc.protocol.packet.ingame.server.world.ServerChunkDataPacket;
 import com.nukkitx.nbt.NBTOutputStream;
 import com.nukkitx.nbt.NbtMap;
@@ -32,8 +33,8 @@ import com.nukkitx.nbt.NbtUtils;
 import com.nukkitx.network.VarInts;
 import com.nukkitx.protocol.bedrock.packet.LevelChunkPacket;
 import io.netty.buffer.ByteBuf;
+import io.netty.buffer.ByteBufAllocator;
 import io.netty.buffer.ByteBufOutputStream;
-import io.netty.buffer.Unpooled;
 import org.geysermc.connector.GeyserConnector;
 import org.geysermc.connector.network.session.GeyserSession;
 import org.geysermc.connector.network.translators.BiomeTranslator;
@@ -66,57 +67,69 @@ public class JavaChunkDataTranslator extends PacketTranslator<ServerChunkDataPac
             return;
         }
 
+        // Merge received column with cache on network thread
+        Column mergedColumn = session.getChunkCache().addToCache(packet.getColumn());
+        if (mergedColumn == null) { // There were no changes?!?
+            return;
+        }
+
+        boolean isNonFullChunk = packet.getColumn().getBiomeData() == null;
+
         GeyserConnector.getInstance().getGeneralThreadPool().execute(() -> {
             try {
-                // Non-full chunks don't have all the chunk data, and Bedrock won't accept that
+                ChunkUtils.ChunkData chunkData = ChunkUtils.translateToBedrock(session, mergedColumn, isNonFullChunk);
-                final boolean isNonFullChunk = (packet.getColumn().getBiomeData() == null);
+                ChunkSection[] sections = chunkData.getSections();
-
-                ChunkUtils.ChunkData chunkData = ChunkUtils.translateToBedrock(session, packet.getColumn(), isNonFullChunk);
-                ByteBuf byteBuf = Unpooled.buffer(32);
-                ChunkSection[] sections = chunkData.sections;
 
+                // Find highest section
                 int sectionCount = sections.length - 1;
-                while (sectionCount >= 0 && sections[sectionCount].isEmpty()) {
+                while (sectionCount >= 0 && sections[sectionCount] == null) {
                     sectionCount--;
                 }
                 sectionCount++;
 
+                // Estimate chunk size
+                int size = 0;
                 for (int i = 0; i < sectionCount; i++) {
-                    ChunkSection section = chunkData.sections[i];
+                    ChunkSection section = sections[i];
-                    section.writeToNetwork(byteBuf);
+                    size += (section != null ? section : ChunkUtils.EMPTY_SECTION).estimateNetworkSize();
                 }
+                size += 256; // Biomes
+                size += 1; // Border blocks
+                size += 1; // Extra data length (always 0)
+                size += chunkData.getBlockEntities().length * 64; // Conservative estimate of 64 bytes per tile entity
 
-                byte[] bedrockBiome;
+                // Allocate output buffer
-                if (packet.getColumn().getBiomeData() == null) {
+                ByteBuf byteBuf = ByteBufAllocator.DEFAULT.buffer(size);
-                    bedrockBiome = BiomeTranslator.toBedrockBiome(session.getConnector().getWorldManager().getBiomeDataAt(session, packet.getColumn().getX(), packet.getColumn().getZ()));
+                byte[] payload;
-                } else {
+                try {
-                    bedrockBiome = BiomeTranslator.toBedrockBiome(packet.getColumn().getBiomeData());
+                    for (int i = 0; i < sectionCount; i++) {
+                        ChunkSection section = sections[i];
+                        (section != null ? section : ChunkUtils.EMPTY_SECTION).writeToNetwork(byteBuf);
+                    }
+
+                    byteBuf.writeBytes(BiomeTranslator.toBedrockBiome(mergedColumn.getBiomeData())); // Biomes - 256 bytes
+                    byteBuf.writeByte(0); // Border blocks - Edu edition only
+                    VarInts.writeUnsignedInt(byteBuf, 0); // extra data length, 0 for now
+
+                    // Encode tile entities into buffer
+                    NBTOutputStream nbtStream = NbtUtils.createNetworkWriter(new ByteBufOutputStream(byteBuf));
+                    for (NbtMap blockEntity : chunkData.getBlockEntities()) {
+                        nbtStream.writeTag(blockEntity);
+                    }
+
+                    // Copy data into byte[], because the protocol lib really likes things that are s l o w
+                    byteBuf.readBytes(payload = new byte[byteBuf.readableBytes()]);
+                } finally {
+                    byteBuf.release(); // Release buffer to allow buffer pooling to be useful
                 }
 
-                byteBuf.writeBytes(bedrockBiome); // Biomes - 256 bytes
-                byteBuf.writeByte(0); // Border blocks - Edu edition only
-                VarInts.writeUnsignedInt(byteBuf, 0); // extra data length, 0 for now
-
-                ByteBufOutputStream stream = new ByteBufOutputStream(Unpooled.buffer());
-                NBTOutputStream nbtStream = NbtUtils.createNetworkWriter(stream);
-                for (NbtMap blockEntity : chunkData.getBlockEntities()) {
-                    nbtStream.writeTag(blockEntity);
-                }
-
-                byteBuf.writeBytes(stream.buffer());
-
-                byte[] payload = new byte[byteBuf.writerIndex()];
-                byteBuf.readBytes(payload);
-
                 LevelChunkPacket levelChunkPacket = new LevelChunkPacket();
                 levelChunkPacket.setSubChunksLength(sectionCount);
                 levelChunkPacket.setCachingEnabled(false);
-                levelChunkPacket.setChunkX(packet.getColumn().getX());
+                levelChunkPacket.setChunkX(mergedColumn.getX());
-                levelChunkPacket.setChunkZ(packet.getColumn().getZ());
+                levelChunkPacket.setChunkZ(mergedColumn.getZ());
                 levelChunkPacket.setData(payload);
                 session.sendUpstreamPacket(levelChunkPacket);
-
-                session.getChunkCache().addToCache(packet.getColumn());
             } catch (Exception ex) {
                 ex.printStackTrace();
             }

connector/src/main/java/org/geysermc/connector/network/translators/world/GeyserWorldManager.java

@@ -25,6 +25,7 @@
 
 package org.geysermc.connector.network.translators.world;
 
+import com.github.steveice10.mc.protocol.data.game.chunk.Chunk;
 import com.github.steveice10.mc.protocol.data.game.chunk.Column;
 import com.github.steveice10.mc.protocol.data.game.entity.player.GameMode;
 import com.github.steveice10.mc.protocol.data.game.setting.Difficulty;
@@ -48,6 +49,31 @@ public class GeyserWorldManager extends WorldManager {
         return 0;
     }
 
+    @Override
+    public void getBlocksInSection(GeyserSession session, int x, int y, int z, Chunk chunk) {
+        ChunkCache chunkCache = session.getChunkCache();
+        Column cachedColumn;
+        Chunk cachedChunk;
+        if (chunkCache == null || (cachedColumn = chunkCache.getChunk(x, z)) == null || (cachedChunk = cachedColumn.getChunks()[y]) == null) {
+            return;
+        }
+
+        // Copy state IDs from cached chunk to output chunk
+        for (int blockY = 0; blockY < 16; blockY++) { // Cache-friendly iteration order
+            for (int blockZ = 0; blockZ < 16; blockZ++) {
+                for (int blockX = 0; blockX < 16; blockX++) {
+                    chunk.set(blockX, blockY, blockZ, cachedChunk.get(blockX, blockY, blockZ));
+                }
+            }
+        }
+    }
+
+    @Override
+    public boolean hasMoreBlockDataThanChunkCache() {
+        // This implementation can only fetch data from the session chunk cache
+        return false;
+    }
+
     @Override
     public int[] getBiomeDataAt(GeyserSession session, int x, int z) {
         if (session.getConnector().getConfig().isCacheChunks()) {

connector/src/main/java/org/geysermc/connector/network/translators/world/WorldManager.java

@@ -25,6 +25,7 @@
 
 package org.geysermc.connector.network.translators.world;
 
+import com.github.steveice10.mc.protocol.data.game.chunk.Chunk;
 import com.github.steveice10.mc.protocol.data.game.entity.metadata.Position;
 import com.github.steveice10.mc.protocol.data.game.entity.player.GameMode;
 import com.github.steveice10.mc.protocol.data.game.setting.Difficulty;
@@ -74,6 +75,27 @@ public abstract class WorldManager {
      */
     public abstract int getBlockAt(GeyserSession session, int x, int y, int z);
 
+    /**
+     * Gets all block states in the specified chunk section.
+     *
+     * @param session the session
+     * @param x the chunk's X coordinate
+     * @param y the chunk's Y coordinate
+     * @param z the chunk's Z coordinate
+     * @param section the chunk section to store the block data in
+     */
+    public abstract void getBlocksInSection(GeyserSession session, int x, int y, int z, Chunk section);
+
+    /**
+     * Checks whether or not this world manager has access to more block data than the chunk cache.
+     * <p>
+     * Some world managers (e.g. Spigot) can provide access to block data outside of the chunk cache, and even with chunk caching disabled. This
+     * method provides a means to check if this manager has this capability.
+     *
+     * @return whether or not this world manager has access to more block data than the chunk cache
+     */
+    public abstract boolean hasMoreBlockDataThanChunkCache();
+
     /**
      * Gets the biome data for the specified chunk.
      *

connector/src/main/java/org/geysermc/connector/network/translators/world/chunk/BlockStorage.java

@@ -29,14 +29,16 @@ import com.nukkitx.network.VarInts;
 import io.netty.buffer.ByteBuf;
 import it.unimi.dsi.fastutil.ints.IntArrayList;
 import it.unimi.dsi.fastutil.ints.IntList;
+import lombok.Getter;
 import org.geysermc.connector.network.translators.world.chunk.bitarray.BitArray;
 import org.geysermc.connector.network.translators.world.chunk.bitarray.BitArrayVersion;
 
 import java.util.function.IntConsumer;
 
+@Getter
 public class BlockStorage {
 
-    private static final int SIZE = 4096;
+    public static final int SIZE = 4096;
 
     private final IntList palette;
     private BitArray bitArray;
@@ -46,12 +48,12 @@ public class BlockStorage {
     }
 
     public BlockStorage(BitArrayVersion version) {
-        this.bitArray = version.createPalette(SIZE);
+        this.bitArray = version.createArray(SIZE);
         this.palette = new IntArrayList(16);
         this.palette.add(0); // Air is at the start of every palette.
     }
 
-    private BlockStorage(BitArray bitArray, IntArrayList palette) {
+    public BlockStorage(BitArray bitArray, IntList palette) {
         this.palette = palette;
         this.bitArray = bitArray;
     }
@@ -64,16 +66,16 @@ public class BlockStorage {
         return BitArrayVersion.get(header >> 1, true);
     }
 
-    public synchronized int getFullBlock(int index) {
+    public int getFullBlock(int index) {
         return this.palette.getInt(this.bitArray.get(index));
     }
 
-    public synchronized void setFullBlock(int index, int runtimeId) {
+    public void setFullBlock(int index, int runtimeId) {
         int idx = this.idFor(runtimeId);
         this.bitArray.set(index, idx);
     }
 
-    public synchronized void writeToNetwork(ByteBuf buffer) {
+    public void writeToNetwork(ByteBuf buffer) {
         buffer.writeByte(getPaletteHeader(bitArray.getVersion(), true));
 
         for (int word : bitArray.getWords()) {
@@ -84,8 +86,18 @@ public class BlockStorage {
         palette.forEach((IntConsumer) id -> VarInts.writeInt(buffer, id));
     }
 
+    public int estimateNetworkSize() {
+        int size = 1; // Palette header
+        size += this.bitArray.getWords().length * 4;
+
+        // We assume that none of the VarInts will be larger than 3 bytes
+        size += 3; // Palette size
+        size += this.palette.size() * 3;
+        return size;
+    }
+
     private void onResize(BitArrayVersion version) {
-        BitArray newBitArray = version.createPalette(SIZE);
+        BitArray newBitArray = version.createArray(SIZE);
 
         for (int i = 0; i < SIZE; i++) {
             newBitArray.set(i, this.bitArray.get(i));

connector/src/main/java/org/geysermc/connector/network/translators/world/chunk/ChunkPosition.java

@@ -1,79 +0,0 @@
-/*
- * Copyright (c) 2019-2020 GeyserMC. http://geysermc.org
- *
- * Permission is hereby granted, free of charge, to any person obtaining a copy
- * of this software and associated documentation files (the "Software"), to deal
- * in the Software without restriction, including without limitation the rights
- * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
- * copies of the Software, and to permit persons to whom the Software is
- * furnished to do so, subject to the following conditions:
- *
- * The above copyright notice and this permission notice shall be included in
- * all copies or substantial portions of the Software.
- *
- * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
- * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
- * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
- * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
- * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
- * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
- * THE SOFTWARE.
- *
- * @author GeyserMC
- * @link https://github.com/GeyserMC/Geyser
- */
-
-package org.geysermc.connector.network.translators.world.chunk;
-
-import com.github.steveice10.mc.protocol.data.game.entity.metadata.Position;
-import lombok.AllArgsConstructor;
-import lombok.Getter;
-import lombok.Setter;
-
-@Getter
-@Setter
-@AllArgsConstructor
-public class ChunkPosition {
-
-    /**
-     * Packs a chunk's X and Z coordinates into a single {@code long}.
-     *
-     * @param x the X coordinate
-     * @param z the Z coordinate
-     * @return the packed coordinates
-     */
-    public static long toLong(int x, int z) {
-        return ((x & 0xFFFFFFFFL) << 32L) | (z & 0xFFFFFFFFL);
-    }
-
-    private int x;
-    private int z;
-
-    public Position getBlock(int x, int y, int z) {
-        return new Position((this.x << 4) + x, y, (this.z << 4) + z);
-    }
-
-    public Position getChunkBlock(int x, int y, int z) {
-        int chunkX = x & 15;
-        int chunkY = y & 15;
-        int chunkZ = z & 15;
-        return new Position(chunkX, chunkY, chunkZ);
-    }
-
-    @Override
-    public boolean equals(Object obj) {
-        if (obj == this)    {
-            return true;
-        } else if (obj instanceof ChunkPosition) {
-            ChunkPosition chunkPosition = (ChunkPosition) obj;
-            return this.x == chunkPosition.x && this.z == chunkPosition.z;
-        } else {
-            return false;
-        }
-    }
-
-    @Override
-    public int hashCode() {
-        return this.x * 2061811133 + this.z * 1424368303;
-    }
-}

connector/src/main/java/org/geysermc/connector/network/translators/world/chunk/ChunkSection.java

@@ -27,42 +27,19 @@ package org.geysermc.connector.network.translators.world.chunk;
 
 import com.nukkitx.network.util.Preconditions;
 import io.netty.buffer.ByteBuf;
-import lombok.Synchronized;
 
 public class ChunkSection {
 
     private static final int CHUNK_SECTION_VERSION = 8;
-    public static final int SIZE = 4096;
 
     private final BlockStorage[] storage;
-    private final NibbleArray blockLight;
-    private final NibbleArray skyLight;
 
     public ChunkSection() {
-        this(new BlockStorage[]{new BlockStorage(), new BlockStorage()}, new NibbleArray(SIZE),
+        this(new BlockStorage[]{new BlockStorage(), new BlockStorage()});
-                new NibbleArray(SIZE));
     }
 
-    public ChunkSection(BlockStorage[] blockStorage) {
+    public ChunkSection(BlockStorage[] storage) {
-        this(blockStorage, new NibbleArray(SIZE), new NibbleArray(SIZE));
-    }
-
-    public ChunkSection(BlockStorage[] storage, byte[] blockLight, byte[] skyLight) {
-        Preconditions.checkNotNull(storage, "storage");
-        Preconditions.checkArgument(storage.length > 1, "Block storage length must be at least 2");
-        for (BlockStorage blockStorage : storage) {
-            Preconditions.checkNotNull(blockStorage, "storage");
-        }
-
         this.storage = storage;
-        this.blockLight = new NibbleArray(blockLight);
-        this.skyLight = new NibbleArray(skyLight);
-    }
-
-    private ChunkSection(BlockStorage[] storage, NibbleArray blockLight, NibbleArray skyLight) {
-        this.storage = storage;
-        this.blockLight = blockLight;
-        this.skyLight = skyLight;
     }
 
     public int getFullBlock(int x, int y, int z, int layer) {
@@ -77,30 +54,6 @@ public class ChunkSection {
         this.storage[layer].setFullBlock(blockPosition(x, y, z), fullBlock);
     }
 
-    @Synchronized("skyLight")
-    public byte getSkyLight(int x, int y, int z) {
-        checkBounds(x, y, z);
-        return this.skyLight.get(blockPosition(x, y, z));
-    }
-
-    @Synchronized("skyLight")
-    public void setSkyLight(int x, int y, int z, byte val) {
-        checkBounds(x, y, z);
-        this.skyLight.set(blockPosition(x, y, z), val);
-    }
-
-    @Synchronized("blockLight")
-    public byte getBlockLight(int x, int y, int z) {
-        checkBounds(x, y, z);
-        return this.blockLight.get(blockPosition(x, y, z));
-    }
-
-    @Synchronized("blockLight")
-    public void setBlockLight(int x, int y, int z, byte val) {
-        checkBounds(x, y, z);
-        this.blockLight.set(blockPosition(x, y, z), val);
-    }
-
     public void writeToNetwork(ByteBuf buffer) {
         buffer.writeByte(CHUNK_SECTION_VERSION);
         buffer.writeByte(this.storage.length);
@@ -109,12 +62,12 @@ public class ChunkSection {
         }
     }
 
-    public NibbleArray getSkyLightArray() {
+    public int estimateNetworkSize() {
-        return skyLight;
+        int size = 2; // Version + storage count
-    }
+        for (BlockStorage blockStorage : this.storage) {
-
+            size += blockStorage.estimateNetworkSize();
-    public NibbleArray getBlockLightArray() {
+        }
-        return blockLight;
+        return size;
     }
 
     public BlockStorage[] getBlockStorageArray() {
@@ -135,7 +88,7 @@ public class ChunkSection {
         for (int i = 0; i < storage.length; i++) {
             storage[i] = this.storage[i].copy();
         }
-        return new ChunkSection(storage, skyLight.copy(), blockLight.copy());
+        return new ChunkSection(storage);
     }
 
     public static int blockPosition(int x, int y, int z) {

connector/src/main/java/org/geysermc/connector/network/translators/world/chunk/bitarray/BitArrayVersion.java

@@ -37,6 +37,8 @@ public enum BitArrayVersion {
     V2(2, 16, V3),
     V1(1, 32, V2);
 
+    private static final BitArrayVersion[] VALUES = values();
+
     final byte bits;
     final byte entriesPerWord;
     final int maxEntryValue;
@@ -58,8 +60,14 @@ public enum BitArrayVersion {
         throw new IllegalArgumentException("Invalid palette version: " + version);
     }
 
-    public BitArray createPalette(int size) {
+    public static BitArrayVersion forBitsCeil(int bits) {
-        return this.createPalette(size, new int[MathUtils.ceil((float) size / entriesPerWord)]);
+        for (int i = VALUES.length - 1; i >= 0; i--)  {
+            BitArrayVersion version = VALUES[i];
+            if (version.bits >= bits)   {
+                return version;
+            }
+        }
+        return null;
     }
 
     public byte getId() {
@@ -78,7 +86,11 @@ public enum BitArrayVersion {
         return next;
     }
 
-    public BitArray createPalette(int size, int[] words) {
+    public BitArray createArray(int size) {
+        return this.createArray(size, new int[MathUtils.ceil((float) size / entriesPerWord)]);
+    }
+
+    public BitArray createArray(int size, int[] words) {
         if (this == V3 || this == V5 || this == V6) {
             // Padded palettes aren't able to use bitwise operations due to their padding.
             return new PaddedBitArray(this, size, words);

connector/src/main/java/org/geysermc/connector/utils/ChunkUtils.java

@@ -25,8 +25,10 @@
 
 package org.geysermc.connector.utils;
 
+import com.github.steveice10.mc.protocol.data.game.chunk.BitStorage;
 import com.github.steveice10.mc.protocol.data.game.chunk.Chunk;
 import com.github.steveice10.mc.protocol.data.game.chunk.Column;
+import com.github.steveice10.mc.protocol.data.game.chunk.palette.Palette;
 import com.github.steveice10.mc.protocol.data.game.entity.metadata.Position;
 import com.github.steveice10.opennbt.tag.builtin.CompoundTag;
 import com.github.steveice10.opennbt.tag.builtin.StringTag;
@@ -36,27 +38,39 @@ import com.nukkitx.math.vector.Vector3i;
 import com.nukkitx.nbt.NBTOutputStream;
 import com.nukkitx.nbt.NbtMap;
 import com.nukkitx.nbt.NbtUtils;
-import com.nukkitx.protocol.bedrock.packet.*;
+import com.nukkitx.protocol.bedrock.packet.LevelChunkPacket;
+import com.nukkitx.protocol.bedrock.packet.NetworkChunkPublisherUpdatePacket;
+import com.nukkitx.protocol.bedrock.packet.UpdateBlockPacket;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
 import it.unimi.dsi.fastutil.objects.Object2IntMap;
 import it.unimi.dsi.fastutil.objects.Object2IntOpenHashMap;
-import it.unimi.dsi.fastutil.objects.ObjectArrayList;
+import lombok.Data;
-import lombok.Getter;
+import lombok.experimental.UtilityClass;
 import org.geysermc.connector.GeyserConnector;
 import org.geysermc.connector.entity.Entity;
 import org.geysermc.connector.entity.ItemFrameEntity;
 import org.geysermc.connector.network.session.GeyserSession;
 import org.geysermc.connector.network.translators.world.block.BlockStateValues;
-import org.geysermc.connector.network.translators.world.block.entity.*;
 import org.geysermc.connector.network.translators.world.block.BlockTranslator;
-import org.geysermc.connector.network.translators.world.chunk.ChunkPosition;
+import org.geysermc.connector.network.translators.world.block.entity.BedrockOnlyBlockEntity;
+import org.geysermc.connector.network.translators.world.block.entity.BlockEntityTranslator;
+import org.geysermc.connector.network.translators.world.block.entity.RequiresBlockState;
+import org.geysermc.connector.network.translators.world.chunk.BlockStorage;
 import org.geysermc.connector.network.translators.world.chunk.ChunkSection;
+import org.geysermc.connector.network.translators.world.chunk.bitarray.BitArray;
+import org.geysermc.connector.network.translators.world.chunk.bitarray.BitArrayVersion;
 
 import java.io.ByteArrayOutputStream;
 import java.io.IOException;
+import java.util.ArrayList;
+import java.util.BitSet;
+import java.util.Collections;
+import java.util.List;
 
-import static org.geysermc.connector.network.translators.world.block.BlockTranslator.AIR;
+import static org.geysermc.connector.network.translators.world.block.BlockTranslator.*;
-import static org.geysermc.connector.network.translators.world.block.BlockTranslator.BEDROCK_WATER_ID;
 
+@UtilityClass
 public class ChunkUtils {
 
     /**
@@ -67,6 +81,9 @@ public class ChunkUtils {
     private static final NbtMap EMPTY_TAG = NbtMap.builder().build();
     public static final byte[] EMPTY_LEVEL_CHUNK_DATA;
 
+    public static final BlockStorage EMPTY_STORAGE = new BlockStorage();
+    public static final ChunkSection EMPTY_SECTION = new ChunkSection(new BlockStorage[]{ EMPTY_STORAGE });
+
     static {
         try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) {
             outputStream.write(new byte[258]); // Biomes + Border Size + Extra Data Size
@@ -76,72 +93,144 @@ public class ChunkUtils {
             }
 
             EMPTY_LEVEL_CHUNK_DATA = outputStream.toByteArray();
-        }catch (IOException e) {
+        } catch (IOException e) {
             throw new AssertionError("Unable to generate empty level chunk data");
         }
     }
 
-    public static ChunkData translateToBedrock(GeyserSession session, Column column, boolean isNonFullChunk) {
+    private static int indexYZXtoXZY(int yzx) {
-        ChunkData chunkData = new ChunkData();
+        return (yzx >> 8) | (yzx & 0x0F0) | ((yzx & 0x00F) << 8);
-        Chunk[] chunks = column.getChunks();
+    }
-        chunkData.sections = new ChunkSection[chunks.length];
 
-        CompoundTag[] blockEntities = column.getTileEntities();
+    public static ChunkData translateToBedrock(GeyserSession session, Column column, boolean isNonFullChunk) {
-        // Temporarily stores positions of BlockState values per chunk load
+        Chunk[] javaSections = column.getChunks();
-        Object2IntMap<Position> blockEntityPositions = new Object2IntOpenHashMap<>();
+        ChunkSection[] sections = new ChunkSection[javaSections.length];
 
         // Temporarily stores compound tags of Bedrock-only block entities
-        ObjectArrayList<NbtMap> bedrockOnlyBlockEntities = new ObjectArrayList<>();
+        List<NbtMap> bedrockOnlyBlockEntities = Collections.emptyList();
 
-        for (int chunkY = 0; chunkY < chunks.length; chunkY++) {
+        BitSet waterloggedPaletteIds = new BitSet();
-            chunkData.sections[chunkY] = new ChunkSection();
+        BitSet pistonOrFlowerPaletteIds = new BitSet();
-            Chunk chunk = chunks[chunkY];
 
-            // Chunk is null and caching chunks is off or this isn't a non-full chunk
+        boolean worldManagerHasMoreBlockDataThanCache = session.getConnector().getWorldManager().hasMoreBlockDataThanChunkCache();
-            if (chunk == null && (!session.getConnector().getConfig().isCacheChunks() || !isNonFullChunk))
+
+        // If the received packet was a full chunk update, null sections in the chunk are guaranteed to also be null in the world manager
+        boolean shouldCheckWorldManagerOnMissingSections = isNonFullChunk && worldManagerHasMoreBlockDataThanCache;
+        Chunk temporarySection = null;
+
+        for (int sectionY = 0; sectionY < javaSections.length; sectionY++) {
+            Chunk javaSection = javaSections[sectionY];
+
+            // Section is null, the cache will not contain anything of use
+            if (javaSection == null) {
+                // The column parameter contains all data currently available from the cache. If the chunk is null and the world manager
+                // reports the ability to access more data than the cache, attempt to fetch from the world manager instead.
+                if (shouldCheckWorldManagerOnMissingSections) {
+                    // Ensure that temporary chunk is set
+                    if (temporarySection == null) {
+                        temporarySection = new Chunk();
+                    }
+
+                    // Read block data in section
+                    session.getConnector().getWorldManager().getBlocksInSection(session, column.getX(), sectionY, column.getZ(), temporarySection);
+
+                    if (temporarySection.isEmpty()) {
+                        // The world manager only contains air for the given section
+                        // We can leave temporarySection as-is to allow it to potentially be re-used for later sections
+                        continue;
+                    } else {
+                        javaSection = temporarySection;
+
+                        // Section contents have been modified, we can't re-use it
+                        temporarySection = null;
+                    }
+                } else {
+                    continue;
+                }
+            }
+
+            // No need to encode an empty section...
+            if (javaSection.isEmpty()) {
                 continue;
+            }
 
-            // If chunk is empty then no need to process
+            Palette javaPalette = javaSection.getPalette();
-            if (chunk != null && chunk.isEmpty())
+            IntList bedrockPalette = new IntArrayList(javaPalette.size());
-                continue;
+            waterloggedPaletteIds.clear();
+            pistonOrFlowerPaletteIds.clear();
 
-            ChunkSection section = chunkData.sections[chunkY];
+            // Iterate through palette and convert state IDs to Bedrock, doing some additional checks as we go
-            for (int x = 0; x < 16; x++) {
+            for (int i = 0; i < javaPalette.size(); i++) {
-                for (int y = 0; y < 16; y++) {
+                int javaId = javaPalette.idToState(i);
-                    for (int z = 0; z < 16; z++) {
+                bedrockPalette.add(BlockTranslator.getBedrockBlockId(javaId));
-                        int blockState;
-                        // If a non-full chunk, then grab the block that should be here to create a 'full' chunk
-                        if (chunk == null) {
-                            blockState = session.getConnector().getWorldManager().getBlockAt(session, (column.getX() << 4) + x, (chunkY << 4) + y, (column.getZ() << 4) + z);
-                        } else {
-                            blockState = chunk.get(x, y, z);
-                        }
-                        int id = BlockTranslator.getBedrockBlockId(blockState);
 
-                        // Check to see if the name is in BlockTranslator.getBlockEntityString, and therefore must be handled differently
+                if (BlockTranslator.isWaterlogged(javaId)) {
-                        if (BlockTranslator.getBlockEntityString(blockState) != null) {
+                    waterloggedPaletteIds.set(i);
-                            Position pos = new ChunkPosition(column.getX(), column.getZ()).getBlock(x, (chunkY << 4) + y, z);
+                }
-                            blockEntityPositions.put(pos, blockState);
-                        }
 
-                        section.getBlockStorageArray()[0].setFullBlock(ChunkSection.blockPosition(x, y, z), id);
+                // Check if block is piston or flower to see if we'll need to create additional block entities, as they're only block entities in Bedrock
+                if (BlockStateValues.getFlowerPotValues().containsKey(javaId) || BlockStateValues.getPistonValues().containsKey(javaId)) {
+                    pistonOrFlowerPaletteIds.set(i);
+                }
+            }
 
-                        // Check if block is piston or flower - only block entities in Bedrock
+            BitStorage javaData = javaSection.getStorage();
-                        if (BlockStateValues.getFlowerPotValues().containsKey(blockState) ||
-                                BlockStateValues.getPistonValues().containsKey(blockState)) {
-                            Position pos = new ChunkPosition(column.getX(), column.getZ()).getBlock(x, (chunkY << 4) + y, z);
-                            bedrockOnlyBlockEntities.add(BedrockOnlyBlockEntity.getTag(Vector3i.from(pos.getX(), pos.getY(), pos.getZ()), blockState));
-                        }
 
-                        if (BlockTranslator.isWaterlogged(blockState)) {
+            // Add Bedrock-exclusive block entities
-                            section.getBlockStorageArray()[1].setFullBlock(ChunkSection.blockPosition(x, y, z), BEDROCK_WATER_ID);
+            // We only if the palette contained any blocks that are Bedrock-exclusive block entities to avoid iterating through the whole block data
-                        }
+            // for no reason, as most sections will not contain any pistons or flower pots
+            if (!pistonOrFlowerPaletteIds.isEmpty()) {
+                bedrockOnlyBlockEntities = new ArrayList<>();
+                for (int yzx = 0; yzx < BlockStorage.SIZE; yzx++) {
+                    int paletteId = javaData.get(yzx);
+                    if (pistonOrFlowerPaletteIds.get(paletteId)) {
+                        bedrockOnlyBlockEntities.add(BedrockOnlyBlockEntity.getTag(
+                                Vector3i.from((column.getX() << 4) + (yzx & 0xF), (sectionY << 4) + ((yzx >> 8) & 0xF), (column.getZ() << 4) + ((yzx >> 4) & 0xF)),
+                                javaPalette.idToState(paletteId)
+                        ));
                     }
                 }
             }
 
+            BitArray bedrockData = BitArrayVersion.forBitsCeil(javaData.getBitsPerEntry()).createArray(BlockStorage.SIZE);
+            BlockStorage layer0 = new BlockStorage(bedrockData, bedrockPalette);
+            BlockStorage[] layers;
+
+            // Convert data array from YZX to XZY coordinate order
+            if (waterloggedPaletteIds.isEmpty()) {
+                // No blocks are waterlogged, simply convert coordinate order
+                // This could probably be optimized further...
+                for (int yzx = 0; yzx < BlockStorage.SIZE; yzx++) {
+                    bedrockData.set(indexYZXtoXZY(yzx), javaData.get(yzx));
+                }
+
+                layers = new BlockStorage[]{ layer0 };
+            } else {
+                // The section contains waterlogged blocks, we need to convert coordinate order AND generate a V1 block storage for
+                // layer 1 with palette ID 1 indicating water
+                int[] layer1Data = new int[BlockStorage.SIZE >> 5];
+                for (int yzx = 0; yzx < BlockStorage.SIZE; yzx++) {
+                    int paletteId = javaData.get(yzx);
+                    int xzy = indexYZXtoXZY(yzx);
+                    bedrockData.set(xzy, paletteId);
+
+                    if (waterloggedPaletteIds.get(paletteId)) {
+                        layer1Data[xzy >> 5] |= 1 << (xzy & 0x1F);
+                    }
+                }
+
+                // V1 palette
+                IntList layer1Palette = new IntArrayList(2);
+                layer1Palette.add(0); // Air
+                layer1Palette.add(BEDROCK_WATER_ID);
+
+                layers = new BlockStorage[]{ layer0, new BlockStorage(BitArrayVersion.V1.createArray(BlockStorage.SIZE, layer1Data), layer1Palette) };
+            }
+
+            sections[sectionY] = new ChunkSection(layers);
         }
 
+        CompoundTag[] blockEntities = column.getTileEntities();
         NbtMap[] bedrockBlockEntities = new NbtMap[blockEntities.length + bedrockOnlyBlockEntities.size()];
         int i = 0;
         while (i < blockEntities.length) {
@@ -155,7 +244,7 @@ public class ChunkUtils {
                 for (Tag subTag : tag) {
                     if (subTag instanceof StringTag) {
                         StringTag stringTag = (StringTag) subTag;
-                        if (stringTag.getValue().equals("")) {
+                        if (stringTag.getValue().isEmpty()) {
                             tagName = stringTag.getName();
                             break;
                         }
@@ -169,17 +258,25 @@ public class ChunkUtils {
             String id = BlockEntityUtils.getBedrockBlockEntityId(tagName);
             BlockEntityTranslator blockEntityTranslator = BlockEntityUtils.getBlockEntityTranslator(id);
             Position pos = new Position((int) tag.get("x").getValue(), (int) tag.get("y").getValue(), (int) tag.get("z").getValue());
-            int blockState = blockEntityPositions.getOrDefault(pos, 0);
+
+            // Get Java blockstate ID from block entity position
+            int blockState = 0;
+            Chunk section = column.getChunks()[pos.getY() >> 4];
+            if (section != null) {
+                blockState = section.get(pos.getX() & 0xF, pos.getY() & 0xF, pos.getZ() & 0xF);
+            }
+
             bedrockBlockEntities[i] = blockEntityTranslator.getBlockEntityTag(tagName, tag, blockState);
             i++;
         }
+
+        // Append Bedrock-exclusive block entities to output array
         for (NbtMap tag : bedrockOnlyBlockEntities) {
             bedrockBlockEntities[i] = tag;
             i++;
         }
 
-        chunkData.blockEntities = bedrockBlockEntities;
+        return new ChunkData(sections, bedrockBlockEntities);
-        return chunkData;
     }
 
     public static void updateChunkPosition(GeyserSession session, Vector3i position) {
@@ -277,10 +374,10 @@ public class ChunkUtils {
         }
     }
 
+    @Data
     public static final class ChunkData {
-        public ChunkSection[] sections;
+        private final ChunkSection[] sections;
 
-        @Getter
+        private final NbtMap[] blockEntities;
-        private NbtMap[] blockEntities = new NbtMap[0];
     }
 }

connector/src/main/java/org/geysermc/connector/utils/MathUtils.java

@@ -74,4 +74,15 @@ public class MathUtils {
         }
         return (Byte) value;
     }
+
+    /**
+     * Packs a chunk's X and Z coordinates into a single {@code long}.
+     *
+     * @param x the X coordinate
+     * @param z the Z coordinate
+     * @return the packed coordinates
+     */
+    public static long chunkPositionToLong(int x, int z) {
+        return ((x & 0xFFFFFFFFL) << 32L) | (z & 0xFFFFFFFFL);
+    }
 }