Many improvements to chunk prioritization and bug fixes

Fixed a few bugs, and made numerous improvements. Fixed issue where a sync chunk load could have its ticket removed and the priority ticket could expire... Still not perfect there but better than before. Also fixed few other misc issues such as watchdog cpu usage, chunk queue update had risk of double enqueue due to it no longer being a set. Added much more information about chunk state to watchdog prints. I see some more room for improvement even, but this is much better than before. Fixes #3407 Fixes #3411 Fixes #3395 Fixes #3389
2025-03-13 19:28:03 +01:00 · 2020-05-20 05:11:57 -04:00 · 2020-05-20 05:11:57 -04:00 · 33246e0ba0
commit 33246e0ba0
parent ade297307f
8 changed files with 247 additions and 73 deletions
--- a/Spigot-Server-Patches/Asynchronous-chunk-IO-and-loading.patch
+++ b/Spigot-Server-Patches/Asynchronous-chunk-IO-and-loading.patch
@ -160,6 +160,44 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
     }
 
     public static Timing getTickList(WorldServer worldserver, String timingsType) {
+diff --git a/src/main/java/com/destroystokyo/paper/PaperCommand.java b/src/main/java/com/destroystokyo/paper/PaperCommand.java
+index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644
+--- a/src/main/java/com/destroystokyo/paper/PaperCommand.java
+++ b/src/main/java/com/destroystokyo/paper/PaperCommand.java
+@@ -0,0 +0,0 @@
+ package com.destroystokyo.paper;
+ 
+import com.destroystokyo.paper.io.chunk.ChunkTaskManager;
+ import com.google.common.base.Functions;
+ import com.google.common.collect.Iterables;
+ import com.google.common.collect.Lists;
+@@ -0,0 +0,0 @@ public class PaperCommand extends Command {
+     public PaperCommand(String name) {
+         super(name);
+         this.description = "Paper related commands";
+-        this.usageMessage = "/paper [heap | entity | reload | version | debug | chunkinfo]";
+        this.usageMessage = "/paper [heap | entity | reload | version | debug | dumpwaiting | chunkinfo]";
+         this.setPermission("bukkit.command.paper");
+     }
+ 
+     @Override
+     public List<String> tabComplete(CommandSender sender, String alias, String[] args, Location location) throws IllegalArgumentException {
+         if (args.length <= 1)
+-            return getListMatchingLast(args, "heap", "entity", "reload", "version", "debug", "chunkinfo");
+            return getListMatchingLast(args, "heap", "entity", "reload", "version", "debug", "dumpwaiting", "chunkinfo");
+ 
+         switch (args[0].toLowerCase(Locale.ENGLISH))
+         {
+@@ -0,0 +0,0 @@ public class PaperCommand extends Command {
+             case "debug":
+                 doDebug(sender, args);
+                 break;
+            case "dumpwaiting":
+                ChunkTaskManager.dumpAllChunkLoadInfo();
+                break;
+             case "chunkinfo":
+                 doChunkInfo(sender, args);
+                 break;
 diff --git a/src/main/java/com/destroystokyo/paper/PaperConfig.java b/src/main/java/com/destroystokyo/paper/PaperConfig.java
 index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644
 --- a/src/main/java/com/destroystokyo/paper/PaperConfig.java
@ -1824,7 +1862,9 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +import net.minecraft.server.IChunkAccess;
 +import net.minecraft.server.MinecraftServer;
 +import net.minecraft.server.NBTTagCompound;
+import net.minecraft.server.PlayerChunk;
 +import net.minecraft.server.WorldServer;
+import org.apache.commons.lang.StringUtils;
 +import org.apache.logging.log4j.Level;
 +import org.bukkit.Bukkit;
 +import org.spigotmc.AsyncCatcher;
@ -1912,20 +1952,28 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +                // log current status of chunk to indicate whether we're waiting on generation or loading
 +                net.minecraft.server.PlayerChunk chunkHolder = chunkInfo.world.getChunkProvider().playerChunkMap.getVisibleChunk(key);
 +
-+                if (chunkHolder == null) {
-+                    PaperFileIOThread.LOGGER.log(Level.ERROR, "Chunk Holder - null");
-+                } else {
-+                    IChunkAccess chunk = chunkHolder.getAvailableChunkNow();
-+                    net.minecraft.server.ChunkStatus holderStatus = chunkHolder.getChunkHolderStatus();
-+                    PaperFileIOThread.LOGGER.log(Level.ERROR, "Chunk Holder - non-null");
-+                    PaperFileIOThread.LOGGER.log(Level.ERROR, "Chunk Status - " + ((chunk == null) ? "null chunk" : chunk.getChunkStatus().toString()));
-+                    PaperFileIOThread.LOGGER.log(Level.ERROR, "Chunk Holder Status - " + ((holderStatus == null) ? "null" : holderStatus.toString()));
-+                }
-+
+                dumpChunkInfo(chunkHolder, chunkInfo.chunkX, chunkInfo.chunkZ);
 +            }
 +        }
 +    }
 +
+    static void dumpChunkInfo(PlayerChunk chunkHolder, int x, int z) {
+        dumpChunkInfo(chunkHolder, x, z, 0);
+    }
+    static void dumpChunkInfo(PlayerChunk chunkHolder, int x, int z, int indent) {
+        String indentStr = StringUtils.repeat("  ", indent);
+        if (chunkHolder == null) {
+            PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Holder - null for (" + x +"," + z +")");
+        } else {
+            IChunkAccess chunk = chunkHolder.getAvailableChunkNow();
+            net.minecraft.server.ChunkStatus holderStatus = chunkHolder.getChunkHolderStatus();
+            PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Holder - non-null");
+            PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Status - " + ((chunk == null) ? "null chunk" : chunk.getChunkStatus().toString()));
+            PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Ticket Status - "  + PlayerChunk.getChunkStatus(chunkHolder.getTicketLevel()));
+            PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Holder Status - " + ((holderStatus == null) ? "null" : holderStatus.toString()));
+        }
+    }
+
 +    public static void initGlobalLoadThreads(int threads) {
 +        if (threads <= 0 || globalWorkers != null) {
 +            return;
--- a/Spigot-Server-Patches/ChunkMapDistance-CME.patch
+++ b/Spigot-Server-Patches/ChunkMapDistance-CME.patch
@ -13,7 +13,16 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
     private final ChunkMapDistance.b f = new ChunkMapDistance.b(8);
     private final ChunkMapDistance.c g = new ChunkMapDistance.c(33);
 -    private final Set<PlayerChunk> pendingChunkUpdates = Sets.newHashSet();
-+    private final java.util.Queue<PlayerChunk> pendingChunkUpdates = new java.util.ArrayDeque<>(); // PAIL pendingChunkUpdates // Paper - use a queue
+    // Paper start use a queue, but still keep unique requirement
+    public final java.util.Queue<PlayerChunk> pendingChunkUpdates = new java.util.ArrayDeque<PlayerChunk>() {
+        @Override
+        public boolean add(PlayerChunk o) {
+            if (o.isUpdateQueued) return true;
+            o.isUpdateQueued = true;
+            return super.add(o);
+        }
+    };
+    // Paper end
     private final ChunkTaskQueueSorter i;
     private final Mailbox<ChunkTaskQueueSorter.a<Runnable>> j;
     private final Mailbox<ChunkTaskQueueSorter.b> k;
@ -43,9 +52,23 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 -            // CraftBukkit end
 -
 +            while(!this.pendingChunkUpdates.isEmpty()) {
-+                this.pendingChunkUpdates.remove().a(playerchunkmap);
+                PlayerChunk remove = this.pendingChunkUpdates.remove();
+                remove.isUpdateQueued = false;
+                remove.a(playerchunkmap);
 +            }
 +            // Paper end
             return true;
         } else {
             if (!this.l.isEmpty()) {
+diff --git a/src/main/java/net/minecraft/server/PlayerChunk.java b/src/main/java/net/minecraft/server/PlayerChunk.java
+index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644
+--- a/src/main/java/net/minecraft/server/PlayerChunk.java
+++ b/src/main/java/net/minecraft/server/PlayerChunk.java
+@@ -0,0 +0,0 @@ public class PlayerChunk {
+     private static final CompletableFuture<Either<Chunk, PlayerChunk.Failure>> UNLOADED_CHUNK_FUTURE = CompletableFuture.completedFuture(PlayerChunk.UNLOADED_CHUNK);
+     private static final List<ChunkStatus> CHUNK_STATUSES = ChunkStatus.a();
+     private static final PlayerChunk.State[] CHUNK_STATES = PlayerChunk.State.values();
+    boolean isUpdateQueued = false; // Paper
+     private final AtomicReferenceArray<CompletableFuture<Either<IChunkAccess, PlayerChunk.Failure>>> statusFutures;
+     private volatile CompletableFuture<Either<Chunk, PlayerChunk.Failure>> fullChunkFuture; private int fullChunkCreateCount; private volatile boolean isFullChunkReady; // Paper - cache chunk ticking stage
+     private volatile CompletableFuture<Either<Chunk, PlayerChunk.Failure>> tickingFuture; private volatile boolean isTickingReady; // Paper - cache chunk ticking stage
--- a/Spigot-Server-Patches/Fix-Light-Command.patch
+++ b/Spigot-Server-Patches/Fix-Light-Command.patch
@ -22,16 +22,16 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
     public PaperCommand(String name) {
         super(name);
         this.description = "Paper related commands";
-        this.usageMessage = "/paper [heap | entity | reload | version | debug | chunkinfo | syncloadinfo]";
-+        this.usageMessage = "/paper [heap | entity | reload | version | debug | chunkinfo | syncloadinfo | fixlight]";
+-        this.usageMessage = "/paper [heap | entity | reload | version | debug | dumpwaiting | chunkinfo | syncloadinfo]";
+        this.usageMessage = "/paper [heap | entity | reload | version | debug | dumpwaiting | chunkinfo | syncloadinfo | fixlight]";
         this.setPermission("bukkit.command.paper");
     }
 
     @Override
     public List<String> tabComplete(CommandSender sender, String alias, String[] args, Location location) throws IllegalArgumentException {
         if (args.length <= 1)
-            return getListMatchingLast(args, "heap", "entity", "reload", "version", "debug", "chunkinfo", "syncloadinfo");
-+            return getListMatchingLast(args, "heap", "entity", "reload", "version", "debug", "chunkinfo", "syncloadinfo", "fixlight");
+-            return getListMatchingLast(args, "heap", "entity", "reload", "version", "debug", "dumpwaiting", "chunkinfo", "syncloadinfo");
+            return getListMatchingLast(args, "heap", "entity", "reload", "version", "debug", "dumpwaiting", "chunkinfo", "syncloadinfo", "fixlight");
 
         switch (args[0].toLowerCase(Locale.ENGLISH))
         {
--- a/Spigot-Server-Patches/Implement-Chunk-Priority-Urgency-System-for-Chunks.patch
+++ b/Spigot-Server-Patches/Implement-Chunk-Priority-Urgency-System-for-Chunks.patch
@ -22,6 +22,54 @@ view distance holds on you.
 Chunks in front of the player have higher priority, to help with
 fast traveling players keep up with their movement.

+diff --git a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java
+index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644
+--- a/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java
+++ b/src/main/java/com/destroystokyo/paper/io/chunk/ChunkTaskManager.java
+@@ -0,0 +0,0 @@ import com.destroystokyo.paper.io.PaperFileIOThread;
+ import com.destroystokyo.paper.io.IOUtil;
+ import com.destroystokyo.paper.io.PrioritizedTaskQueue;
+ import com.destroystokyo.paper.io.QueueExecutorThread;
+import it.unimi.dsi.fastutil.longs.Long2ObjectMap;
+import net.minecraft.server.ChunkCoordIntPair;
+ import net.minecraft.server.ChunkRegionLoader;
+import net.minecraft.server.ChunkStatus;
+ import net.minecraft.server.IAsyncTaskHandler;
+ import net.minecraft.server.IChunkAccess;
+ import net.minecraft.server.MinecraftServer;
+@@ -0,0 +0,0 @@ public final class ChunkTaskManager {
+             PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Status - " + ((chunk == null) ? "null chunk" : chunk.getChunkStatus().toString()));
+             PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Ticket Status - "  + PlayerChunk.getChunkStatus(chunkHolder.getTicketLevel()));
+             PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Holder Status - " + ((holderStatus == null) ? "null" : holderStatus.toString()));
+            PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Holder Priority - " + chunkHolder.getCurrentPriority());
+            synchronized (chunkHolder.neighborPriorities) {
+                if (!chunkHolder.neighborPriorities.isEmpty()) {
+                    PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Neighbors Requested Priority: ");
+                    for (Long2ObjectMap.Entry<Integer> entry : chunkHolder.neighborPriorities.long2ObjectEntrySet()) {
+                        ChunkCoordIntPair r = new ChunkCoordIntPair(entry.getLongKey());
+                        PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + " (" + r.x + "," + r.z + "): " + entry.getValue());
+                    }
+                }
+            }
+
+            synchronized (chunkHolder.neighbors) {
+                if (!chunkHolder.neighbors.isEmpty()) {
+                    PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Neighbors: ");
+                    for (PlayerChunk neighbor : chunkHolder.neighbors.keySet()) {
+                        ChunkStatus status = neighbor.getChunkHolderStatus();
+                        if (status != null && status.isAtLeastStatus(PlayerChunk.getChunkStatus(neighbor.getTicketLevel()))) {
+                            continue;
+                        }
+                        int nx = neighbor.location.x;
+                        int nz = neighbor.location.z;
+                        PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "  " + nx + "," + nz + " in " + chunkHolder.getWorld().getWorld().getName() + ":");
+                        dumpChunkInfo(neighbor, nx, nz, indent + 1);
+                    }
+                }
+            }
+         }
+     }
+ 
 diff --git a/src/main/java/net/minecraft/server/ChunkMapDistance.java b/src/main/java/net/minecraft/server/ChunkMapDistance.java
 index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644
 --- a/src/main/java/net/minecraft/server/ChunkMapDistance.java
@ -31,7 +79,7 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 
         ticket1.a(this.currentTick);
 -        if (ticket.b() < j) {
-+        if (ticket.b() < j || (ticket.getTicketType() == TicketType.PRIORITY && ((Ticket<Integer>) ticket).getObjectReason() < j)) { // Paper - check priority tickets too
+        if (ticket.b() < j || (ticket.getTicketType() == TicketType.PRIORITY && (30 - ticket.priority) < j)) { // Paper - check priority tickets too
             this.e.b(i, ticket.b(), true);
         }
 
@ -45,8 +93,15 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +    }
 +    public boolean markHighPriority(ChunkCoordIntPair coords, int priority) {
 +        priority = Math.min(30, Math.max(1, priority));
-+        Ticket<Integer> ticket = new Ticket<Integer>(TicketType.PRIORITY, 31, priority);
-+        return this.addTicket(coords.pair(), ticket);
+        long pair = coords.pair();
+        int currentPriority = getChunkPriority(coords);
+        if (currentPriority > priority) {
+            return false;
+        }
+        Ticket<Integer> ticket = new Ticket<Integer>(TicketType.PRIORITY, 31, 0);
+        ticket.priority = priority;
+        this.removeTicket(pair, ticket);
+        return this.addTicket(pair, ticket);
 +    }
 +    public int getChunkPriority(ChunkCoordIntPair coords) {
 +        int priority = 0;
@ -55,31 +110,30 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +            return priority;
 +        }
 +        for (Ticket<?> ticket : tickets) {
-+            if (ticket.getTicketType() != TicketType.PRIORITY) {
-+                continue;
-+            }
-+            //noinspection unchecked
-+            Ticket<Integer> prioTicket = (Ticket<Integer>) ticket;
-+            if (prioTicket.getObjectReason() > priority) {
-+                priority = prioTicket.getObjectReason();
+            if (ticket.getTicketType() == TicketType.PRIORITY && ticket.priority > 0) {
+                return ticket.priority;
 +            }
 +        }
 +        return priority;
 +    }
-+    public void clearPriorityTickets(ChunkCoordIntPair coords) {
+
+    public void refreshUrgentTicket(ChunkCoordIntPair coords) {
 +        ArraySetSorted<Ticket<?>> tickets = this.tickets.get(coords.pair());
-+        java.util.List<Ticket<?>> toRemove = new java.util.ArrayList<>();
-+        if (tickets == null) return;
+        if (tickets == null) {
+            markUrgent(coords);
+            return;
+        }
 +        for (Ticket<?> ticket : tickets) {
 +            if (ticket.getTicketType() == TicketType.PRIORITY) {
-+                toRemove.add(ticket);
+                ticket.setCurrentTick(this.currentTick);
+                return;
 +            }
 +        }
-+        for (Ticket<?> ticket : toRemove) {
-+            this.removeTicket(coords.pair(), ticket);
-+        }
 +
 +    }
+    public void clearPriorityTickets(ChunkCoordIntPair coords) {
+        this.removeTicket(coords.pair(), new Ticket<Integer>(TicketType.PRIORITY, 31, 0));
+    }
 +    // Paper end
     public <T> boolean addTicketAtLevel(TicketType<T> ticketType, ChunkCoordIntPair chunkcoordintpair, int level, T identifier) {
         return this.addTicket(chunkcoordintpair.pair(), new Ticket<>(ticketType, level, identifier));
@ -124,8 +178,15 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
                 this.world.asyncChunkTaskManager.raisePriority(x, z, com.destroystokyo.paper.io.PrioritizedTaskQueue.HIGHEST_PRIORITY);
                 com.destroystokyo.paper.io.chunk.ChunkTaskManager.pushChunkWait(this.world, x, z);
                 // Paper end
-@@ -0,0 +0,0 @@ public class ChunkProviderServer extends IChunkProvider {
-             this.serverThreadQueue.awaitTasks(completablefuture::isDone);
+                 com.destroystokyo.paper.io.SyncLoadFinder.logSyncLoad(this.world, x, z); // Paper - sync load info
+                 this.world.timings.syncChunkLoad.startTiming(); // Paper
+-            this.serverThreadQueue.awaitTasks(completablefuture::isDone);
+                // Paper start - keep priority ticket refreshed
+                this.serverThreadQueue.awaitTasks(() -> {
+                    this.chunkMapDistance.refreshUrgentTicket(pair);
+                    return completablefuture.isDone();
+                });
+                // PAper end
                 com.destroystokyo.paper.io.chunk.ChunkTaskManager.popChunkWait(); // Paper - async chunk debug
                 this.world.timings.syncChunkLoad.stopTiming(); // Paper
 +                this.clearPriorityTickets(pair); // Paper
@ -189,18 +250,28 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
     public int oldTicketLevel;
     private int ticketLevel;
 -    private int n;
-+    private int n; public final int getCurrentPriority() { return n; } // Paper - OBFHELPER
-     final ChunkCoordIntPair location; // Paper - private -> package
+-    final ChunkCoordIntPair location; // Paper - private -> package
+    volatile int n; public final int getCurrentPriority() { return n; } // Paper - OBFHELPER - make volatile since this is concurrently accessed
+    public final ChunkCoordIntPair location; // Paper - private -> public
     private final short[] dirtyBlocks;
     private int dirtyCount;
+     private int r;
+@@ -0,0 +0,0 @@ public class PlayerChunk {
+     private boolean hasBeenLoaded;
+ 
+     private final PlayerChunkMap chunkMap; // Paper
+    public WorldServer getWorld() { return chunkMap.world; } // Paper
+ 
+     long lastAutoSaveTime; // Paper - incremental autosave
+     long inactiveTimeStart; // Paper - incremental autosave
@@ -0,0 +0,0 @@ public class PlayerChunk {
         return null;
     }
     // Paper end - no-tick view distance
 +    // Paper start - Chunk gen/load priority system
 +    volatile int neighborPriority = -1;
-+    final java.util.concurrent.ConcurrentHashMap<PlayerChunk, Integer> neighbors = new java.util.concurrent.ConcurrentHashMap<>();
-+    final it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap<Integer> neighborPriorities = new it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap<>();
+    public final java.util.concurrent.ConcurrentHashMap<PlayerChunk, ChunkStatus> neighbors = new java.util.concurrent.ConcurrentHashMap<>();
+    public final it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap<Integer> neighborPriorities = new it.unimi.dsi.fastutil.longs.Long2ObjectOpenHashMap<>();
 +
 +    public int getPreferredPriority() {
 +        int priority = neighborPriority; // if we have a neighbor priority, use it
@ -221,10 +292,10 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +        return Math.max(1, Math.min(PlayerChunkMap.GOLDEN_TICKET, priority));
 +    }
 +    public void onNeighborRequest(PlayerChunk neighbor, ChunkStatus status) {
-+        int currentPriority = getCurrentPriority();
-+        if (!neighborPriorities.containsKey(neighbor.location.pair()) && (neighbor.neighborPriority == -1 || neighbor.neighborPriority > currentPriority)) {
-+            this.neighbors.put(neighbor, currentPriority);
-+            neighbor.setNeighborPriority(this, Math.max(1, currentPriority));
+        int priority = getCurrentPriority() + 1;
+        if (!neighborPriorities.containsKey(neighbor.location.pair()) && (neighbor.neighborPriority == -1 || neighbor.neighborPriority > priority)) {
+            this.neighbors.put(neighbor, status);
+            neighbor.setNeighborPriority(this, Math.max(1, priority));
 +        }
 +    }
 +
@ -305,19 +376,8 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 
 +    private void setPriority(int i) { d(i); } // Paper - OBFHELPER
     private void d(int i) {
-+        if (i == n) return; // Paper
         this.n = i;
-+        // Paper start
-+        this.neighbors.keySet().forEach(neighbor -> {
-+            if (neighbor.getCurrentPriority() > i) {
-+                neighbor.setNeighborPriority(this, i);
-+                this.w.changePriority(neighbor.location, neighbor::getCurrentPriority, neighbor.getCurrentPriority(), neighbor::setPriority);
-+            }
-+        });
-+        // Paper end
     }
- 
-     public void a(int i) {
@@ -0,0 +0,0 @@ public class PlayerChunk {
                     Chunk fullChunk = either.left().get();
                     PlayerChunk.this.isFullChunkReady = true;
@ -327,11 +387,35 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 
                 }
@@ -0,0 +0,0 @@ public class PlayerChunk {
+             this.entityTickingFuture.complete(PlayerChunk.UNLOADED_CHUNK); this.isEntityTickingReady = false; // Paper - cache chunk ticking stage
             this.entityTickingFuture = PlayerChunk.UNLOADED_CHUNK_FUTURE;
         }
- 
+-
 -        this.w.a(this.location, this::k, this.ticketLevel, this::d);
-+        this.w.a(this.location, this::k, getPreferredPriority(), this::d); // Paper - preferred priority
+        // Paper start - raise IO/load priority if priority changes, use our preferred priority
+        int priority = getPreferredPriority();
+        if (getCurrentPriority() > priority) {
+            int ioPriority = com.destroystokyo.paper.io.PrioritizedTaskQueue.NORMAL_PRIORITY;
+            if (priority <= 10) {
+                ioPriority = com.destroystokyo.paper.io.PrioritizedTaskQueue.HIGHEST_PRIORITY;
+            } else if (priority <= 20) {
+                ioPriority = com.destroystokyo.paper.io.PrioritizedTaskQueue.HIGH_PRIORITY;
+            }
+            chunkMap.world.asyncChunkTaskManager.raisePriority(location.x, location.z, ioPriority);
+        }
+        this.w.a(this.location, this::getCurrentPriority, priority, this::setPriority); // use preferred priority
+        this.neighbors.forEach((neighbor, neighborDesired) -> {
+            ChunkStatus neighborCurrent = neighbor.getChunkHolderStatus();
+            if (neighborCurrent == null || !neighborCurrent.isAtLeastStatus(neighborDesired)) {
+                if (neighbor.getCurrentPriority() > priority + 1 && neighbor.neighborPriority > priority + 1) {
+                    neighbor.setNeighborPriority(this, priority + 1);
+                    // Pending chunk update will run this same code here for the neighbor to update their priority
+                    // And since we are in the poll loop when this method runs, it should happen immediately after this.
+                    chunkMap.chunkDistanceManager.pendingChunkUpdates.add(neighbor);
+                }
+            }
+        });
+        // Paper end
         this.oldTicketLevel = this.ticketLevel;
         // CraftBukkit start
         // ChunkLoadEvent: Called after the chunk is loaded: isChunkLoaded returns true and chunk is ready to be modified by plugins.
@ -489,6 +573,26 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
         });
     }
 
+diff --git a/src/main/java/net/minecraft/server/Ticket.java b/src/main/java/net/minecraft/server/Ticket.java
+index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644
+--- a/src/main/java/net/minecraft/server/Ticket.java
+++ b/src/main/java/net/minecraft/server/Ticket.java
+@@ -0,0 +0,0 @@ public final class Ticket<T> implements Comparable<Ticket<?>> {
+     private final int b;
+     public final T identifier; public final T getObjectReason() { return this.identifier; } // Paper - OBFHELPER
+     private long d; public final long getCreationTick() { return this.d; } // Paper - OBFHELPER
+    public int priority = 0; // Paper
+ 
+     protected Ticket(TicketType<T> tickettype, int i, T t0) {
+         this.a = tickettype;
+@@ -0,0 +0,0 @@ public final class Ticket<T> implements Comparable<Ticket<?>> {
+         return this.b;
+     }
+ 
+    protected final void setCurrentTick(long i) { a(i); } // Paper - OBFHELPER
+     protected void a(long i) {
+         this.d = i;
+     }
 diff --git a/src/main/java/net/minecraft/server/TicketType.java b/src/main/java/net/minecraft/server/TicketType.java
 index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644
 --- a/src/main/java/net/minecraft/server/TicketType.java
--- a/Spigot-Server-Patches/Improved-Watchdog-Support.patch
+++ b/Spigot-Server-Patches/Improved-Watchdog-Support.patch
@ -316,6 +316,14 @@ diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/
 index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644
 --- a/src/main/java/org/spigotmc/WatchdogThread.java
 +++ b/src/main/java/org/spigotmc/WatchdogThread.java
+@@ -0,0 +0,0 @@ import org.bukkit.Bukkit;
+ public class WatchdogThread extends Thread
+ {
+ 
+    public static final boolean DISABLE_WATCHDOG = Boolean.getBoolean("disable.watchdog"); // Paper
+     private static WatchdogThread instance;
+     private final long timeoutTime;
+     private final long earlyWarningEvery; // Paper - Timeout time for just printing a dump but not restarting
@@ -0,0 +0,0 @@ public class WatchdogThread extends Thread
     {
         if ( instance == null )
@ -330,7 +338,7 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
             long currentTime = monotonicMillis();
 -            if ( lastTick != 0 && currentTime > lastTick + earlyWarningEvery && !Boolean.getBoolean("disable.watchdog") )
 +            MinecraftServer server = MinecraftServer.getServer();
-+            if (lastTick != 0 && hasStarted && (!server.isRunning() || (currentTime > lastTick + earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) ))
+            if (lastTick != 0 && hasStarted && (!server.isRunning() || (currentTime > lastTick + earlyWarningEvery && !DISABLE_WATCHDOG) ))
             {
 -                boolean isLongTimeout = currentTime > lastTick + timeoutTime;
 +                boolean isLongTimeout = currentTime > lastTick + timeoutTime || (!server.isRunning() && !server.hasStopped() && currentTime > lastTick + 1000);
--- a/Spigot-Server-Patches/No-Tick-view-distance-implementation.patch
+++ b/Spigot-Server-Patches/No-Tick-view-distance-implementation.patch
@ -118,16 +118,6 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 --- a/src/main/java/net/minecraft/server/PlayerChunk.java
 +++ b/src/main/java/net/minecraft/server/PlayerChunk.java
@@ -0,0 +0,0 @@ public class PlayerChunk {
-     // cached here to avoid a map lookup
-     com.destroystokyo.paper.util.misc.PooledLinkedHashSets.PooledObjectLinkedOpenHashSet<EntityPlayer> playersInMobSpawnRange;
-     com.destroystokyo.paper.util.misc.PooledLinkedHashSets.PooledObjectLinkedOpenHashSet<EntityPlayer> playersInChunkTickRange;
-+    com.destroystokyo.paper.util.misc.PooledLinkedHashSets.PooledObjectLinkedOpenHashSet<EntityPlayer> playersInTickingRange;
- 
-     void updateRanges() {
-         long key = net.minecraft.server.MCUtil.getCoordinateKey(this.location);
-         this.playersInMobSpawnRange = this.chunkMap.playerMobSpawnMap.getObjectsInRange(key);
-         this.playersInChunkTickRange = this.chunkMap.playerChunkTickRangeMap.getObjectsInRange(key);
-+        this.playersInTickingRange = this.chunkMap.playerViewDistanceTickMap.getObjectsInRange(key);
     }
     // Paper end - optimise isOutsideOfRange
 
--- a/Spigot-Server-Patches/Optimize-isOutsideRange-to-use-distance-maps.patch
+++ b/Spigot-Server-Patches/Optimize-isOutsideRange-to-use-distance-maps.patch
@ -16,8 +16,8 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 -    private final ChunkMapDistance.b f = new ChunkMapDistance.b(8);
 +    public static final int MOB_SPAWN_RANGE = 8; //private final ChunkMapDistance.b f = new ChunkMapDistance.b(8); // Paper - no longer used
     private final ChunkMapDistance.c g = new ChunkMapDistance.c(33);
-     private final java.util.Queue<PlayerChunk> pendingChunkUpdates = new java.util.ArrayDeque<>(); // PAIL pendingChunkUpdates // Paper - use a queue
-     private final ChunkTaskQueueSorter i;
+     // Paper start use a queue, but still keep unique requirement
+     public final java.util.Queue<PlayerChunk> pendingChunkUpdates = new java.util.ArrayDeque<PlayerChunk>() {
@@ -0,0 +0,0 @@ public abstract class ChunkMapDistance {
     private final Executor m;
     private long currentTick;
--- a/Spigot-Server-Patches/Reduce-sync-loads.patch
+++ b/Spigot-Server-Patches/Reduce-sync-loads.patch
@ -17,6 +17,7 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
@@ -0,0 +0,0 @@
 package com.destroystokyo.paper;
 
+ import com.destroystokyo.paper.io.chunk.ChunkTaskManager;
 +import com.destroystokyo.paper.io.SyncLoadFinder;
 import com.google.common.base.Functions;
 import com.google.common.collect.Iterables;
@ -42,16 +43,16 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
     public PaperCommand(String name) {
         super(name);
         this.description = "Paper related commands";
-        this.usageMessage = "/paper [heap | entity | reload | version | debug | chunkinfo]";
-+        this.usageMessage = "/paper [heap | entity | reload | version | debug | chunkinfo | syncloadinfo]";
+-        this.usageMessage = "/paper [heap | entity | reload | version | debug | dumpwaiting | chunkinfo]";
+        this.usageMessage = "/paper [heap | entity | reload | version | debug | dumpwaiting | chunkinfo | syncloadinfo]";
         this.setPermission("bukkit.command.paper");
     }
 
     @Override
     public List<String> tabComplete(CommandSender sender, String alias, String[] args, Location location) throws IllegalArgumentException {
         if (args.length <= 1)
-            return getListMatchingLast(args, "heap", "entity", "reload", "version", "debug", "chunkinfo");
-+            return getListMatchingLast(args, "heap", "entity", "reload", "version", "debug", "chunkinfo", "syncloadinfo");
+-            return getListMatchingLast(args, "heap", "entity", "reload", "version", "debug", "dumpwaiting", "chunkinfo");
+            return getListMatchingLast(args, "heap", "entity", "reload", "version", "debug", "dumpwaiting", "chunkinfo", "syncloadinfo");
 
         switch (args[0].toLowerCase(Locale.ENGLISH))
         {