From 5978d3cdd8cdf0379258047a17a0e61898280269 Mon Sep 17 00:00:00 2001
From: Aikar <aikar@aikar.co>
Date: Fri, 22 May 2020 19:03:48 -0400
Subject: [PATCH] More Improvements to Chunks

Fixed issues where urgent and prioritized chunks didn't actually
always get their priority boosted correctly....

Properly deprioritize non ticking chunks.
Limit recursion on watchdog prints to stop flooding as much
Remove neighbor priorities from watchdog to reduce information
reduce synchronization duration so that watch dog won't block main should main actually wake up
probably fixed a deadlock risk in watchdog printing also that was leading to crashes
fixed chunk holder enqueues not being processed correctly
added async catchers in some locations that should not be ran async

Fixed upstream bug where VITAL callbacks that must run on main actually could
sometimes run on the server thread pool causing alot of these nasty bugs we've seen lately!

This build will provide massive improvements to stability as well as even faster
sync chunk load/gens now that priority is correctly set.

Fixes #3435
---
 .../Asynchronous-chunk-IO-and-loading.patch   |  26 +--
 .../Basic-PlayerProfile-API.patch             |   3 +-
 .../Chunk-debug-command.patch                 |   6 +-
 ...k-Priority-Urgency-System-for-Chunks.patch | 197 ++++++++++++++----
 Spigot-Server-Patches/MC-Utils.patch          |  67 +++++-
 .../String-based-Action-Bar-API.patch         |  37 ----
 6 files changed, 229 insertions(+), 107 deletions(-)

diff --git a/Spigot-Server-Patches/Asynchronous-chunk-IO-and-loading.patch b/Spigot-Server-Patches/Asynchronous-chunk-IO-and-loading.patch
index 1ae2641828..3e019445a5 100644
--- a/Spigot-Server-Patches/Asynchronous-chunk-IO-and-loading.patch
+++ b/Spigot-Server-Patches/Asynchronous-chunk-IO-and-loading.patch
@@ -1929,22 +1929,20 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +        }
 +    }
 +
-+    public static String getChunkWaitInfo() {
++    private static ChunkInfo[] getChunkInfos() {
++        ChunkInfo[] chunks;
 +        synchronized (WAITING_CHUNKS) {
-+            return WAITING_CHUNKS.toString();
++            chunks = WAITING_CHUNKS.toArray(new ChunkInfo[0]);
 +        }
++        return chunks;
 +    }
 +
 +    public static void dumpAllChunkLoadInfo() {
-+        synchronized (WAITING_CHUNKS) {
-+            if (WAITING_CHUNKS.isEmpty()) {
-+                return;
-+            }
-+
++        ChunkInfo[] chunks = getChunkInfos();
++        if (chunks.length > 0) {
 +            PaperFileIOThread.LOGGER.log(Level.ERROR, "Chunk wait task info below: ");
-+            Set<PlayerChunk> seenChunks = new HashSet<>();
 +
-+            for (final ChunkInfo chunkInfo : WAITING_CHUNKS) {
++            for (final ChunkInfo chunkInfo : chunks) {
 +                final long key = IOUtil.getCoordinateKey(chunkInfo.chunkX, chunkInfo.chunkZ);
 +                final ChunkLoadTask loadTask = chunkInfo.world.asyncChunkTaskManager.chunkLoadTasks.get(key);
 +                final ChunkSaveTask saveTask = chunkInfo.world.asyncChunkTaskManager.chunkSaveTasks.get(key);
@@ -1955,18 +1953,22 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +                // log current status of chunk to indicate whether we're waiting on generation or loading
 +                net.minecraft.server.PlayerChunk chunkHolder = chunkInfo.world.getChunkProvider().playerChunkMap.getVisibleChunk(key);
 +
-+                dumpChunkInfo(seenChunks, chunkHolder, chunkInfo.chunkX, chunkInfo.chunkZ);
++                dumpChunkInfo(new HashSet<>(), chunkHolder, chunkInfo.chunkX, chunkInfo.chunkZ);
 +            }
 +        }
 +    }
 +
 +    static void dumpChunkInfo(Set<PlayerChunk> seenChunks, PlayerChunk chunkHolder, int x, int z) {
-+        dumpChunkInfo(seenChunks, chunkHolder, x, z, 0);
++        dumpChunkInfo(seenChunks, chunkHolder, x, z, 0, 1);
 +    }
-+    static void dumpChunkInfo(Set<PlayerChunk> seenChunks, PlayerChunk chunkHolder, int x, int z, int indent) {
++
++    static void dumpChunkInfo(Set<PlayerChunk> seenChunks, PlayerChunk chunkHolder, int x, int z, int indent, int maxDepth) {
 +        if (seenChunks.contains(chunkHolder)) {
 +            return;
 +        }
++        if (indent > maxDepth) {
++            return;
++        }
 +        seenChunks.add(chunkHolder);
 +        String indentStr = StringUtils.repeat("  ", indent);
 +        if (chunkHolder == null) {
diff --git a/Spigot-Server-Patches/Basic-PlayerProfile-API.patch b/Spigot-Server-Patches/Basic-PlayerProfile-API.patch
index 7d2607e358..f044ed7117 100644
--- a/Spigot-Server-Patches/Basic-PlayerProfile-API.patch
+++ b/Spigot-Server-Patches/Basic-PlayerProfile-API.patch
@@ -413,10 +413,11 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +import com.destroystokyo.paper.profile.CraftPlayerProfile;
 +import com.destroystokyo.paper.profile.PlayerProfile;
  import com.google.common.util.concurrent.ThreadFactoryBuilder;
-+import com.mojang.authlib.GameProfile;
  import org.apache.commons.lang.exception.ExceptionUtils;
++import com.mojang.authlib.GameProfile;
  import org.bukkit.Location;
  import org.bukkit.block.BlockFace;
+ import org.bukkit.craftbukkit.CraftWorld;
 @@ -0,0 +0,0 @@ public final class MCUtil {
          return run.get();
      }
diff --git a/Spigot-Server-Patches/Chunk-debug-command.patch b/Spigot-Server-Patches/Chunk-debug-command.patch
index a8e50f79e7..843227c30d 100644
--- a/Spigot-Server-Patches/Chunk-debug-command.patch
+++ b/Spigot-Server-Patches/Chunk-debug-command.patch
@@ -201,10 +201,10 @@ diff --git a/src/main/java/net/minecraft/server/MCUtil.java b/src/main/java/net/
 index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644
 --- a/src/main/java/net/minecraft/server/MCUtil.java
 +++ b/src/main/java/net/minecraft/server/MCUtil.java
-@@ -0,0 +0,0 @@ import com.destroystokyo.paper.block.TargetBlockInfo;
- import com.destroystokyo.paper.profile.CraftPlayerProfile;
+@@ -0,0 +0,0 @@ import com.destroystokyo.paper.profile.CraftPlayerProfile;
  import com.destroystokyo.paper.profile.PlayerProfile;
  import com.google.common.util.concurrent.ThreadFactoryBuilder;
+ import org.apache.commons.lang.exception.ExceptionUtils;
 +import com.google.gson.JsonArray;
 +import com.google.gson.JsonObject;
 +import com.google.gson.internal.Streams;
@@ -212,9 +212,9 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
  import com.mojang.authlib.GameProfile;
 +import com.mojang.datafixers.util.Either;
 +import it.unimi.dsi.fastutil.longs.Long2ObjectLinkedOpenHashMap;
- import org.apache.commons.lang.exception.ExceptionUtils;
  import org.bukkit.Location;
  import org.bukkit.block.BlockFace;
+ import org.bukkit.craftbukkit.CraftWorld;
 @@ -0,0 +0,0 @@ import org.spigotmc.AsyncCatcher;
  
  import javax.annotation.Nonnull;
diff --git a/Spigot-Server-Patches/Implement-Chunk-Priority-Urgency-System-for-Chunks.patch b/Spigot-Server-Patches/Implement-Chunk-Priority-Urgency-System-for-Chunks.patch
index c9be3f60e5..a0ae6b33aa 100644
--- a/Spigot-Server-Patches/Implement-Chunk-Priority-Urgency-System-for-Chunks.patch
+++ b/Spigot-Server-Patches/Implement-Chunk-Priority-Urgency-System-for-Chunks.patch
@@ -37,40 +37,43 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
  import net.minecraft.server.IAsyncTaskHandler;
  import net.minecraft.server.IChunkAccess;
  import net.minecraft.server.MinecraftServer;
+@@ -0,0 +0,0 @@ public final class ChunkTaskManager {
+     }
+ 
+     static void dumpChunkInfo(Set<PlayerChunk> seenChunks, PlayerChunk chunkHolder, int x, int z) {
+-        dumpChunkInfo(seenChunks, chunkHolder, x, z, 0, 1);
++        dumpChunkInfo(seenChunks, chunkHolder, x, z, 0, 4);
+     }
+ 
+     static void dumpChunkInfo(Set<PlayerChunk> seenChunks, PlayerChunk chunkHolder, int x, int z, int indent, int maxDepth) {
 @@ -0,0 +0,0 @@ public final class ChunkTaskManager {
              PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Status - " + ((chunk == null) ? "null chunk" : chunk.getChunkStatus().toString()));
              PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Ticket Status - "  + PlayerChunk.getChunkStatus(chunkHolder.getTicketLevel()));
              PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Holder Status - " + ((holderStatus == null) ? "null" : holderStatus.toString()));
 +            PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Holder Priority - " + chunkHolder.getCurrentPriority());
-+            synchronized (chunkHolder.neighborPriorities) {
-+                if (!chunkHolder.neighborPriorities.isEmpty()) {
-+                    PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Neighbors Requested Priority: ");
-+                    for (Long2ObjectMap.Entry<Integer> entry : chunkHolder.neighborPriorities.long2ObjectEntrySet()) {
-+                        ChunkCoordIntPair r = new ChunkCoordIntPair(entry.getLongKey());
-+                        PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + " (" + r.x + "," + r.z + "): " + entry.getValue());
++
++            if (!chunkHolder.neighbors.isEmpty()) {
++                if (indent >= maxDepth) {
++                    PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Neighbors: (Can't show, too deeply nested)");
++                    return;
++                }
++                PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Neighbors: ");
++                for (PlayerChunk neighbor : chunkHolder.neighbors.keySet()) {
++                    ChunkStatus status = neighbor.getChunkHolderStatus();
++                    if (status != null && status.isAtLeastStatus(PlayerChunk.getChunkStatus(neighbor.getTicketLevel()))) {
++                        continue;
 +                    }
++                    int nx = neighbor.location.x;
++                    int nz = neighbor.location.z;
++                    if (seenChunks.contains(neighbor)) {
++                        PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "  " + nx + "," + nz + " in " + chunkHolder.getWorld().getWorld().getName() + " (CIRCULAR)");
++                        continue;
++                    }
++                    PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "  " + nx + "," + nz + " in " + chunkHolder.getWorld().getWorld().getName() + ":");
++                    dumpChunkInfo(seenChunks, neighbor, nx, nz, indent + 1, maxDepth);
 +                }
 +            }
 +
-+            synchronized (chunkHolder.neighbors) {
-+                if (!chunkHolder.neighbors.isEmpty()) {
-+                    PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "Chunk Neighbors: ");
-+                    for (PlayerChunk neighbor : chunkHolder.neighbors.keySet()) {
-+                        ChunkStatus status = neighbor.getChunkHolderStatus();
-+                        if (status != null && status.isAtLeastStatus(PlayerChunk.getChunkStatus(neighbor.getTicketLevel()))) {
-+                            continue;
-+                        }
-+                        int nx = neighbor.location.x;
-+                        int nz = neighbor.location.z;
-+                        if (seenChunks.contains(neighbor)) {
-+                            PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "  " + nx + "," + nz + " in " + chunkHolder.getWorld().getWorld().getName() + " (CIRCULAR)");
-+                            continue;
-+                        }
-+                        PaperFileIOThread.LOGGER.log(Level.ERROR, indentStr + "  " + nx + "," + nz + " in " + chunkHolder.getWorld().getWorld().getName() + ":");
-+                        dumpChunkInfo(seenChunks, neighbor, nx, nz, indent + 1);
-+                    }
-+                }
-+            }
          }
      }
  
@@ -94,6 +97,14 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
          return !arraysetsorted.isEmpty() ? ((Ticket) arraysetsorted.b()).b() : PlayerChunkMap.GOLDEN_TICKET + 1;
      }
  
+@@ -0,0 +0,0 @@ public abstract class ChunkMapDistance {
+ 
+     public boolean a(PlayerChunkMap playerchunkmap) {
+         //this.f.a(); // Paper - no longer used
++        AsyncCatcher.catchOp("DistanceManagerTick");
+         this.g.a();
+         int i = Integer.MAX_VALUE - this.e.a(Integer.MAX_VALUE);
+         boolean flag = i != 0;
 @@ -0,0 +0,0 @@ public abstract class ChunkMapDistance {
  
          // Paper start
@@ -139,6 +150,7 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
      }
  
 +    // Paper start
++    public static final int PRIORITY_TICKET_LEVEL = 33;
 +    public boolean markUrgent(ChunkCoordIntPair coords) {
 +        return addPriorityTicket(coords, TicketType.URGENT, 30);
 +    }
@@ -150,16 +162,38 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +    private boolean addPriorityTicket(ChunkCoordIntPair coords, TicketType<ChunkCoordIntPair> ticketType, int priority) {
 +        AsyncCatcher.catchOp("ChunkMapDistance::addPriorityTicket");
 +        long pair = coords.pair();
-+        Ticket<ChunkCoordIntPair> ticket = new Ticket<ChunkCoordIntPair>(ticketType, 34, coords);
-+        ticket.priority = priority;
 +
-+        this.removeTicket(pair, ticket);
-+        boolean added = this.addTicket(pair, ticket);
-+        PlayerChunk updatingChunk = chunkMap.getUpdatingChunk(pair);
-+        if (updatingChunk != null) {
-+            chunkMap.queueHolderUpdate(updatingChunk);
++        boolean success;
++        if (!(success = updatePriorityTicket(coords, ticketType, priority))) {
++            Ticket<ChunkCoordIntPair> ticket = new Ticket<ChunkCoordIntPair>(ticketType, PRIORITY_TICKET_LEVEL, coords);
++            ticket.priority = priority;
++            success = this.addTicket(pair, ticket);
 +        }
-+        return added;
++
++        chunkMap.world.getChunkProvider().tickDistanceManager();
++        PlayerChunk updatingChunk = chunkMap.getUpdatingChunk(pair);
++        if (updatingChunk != null && updatingChunk.priorityBoost < priority) {
++            // May not be enqueued, enqueue it if not and tick distance manager
++            chunkMap.queueHolderUpdate(updatingChunk);
++            chunkMap.world.getChunkProvider().tickDistanceManager();
++        }
++        return success;
++    }
++
++    private boolean updatePriorityTicket(ChunkCoordIntPair coords, TicketType<ChunkCoordIntPair> type, int priority) {
++        ArraySetSorted<Ticket<?>> tickets = this.tickets.get(coords.pair());
++        if (tickets == null) {
++            return false;
++        }
++        for (Ticket<?> ticket : tickets) {
++            if (ticket.getTicketType() == type) {
++                // We only support increasing, not decreasing, too complicated
++                ticket.priority = Math.max(ticket.priority, priority);
++                return true;
++            }
++        }
++
++        return false;
 +    }
 +
 +    public int getChunkPriority(ChunkCoordIntPair coords) {
@@ -183,12 +217,12 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +
 +    public void clearPriorityTickets(ChunkCoordIntPair coords) {
 +        AsyncCatcher.catchOp("ChunkMapDistance::clearPriority");
-+        this.removeTicket(coords.pair(), new Ticket<ChunkCoordIntPair>(TicketType.PRIORITY, 34, coords));
++        this.removeTicket(coords.pair(), new Ticket<ChunkCoordIntPair>(TicketType.PRIORITY, PRIORITY_TICKET_LEVEL, coords));
 +    }
 +
 +    public void clearUrgent(ChunkCoordIntPair coords) {
 +        AsyncCatcher.catchOp("ChunkMapDistance::clearUrgent");
-+        this.removeTicket(coords.pair(), new Ticket<ChunkCoordIntPair>(TicketType.URGENT, 34, coords));
++        this.removeTicket(coords.pair(), new Ticket<ChunkCoordIntPair>(TicketType.URGENT, PRIORITY_TICKET_LEVEL, coords));
 +    }
 +    // Paper end
      public <T> boolean addTicketAtLevel(TicketType<T> ticketType, ChunkCoordIntPair chunkcoordintpair, int level, T identifier) {
@@ -345,7 +379,7 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +        int priority = neighborPriority; // if we have a neighbor priority, use it
 +        int myPriority = getMyPriority();
 +
-+        if (priority == -1 || priority > myPriority) {
++        if (priority == -1 || (ticketLevel <= 33 && priority > myPriority)) {
 +            priority = myPriority;
 +        }
 +
@@ -357,7 +391,7 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +            return 1; // Urgent - ticket level isn't always 31 so 33-30 = 3
 +        }
 +        int basePriority = ticketLevel - priorityBoost;
-+        if (ticketLevel >= 34 && priorityBoost == 0 && neighborPriorities.isEmpty()) {
++        if (ticketLevel >= 33 && priorityBoost == 0 && (neighborPriority >= 34 || neighborPriorities.isEmpty())) {
 +            basePriority += 5;
 +        }
 +        return basePriority;
@@ -469,6 +503,9 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +            return status;
 +        }
 +        return CHUNK_STATUSES.get(status.getStatusIndex() + 1);
++    }
++    public CompletableFuture<Either<IChunkAccess, PlayerChunk.Failure>> getStatusFutureUncheckedMain(ChunkStatus chunkstatus) {
++        return MCUtil.ensureMain(getStatusFutureUnchecked(chunkstatus));
 +    }
      // Paper end
  
@@ -482,6 +519,22 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
          this.n = i;
      }
 @@ -0,0 +0,0 @@ public class PlayerChunk {
+         // CraftBukkit start
+         // ChunkUnloadEvent: Called before the chunk is unloaded: isChunkLoaded is still true and chunk can still be modified by plugins.
+         if (playerchunk_state.isAtLeast(PlayerChunk.State.BORDER) && !playerchunk_state1.isAtLeast(PlayerChunk.State.BORDER)) {
+-            this.getStatusFutureUnchecked(ChunkStatus.FULL).thenAccept((either) -> {
++            this.getStatusFutureUncheckedMain(ChunkStatus.FULL).thenAccept((either) -> { // Paper - ensure main
+                 Chunk chunk = (Chunk)either.left().orElse(null);
+                 if (chunk != null) {
+                     playerchunkmap.callbackExecutor.execute(() -> {
+@@ -0,0 +0,0 @@ public class PlayerChunk {
+         if (!flag2 && flag3) {
+             // Paper start - cache ticking ready status
+             int expectCreateCount = ++this.fullChunkCreateCount;
+-            this.fullChunkFuture = playerchunkmap.b(this); this.fullChunkFuture.thenAccept((either) -> {
++            this.fullChunkFuture = playerchunkmap.b(this); MCUtil.ensureMain(this.fullChunkFuture).thenAccept((either) -> { // Paper - ensure main
+                 if (either.left().isPresent() && PlayerChunk.this.fullChunkCreateCount == expectCreateCount) {
+                     // note: Here is a very good place to add callbacks to logic waiting on this.
                      Chunk fullChunk = either.left().get();
                      PlayerChunk.this.isFullChunkReady = true;
                      fullChunk.playerChunk = PlayerChunk.this;
@@ -490,6 +543,24 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
  
                  }
 @@ -0,0 +0,0 @@ public class PlayerChunk {
+ 
+         if (!flag4 && flag5) {
+             // Paper start - cache ticking ready status
+-            this.tickingFuture = playerchunkmap.a(this); this.tickingFuture.thenAccept((either) -> {
++            this.tickingFuture = playerchunkmap.a(this); MCUtil.ensureMain(this.tickingFuture).thenAccept((either) -> { // Paper - ensure main
+                 if (either.left().isPresent()) {
+                     // note: Here is a very good place to add callbacks to logic waiting on this.
+                     Chunk tickingChunk = either.left().get();
+@@ -0,0 +0,0 @@ public class PlayerChunk {
+             }
+ 
+             // Paper start - cache ticking ready status
+-            this.entityTickingFuture = playerchunkmap.b(this.location); this.entityTickingFuture.thenAccept((either) -> {
++            this.entityTickingFuture = playerchunkmap.b(this.location); MCUtil.ensureMain(this.entityTickingFuture).thenAccept((either) -> { // Paper ensureMain
+                 if (either.left().isPresent()) {
+                     // note: Here is a very good place to add callbacks to logic waiting on this.
+                     Chunk entityTickingChunk = either.left().get();
+@@ -0,0 +0,0 @@ public class PlayerChunk {
              this.entityTickingFuture.complete(PlayerChunk.UNLOADED_CHUNK); this.isEntityTickingReady = false; // Paper - cache chunk ticking stage
              this.entityTickingFuture = PlayerChunk.UNLOADED_CHUNK_FUTURE;
          }
@@ -507,13 +578,21 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +            }
 +            chunkMap.world.asyncChunkTaskManager.raisePriority(location.x, location.z, ioPriority);
 +        }
-+        this.w.a(this.location, this::getCurrentPriority, priority, this::setPriority); // use preferred priority
-+        int neighborsPriority = getNeighborsPriority();
-+        this.neighbors.forEach((neighbor, neighborDesired) -> neighbor.setNeighborPriority(this, neighborsPriority));
++        if (getCurrentPriority() != priority) {
++            this.w.a(this.location, this::getCurrentPriority, priority, this::setPriority); // use preferred priority
++            int neighborsPriority = getNeighborsPriority();
++            this.neighbors.forEach((neighbor, neighborDesired) -> neighbor.setNeighborPriority(this, neighborsPriority));
++        }
 +        // Paper end
          this.oldTicketLevel = this.ticketLevel;
          // CraftBukkit start
          // ChunkLoadEvent: Called after the chunk is loaded: isChunkLoaded returns true and chunk is ready to be modified by plugins.
+         if (!playerchunk_state.isAtLeast(PlayerChunk.State.BORDER) && playerchunk_state1.isAtLeast(PlayerChunk.State.BORDER)) {
+-            this.getStatusFutureUnchecked(ChunkStatus.FULL).thenAccept((either) -> {
++            this.getStatusFutureUncheckedMain(ChunkStatus.FULL).thenAccept((either) -> { // Paper - ensure main
+                 Chunk chunk = (Chunk)either.left().orElse(null);
+                 if (chunk != null) {
+                     playerchunkmap.callbackExecutor.execute(() -> {
 @@ -0,0 +0,0 @@ public class PlayerChunk {
  
      public interface c {
@@ -526,6 +605,30 @@ diff --git a/src/main/java/net/minecraft/server/PlayerChunkMap.java b/src/main/j
 index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644
 --- a/src/main/java/net/minecraft/server/PlayerChunkMap.java
 +++ b/src/main/java/net/minecraft/server/PlayerChunkMap.java
+@@ -0,0 +0,0 @@ import org.apache.commons.lang3.mutable.MutableBoolean;
+ import org.apache.logging.log4j.LogManager;
+ import org.apache.logging.log4j.Logger;
+ import org.bukkit.entity.Player; // CraftBukkit
++import org.spigotmc.AsyncCatcher;
+ 
+ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
+ 
+@@ -0,0 +0,0 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
+ 
+         @Override
+         public void execute(Runnable runnable) {
++            AsyncCatcher.catchOp("Callback Executor execute");
+             if (queued == null) {
+                 queued = new java.util.ArrayDeque<>();
+             }
+@@ -0,0 +0,0 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
+ 
+         @Override
+         public void run() {
++            AsyncCatcher.catchOp("Callback Executor run");
+             if (queued == null) {
+                 return;
+             }
 @@ -0,0 +0,0 @@ public class PlayerChunkMap extends IChunkLoader implements PlayerChunk.d {
          this.playerViewDistanceTickMap = new com.destroystokyo.paper.util.misc.PlayerAreaMap(this.pooledLinkedPlayerHashSets,
              (EntityPlayer player, int rangeX, int rangeZ, int currPosX, int currPosZ, int prevPosX, int prevPosZ,
@@ -551,13 +654,21 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +    // Paper start - Chunk Prioritization
 +    private static final int[][] neighborMatrix = {{-1, 0}, {0, -1}, {0, 1}, {1, 0}};
 +    public void queueHolderUpdate(PlayerChunk playerchunk) {
-+        executor.execute(() -> {
-+            if (isUnloading(playerchunk)) return; // unloaded
++        Runnable runnable = () -> {
++            if (isUnloading(playerchunk)) {
++                return; // unloaded
++            }
 +            chunkDistanceManager.pendingChunkUpdates.add(playerchunk);
 +            if (!chunkDistanceManager.pollingPendingChunkUpdates) {
 +                world.getChunkProvider().tickDistanceManager();
 +            }
-+        });
++        };
++        if (MCUtil.isMainThread()) {
++            // We can't use executor here because it will not execute tasks if its currently in the middle of executing tasks...
++            runnable.run();
++        } else {
++            executor.execute(runnable);
++        }
 +    }
 +
 +    public boolean isUnloading(PlayerChunk playerchunk) {
diff --git a/Spigot-Server-Patches/MC-Utils.patch b/Spigot-Server-Patches/MC-Utils.patch
index 63df3a5256..be1fee685f 100644
--- a/Spigot-Server-Patches/MC-Utils.patch
+++ b/Spigot-Server-Patches/MC-Utils.patch
@@ -3273,6 +3273,7 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +
 +import com.destroystokyo.paper.block.TargetBlockInfo;
 +import com.google.common.util.concurrent.ThreadFactoryBuilder;
++import org.apache.commons.lang.exception.ExceptionUtils;
 +import org.bukkit.Location;
 +import org.bukkit.block.BlockFace;
 +import org.bukkit.craftbukkit.CraftWorld;
@@ -3290,6 +3291,7 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +import java.util.concurrent.TimeUnit;
 +import java.util.concurrent.TimeoutException;
 +import java.util.concurrent.atomic.AtomicBoolean;
++import java.util.function.BiConsumer;
 +import java.util.function.Consumer;
 +import java.util.function.Supplier;
 +
@@ -3317,6 +3319,12 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +        };
 +    }
 +
++    public static <T> Runnable once(List<T> list, Consumer<T> cb) {
++        return once(() -> {
++            list.forEach(cb);
++        });
++    }
++
 +    private static Runnable makeCleanerCallback(Runnable run) {
 +        return once(() -> cleanerExecutor.execute(run));
 +    }
@@ -3384,19 +3392,16 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +        return list;
 +    }
 +
-+    public static long getCoordinateKey(final BlockPosition blockPos) {
-+        return ((long)(blockPos.getZ() >> 4) << 32) | ((blockPos.getX() >> 4) & 0xFFFFFFFFL);
-+    }
-+
-+    public static long getCoordinateKey(final Entity entity) {
-+        return ((long)(MCUtil.fastFloor(entity.locZ()) >> 4) << 32) | ((MCUtil.fastFloor(entity.locX()) >> 4) & 0xFFFFFFFFL);
-+    }
-+
 +    public static int fastFloor(double x) {
 +        int truncated = (int)x;
 +        return x < (double)truncated ? truncated - 1 : truncated;
 +    }
 +
++    public static int fastFloor(float x) {
++        int truncated = (int)x;
++        return x < (double)truncated ? truncated - 1 : truncated;
++    }
++
 +    public static float normalizeYaw(float f) {
 +        float f1 = f % 360.0F;
 +
@@ -3411,9 +3416,31 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +        return f1;
 +    }
 +
-+    public static int fastFloor(float x) {
-+        int truncated = (int)x;
-+        return x < (double)truncated ? truncated - 1 : truncated;
++    /**
++     * Quickly generate a stack trace for current location
++     *
++     * @return Stacktrace
++     */
++    public static String stack() {
++        return ExceptionUtils.getFullStackTrace(new Throwable());
++    }
++
++    /**
++     * Quickly generate a stack trace for current location with message
++     *
++     * @param str
++     * @return Stacktrace
++     */
++    public static String stack(String str) {
++        return ExceptionUtils.getFullStackTrace(new Throwable(str));
++    }
++
++    public static long getCoordinateKey(final BlockPosition blockPos) {
++        return ((long)(blockPos.getZ() >> 4) << 32) | ((blockPos.getX() >> 4) & 0xFFFFFFFFL);
++    }
++
++    public static long getCoordinateKey(final Entity entity) {
++        return ((long)(MCUtil.fastFloor(entity.locZ()) >> 4) << 32) | ((MCUtil.fastFloor(entity.locX()) >> 4) & 0xFFFFFFFFL);
 +    }
 +
 +    public static long getCoordinateKey(final ChunkCoordIntPair pair) {
@@ -3466,6 +3493,24 @@ index 0000000000000000000000000000000000000000..00000000000000000000000000000000
 +
 +    private MCUtil() {}
 +
++    public static final java.util.concurrent.Executor MAIN_EXECUTOR = (run) -> {
++        if (!isMainThread()) {
++            MinecraftServer.getServer().execute(run);
++        } else {
++            run.run();
++        }
++    };
++
++    public static <T> CompletableFuture<T> ensureMain(CompletableFuture<T> future) {
++        return future.thenApplyAsync(r -> r, MAIN_EXECUTOR);
++    }
++
++    public static <T> void thenOnMain(CompletableFuture<T> future, Consumer<T> consumer) {
++        future.thenAcceptAsync(consumer, MAIN_EXECUTOR);
++    }
++    public static <T> void thenOnMain(CompletableFuture<T> future, BiConsumer<T, Throwable> consumer) {
++        future.whenCompleteAsync(consumer, MAIN_EXECUTOR);
++    }
 +
 +    public static boolean isMainThread() {
 +        return MinecraftServer.getServer().isMainThread();
diff --git a/Spigot-Server-Patches/String-based-Action-Bar-API.patch b/Spigot-Server-Patches/String-based-Action-Bar-API.patch
index 7331a65557..66111022b2 100644
--- a/Spigot-Server-Patches/String-based-Action-Bar-API.patch
+++ b/Spigot-Server-Patches/String-based-Action-Bar-API.patch
@@ -4,43 +4,6 @@ Date: Tue, 27 Dec 2016 15:02:42 -0500
 Subject: [PATCH] String based Action Bar API
 
 
-diff --git a/src/main/java/net/minecraft/server/MCUtil.java b/src/main/java/net/minecraft/server/MCUtil.java
-index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644
---- a/src/main/java/net/minecraft/server/MCUtil.java
-+++ b/src/main/java/net/minecraft/server/MCUtil.java
-@@ -0,0 +0,0 @@ package net.minecraft.server;
- 
- import com.destroystokyo.paper.block.TargetBlockInfo;
- import com.google.common.util.concurrent.ThreadFactoryBuilder;
-+import org.apache.commons.lang.exception.ExceptionUtils;
- import org.bukkit.Location;
- import org.bukkit.block.BlockFace;
- import org.bukkit.craftbukkit.CraftWorld;
-@@ -0,0 +0,0 @@ public final class MCUtil {
- 
-     private MCUtil() {}
- 
-+    /**
-+     * Quickly generate a stack trace for current location
-+     *
-+     * @return Stacktrace
-+     */
-+    public static String stack() {
-+        return ExceptionUtils.getFullStackTrace(new Throwable());
-+    }
-+
-+    /**
-+     * Quickly generate a stack trace for current location with message
-+     *
-+     * @param str
-+     * @return Stacktrace
-+     */
-+    public static String stack(String str) {
-+        return ExceptionUtils.getFullStackTrace(new Throwable(str));
-+    }
- 
-     public static boolean isMainThread() {
-         return MinecraftServer.getServer().isMainThread();
 diff --git a/src/main/java/org/bukkit/craftbukkit/entity/CraftPlayer.java b/src/main/java/org/bukkit/craftbukkit/entity/CraftPlayer.java
 index 0000000000000000000000000000000000000000..0000000000000000000000000000000000000000 100644
 --- a/src/main/java/org/bukkit/craftbukkit/entity/CraftPlayer.java