PaperMC/patches/server/1061-Improved-Watchdog-Support.patch

From 0000000000000000000000000000000000000000 Mon Sep 17 00:00:00 2001
From: Aikar <aikar@aikar.co>
Date: Sun, 12 Apr 2020 15:50:48 -0400
Subject: [PATCH] Improved Watchdog Support

Forced Watchdog Crash support and Improve Async Shutdown

If the request to shut down the server is received while we are in
a watchdog hang, immediately treat it as a crash and begin the shutdown
process. Shutdown process is now improved to also shutdown cleanly when
not using restart scripts either.

If a server is deadlocked, a server owner can send SIGUP (or any other signal
the JVM understands to shut down as it currently does) and the watchdog
will no longer need to wait until the full timeout, allowing you to trigger
a close process and try to shut the server down gracefully, saving player and
world data.

Previously there was no way to trigger this outside of waiting for a full watchdog
timeout, which may be set to a really long time...

Additionally, fix everything to do with shutting the server down asynchronously.

Previously, nearly everything about the process was fragile and unsafe. Main might
not have actually been frozen, and might still be manipulating state.

Or, some reuest might ask main to do something in the shutdown but main is dead.

Or worse, other things might start closing down items such as the Console or Thread Pool
before we are fully shutdown.

This change tries to resolve all of these issues by moving everything into the stop
method and guaranteeing only one thread is stopping the server.

We then issue Thread Death to the main thread of another thread initiates the stop process.
We have to ensure Thread Death propagates correctly though to stop main completely.

This is to ensure that if main isn't truely stuck, it's not manipulating state we are trying to save.

This also moves all plugins who register "delayed init" tasks to occur just before "Done" so they
are properly accounted for and wont trip watchdog on init.

diff --git a/src/main/java/io/papermc/paper/util/LogManagerShutdownThread.java b/src/main/java/io/papermc/paper/util/LogManagerShutdownThread.java
new file mode 100644
index 0000000000000000000000000000000000000000..183e141d0c13190c6905dc4510d891992afef878
--- /dev/null
+++ b/src/main/java/io/papermc/paper/util/LogManagerShutdownThread.java
@@ -0,0 +1,26 @@
+package io.papermc.paper.util;
+
+public class LogManagerShutdownThread extends Thread {
+
+    static LogManagerShutdownThread INSTANCE = new LogManagerShutdownThread();
+    public static final void hook() {
+        if (INSTANCE == null) {
+            throw new IllegalStateException("Cannot re-hook after being unhooked");
+        }
+        Runtime.getRuntime().addShutdownHook(INSTANCE);
+    }
+
+    public static final void unhook() {
+        Runtime.getRuntime().removeShutdownHook(INSTANCE);
+        INSTANCE = null;
+    }
+
+    private LogManagerShutdownThread() {
+        super("Log4j2 Shutdown Thread");
+    }
+
+    @Override
+    public void run() {
+        org.apache.logging.log4j.LogManager.shutdown();
+    }
+}
diff --git a/src/main/java/net/minecraft/CrashReport.java b/src/main/java/net/minecraft/CrashReport.java
index 589a8bf75be6ccc59f1e5dd5d8d9afed41c4772d..b24265573fdef5d9a964bcd76146f34542c420cf 100644
--- a/src/main/java/net/minecraft/CrashReport.java
+++ b/src/main/java/net/minecraft/CrashReport.java
@@ -237,6 +237,7 @@ public class CrashReport {
     }
 
     public static CrashReport forThrowable(Throwable cause, String title) {
+        if (cause instanceof ThreadDeath) com.destroystokyo.paper.util.SneakyThrow.sneaky(cause); // Paper
         while (cause instanceof CompletionException && cause.getCause() != null) {
             cause = cause.getCause();
         }
diff --git a/src/main/java/net/minecraft/server/Main.java b/src/main/java/net/minecraft/server/Main.java
index 42d46c7a7437bea5335a23cbee5708ac57131474..300a044bb0f0e377133f24469cea1a9669de6e58 100644
--- a/src/main/java/net/minecraft/server/Main.java
+++ b/src/main/java/net/minecraft/server/Main.java
@@ -79,6 +79,7 @@ public class Main {
     @SuppressForbidden(reason = "System.out needed before bootstrap") // CraftBukkit - decompile error
     @DontObfuscate
     public static void main(final OptionSet optionset) { // CraftBukkit - replaces main(String[] astring)
+        io.papermc.paper.util.LogManagerShutdownThread.hook(); // Paper
         SharedConstants.tryDetectVersion();
         /* CraftBukkit start - Replace everything
         OptionParser optionparser = new OptionParser();
diff --git a/src/main/java/net/minecraft/server/MinecraftServer.java b/src/main/java/net/minecraft/server/MinecraftServer.java
index 5e7ba47247fc9b6bc8da86d8f67c6cd923cd0b1e..73482b0da01b26e68b58fc57093b3c82cfa12ea7 100644
--- a/src/main/java/net/minecraft/server/MinecraftServer.java
+++ b/src/main/java/net/minecraft/server/MinecraftServer.java
@@ -317,7 +317,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
     public java.util.Queue<Runnable> processQueue = new java.util.concurrent.ConcurrentLinkedQueue<Runnable>();
     public int autosavePeriod;
     // Paper - don't store the vanilla dispatcher
-    private boolean forceTicks;
+    public boolean forceTicks; // Paper - Improved watchdog support
     // CraftBukkit end
     // Spigot start
     public static final int TPS = 20;
@@ -330,6 +330,9 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
     public boolean isIteratingOverLevels = false; // Paper - Throw exception on world create while being ticked
     private final Set<String> pluginsBlockingSleep = new java.util.HashSet<>(); // Paper - API to allow/disallow tick sleeping
 
+    public volatile Thread shutdownThread; // Paper
+    public volatile boolean abnormalExit = false; // Paper
+
     public static <S extends MinecraftServer> S spin(Function<Thread, S> serverFactory) {
         ca.spottedleaf.dataconverter.minecraft.datatypes.MCTypeRegistry.init(); // Paper - rewrite data converter system
         AtomicReference<S> atomicreference = new AtomicReference();
@@ -504,6 +507,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
         }
         */
         // Paper end
+        io.papermc.paper.util.LogManagerShutdownThread.unhook(); // Paper
         Runtime.getRuntime().addShutdownHook(new org.bukkit.craftbukkit.util.ServerShutdownThread(this));
         // CraftBukkit end
         this.paperConfigurations = services.paperConfigurations(); // Paper - add paper configuration files
@@ -1023,6 +1027,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
     // CraftBukkit start
     private boolean hasStopped = false;
     private boolean hasLoggedStop = false; // Paper - Debugging
+    public volatile boolean hasFullyShutdown = false; // Paper
     private final Object stopLock = new Object();
     public final boolean hasStopped() {
         synchronized (this.stopLock) {
@@ -1038,6 +1043,10 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
             this.hasStopped = true;
         }
         if (!hasLoggedStop && isDebugging()) io.papermc.paper.util.TraceUtil.dumpTraceForThread("Server stopped"); // Paper - Debugging
+        // Paper start - kill main thread, and kill it hard
+        shutdownThread = Thread.currentThread();
+        org.spigotmc.WatchdogThread.doStop(); // Paper
+        // Paper end
         // CraftBukkit end
         if (this.metricsRecorder.isRecording()) {
             this.cancelRecordingMetrics();
@@ -1119,6 +1128,15 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
             ca.spottedleaf.moonrise.common.util.MoonriseCommon.haltExecutors();
         }
         // Paper end - rewrite chunk system
+        // Paper start - Improved watchdog support - move final shutdown items here
+        Util.shutdownExecutors();
+        try {
+            net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+        } catch (final Exception ignored) {
+        }
+        io.papermc.paper.log.CustomLogManager.forceReset(); // Paper - Reset loggers after shutdown
+        this.onServerExit();
+        // Paper end - Improved watchdog support - move final shutdown items here
     }
 
     public String getLocalIp() {
@@ -1213,6 +1231,7 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
 
     protected void runServer() {
         try {
+            long serverStartTime = Util.getNanos(); // Paper
             if (!this.initServer()) {
                 throw new IllegalStateException("Failed to initialize server");
             }
@@ -1223,6 +1242,17 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
 
             this.server.spark.enableBeforePlugins(); // Paper - spark
             // Spigot start
+            // Paper start - Improved Watchdog Support
+            LOGGER.info("Running delayed init tasks");
+            this.server.getScheduler().mainThreadHeartbeat(); // run all 1 tick delay tasks during init,
+            // this is going to be the first thing the tick process does anyways, so move done and run it after
+            // everything is init before watchdog tick.
+            // anything at 3+ won't be caught here but also will trip watchdog....
+            // tasks are default scheduled at -1 + delay, and first tick will tick at 1
+            final long actualDoneTimeMs = System.currentTimeMillis() - org.bukkit.craftbukkit.Main.BOOT_TIME.toEpochMilli(); // Paper - Add total time
+            LOGGER.info("Done ({})! For help, type \"help\"", String.format(java.util.Locale.ROOT, "%.3fs", actualDoneTimeMs / 1000.00D)); // Paper - Add total time
+            org.spigotmc.WatchdogThread.tick();
+            // Paper end - Improved Watchdog Support
             org.spigotmc.WatchdogThread.hasStarted = true; // Paper
             Arrays.fill( this.recentTps, 20 );
             // Paper start - further improve server tick loop
@@ -1343,6 +1373,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
                 JvmProfiler.INSTANCE.onServerTick(this.smoothedTickTimeMillis);
             }
         } catch (Throwable throwable2) {
+            // Paper start
+            if (throwable2 instanceof ThreadDeath) {
+                MinecraftServer.LOGGER.error("Main thread terminated by WatchDog due to hard crash", throwable2);
+                return;
+            }
+            // Paper end
             MinecraftServer.LOGGER.error("Encountered an unexpected exception", throwable2);
             CrashReport crashreport = MinecraftServer.constructOrExtractCrashReport(throwable2);
 
@@ -1367,15 +1403,15 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
                     this.services.profileCache().clearExecutor();
                 }
 
-                org.spigotmc.WatchdogThread.doStop(); // Spigot
+                //org.spigotmc.WatchdogThread.doStop(); // Spigot // Paper - move into stop
                 // CraftBukkit start - Restore terminal to original settings
                 try {
-                    net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+                    //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop
                 } catch (Exception ignored) {
                 }
                 // CraftBukkit end
-                io.papermc.paper.log.CustomLogManager.forceReset(); // Paper - Reset loggers after shutdown
-                this.onServerExit();
+                //io.papermc.paper.log.CustomLogManager.forceReset(); // Paper - Reset loggers after shutdown
+                //this.onServerExit(); // Paper - moved into stop
             }
 
         }
@@ -1494,6 +1530,12 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
 
     @Override
     public TickTask wrapRunnable(Runnable runnable) {
+        // Paper start - anything that does try to post to main during watchdog crash, run on watchdog
+        if (this.hasStopped && Thread.currentThread().equals(shutdownThread)) {
+            runnable.run();
+            runnable = () -> {};
+        }
+        // Paper end
         return new TickTask(this.tickCount, runnable);
     }
 
@@ -2333,7 +2375,15 @@ public abstract class MinecraftServer extends ReentrantBlockableEventLoop<TickTa
             this.resources.managers.updateStaticRegistryTags();
             this.resources.managers.getRecipeManager().finalizeRecipeLoading(this.worldData.enabledFeatures());
             this.potionBrewing = this.potionBrewing.reload(this.worldData.enabledFeatures()); // Paper - Custom Potion Mixes
-            this.getPlayerList().saveAll();
+            // Paper start
+            if (Thread.currentThread() != this.serverThread) {
+                return;
+            }
+            // this.getPlayerList().saveAll(); // Paper - we don't need to save everything, just advancements // TODO Move this to a different patch
+            for (ServerPlayer player : this.getPlayerList().getPlayers()) {
+                player.getAdvancements().save();
+            }
+            // Paper end
             this.getPlayerList().reloadResources();
             this.functionManager.replaceLibrary(this.resources.managers.getFunctionLibrary());
             this.structureTemplateManager.onResourceManagerReload(this.resources.resourceManager);
diff --git a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
index 0dd9ed7465d222505d5368781654ec4954f6e5c3..17a158ff6ce6520b69a5a0032ba4c05449dd0cf8 100644
--- a/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
+++ b/src/main/java/net/minecraft/server/dedicated/DedicatedServer.java
@@ -327,7 +327,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
             long j = Util.getNanos() - i;
             String s = String.format(Locale.ROOT, "%.3fs", (double) j / 1.0E9D);
 
-            DedicatedServer.LOGGER.info("Done ({})! For help, type \"help\"", s);
+            DedicatedServer.LOGGER.info("Done preparing level \"{}\" ({})", this.getLevelIdName(), s); // Paper - clarify startup log messages & add total time
             if (dedicatedserverproperties.announcePlayerAchievements != null) {
                 ((GameRules.BooleanValue) this.getGameRules().getRule(GameRules.RULE_ANNOUNCE_ADVANCEMENTS)).set(dedicatedserverproperties.announcePlayerAchievements, this.overworld()); // CraftBukkit - per-world
             }
@@ -444,7 +444,8 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
             // this.remoteStatusListener.stop(); // Paper - don't wait for remote connections
         }
 
-        System.exit(0); // CraftBukkit
+        hasFullyShutdown = true; // Paper
+        System.exit(this.abnormalExit ? 70 : 0); // CraftBukkit // Paper
     }
 
     @Override
@@ -790,7 +791,7 @@ public class DedicatedServer extends MinecraftServer implements ServerInterface
     @Override
     public void stopServer() {
         super.stopServer();
-        Util.shutdownExecutors();
+        //Util.shutdownExecutors(); // Paper - moved into super
         SkullBlockEntity.clear();
     }
 
diff --git a/src/main/java/net/minecraft/server/players/PlayerList.java b/src/main/java/net/minecraft/server/players/PlayerList.java
index 7913c41aac1f9dd53a2b49da2a17fd894bcb6b3a..b170e506d9b1b21dd548ba8cd7ac34510184daa3 100644
--- a/src/main/java/net/minecraft/server/players/PlayerList.java
+++ b/src/main/java/net/minecraft/server/players/PlayerList.java
@@ -553,7 +553,7 @@ public abstract class PlayerList {
         this.cserver.getPluginManager().callEvent(playerQuitEvent);
         entityplayer.getBukkitEntity().disconnect(playerQuitEvent.getQuitMessage());
 
-        entityplayer.doTick(); // SPIGOT-924
+        if (server.isSameThread()) entityplayer.doTick(); // SPIGOT-924 // Paper - don't tick during emergency shutdowns (Watchdog)
         // CraftBukkit end
 
         // Paper start - Configurable player collision; Remove from collideRule team if needed
diff --git a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
index d6e942aca1bcc769c390504a4119d6619872c4d4..9b706276dc5b5f55b966c5472c6c4e864342b916 100644
--- a/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
+++ b/src/main/java/net/minecraft/util/thread/BlockableEventLoop.java
@@ -168,6 +168,6 @@ public abstract class BlockableEventLoop<R extends Runnable> implements Profiler
     public static boolean isNonRecoverable(Throwable exception) {
         return exception instanceof ReportedException reportedException
             ? isNonRecoverable(reportedException.getCause())
-            : exception instanceof OutOfMemoryError || exception instanceof StackOverflowError;
+            : exception instanceof OutOfMemoryError || exception instanceof StackOverflowError || exception instanceof ThreadDeath; // Paper
     }
 }
diff --git a/src/main/java/net/minecraft/world/level/Level.java b/src/main/java/net/minecraft/world/level/Level.java
index f477c5817f022ce7c4ad25e9b827401434bcfff1..d518493ecf3853b9f2aefceb72e1a4d2e9bf1184 100644
--- a/src/main/java/net/minecraft/world/level/Level.java
+++ b/src/main/java/net/minecraft/world/level/Level.java
@@ -1489,6 +1489,7 @@ public abstract class Level implements LevelAccessor, AutoCloseable, ca.spottedl
         try {
             tickConsumer.accept(entity);
         } catch (Throwable throwable) {
+            if (throwable instanceof ThreadDeath) throw throwable; // Paper
             // Paper start - Prevent block entity and entity crashes
             final String msg = String.format("Entity threw exception at %s:%s,%s,%s", entity.level().getWorld().getName(), entity.getX(), entity.getY(), entity.getZ());
             MinecraftServer.LOGGER.error(msg, throwable);
diff --git a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
index 134d63076f231791988e67a5bdf191005112080b..97937e3bd211997f0a0a3e9e671a1c59712d0003 100644
--- a/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
+++ b/src/main/java/net/minecraft/world/level/chunk/LevelChunk.java
@@ -1083,6 +1083,7 @@ public class LevelChunk extends ChunkAccess implements ca.spottedleaf.moonrise.p
 
                         gameprofilerfiller.pop();
                     } catch (Throwable throwable) {
+                        if (throwable instanceof ThreadDeath) throw throwable; // Paper
                         // Paper start - Prevent block entity and entity crashes
                         final String msg = String.format("BlockEntity threw exception at %s:%s,%s,%s", LevelChunk.this.getLevel().getWorld().getName(), this.getPos().getX(), this.getPos().getY(), this.getPos().getZ());
                         net.minecraft.server.MinecraftServer.LOGGER.error(msg, throwable);
diff --git a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
index c6e8441e299f477ddb22c1ce2618710763978f1a..e8e93538dfd71de86515d9405f728db1631e949a 100644
--- a/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
+++ b/src/main/java/org/bukkit/craftbukkit/util/ServerShutdownThread.java
@@ -12,11 +12,27 @@ public class ServerShutdownThread extends Thread {
     @Override
     public void run() {
         try {
+            // Paper start - try to shutdown on main
+            server.safeShutdown(false, false);
+            for (int i = 1000; i > 0 && !server.hasStopped(); i -= 100) {
+                Thread.sleep(100);
+            }
+            if (server.hasStopped()) {
+                while (!server.hasFullyShutdown) Thread.sleep(1000);
+                return;
+            }
+            // Looks stalled, close async
             org.spigotmc.AsyncCatcher.enabled = false; // Spigot
+            server.forceTicks = true;
             this.server.close();
+            while (!server.hasFullyShutdown) Thread.sleep(1000);
+        } catch (InterruptedException e) {
+            e.printStackTrace();
+            // Paper end
         } finally {
+            org.apache.logging.log4j.LogManager.shutdown(); // Paper
             try {
-                net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Use TerminalConsoleAppender
+                //net.minecrell.terminalconsole.TerminalConsoleAppender.close(); // Paper - Move into stop
             } catch (Exception e) {
             }
         }
diff --git a/src/main/java/org/spigotmc/RestartCommand.java b/src/main/java/org/spigotmc/RestartCommand.java
index 39e56b95aaafbcd8ebe68fdefaace83702e9510d..3ba27955548a26367a87d6b87c3c61beb299dfb9 100644
--- a/src/main/java/org/spigotmc/RestartCommand.java
+++ b/src/main/java/org/spigotmc/RestartCommand.java
@@ -139,7 +139,7 @@ public class RestartCommand extends Command
     // Paper end
 
     // Paper start - copied from above and modified to return if the hook registered
-    private static boolean addShutdownHook(String restartScript)
+    public static boolean addShutdownHook(String restartScript) // Paper
     {
         String[] split = restartScript.split( " " );
         if ( split.length > 0 && new File( split[0] ).isFile() )
diff --git a/src/main/java/org/spigotmc/WatchdogThread.java b/src/main/java/org/spigotmc/WatchdogThread.java
index 529df2a41dd93d6e1505053bd04032dbf0cdaa31..c9e17225bc52fe5e7b2dc0908db225a86c6e94d1 100644
--- a/src/main/java/org/spigotmc/WatchdogThread.java
+++ b/src/main/java/org/spigotmc/WatchdogThread.java
@@ -11,6 +11,7 @@ import org.bukkit.Bukkit;
 public class WatchdogThread extends ca.spottedleaf.moonrise.common.util.TickThread // Paper - rewrite chunk system
 {
 
+    public static final boolean DISABLE_WATCHDOG = Boolean.getBoolean("disable.watchdog"); // Paper - Improved watchdog support
     private static WatchdogThread instance;
     private long timeoutTime;
     private boolean restart;
@@ -39,6 +40,7 @@ public class WatchdogThread extends ca.spottedleaf.moonrise.common.util.TickThre
     {
         if ( WatchdogThread.instance == null )
         {
+            if (timeoutTime <= 0) timeoutTime = 300; // Paper
             WatchdogThread.instance = new WatchdogThread( timeoutTime * 1000L, restart );
             WatchdogThread.instance.start();
         } else
@@ -70,12 +72,13 @@ public class WatchdogThread extends ca.spottedleaf.moonrise.common.util.TickThre
             // Paper start
             Logger log = Bukkit.getServer().getLogger();
             long currentTime = WatchdogThread.monotonicMillis();
-            if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
+            MinecraftServer server = MinecraftServer.getServer();
+            if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.hasStarted && (!server.isRunning() || (currentTime > this.lastTick + this.earlyWarningEvery && !DISABLE_WATCHDOG) )) // Paper - add property to disable
             {
-                boolean isLongTimeout = currentTime > lastTick + timeoutTime;
+                boolean isLongTimeout = currentTime > lastTick + timeoutTime || (!server.isRunning() && !server.hasStopped() && currentTime > lastTick + 1000);
                 // Don't spam early warning dumps
                 if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
-                if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
+                if ( !isLongTimeout && server.hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
                 lastEarlyWarning = currentTime;
                 if (isLongTimeout) {
                 // Paper end
@@ -136,9 +139,24 @@ public class WatchdogThread extends ca.spottedleaf.moonrise.common.util.TickThre
 
                 if ( isLongTimeout )
                 {
-                if ( this.restart && !MinecraftServer.getServer().hasStopped() )
+                if ( !server.hasStopped() )
                 {
-                    RestartCommand.restart();
+                    AsyncCatcher.enabled = false; // Disable async catcher incase it interferes with us
+                    server.forceTicks = true;
+                    if (restart) {
+                        RestartCommand.addShutdownHook( SpigotConfig.restartScript );
+                    }
+                    // try one last chance to safe shutdown on main incase it 'comes back'
+                    server.abnormalExit = true;
+                    server.safeShutdown(false, restart);
+                    try {
+                        Thread.sleep(1000);
+                    } catch (InterruptedException e) {
+                        e.printStackTrace();
+                    }
+                    if (!server.hasStopped()) {
+                        server.close();
+                    }
                 }
                 break;
                 } // Paper end
diff --git a/src/main/resources/log4j2.xml b/src/main/resources/log4j2.xml
index 637d64da9938e51a97338b9253b43889585c67bb..d2a75850af9c6ad2aca66a5f994f1b587d73eac4 100644
--- a/src/main/resources/log4j2.xml
+++ b/src/main/resources/log4j2.xml
@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="UTF-8"?>
-<Configuration status="WARN">
+<Configuration status="WARN" shutdownHook="disable">
     <Appenders>
         <Queue name="ServerGuiConsole">
             <PatternLayout pattern="[%d{HH:mm:ss} %level]: %msg{nolookups}%n" />