OpenSource/PaperMC
OpenSource/PaperMC
1
0
Fork 0
mirror of https://github.com/PaperMC/Paper.git synced 2025-04-16 02:30:53 +02:00
Code Issues Projects Releases Packages Wiki Activity

Add Early Warning Feature to WatchDog

Browse source 
Detect when the server has been hung for a long duration, and start printing
thread dumps at an interval until the point of crash.

This will help diagnose what was going on in that time before the crash.
This commit is contained in:
miclebrick 2018-08-08 15:30:52 -04:00
parent 2f4d83a219
commit 31fc02af68
5 changed files with 77 additions and 42 deletions
paper-server
patches/sources/net/minecraft/server
MinecraftServer.java.patch
dedicated
DedicatedServer.java.patch
src/main/java/org
bukkit/craftbukkit
CraftServer.java
spigotmc
SpigotConfig.javaWatchdogThread.java

53
paper-server/patches/sources/net/minecraft/server/MinecraftServer.java.patch
View file

@ -822,7 +822,7 @@
protected void runServer() {
try {
if (!this.initServer()) {
@@ -727,9 +1116,15 @@
@@ -727,9 +1116,16 @@
}
this.nextTickTimeNanos = Util.getNanos();
@ -831,6 +831,7 @@
this.status = this.buildServerStatus();
+ // Spigot start
+ org.spigotmc.WatchdogThread.hasStarted = true; // Paper
+ Arrays.fill( this.recentTps, 20 );
+ // Paper start - further improve server tick loop
+ long tickSection = Util.getNanos();
@ -839,7 +840,7 @@
while (this.running) {
long i;
@@ -744,11 +1139,30 @@
@@ -744,11 +1140,30 @@
if (j > MinecraftServer.OVERLOADED_THRESHOLD_NANOS + 20L * i && this.nextTickTimeNanos - this.lastOverloadWarningNanos >= MinecraftServer.OVERLOADED_WARNING_INTERVAL_NANOS + 100L * i) {
long k = j / i;
@ -870,7 +871,7 @@
boolean flag = i == 0L;
@@ -757,6 +1171,8 @@
@@ -757,6 +1172,8 @@
this.debugCommandProfiler = new MinecraftServer.TimeProfiler(Util.getNanos(), this.tickCount);
}
@ -879,7 +880,7 @@
this.nextTickTimeNanos += i;
try {
@@ -830,6 +1246,13 @@
@@ -830,6 +1247,13 @@
this.services.profileCache().clearExecutor();
}
@ -893,7 +894,7 @@
this.onServerExit();
}
@@ -889,9 +1312,16 @@
@@ -889,9 +1313,16 @@
}
private boolean haveTime() {
@ -911,7 +912,7 @@
public static boolean throwIfFatalException() {
RuntimeException runtimeexception = (RuntimeException) MinecraftServer.fatalException.get();
@@ -903,7 +1333,7 @@
@@ -903,7 +1334,7 @@
}
public static void setFatalException(RuntimeException exception) {
@ -920,7 +921,7 @@
}
@Override
@@ -977,7 +1407,7 @@
@@ -977,7 +1408,7 @@
}
}
@ -929,7 +930,7 @@
Profiler.get().incrementCounter("runTask");
super.doRunTask(ticktask);
}
@@ -1025,6 +1455,7 @@
@@ -1025,6 +1456,7 @@
}
public void tickServer(BooleanSupplier shouldKeepTicking) {
@ -937,7 +938,7 @@
long i = Util.getNanos();
int j = this.pauseWhileEmptySeconds() * 20;
@@ -1041,6 +1472,7 @@
@@ -1041,6 +1473,7 @@
this.autoSave();
}
@ -945,7 +946,7 @@
this.tickConnection();
return;
}
@@ -1055,12 +1487,13 @@
@@ -1055,12 +1488,13 @@
}
--this.ticksUntilAutosave;
@ -960,7 +961,7 @@
gameprofilerfiller.push("tallying");
long k = Util.getNanos() - i;
int l = this.tickCount % 100;
@@ -1074,7 +1507,7 @@
@@ -1074,7 +1508,7 @@
}
private void autoSave() {
@ -969,7 +970,7 @@
MinecraftServer.LOGGER.debug("Autosave started");
ProfilerFiller gameprofilerfiller = Profiler.get();
@@ -1123,7 +1556,7 @@
@@ -1123,7 +1557,7 @@
private ServerStatus buildServerStatus() {
ServerStatus.Players serverping_serverpingplayersample = this.buildPlayerStatus();
@ -978,7 +979,7 @@
}
private ServerStatus.Players buildPlayerStatus() {
@@ -1133,7 +1566,7 @@
@@ -1133,7 +1567,7 @@
if (this.hidesOnlinePlayers()) {
return new ServerStatus.Players(i, list.size(), List.of());
} else {
@ -987,7 +988,7 @@
ObjectArrayList<GameProfile> objectarraylist = new ObjectArrayList(j);
int k = Mth.nextInt(this.random, 0, list.size() - j);
@@ -1154,24 +1587,43 @@
@@ -1154,24 +1588,43 @@
this.getPlayerList().getPlayers().forEach((entityplayer) -> {
entityplayer.connection.suspendFlushing();
});
@ -1031,7 +1032,7 @@
gameprofilerfiller.push("tick");
@@ -1186,6 +1638,7 @@
@@ -1186,6 +1639,7 @@
gameprofilerfiller.pop();
gameprofilerfiller.pop();
@ -1039,7 +1040,7 @@
}
gameprofilerfiller.popPush("connection");
@@ -1265,8 +1718,24 @@
@@ -1265,8 +1719,24 @@
@Nullable
public ServerLevel getLevel(ResourceKey<Level> key) {
return (ServerLevel) this.levels.get(key);
@ -1064,7 +1065,7 @@
public Set<ResourceKey<Level>> levelKeys() {
return this.levels.keySet();
}
@@ -1296,7 +1765,7 @@
@@ -1296,7 +1766,7 @@
@DontObfuscate
public String getServerModName() {
@ -1073,7 +1074,7 @@
}
public SystemReport fillSystemReport(SystemReport details) {
@@ -1347,7 +1816,7 @@
@@ -1347,7 +1817,7 @@
@Override
public void sendSystemMessage(Component message) {
@ -1082,7 +1083,7 @@
}
public KeyPair getKeyPair() {
@@ -1481,10 +1950,20 @@
@@ -1481,10 +1951,20 @@
@Override
public String getMotd() {
@ -1104,7 +1105,7 @@
this.motd = motd;
}
@@ -1507,7 +1986,7 @@
@@ -1507,7 +1987,7 @@
}
public ServerConnectionListener getConnection() {
@ -1113,7 +1114,7 @@
}
public boolean isReady() {
@@ -1634,11 +2113,11 @@
@@ -1634,11 +2114,11 @@
public CompletableFuture<Void> reloadResources(Collection<String> dataPacks) {
CompletableFuture<Void> completablefuture = CompletableFuture.supplyAsync(() -> {
@ -1127,7 +1128,7 @@
}, this).thenCompose((immutablelist) -> {
MultiPackResourceManager resourcemanager = new MultiPackResourceManager(PackType.SERVER_DATA, immutablelist);
List<Registry.PendingTags<?>> list = TagLoader.loadTagsForExistingRegistries(resourcemanager, this.registries.compositeAccess());
@@ -1654,6 +2133,7 @@
@@ -1654,6 +2134,7 @@
}).thenAcceptAsync((minecraftserver_reloadableresources) -> {
this.resources.close();
this.resources = minecraftserver_reloadableresources;
@ -1135,7 +1136,7 @@
this.packRepository.setSelected(dataPacks);
WorldDataConfiguration worlddataconfiguration = new WorldDataConfiguration(MinecraftServer.getSelectedPacks(this.packRepository, true), this.worldData.enabledFeatures());
@@ -1952,7 +2432,7 @@
@@ -1952,7 +2433,7 @@
final List<String> list = Lists.newArrayList();
final GameRules gamerules = this.getGameRules();
@ -1144,7 +1145,7 @@
@Override
public <T extends GameRules.Value<T>> void visit(GameRules.Key<T> key, GameRules.Type<T> type) {
list.add(String.format(Locale.ROOT, "%s=%s\n", key.getId(), gamerules.getRule(key)));
@@ -2058,7 +2538,7 @@
@@ -2058,7 +2539,7 @@
try {
label51:
{
@ -1153,7 +1154,7 @@
try {
arraylist = Lists.newArrayList(NativeModuleLister.listModules());
@@ -2108,6 +2588,21 @@
@@ -2108,6 +2589,21 @@
}
@ -1175,7 +1176,7 @@
private ProfilerFiller createProfiler() {
if (this.willStartRecordingMetrics) {
this.metricsRecorder = ActiveMetricsRecorder.createStarted(new ServerMetricsSamplersProvider(Util.timeSource, this.isDedicatedServer()), Util.timeSource, Util.ioPool(), new MetricsPersister("server"), this.onMetricsRecordingStopped, (path) -> {
@@ -2225,18 +2720,24 @@
@@ -2225,18 +2721,24 @@
}
public void logChatMessage(Component message, ChatType.Bound params, @Nullable String prefix) {

25
paper-server/patches/sources/net/minecraft/server/dedicated/DedicatedServer.java.patch
View file

@ -142,7 +142,7 @@
thread.setDaemon(true);
thread.setUncaughtExceptionHandler(new DefaultUncaughtExceptionHandler(DedicatedServer.LOGGER));
thread.start();
@@ -126,13 +203,26 @@
@@ -126,13 +203,27 @@
this.setPreventProxyConnections(dedicatedserverproperties.preventProxyConnections);
this.setLocalIp(dedicatedserverproperties.serverIp);
}
@ -156,6 +156,7 @@
+ this.paperConfigurations.initializeGlobalConfiguration(this.registryAccess());
+ this.paperConfigurations.initializeWorldDefaultsConfiguration(this.registryAccess());
+ // Paper end - initialize global and world-defaults configuration
+ org.spigotmc.WatchdogThread.doStart(org.spigotmc.SpigotConfig.timeoutTime, org.spigotmc.SpigotConfig.restartOnCrash); // Paper - start watchdog thread
+ io.papermc.paper.command.PaperCommands.registerCommands(this); // Paper - setup /paper command
+ com.destroystokyo.paper.Metrics.PaperMetrics.startMetrics(); // Paper - start metrics
+ com.destroystokyo.paper.VersionHistoryManager.INSTANCE.getClass(); // Paper - load version history now
@ -170,7 +171,7 @@
DedicatedServer.LOGGER.info("Default game type: {}", dedicatedserverproperties.gamemode);
InetAddress inetaddress = null;
@@ -156,21 +246,34 @@
@@ -156,21 +247,34 @@
return false;
}
@ -208,7 +209,7 @@
this.debugSampleSubscriptionTracker = new DebugSampleSubscriptionTracker(this.getPlayerList());
this.tickTimeLogger = new RemoteSampleLogger(TpsDebugDimensions.values().length, this.debugSampleSubscriptionTracker, RemoteDebugSampleType.TICK_TIME);
long i = Util.getNanos();
@@ -178,13 +281,13 @@
@@ -178,13 +282,13 @@
SkullBlockEntity.setup(this.services, this);
GameProfileCache.setUsesAuthentication(this.usesAuthentication());
DedicatedServer.LOGGER.info("Preparing level \"{}\"", this.getLevelIdName());
@ -224,7 +225,7 @@
}
if (dedicatedserverproperties.enableQuery) {
@@ -197,7 +300,7 @@
@@ -197,7 +301,7 @@
this.rconThread = RconThread.create(this);
}
@ -233,7 +234,7 @@
Thread thread1 = new Thread(new ServerWatchdog(this));
thread1.setUncaughtExceptionHandler(new DefaultUncaughtExceptionHandlerWithName(DedicatedServer.LOGGER));
@@ -213,7 +316,13 @@
@@ -213,7 +317,13 @@
return true;
}
@ -247,7 +248,7 @@
@Override
public boolean isSpawningMonsters() {
@@ -293,6 +402,7 @@
@@ -293,6 +403,7 @@
this.queryThreadGs4.stop();
}
@ -255,7 +256,7 @@
}
@Override
@@ -302,8 +412,8 @@
@@ -302,8 +413,8 @@
}
@Override
@ -266,7 +267,7 @@
}
public void handleConsoleInput(String command, CommandSourceStack commandSource) {
@@ -314,7 +424,15 @@
@@ -314,7 +425,15 @@
while (!this.consoleInput.isEmpty()) {
ConsoleInput servercommand = (ConsoleInput) this.consoleInput.remove(0);
@ -283,7 +284,7 @@
}
}
@@ -383,7 +501,7 @@
@@ -383,7 +502,7 @@
@Override
public boolean isUnderSpawnProtection(ServerLevel world, BlockPos pos, Player player) {
@ -292,7 +293,7 @@
return false;
} else if (this.getPlayerList().getOps().isEmpty()) {
return false;
@@ -453,7 +571,11 @@
@@ -453,7 +572,11 @@
public boolean enforceSecureProfile() {
DedicatedServerProperties dedicatedserverproperties = this.getProperties();
@ -305,7 +306,7 @@
}
@Override
@@ -541,16 +663,52 @@
@@ -541,16 +664,52 @@
@Override
public String getPluginNames() {
@ -362,7 +363,7 @@
}
public void storeUsingWhiteList(boolean useWhitelist) {
@@ -660,4 +818,15 @@
@@ -660,4 +819,15 @@
}
}
}

2
paper-server/src/main/java/org/bukkit/craftbukkit/CraftServer.java
View file

@ -965,6 +965,7 @@ public final class CraftServer implements Server {
@Override
public void reload() {
org.spigotmc.WatchdogThread.hasStarted = false; // Paper - Disable watchdog early timeout on reload
this.reloadCount++;
this.configuration = YamlConfiguration.loadConfiguration(this.getConfigFile());
this.commandsConfiguration = YamlConfiguration.loadConfiguration(this.getCommandsConfigFile());
@ -1057,6 +1058,7 @@ public final class CraftServer implements Server {
this.enablePlugins(PluginLoadOrder.POSTWORLD);
if (io.papermc.paper.plugin.PluginInitializerManager.instance().pluginRemapper != null) io.papermc.paper.plugin.PluginInitializerManager.instance().pluginRemapper.pluginsEnabled(); // Paper - Remap plugins
this.getPluginManager().callEvent(new ServerLoadEvent(ServerLoadEvent.LoadType.RELOAD));
org.spigotmc.WatchdogThread.hasStarted = true; // Paper - Disable watchdog early timeout on reload
}
@Override

2
paper-server/src/main/java/org/spigotmc/SpigotConfig.java
View file

@ -225,7 +225,7 @@ public class SpigotConfig
SpigotConfig.restartScript = SpigotConfig.getString( "settings.restart-script", SpigotConfig.restartScript );
SpigotConfig.restartMessage = SpigotConfig.transform( SpigotConfig.getString( "messages.restart", "Server is restarting" ) );
SpigotConfig.commands.put( "restart", new RestartCommand( "restart" ) );
WatchdogThread.doStart( SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash );
// WatchdogThread.doStart( SpigotConfig.timeoutTime, SpigotConfig.restartOnCrash ); // Paper - moved to after paper config initialization
}
public static boolean bungee;

37
paper-server/src/main/java/org/spigotmc/WatchdogThread.java
View file

@ -14,6 +14,10 @@ public class WatchdogThread extends Thread
private static WatchdogThread instance;
private long timeoutTime;
private boolean restart;
private final long earlyWarningEvery; // Paper - Timeout time for just printing a dump but not restarting
private final long earlyWarningDelay; // Paper
public static volatile boolean hasStarted; // Paper
private long lastEarlyWarning; // Paper - Keep track of short dump times to avoid spamming console with short dumps
private volatile long lastTick;
private volatile boolean stopping;
@ -22,6 +26,8 @@ public class WatchdogThread extends Thread
super( "Paper Watchdog Thread" );
this.timeoutTime = timeoutTime;
this.restart = restart;
earlyWarningEvery = Math.min(io.papermc.paper.configuration.GlobalConfiguration.get().watchdog.earlyWarningEvery, timeoutTime); // Paper
earlyWarningDelay = Math.min(io.papermc.paper.configuration.GlobalConfiguration.get().watchdog.earlyWarningDelay, timeoutTime); // Paper
}
private static long monotonicMillis()
@ -61,9 +67,18 @@ public class WatchdogThread extends Thread
while ( !this.stopping )
{
//
if ( this.lastTick != 0 && this.timeoutTime > 0 && WatchdogThread.monotonicMillis() > this.lastTick + this.timeoutTime && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
// Paper start
Logger log = Bukkit.getServer().getLogger();
long currentTime = WatchdogThread.monotonicMillis();
if ( this.lastTick != 0 && this.timeoutTime > 0 && currentTime > this.lastTick + this.earlyWarningEvery && !Boolean.getBoolean("disable.watchdog")) // Paper - Add property to disable
{
Logger log = Bukkit.getServer().getLogger();
boolean isLongTimeout = currentTime > lastTick + timeoutTime;
// Don't spam early warning dumps
if ( !isLongTimeout && (earlyWarningEvery <= 0 || !hasStarted || currentTime < lastEarlyWarning + earlyWarningEvery || currentTime < lastTick + earlyWarningDelay)) continue;
if ( !isLongTimeout && MinecraftServer.getServer().hasStopped()) continue; // Don't spam early watchdog warnings during shutdown, we'll come back to this...
lastEarlyWarning = currentTime;
if (isLongTimeout) {
// Paper end
log.log( Level.SEVERE, "------------------------------" );
log.log( Level.SEVERE, "The server has stopped responding! This is (probably) not a Paper bug." ); // Paper
log.log( Level.SEVERE, "If you see a plugin in the Server thread dump below, then please report it to that author" );
@ -92,29 +107,45 @@ public class WatchdogThread extends Thread
}
}
// Paper end
} else
{
log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH - " + Bukkit.getServer().getVersion() + " ---");
log.log(Level.SEVERE, "The server has not responded for " + (currentTime - lastTick) / 1000 + " seconds! Creating thread dump");
}
// Paper end - Different message for short timeout
log.log( Level.SEVERE, "------------------------------" );
log.log( Level.SEVERE, "Server thread dump (Look for plugins here before reporting to Paper!):" ); // Paper
WatchdogThread.dumpThread( ManagementFactory.getThreadMXBean().getThreadInfo( MinecraftServer.getServer().serverThread.getId(), Integer.MAX_VALUE ), log );
log.log( Level.SEVERE, "------------------------------" );
//
// Paper start - Only print full dump on long timeouts
if ( isLongTimeout )
{
log.log( Level.SEVERE, "Entire Thread Dump:" );
ThreadInfo[] threads = ManagementFactory.getThreadMXBean().dumpAllThreads( true, true );
for ( ThreadInfo thread : threads )
{
WatchdogThread.dumpThread( thread, log );
}
} else {
log.log(Level.SEVERE, "--- DO NOT REPORT THIS TO PAPER - THIS IS NOT A BUG OR A CRASH ---");
}
log.log( Level.SEVERE, "------------------------------" );
if ( isLongTimeout )
{
if ( this.restart && !MinecraftServer.getServer().hasStopped() )
{
RestartCommand.restart();
}
break;
} // Paper end
}
try
{
sleep( 10000 );
sleep( 1000 ); // Paper - Reduce check time to every second instead of every ten seconds, more consistent and allows for short timeout
} catch ( InterruptedException ex )
{
this.interrupt();