/// <inheritdoc /> protected sealed override async Task <IDmbProvider> PrepServerForLaunch(IDmbProvider dmbToUse, CancellationToken cancellationToken) { if (ActiveSwappable != null) { throw new InvalidOperationException("Expected activeSwappable to be null!"); } if (startupDmbProvider != null) { throw new InvalidOperationException("Expected startupDmbProvider to be null!"); } Logger.LogTrace("Prep for server launch. pendingSwappable is {0}available", pendingSwappable == null ? "not " : String.Empty); // Add another lock to the startup DMB because it'll be used throughout the lifetime of the watchdog startupDmbProvider = await DmbFactory.FromCompileJob(dmbToUse.CompileJob, cancellationToken).ConfigureAwait(false); pendingSwappable ??= new SwappableDmbProvider(dmbToUse, GameIOManager, symlinkFactory); ActiveSwappable = pendingSwappable; pendingSwappable = null; try { await InitialLink(cancellationToken).ConfigureAwait(false); } catch { // We won't worry about disposing activeSwappable here as we can't dispose dmbToUse here. ActiveSwappable = null; throw; } return(ActiveSwappable); }
/// <inheritdoc /> protected override async Task InitControllers( Task chatTask, ReattachInformation reattachInfo, CancellationToken cancellationToken) { // don't need a new dmb if reattaching var reattachInProgress = reattachInfo != null; var dmbToUse = reattachInProgress ? null : DmbFactory.LockNextDmb(1); // if this try catches something, both servers are killed try { // start the alpha server task, either by launch a new process or attaching to an existing one // The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail // The tasks pertaining to server startup times are in the ISessionControllers Task <ISessionController> serverLaunchTask; if (!reattachInProgress) { dmbToUse = await PrepServerForLaunch(dmbToUse, cancellationToken).ConfigureAwait(false); serverLaunchTask = SessionControllerFactory.LaunchNew( dmbToUse, null, ActiveLaunchParameters, false, cancellationToken); } else { serverLaunchTask = SessionControllerFactory.Reattach(reattachInfo, cancellationToken); } // retrieve the session controller Server = await serverLaunchTask.ConfigureAwait(false); // failed reattaches will return null Server?.SetHighPriority(); // possiblity of null servers due to failed reattaches if (Server == null) { await ReattachFailure( chatTask, cancellationToken) .ConfigureAwait(false); return; } await CheckLaunchResult(Server, "Server", cancellationToken).ConfigureAwait(false); Server.EnableCustomChatCommands(); } catch { // kill the controllers bool serverWasActive = Server != null; // DCT: Operation must always run await DisposeAndNullControllers(default).ConfigureAwait(false);
/// <inheritdoc /> protected override async Task HandleNewDmbAvailable(CancellationToken cancellationToken) { IDmbProvider compileJobProvider = DmbFactory.LockNextDmb(1); WindowsSwappableDmbProvider windowsProvider = null; try { windowsProvider = new WindowsSwappableDmbProvider(compileJobProvider, ioManager, symlinkFactory); Logger.LogDebug("Swapping to compile job {0}...", windowsProvider.CompileJob.Id); Server.Suspend(); await windowsProvider.MakeActive(cancellationToken).ConfigureAwait(false); Server.Resume(); } catch { IDmbProvider providerToDispose = windowsProvider ?? compileJobProvider; providerToDispose.Dispose(); throw; } pendingSwappable?.Dispose(); pendingSwappable = windowsProvider; }
/// <inheritdoc /> protected override async Task HandleNewDmbAvailable(CancellationToken cancellationToken) { IDmbProvider compileJobProvider = DmbFactory.LockNextDmb(1); if (compileJobProvider.CompileJob.ByondVersion != ActiveCompileJob.ByondVersion) { // have to do a graceful restart Logger.LogDebug( "Not swapping to new compile job {0} as it uses a different BYOND version ({1}) than what is currently active {2}. Queueing graceful restart instead...", compileJobProvider.CompileJob.Id, compileJobProvider.CompileJob.ByondVersion, ActiveCompileJob.ByondVersion); compileJobProvider.Dispose(); await base.HandleNewDmbAvailable(cancellationToken).ConfigureAwait(false); return; } WindowsSwappableDmbProvider windowsProvider = null; bool suspended = false; try { windowsProvider = new WindowsSwappableDmbProvider(compileJobProvider, ioManager, symlinkFactory); Logger.LogDebug("Swapping to compile job {0}...", windowsProvider.CompileJob.Id); try { Server.Suspend(); suspended = true; } catch (Exception ex) { Logger.LogWarning("Exception while suspending server: {0}", ex); } await windowsProvider.MakeActive(cancellationToken).ConfigureAwait(false); } catch (Exception ex) { Logger.LogError("Exception while swapping: {0}", ex); IDmbProvider providerToDispose = windowsProvider ?? compileJobProvider; providerToDispose.Dispose(); throw; } // Let this throw hard if it fails if (suspended) { Server.Resume(); } pendingSwappable?.Dispose(); pendingSwappable = windowsProvider; }
/// <inheritdoc /> #pragma warning disable CA1506 // TODO: Decomplexify public async Task <IInstance> CreateInstance(IBridgeRegistrar bridgeRegistrar, Models.Instance metadata) { // Create the ioManager for the instance var instanceIoManager = new ResolvingIOManager(ioManager, metadata.Path); // various other ioManagers var repoIoManager = new ResolvingIOManager(instanceIoManager, "Repository"); var byondIOManager = new ResolvingIOManager(instanceIoManager, "Byond"); var gameIoManager = new ResolvingIOManager(instanceIoManager, "Game"); var diagnosticsIOManager = new ResolvingIOManager(instanceIoManager, "Diagnostics"); var configurationIoManager = new ResolvingIOManager(instanceIoManager, "Configuration"); var configuration = new StaticFiles.Configuration( configurationIoManager, synchronousIOManager, symlinkFactory, processExecutor, postWriteHandler, platformIdentifier, fileTransferService, loggerFactory.CreateLogger <StaticFiles.Configuration>()); var eventConsumer = new EventConsumer(configuration); var repoManager = new RepositoryManager( repositoryFactory, repositoryCommands, repoIoManager, eventConsumer, gitRemoteFeaturesFactory, loggerFactory.CreateLogger <Repository.Repository>(), loggerFactory.CreateLogger <RepositoryManager>()); try { var byond = new ByondManager(byondIOManager, byondInstaller, eventConsumer, loggerFactory.CreateLogger <ByondManager>()); var commandFactory = new CommandFactory(assemblyInformationProvider, byond, repoManager, databaseContextFactory, metadata); var chatManager = chatFactory.CreateChatManager(instanceIoManager, commandFactory, metadata.ChatSettings); try { var sessionControllerFactory = new SessionControllerFactory( processExecutor, byond, topicClientFactory, cryptographySuite, assemblyInformationProvider, gameIoManager, chatManager, networkPromptReaper, platformIdentifier, bridgeRegistrar, serverPortProvider, eventConsumer, loggerFactory, loggerFactory.CreateLogger <SessionControllerFactory>(), metadata); var dmbFactory = new DmbFactory( databaseContextFactory, gameIoManager, remoteDeploymentManagerFactory, loggerFactory.CreateLogger <DmbFactory>(), metadata); try { var reattachInfoHandler = new SessionPersistor( databaseContextFactory, dmbFactory, processExecutor, loggerFactory.CreateLogger <SessionPersistor>(), metadata); var watchdog = watchdogFactory.CreateWatchdog( chatManager, dmbFactory, reattachInfoHandler, sessionControllerFactory, gameIoManager, diagnosticsIOManager, eventConsumer, remoteDeploymentManagerFactory, metadata, metadata.DreamDaemonSettings); eventConsumer.SetWatchdog(watchdog); commandFactory.SetWatchdog(watchdog); try { Instance instance = null; var dreamMaker = new DreamMaker( byond, gameIoManager, configuration, sessionControllerFactory, eventConsumer, chatManager, processExecutor, dmbFactory, repoManager, remoteDeploymentManagerFactory, loggerFactory.CreateLogger <DreamMaker>(), metadata); instance = new Instance( metadata, repoManager, byond, dreamMaker, watchdog, chatManager, configuration, dmbFactory, jobManager, eventConsumer, remoteDeploymentManagerFactory, loggerFactory.CreateLogger <Instance>()); return(instance); } catch { await watchdog.DisposeAsync().ConfigureAwait(false); throw; } } catch { dmbFactory.Dispose(); throw; } } catch { await chatManager.DisposeAsync().ConfigureAwait(false); throw; } } catch { repoManager.Dispose(); throw; } }
/// <inheritdoc /> public IInstance CreateInstance(Models.Instance metadata) { //Create the ioManager for the instance var instanceIoManager = new ResolvingIOManager(ioManager, metadata.Path); //various other ioManagers var repoIoManager = new ResolvingIOManager(instanceIoManager, "Repository"); var byondIOManager = new ResolvingIOManager(instanceIoManager, "Byond"); var gameIoManager = new ResolvingIOManager(instanceIoManager, "Game"); var configurationIoManager = new ResolvingIOManager(instanceIoManager, "Configuration"); var configuration = new StaticFiles.Configuration(configurationIoManager, synchronousIOManager, symlinkFactory, processExecutor, postWriteHandler, loggerFactory.CreateLogger <StaticFiles.Configuration>()); var eventConsumer = new EventConsumer(configuration); var dmbFactory = new DmbFactory(databaseContextFactory, gameIoManager, loggerFactory.CreateLogger <DmbFactory>(), metadata.CloneMetadata()); try { var repoManager = new RepositoryManager(metadata.RepositorySettings, repoIoManager, eventConsumer, credentialsProvider, loggerFactory.CreateLogger <Repository.Repository>(), loggerFactory.CreateLogger <RepositoryManager>()); try { var byond = new ByondManager(byondIOManager, byondInstaller, eventConsumer, loggerFactory.CreateLogger <ByondManager>()); var commandFactory = new CommandFactory(application, byond, repoManager, databaseContextFactory, metadata); var chat = chatFactory.CreateChat(instanceIoManager, commandFactory, metadata.ChatSettings); try { var sessionControllerFactory = new SessionControllerFactory(processExecutor, byond, byondTopicSender, cryptographySuite, application, gameIoManager, chat, networkPromptReaper, loggerFactory, metadata.CloneMetadata()); var reattachInfoHandler = new ReattachInfoHandler(databaseContextFactory, dmbFactory, loggerFactory.CreateLogger <ReattachInfoHandler>(), metadata.CloneMetadata()); var watchdog = watchdogFactory.CreateWatchdog(chat, dmbFactory, reattachInfoHandler, configuration, sessionControllerFactory, metadata.CloneMetadata(), metadata.DreamDaemonSettings); eventConsumer.SetWatchdog(watchdog); commandFactory.SetWatchdog(watchdog); try { var dreamMaker = new DreamMaker(byond, gameIoManager, configuration, sessionControllerFactory, dmbFactory, application, eventConsumer, chat, processExecutor, watchdog, loggerFactory.CreateLogger <DreamMaker>()); return(new Instance(metadata.CloneMetadata(), repoManager, byond, dreamMaker, watchdog, chat, configuration, dmbFactory, databaseContextFactory, dmbFactory, jobManager, eventConsumer, gitHubClientFactory, loggerFactory.CreateLogger <Instance>())); } catch { watchdog.Dispose(); throw; } } catch { chat.Dispose(); throw; } } catch { repoManager.Dispose(); throw; } } catch { dmbFactory.Dispose(); throw; } }
/// <inheritdoc /> protected sealed override async Task InitControllers(Action callBeforeRecurse, Task chatTask, DualReattachInformation reattachInfo, CancellationToken cancellationToken) { var serverToReattach = reattachInfo?.Alpha ?? reattachInfo?.Bravo; var serverToKill = reattachInfo?.Bravo ?? reattachInfo?.Alpha; // vice versa if (serverToKill == serverToReattach) { serverToKill = null; } if (reattachInfo?.AlphaIsActive == false) { var temp = serverToReattach; serverToReattach = serverToKill; serverToKill = temp; } // don't need a new dmb if reattaching var doesntNeedNewDmb = serverToReattach != null; var dmbToUse = doesntNeedNewDmb ? null : DmbFactory.LockNextDmb(1); // if this try catches something, both servers are killed bool inactiveServerWasKilled = false; try { // start the alpha server task, either by launch a new process or attaching to an existing one // The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail // The tasks pertaining to server startup times are in the ISessionControllers Task <ISessionController> serverLaunchTask, inactiveReattachTask; if (!doesntNeedNewDmb) { dmbToUse = await PrepServerForLaunch(dmbToUse, cancellationToken).ConfigureAwait(false); serverLaunchTask = SessionControllerFactory.LaunchNew( dmbToUse, null, ActiveLaunchParameters, true, true, false, cancellationToken); } else { serverLaunchTask = SessionControllerFactory.Reattach(serverToReattach, cancellationToken); } bool thereIsAnInactiveServerToKill = serverToKill != null; if (thereIsAnInactiveServerToKill) { inactiveReattachTask = SessionControllerFactory.Reattach(serverToKill, cancellationToken); } else { inactiveReattachTask = Task.FromResult <ISessionController>(null); } // retrieve the session controller Server = await serverLaunchTask.ConfigureAwait(false); // failed reattaches will return null Server?.SetHighPriority(); var inactiveServerController = await inactiveReattachTask.ConfigureAwait(false); inactiveServerController?.Dispose(); inactiveServerWasKilled = inactiveServerController != null; // possiblity of null servers due to failed reattaches if (Server == null) { callBeforeRecurse(); await NotifyOfFailedReattach(thereIsAnInactiveServerToKill && !inactiveServerWasKilled, cancellationToken).ConfigureAwait(false); return; } await CheckLaunchResult(Server, "Server", cancellationToken).ConfigureAwait(false); Server.EnableCustomChatCommands(); } catch { // kill the controllers bool serverWasActive = Server != null; DisposeAndNullControllers(); // server didn't get control of this dmb if (dmbToUse != null && !serverWasActive) { dmbToUse.Dispose(); } if (serverToKill != null && !inactiveServerWasKilled) { serverToKill.Dmb.Dispose(); } throw; } }
/// <summary> /// Handles the actions to take when the monitor has to "wake up" /// </summary> /// <param name="activationReason">The <see cref="MonitorActivationReason"/> that caused the invocation</param> /// <param name="monitorState">The current <see cref="MonitorState"/>. Will be modified upon retrn</param> /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param> /// <returns>A <see cref="Task"/> representing the running operation</returns> #pragma warning disable CA1502 // TODO: Decomplexify async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken) { Logger.LogDebug("Monitor activation. Reason: {0}", activationReason); // this is where the bulk of the watchdog handling code lives and is fraught with lambdas, sorry not sorry // I'll do my best to walk you through it // returns true if the inactive server can't be used immediately // also sets monitor to restart if the above holds bool FullRestartDeadInactive() { if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail) { Logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting"); monitorState.NextAction = MonitorAction.Restart; // will dispose server return(true); } return(false); } // trys to set inactive server's port to the public game port // doesn't handle closing active server's port // returns true on success and swaps inactiveserver and activeserver also sets LastLaunchParameters to ActiveLaunchParameters // on failure, sets monitor to restart async Task <bool> MakeInactiveActive() { Logger.LogDebug("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value); var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); if (!result) { Logger.LogWarning("Failed to activate inactive server! Restarting monitor..."); monitorState.NextAction = MonitorAction.Restart; // will dispose server return(false); } // inactive server should always be using active launch parameters LastLaunchParameters = ActiveLaunchParameters; var tmp = monitorState.ActiveServer; monitorState.ActiveServer = monitorState.InactiveServer; monitorState.InactiveServer = tmp; alphaIsActive = !AlphaIsActive; monitorState.ActiveServer.EnableCustomChatCommands(); return(true); } // Kills and tries to launch inactive server with the latest dmb // falls back to current dmb on failure // Sets critfail on inactive server failing that // returns false if the backup dmb was used successfully, true otherwise async Task UpdateAndRestartInactiveServer(bool breakAfter) { ActiveParametersUpdated = new TaskCompletionSource <object>(); monitorState.InactiveServer.Dispose(); // kill or recycle it var desiredNextAction = breakAfter ? MonitorAction.Break : MonitorAction.Continue; monitorState.NextAction = desiredNextAction; Logger.LogInformation("Rebooting inactive server..."); var newDmb = DmbFactory.LockNextDmb(1); try { monitorState.InactiveServer = await SessionControllerFactory.LaunchNew(ActiveLaunchParameters, newDmb, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); monitorState.InactiveServer.SetHighPriority(); } catch (OperationCanceledException) { throw; } catch (Exception e) { Logger.LogError("Error occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); // ahh jeez, what do we do here? // this is our fault, so it should never happen but // idk maybe a database error while handling the newest dmb? // either way try to start it using the active server's dmb as a backup try { var dmbBackup = await DmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob, cancellationToken).ConfigureAwait(false); if (dmbBackup == null) // NANI!? { throw new JobException("Creating backup DMB provider failed!"); // just give up, if THAT compile job is failing then the ActiveServer is gonna crash soon too or already has } monitorState.InactiveServer = await SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false); monitorState.InactiveServer.SetHighPriority(); await Chat.SendWatchdogMessage("Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...", cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { throw; } catch (Exception e2) { // fuuuuucckkk Logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! Exception: {0}", e2.ToString()); monitorState.InactiveServerCritFail = true; await Chat.SendWatchdogMessage("Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", cancellationToken).ConfigureAwait(false); return; } } Logger.LogInformation("Successfully relaunched inactive server!"); monitorState.RebootingInactiveServer = true; } string ExitWord(ISessionController controller) => controller.TerminationWasRequested ? "exited" : "crashed"; // reason handling switch (activationReason) { case MonitorActivationReason.ActiveServerCrashed: if (monitorState.ActiveServer.RebootState == Watchdog.RebootState.Shutdown) { // the time for graceful shutdown is now await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Exiting due to graceful termination request...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); DisposeAndNullControllers(); monitorState.NextAction = MonitorAction.Exit; break; } if (FullRestartDeadInactive()) { // tell chat about it and go ahead await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Inactive server unable to online!", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); // we've already been set to restart break; } // tell chat about it await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Onlining inactive server...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false); // try to activate the inactive server if (!await MakeInactiveActive().ConfigureAwait(false)) { break; // failing that, we've already been set to restart } // bring up another inactive server await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); break; case MonitorActivationReason.InactiveServerCrashed: // just announce and try to bring it back await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Inactive server {0}! Rebooting...", ExitWord(monitorState.InactiveServer)), cancellationToken).ConfigureAwait(false); await UpdateAndRestartInactiveServer(false).ConfigureAwait(false); break; case MonitorActivationReason.ActiveServerRebooted: // ideal goal: active server just closed its port // tell inactive server to open it's port and that's now the active server var rebootState = monitorState.ActiveServer.RebootState; monitorState.ActiveServer.ResetRebootState(); // the DMAPI has already done this internally if (FullRestartDeadInactive() && rebootState != Watchdog.RebootState.Shutdown) { break; // full restart if the inactive server is being fucky } // what matters here is the RebootState var restartOnceSwapped = false; switch (rebootState) { case Watchdog.RebootState.Normal: // life as normal break; case Watchdog.RebootState.Restart: // reboot the current active server once the inactive one activates restartOnceSwapped = true; break; case Watchdog.RebootState.Shutdown: // graceful shutdown time await Chat.SendWatchdogMessage("Active server rebooted! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false); DisposeAndNullControllers(); monitorState.NextAction = MonitorAction.Exit; return; default: throw new InvalidOperationException($"Invalid reboot state: {rebootState}"); } // are both servers now running the same CompileJob? var sameCompileJob = monitorState.InactiveServer.Dmb.CompileJob.Id == monitorState.ActiveServer.Dmb.CompileJob.Id; if (!sameCompileJob || ActiveLaunchParameters != LastLaunchParameters) { restartOnceSwapped = true; // need a new launch to update either settings or compile job } if (restartOnceSwapped) { /* * we need to manually restart active server * just kill it here, easier that way */ monitorState.ActiveServer.Dispose(); } var activeServerStillHasPortOpen = !restartOnceSwapped && !monitorState.ActiveServer.ClosePortOnReboot; if (activeServerStillHasPortOpen) { /* we didn't want active server to swap for some reason and it still has it's port open * just continue as normal */ break; } if (!await MakeInactiveActive().ConfigureAwait(false)) { break; // monitor will restart } // servers now swapped // enable this now if inactive server is not still valid monitorState.ActiveServer.ClosePortOnReboot = restartOnceSwapped; if (!restartOnceSwapped) { /* * now try to reopen it on the private port * failing that, just reboot it */ restartOnceSwapped = !await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false); } // break either way because any issues past this point would be solved by the reboot if (restartOnceSwapped) // for one reason or another { await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); // update and reboot } else { monitorState.NextAction = MonitorAction.Skip; // only skip checking inactive server rebooted, it's guaranteed InactiveServerStartup complete wouldn't fire this iteration } break; case MonitorActivationReason.InactiveServerRebooted: // just don't let the active server close it's port if the inactive server isn't ready monitorState.RebootingInactiveServer = true; monitorState.InactiveServer.ResetRebootState(); monitorState.ActiveServer.ClosePortOnReboot = false; monitorState.NextAction = MonitorAction.Continue; break; case MonitorActivationReason.InactiveServerStartupComplete: // opposite of above case monitorState.RebootingInactiveServer = false; monitorState.ActiveServer.ClosePortOnReboot = true; monitorState.NextAction = MonitorAction.Continue; break; case MonitorActivationReason.NewDmbAvailable: case MonitorActivationReason.ActiveLaunchParametersUpdated: // just reload the inactive server and wait for a swap to apply the changes await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); break; default: Trace.Assert(false, String.Format(CultureInfo.InvariantCulture, "Invalid monitor activation reason: {0}!", activationReason)); break; } }
#pragma warning restore CA1502 /// <inheritdoc /> #pragma warning disable CA1502 // TODO: Decomplexify protected override async Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken) { Debug.Assert(alphaServer == null && bravoServer == null, "Entered LaunchNoLock with one or more of the servers not being null!"); // don't need a new dmb if reattaching var doesntNeedNewDmb = reattachInfo?.Alpha != null && reattachInfo?.Bravo != null; var dmbToUse = doesntNeedNewDmb ? null : DmbFactory.LockNextDmb(2); // if this try catches something, both servers are killed try { // start the alpha server task, either by launch a new process or attaching to an existing one // The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail // The tasks pertaining to server startup times are in the ISessionControllers Task <ISessionController> alphaServerTask; if (!doesntNeedNewDmb) { alphaServerTask = SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, true, true, false, cancellationToken); } else { alphaServerTask = SessionControllerFactory.Reattach(reattachInfo.Alpha, cancellationToken); } // retrieve the session controller var startTime = DateTimeOffset.Now; alphaServer = await alphaServerTask.ConfigureAwait(false); // failed reattaches will return null alphaServer?.SetHighPriority(); // extra delay for total ordering var now = DateTimeOffset.Now; var delay = now - startTime; // definitely not if reattaching though if (reattachInfo == null && delay.TotalSeconds < AlphaBravoStartupSeperationInterval) { await AsyncDelayer.Delay(startTime.AddSeconds(AlphaBravoStartupSeperationInterval) - now, cancellationToken).ConfigureAwait(false); } // now bring bravo up if (!doesntNeedNewDmb) { bravoServer = await SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, false, false, false, cancellationToken).ConfigureAwait(false); } else { bravoServer = await SessionControllerFactory.Reattach(reattachInfo.Bravo, cancellationToken).ConfigureAwait(false); } // failed reattaches will return null bravoServer?.SetHighPriority(); // possiblity of null servers due to failed reattaches if (alphaServer == null || bravoServer == null) { await chatTask.ConfigureAwait(false); var bothServersDead = alphaServer == null && bravoServer == null; if (bothServersDead || (alphaServer == null && reattachInfo.AlphaIsActive) || (bravoServer == null && !reattachInfo.AlphaIsActive)) { // we lost the active server, just restart entirely DisposeAndNullControllers(); const string FailReattachMessage = "Unable to properly reattach to active server! Restarting..."; Logger.LogWarning(FailReattachMessage); Logger.LogDebug(bothServersDead ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!"); chatTask = Chat.SendWatchdogMessage(FailReattachMessage, cancellationToken); callBeforeRecurse(); await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false); await chatTask.ConfigureAwait(false); return; } // we still have the active server but the other one is dead to us, hand it off to the monitor to restart const string InactiveReattachFailureMessage = "Unable to reattach to inactive server. Leaving for monitor to reboot..."; chatTask = Chat.SendWatchdogMessage(InactiveReattachFailureMessage, cancellationToken); Logger.LogWarning(InactiveReattachFailureMessage); if (reattachInfo.AlphaIsActive) { bravoServer = SessionControllerFactory.CreateDeadSession(reattachInfo.Bravo.Dmb); } else { alphaServer = SessionControllerFactory.CreateDeadSession(reattachInfo.Alpha.Dmb); } } var alphaLrt = CheckLaunchResult(alphaServer, "Alpha", cancellationToken); var bravoLrt = CheckLaunchResult(bravoServer, "Bravo", cancellationToken); // this task completes when both serers have finished booting var allTask = Task.WhenAll(alphaLrt, bravoLrt); await allTask.ConfigureAwait(false); // both servers are now running, alpha is the active server(unless reattach), huzzah alphaIsActive = reattachInfo?.AlphaIsActive ?? true; var activeServer = AlphaIsActive ? alphaServer : bravoServer; activeServer.EnableCustomChatCommands(); activeServer.ClosePortOnReboot = true; } catch { if (dmbToUse != null) { // we locked 2 dmbs if (bravoServer == null) { // bravo didn't get control of his dmbToUse.Dispose(); if (alphaServer == null) { dmbToUse.Dispose(); // alpha didn't get control of his } } } else if (doesntNeedNewDmb) // we have reattachInfo { if (bravoServer == null) { // bravo didn't get control of his reattachInfo.Bravo?.Dmb.Dispose(); if (alphaServer == null) { reattachInfo.Alpha?.Dmb.Dispose(); // alpha didn't get control of his } } } // kill the controllers DisposeAndNullControllers(); throw; } }
/// <inheritdoc /> #pragma warning disable CA1502 // TODO: Decomplexify protected override async Task HandleMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken) { Logger.LogDebug("Monitor activation. Reason: {0}", activationReason); // this is where the bulk of the watchdog handling code lives and is fraught with lambdas, sorry not sorry // I'll do my best to walk you through it // returns true if the inactive server can't be used immediately // also sets monitor to restart if the above holds bool FullRestartDeadInactive() { if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail) { Logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting"); monitorState.NextAction = MonitorAction.Restart; // will dispose server return(true); } return(false); } // trys to set inactive server's port to the public game port // doesn't handle closing active server's port // returns true on success and swaps inactiveserver and activeserver also sets LastLaunchParameters to ActiveLaunchParameters // on failure, sets monitor to restart async Task <bool> MakeInactiveActive() { Logger.LogDebug("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value); var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false); if (!result) { Logger.LogWarning("Failed to activate inactive server! Restarting monitor..."); monitorState.NextAction = MonitorAction.Restart; // will dispose server return(false); } // inactive server should always be using active launch parameters LastLaunchParameters = ActiveLaunchParameters; var tmp = monitorState.ActiveServer; monitorState.ActiveServer = monitorState.InactiveServer; monitorState.InactiveServer = tmp; alphaIsActive = !AlphaIsActive; monitorState.ActiveServer.EnableCustomChatCommands(); return(true); } // Kills and tries to launch inactive server with the latest dmb // falls back to current dmb on failure // Sets critfail on inactive server failing that // returns false if the backup dmb was used successfully, true otherwise async Task UpdateAndRestartInactiveServer(bool breakAfter) { ActiveParametersUpdated = new TaskCompletionSource <object>(); monitorState.InactiveServer.Dispose(); // kill or recycle it var desiredNextAction = breakAfter ? MonitorAction.Break : MonitorAction.Continue; monitorState.NextAction = desiredNextAction; Logger.LogInformation("Rebooting inactive server..."); var newDmb = DmbFactory.LockNextDmb(1); try { monitorState.InactiveServer = await SessionControllerFactory.LaunchNew( newDmb, null, ActiveLaunchParameters, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken) .ConfigureAwait(false); monitorState.InactiveServer.SetHighPriority(); } catch (OperationCanceledException) { throw; } catch (Exception e) { Logger.LogError("Error occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString()); // ahh jeez, what do we do here? // this is our fault, so it should never happen but // idk maybe a database error while handling the newest dmb? // either way try to start it using the active server's dmb as a backup try { var dmbBackup = await DmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob, cancellationToken).ConfigureAwait(false); if (dmbBackup == null) // NANI!? { throw new InvalidOperationException("Watchdog double crit-fail!"); // just give up, if THAT compile job is failing then the ActiveServer is gonna crash soon too or already has } monitorState.InactiveServer = await SessionControllerFactory.LaunchNew( dmbBackup, null, ActiveLaunchParameters, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken) .ConfigureAwait(false); monitorState.InactiveServer.SetHighPriority(); await Chat.SendWatchdogMessage( "Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...", false, cancellationToken).ConfigureAwait(false); } catch (OperationCanceledException) { throw; } catch (Exception e2) { // fuuuuucckkk Logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! Exception: {0}", e2.ToString()); monitorState.InactiveServerCritFail = true; await Chat.SendWatchdogMessage( "Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", false, cancellationToken).ConfigureAwait(false); return; } } Logger.LogInformation("Successfully relaunched inactive server!"); monitorState.RebootingInactiveServer = true; }