public async Task TestCancel() { var delayer = new AsyncDelayer(); using var cts = new CancellationTokenSource(); cts.Cancel(); await Assert.ThrowsExceptionAsync <TaskCanceledException>(() => delayer.Delay(TimeSpan.FromSeconds(1), cts.Token)).ConfigureAwait(false); }
public async Task TestDelay() { var delayer = new AsyncDelayer(); var startDelay = delayer.Delay(TimeSpan.FromSeconds(1), default); var checkDelay = Task.Delay(TimeSpan.FromSeconds(1) - TimeSpan.FromMilliseconds(10), default); await startDelay.ConfigureAwait(false); Assert.IsTrue(checkDelay.IsCompleted); }
/// <inheritdoc /> protected sealed override async Task MonitorLifetimes(CancellationToken cancellationToken) { Logger.LogTrace("Entered MonitorLifetimes"); // this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState var iteration = 1; for (MonitorAction nextAction = MonitorAction.Continue; nextAction != MonitorAction.Exit; ++iteration) { // always start out with continue nextAction = MonitorAction.Continue; // dump some info to the logs Logger.LogDebug("Iteration {0} of monitor loop", iteration); try { Logger.LogDebug("Server Compile Job ID: {0}", Server.Dmb.CompileJob.Id); // load the activation tasks into local variables Task activeServerLifetime = Server.Lifetime; var activeServerReboot = Server.OnReboot; Task activeLaunchParametersChanged = ActiveParametersUpdated.Task; var newDmbAvailable = DmbFactory.OnNewerDmb; // cancel waiting if requested var cancelTcs = new TaskCompletionSource <object>(); using (cancellationToken.Register(() => cancelTcs.SetCanceled())) { var toWaitOn = Task.WhenAny(activeServerLifetime, activeServerReboot, newDmbAvailable, cancelTcs.Task, activeLaunchParametersChanged); // wait for something to happen await toWaitOn.ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); } var chatTask = Task.CompletedTask; using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) { // always run HandleMonitorWakeup from the context of the semaphore lock // multiple things may have happened, handle them one at a time for (var moreActivationsToProcess = true; moreActivationsToProcess && (nextAction == MonitorAction.Continue || nextAction == MonitorAction.Skip);) { MonitorActivationReason activationReason = default; // this will always be assigned before being used // process the tasks in this order and call HandlerMonitorWakup for each bool CheckActivationReason(ref Task task, MonitorActivationReason testActivationReason) { var taskCompleted = task?.IsCompleted == true; task = null; if (nextAction == MonitorAction.Skip) { nextAction = MonitorAction.Continue; } else if (taskCompleted) { activationReason = testActivationReason; return(true); } return(false); } if (CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed) || CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted) || CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable) || CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated)) { Logger.LogTrace("Monitor activation: {0}", activationReason); nextAction = await HandleMonitorWakeup(activationReason, cancellationToken).ConfigureAwait(false); } else { moreActivationsToProcess = false; } } } // full reboot required if (nextAction == MonitorAction.Restart) { Logger.LogDebug("Next state action is to restart"); DisposeAndNullControllers(); for (var retryAttempts = 1; nextAction == MonitorAction.Restart; ++retryAttempts) { Exception launchException = null; using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false)) try { // use LaunchImplNoLock without announcements or restarting the monitor await LaunchImplNoLock(false, false, null, cancellationToken).ConfigureAwait(false); if (Running) { Logger.LogDebug("Relaunch successful, resetting monitor state..."); break; // continue on main loop } } catch (OperationCanceledException) { throw; } catch (Exception e) { launchException = e; } await chatTask.ConfigureAwait(false); if (!Running) { if (launchException == null) { Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}", retryAttempts); } else { Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException); } var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); // max of one hour, increasing by a power of 2 each time chatTask = Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken); await Task.WhenAll(AsyncDelayer.Delay(TimeSpan.FromSeconds(retryDelay), cancellationToken), chatTask).ConfigureAwait(false); } } } } catch (OperationCanceledException) { Logger.LogDebug("Monitor cancelled"); break; } catch (Exception e) { // really, this should NEVER happen Logger.LogError("Monitor crashed! Iteration: {0}, NextAction: {1}, Exception: {2}", iteration, nextAction, e); await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Monitor crashed, this should NEVER happen! Please report this, full details in logs! Restarting monitor... Error: {0}", e.Message), cancellationToken).ConfigureAwait(false); } } Logger.LogTrace("Monitor exiting..."); }
#pragma warning restore CA1502 /// <inheritdoc /> #pragma warning disable CA1502 // TODO: Decomplexify protected override async Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken) { Debug.Assert(alphaServer == null && bravoServer == null, "Entered LaunchNoLock with one or more of the servers not being null!"); // don't need a new dmb if reattaching var doesntNeedNewDmb = reattachInfo?.Alpha != null && reattachInfo?.Bravo != null; var dmbToUse = doesntNeedNewDmb ? null : DmbFactory.LockNextDmb(2); // if this try catches something, both servers are killed try { // start the alpha server task, either by launch a new process or attaching to an existing one // The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail // The tasks pertaining to server startup times are in the ISessionControllers Task <ISessionController> alphaServerTask; if (!doesntNeedNewDmb) { alphaServerTask = SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, true, true, false, cancellationToken); } else { alphaServerTask = SessionControllerFactory.Reattach(reattachInfo.Alpha, cancellationToken); } // retrieve the session controller var startTime = DateTimeOffset.Now; alphaServer = await alphaServerTask.ConfigureAwait(false); // failed reattaches will return null alphaServer?.SetHighPriority(); // extra delay for total ordering var now = DateTimeOffset.Now; var delay = now - startTime; // definitely not if reattaching though if (reattachInfo == null && delay.TotalSeconds < AlphaBravoStartupSeperationInterval) { await AsyncDelayer.Delay(startTime.AddSeconds(AlphaBravoStartupSeperationInterval) - now, cancellationToken).ConfigureAwait(false); } // now bring bravo up if (!doesntNeedNewDmb) { bravoServer = await SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, false, false, false, cancellationToken).ConfigureAwait(false); } else { bravoServer = await SessionControllerFactory.Reattach(reattachInfo.Bravo, cancellationToken).ConfigureAwait(false); } // failed reattaches will return null bravoServer?.SetHighPriority(); // possiblity of null servers due to failed reattaches if (alphaServer == null || bravoServer == null) { await chatTask.ConfigureAwait(false); var bothServersDead = alphaServer == null && bravoServer == null; if (bothServersDead || (alphaServer == null && reattachInfo.AlphaIsActive) || (bravoServer == null && !reattachInfo.AlphaIsActive)) { // we lost the active server, just restart entirely DisposeAndNullControllers(); const string FailReattachMessage = "Unable to properly reattach to active server! Restarting..."; Logger.LogWarning(FailReattachMessage); Logger.LogDebug(bothServersDead ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!"); chatTask = Chat.SendWatchdogMessage(FailReattachMessage, cancellationToken); callBeforeRecurse(); await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false); await chatTask.ConfigureAwait(false); return; } // we still have the active server but the other one is dead to us, hand it off to the monitor to restart const string InactiveReattachFailureMessage = "Unable to reattach to inactive server. Leaving for monitor to reboot..."; chatTask = Chat.SendWatchdogMessage(InactiveReattachFailureMessage, cancellationToken); Logger.LogWarning(InactiveReattachFailureMessage); if (reattachInfo.AlphaIsActive) { bravoServer = SessionControllerFactory.CreateDeadSession(reattachInfo.Bravo.Dmb); } else { alphaServer = SessionControllerFactory.CreateDeadSession(reattachInfo.Alpha.Dmb); } } var alphaLrt = CheckLaunchResult(alphaServer, "Alpha", cancellationToken); var bravoLrt = CheckLaunchResult(bravoServer, "Bravo", cancellationToken); // this task completes when both serers have finished booting var allTask = Task.WhenAll(alphaLrt, bravoLrt); await allTask.ConfigureAwait(false); // both servers are now running, alpha is the active server(unless reattach), huzzah alphaIsActive = reattachInfo?.AlphaIsActive ?? true; var activeServer = AlphaIsActive ? alphaServer : bravoServer; activeServer.EnableCustomChatCommands(); activeServer.ClosePortOnReboot = true; } catch { if (dmbToUse != null) { // we locked 2 dmbs if (bravoServer == null) { // bravo didn't get control of his dmbToUse.Dispose(); if (alphaServer == null) { dmbToUse.Dispose(); // alpha didn't get control of his } } } else if (doesntNeedNewDmb) // we have reattachInfo { if (bravoServer == null) { // bravo didn't get control of his reattachInfo.Bravo?.Dmb.Dispose(); if (alphaServer == null) { reattachInfo.Alpha?.Dmb.Dispose(); // alpha didn't get control of his } } } // kill the controllers DisposeAndNullControllers(); throw; } }