/// <inheritdoc />
public async Task StopAsync(CancellationToken cancellationToken)
{
if (releaseServers && Running)
{
var reattachInformation = new WatchdogReattachInformation
{
AlphaIsActive = AlphaIsActive
};
reattachInformation.Alpha = alphaServer?.Release();
reattachInformation.Bravo = bravoServer?.Release();
await reattachInfoHandler.Save(reattachInformation, cancellationToken).ConfigureAwait(false);
}
await Terminate(false, cancellationToken).ConfigureAwait(false);
}
/// <summary> /// Starts all <see cref="ISessionController"/>s. /// </summary> /// <param name="callBeforeRecurse">An <see cref="Action"/> that must be run before making a recursive call to <see cref="LaunchImplNoLock(bool, bool, WatchdogReattachInformation, CancellationToken)"/>.</param> /// <param name="chatTask">A, possibly active, <see cref="Task"/> for an outgoing chat message.</param> /// <param name="reattachInfo"><see cref="WatchdogReattachInformation"/> to use, if any</param> /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param> /// <returns>A <see cref="Task"/> representing the running operation</returns> protected abstract Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken);
/// <inheritdoc />
protected sealed override async Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken)
{
Debug.Assert(Server == null, "Entered LaunchNoLock with server not being null!");
// don't need a new dmb if reattaching
var doesntNeedNewDmb = reattachInfo?.Alpha != null && reattachInfo?.Bravo != null;
var dmbToUse = doesntNeedNewDmb ? null : DmbFactory.LockNextDmb(1);
var serverToReattach = reattachInfo?.Alpha ?? reattachInfo?.Bravo;
var serverToKill = reattachInfo?.Bravo ?? reattachInfo?.Alpha;
// vice versa
if (reattachInfo?.AlphaIsActive == false)
{
var temp = serverToReattach;
serverToReattach = serverToKill;
serverToKill = temp;
}
// if this try catches something, both servers are killed
bool inactiveServerWasKilled = false;
try
{
// start the alpha server task, either by launch a new process or attaching to an existing one
// The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail
// The tasks pertaining to server startup times are in the ISessionControllers
Task <ISessionController> serverLaunchTask, inactiveReattachTask;
if (!doesntNeedNewDmb)
{
dmbToUse = await PrepServerForLaunch(dmbToUse, cancellationToken).ConfigureAwait(false);
serverLaunchTask = SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, true, true, false, cancellationToken);
}
else
{
serverLaunchTask = SessionControllerFactory.Reattach(serverToReattach, cancellationToken);
}
bool thereIsAnInactiveServerToKill = serverToKill != null;
if (thereIsAnInactiveServerToKill)
{
inactiveReattachTask = SessionControllerFactory.Reattach(serverToKill, cancellationToken);
}
else
{
inactiveReattachTask = Task.FromResult <ISessionController>(null);
}
// retrieve the session controller
Server = await serverLaunchTask.ConfigureAwait(false);
// failed reattaches will return null
Server?.SetHighPriority();
var inactiveServerController = await inactiveReattachTask.ConfigureAwait(false);
inactiveServerController?.Dispose();
inactiveServerWasKilled = inactiveServerController != null;
// possiblity of null servers due to failed reattaches
if (Server == null)
{
callBeforeRecurse();
await NotifyOfFailedReattach(thereIsAnInactiveServerToKill && !inactiveServerWasKilled, cancellationToken).ConfigureAwait(false);
return;
}
await CheckLaunchResult(Server, "Server", cancellationToken).ConfigureAwait(false);
Server.EnableCustomChatCommands();
}
catch
{
// kill the controllers
bool serverWasActive = Server != null;
DisposeAndNullControllers();
// server didn't get control of this dmb
if (dmbToUse != null && !serverWasActive)
{
dmbToUse.Dispose();
}
if (serverToKill != null && !inactiveServerWasKilled)
{
serverToKill.Dmb.Dispose();
}
throw;
}
}
/// <summary>
/// Launches the watchdog.
/// </summary>
/// <param name="startMonitor">If <see cref="MonitorLifetimes(CancellationToken)"/> should be started by this function</param>
/// <param name="announce">If the launch should be announced to chat by this function</param>
/// <param name="reattachInfo"><see cref="WatchdogReattachInformation"/> to use, if any</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
protected async Task LaunchImplNoLock(bool startMonitor, bool announce, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken)
{
Logger.LogTrace("Begin LaunchNoLock");
if (Running)
{
throw new JobException("Watchdog already running!");
}
if (!DmbFactory.DmbAvailable)
{
throw new JobException("Corrupted compilation, please redeploy!");
}
// this is necessary, the monitor could be in it's sleep loop trying to restart, if so cancel THAT monitor and start our own with blackjack and hookers
Task chatTask;
if (startMonitor && await StopMonitor().ConfigureAwait(false))
{
chatTask = Chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", cancellationToken);
}
else if (announce)
{
chatTask = Chat.SendWatchdogMessage(reattachInfo == null ? "Starting..." : "Reattaching...", cancellationToken); // simple announce
}
else
{
chatTask = Task.CompletedTask; // no announce
}
// since neither server is running, this is safe to do
LastLaunchParameters = ActiveLaunchParameters;
// for when we call ourself and want to not catch thrown exceptions
var ignoreNestedException = false;
try
{
await InitControllers(() => ignoreNestedException = true, chatTask, reattachInfo, cancellationToken).ConfigureAwait(false);
await chatTask.ConfigureAwait(false);
Logger.LogInformation("Launched servers successfully");
Running = true;
if (startMonitor)
{
StartMonitor();
}
}
catch (Exception e)
{
// don't try to send chat tasks or warning logs if were suppressing exceptions or cancelled
if (!ignoreNestedException && !cancellationToken.IsCancellationRequested)
{
var originalChatTask = chatTask;
async Task ChainChatTaskWithErrorMessage()
{
await originalChatTask.ConfigureAwait(false);
await Chat.SendWatchdogMessage("Startup failed!", cancellationToken).ConfigureAwait(false);
}
chatTask = ChainChatTaskWithErrorMessage();
Logger.LogWarning("Failed to start watchdog: {0}", e.ToString());
}
throw;
}
finally
{
// finish the chat task that's in flight
try
{
await chatTask.ConfigureAwait(false);
}
catch (OperationCanceledException) { }
}
}
#pragma warning restore CA1502
/// <inheritdoc />
#pragma warning disable CA1502 // TODO: Decomplexify
protected override async Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken)
{
Debug.Assert(alphaServer == null && bravoServer == null, "Entered LaunchNoLock with one or more of the servers not being null!");
// don't need a new dmb if reattaching
var doesntNeedNewDmb = reattachInfo?.Alpha != null && reattachInfo?.Bravo != null;
var dmbToUse = doesntNeedNewDmb ? null : DmbFactory.LockNextDmb(2);
// if this try catches something, both servers are killed
try
{
// start the alpha server task, either by launch a new process or attaching to an existing one
// The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail
// The tasks pertaining to server startup times are in the ISessionControllers
Task <ISessionController> alphaServerTask;
if (!doesntNeedNewDmb)
{
alphaServerTask = SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, true, true, false, cancellationToken);
}
else
{
alphaServerTask = SessionControllerFactory.Reattach(reattachInfo.Alpha, cancellationToken);
}
// retrieve the session controller
var startTime = DateTimeOffset.Now;
alphaServer = await alphaServerTask.ConfigureAwait(false);
// failed reattaches will return null
alphaServer?.SetHighPriority();
// extra delay for total ordering
var now = DateTimeOffset.Now;
var delay = now - startTime;
// definitely not if reattaching though
if (reattachInfo == null && delay.TotalSeconds < AlphaBravoStartupSeperationInterval)
{
await AsyncDelayer.Delay(startTime.AddSeconds(AlphaBravoStartupSeperationInterval) - now, cancellationToken).ConfigureAwait(false);
}
// now bring bravo up
if (!doesntNeedNewDmb)
{
bravoServer = await SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, false, false, false, cancellationToken).ConfigureAwait(false);
}
else
{
bravoServer = await SessionControllerFactory.Reattach(reattachInfo.Bravo, cancellationToken).ConfigureAwait(false);
}
// failed reattaches will return null
bravoServer?.SetHighPriority();
// possiblity of null servers due to failed reattaches
if (alphaServer == null || bravoServer == null)
{
await chatTask.ConfigureAwait(false);
var bothServersDead = alphaServer == null && bravoServer == null;
if (bothServersDead ||
(alphaServer == null && reattachInfo.AlphaIsActive) ||
(bravoServer == null && !reattachInfo.AlphaIsActive))
{
// we lost the active server, just restart entirely
DisposeAndNullControllers();
const string FailReattachMessage = "Unable to properly reattach to active server! Restarting...";
Logger.LogWarning(FailReattachMessage);
Logger.LogDebug(bothServersDead ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!");
chatTask = Chat.SendWatchdogMessage(FailReattachMessage, cancellationToken);
callBeforeRecurse();
await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false);
await chatTask.ConfigureAwait(false);
return;
}
// we still have the active server but the other one is dead to us, hand it off to the monitor to restart
const string InactiveReattachFailureMessage = "Unable to reattach to inactive server. Leaving for monitor to reboot...";
chatTask = Chat.SendWatchdogMessage(InactiveReattachFailureMessage, cancellationToken);
Logger.LogWarning(InactiveReattachFailureMessage);
if (reattachInfo.AlphaIsActive)
{
bravoServer = SessionControllerFactory.CreateDeadSession(reattachInfo.Bravo.Dmb);
}
else
{
alphaServer = SessionControllerFactory.CreateDeadSession(reattachInfo.Alpha.Dmb);
}
}
var alphaLrt = CheckLaunchResult(alphaServer, "Alpha", cancellationToken);
var bravoLrt = CheckLaunchResult(bravoServer, "Bravo", cancellationToken);
// this task completes when both serers have finished booting
var allTask = Task.WhenAll(alphaLrt, bravoLrt);
await allTask.ConfigureAwait(false);
// both servers are now running, alpha is the active server(unless reattach), huzzah
alphaIsActive = reattachInfo?.AlphaIsActive ?? true;
var activeServer = AlphaIsActive ? alphaServer : bravoServer;
activeServer.EnableCustomChatCommands();
activeServer.ClosePortOnReboot = true;
}
catch
{
if (dmbToUse != null)
{
// we locked 2 dmbs
if (bravoServer == null)
{
// bravo didn't get control of his
dmbToUse.Dispose();
if (alphaServer == null)
{
dmbToUse.Dispose(); // alpha didn't get control of his
}
}
}
else if (doesntNeedNewDmb) // we have reattachInfo
{
if (bravoServer == null)
{
// bravo didn't get control of his
reattachInfo.Bravo?.Dmb.Dispose();
if (alphaServer == null)
{
reattachInfo.Alpha?.Dmb.Dispose(); // alpha didn't get control of his
}
}
}
// kill the controllers
DisposeAndNullControllers();
throw;
}
}