/// <inheritdoc />
protected override void DisposeAndNullControllersImpl()
{
Server?.Dispose();
Server = null;
Running = false;
gracefulRebootRequired = false;
}
/// <inheritdoc />
protected override async Task InitControllers(
Task chatTask,
ReattachInformation reattachInfo,
CancellationToken cancellationToken)
{
// don't need a new dmb if reattaching
var reattachInProgress = reattachInfo != null;
var dmbToUse = reattachInProgress ? null : DmbFactory.LockNextDmb(1);
// if this try catches something, both servers are killed
try
{
// start the alpha server task, either by launch a new process or attaching to an existing one
// The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail
// The tasks pertaining to server startup times are in the ISessionControllers
Task <ISessionController> serverLaunchTask;
if (!reattachInProgress)
{
dmbToUse = await PrepServerForLaunch(dmbToUse, cancellationToken).ConfigureAwait(false);
serverLaunchTask = SessionControllerFactory.LaunchNew(
dmbToUse,
null,
ActiveLaunchParameters,
false,
cancellationToken);
}
else
{
serverLaunchTask = SessionControllerFactory.Reattach(reattachInfo, cancellationToken);
}
// retrieve the session controller
Server = await serverLaunchTask.ConfigureAwait(false);
// failed reattaches will return null
Server?.SetHighPriority();
// possiblity of null servers due to failed reattaches
if (Server == null)
{
await ReattachFailure(
chatTask,
cancellationToken)
.ConfigureAwait(false);
return;
}
await CheckLaunchResult(Server, "Server", cancellationToken).ConfigureAwait(false);
Server.EnableCustomChatCommands();
}
catch
{
// kill the controllers
bool serverWasActive = Server != null;
// DCT: Operation must always run
await DisposeAndNullControllers(default).ConfigureAwait(false);
internal NetworkHandler(
IHandleProvider provider,
ISessionController sessionController)
{
_handleProvider = provider;
_sessionController = sessionController;
}
public void Dispose()
{
_disposed = true;
StopAutomaticProgress();
var controllers = SessionControllers.ToList();
SessionControllers = new ISessionController[] { };
foreach (var controller in controllers)
{
var disposable = controller as IDisposable;
if (disposable != null)
{
_logger.Debug("Disposing session controller {0}", disposable.GetType().Name);
try
{
disposable.Dispose();
}
catch (Exception ex)
{
_logger.ErrorException("Error disposing session controller", ex);
}
}
}
_sessionManager = null;
}
public CharacterLayoutHostView()
{
InitializeComponent();
_lastToken = null;
_session = new MMXSessionController(new CharacterInfo(), new MudRevBBS());
_socket = new TelnetSocket("MajorMud.DontExist.com");
}
public void AddController(ISessionController controller)
{
var controllers = SessionControllers.ToList();
controllers.Add(controller);
SessionControllers = controllers.ToArray();
}
/// <summary>
/// Call <see cref="IDisposable.Dispose"/> on <see cref="alphaServer"/> and <see cref="bravoServer"/> and set them to <see langword="null"/>
/// </summary>
void DisposeAndNullControllers()
{
alphaServer?.Dispose();
alphaServer = null;
bravoServer?.Dispose();
bravoServer = null;
Running = false;
}
internal HandshakeDecoder(
IPlayerController playerController,
ISessionController sessionController)
{
_playerController = playerController;
_sessionController = sessionController;
status = HandshakeStatus.Handshake;
}
public SessionInfo(ISessionManager sessionManager, ILogger logger)
{
_sessionManager = sessionManager;
_logger = logger;
AdditionalUsers = new SessionUserInfo[] { };
PlayState = new PlayerStateInfo();
SessionControllers = new ISessionController[] { };
}
public NetworkInitializer(
IHandleProvider provider,
IPlayerController playerController,
ISessionController sessionController)
{
_handleProvider = provider;
_playerController = playerController;
_sessionController = sessionController;
}
/// <summary>
/// Initialises a new instance of the <see cref="AccountController"/> class.
/// </summary>
/// <param name="log">Logging module</param>
/// <param name="memberDataAccess">Member data access</param>
/// <param name="sessionController">Session controller</param>
/// <param name="forgotPassword">Forgot password</param>
/// <param name="dealDataAccess">Deal data access</param>
/// <param name="hash">Hasher</param>
/// <param name="emailSender">Email sender</param>
/// <param name="currentUser">Current user</param>
public AccountController(ILogger log, IMemberDataAccess memberDataAccess, ISessionController sessionController, IRecoverPassword forgotPassword, IDealDataAccess dealDataAccess, IHash hash, IEmailSender emailSender, ICurrentUser currentUser)
{
this.log = log;
this.memberDataAccess = memberDataAccess;
this.sessionController = sessionController;
this.forgotPassword = forgotPassword;
this.dealDataAccess = dealDataAccess;
this.hash = hash;
this.emailSender = emailSender;
userName = currentUser.GetCurrentUser();
}
public RealSpaceEngineers(
IObserver observer,
ICharacterController controller,
ISessionController sessionController,
IItems items,
IDefinitions definitions
)
{
Observer = observer;
Character = controller;
Session = sessionController;
Items = items;
Definitions = definitions;
}
/// <summary>
/// Check the <see cref="LaunchResult"/> of a given <paramref name="controller"/> for errors and throw a <see cref="JobException"/> if any are detected.
/// </summary>
/// <param name="controller">The <see cref="ISessionController"/> to checkou.</param>
/// <param name="serverName">The name of the server being checked.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation.</param>
/// <returns>A <see cref="Task"/> representing the running operation.</returns>
protected async Task CheckLaunchResult(ISessionController controller, string serverName, CancellationToken cancellationToken)
{
var launchResult = await controller.LaunchResult.WithToken(cancellationToken).ConfigureAwait(false);
// Dead sessions won't trigger this
if (launchResult.ExitCode.HasValue) // you killed us ray...
{
throw new JobException(String.Format(CultureInfo.InvariantCulture, "{0} failed to start: {1}", serverName, launchResult));
}
if (!launchResult.StartupTime.HasValue)
{
throw new JobException(String.Format(CultureInfo.InvariantCulture, "{0} timed out on startup: {1}s", serverName, ActiveLaunchParameters.StartupTimeout.Value));
}
}
/// <inheritdoc />
protected override async Task DisposeAndNullControllersImpl()
{
var disposeTask = Server?.DisposeAsync();
gracefulRebootRequired = false;
if (!disposeTask.HasValue)
{
return;
}
await disposeTask.Value.ConfigureAwait(false);
Server = null;
}
public void Start()
{
if (this.session != null)
{
throw new InvalidOperationException("The session is already started.");
}
RealTimeTraceCollectorInfo info = new RealTimeTraceCollectorInfo(this.name);
info.Providers.Add(new ProviderInfo(KernelProcessProviderId) { KeywordsAll = 0x10, Level = 4 });
this.session = info.Create();
this.session.Start();
IObservable<EtwNativeEvent> stream = EtwObservable.FromSession(this.name);
this.subscription = stream.Subscribe(e => this.OnNext(e));
}
public void Start()
{
if (this.session != null)
{
throw new InvalidOperationException("The session is already started.");
}
RealTimeTraceCollectorInfo info = new RealTimeTraceCollectorInfo(this.name);
info.Providers.Add(new ProviderInfo(KernelProcessProviderId)
{
KeywordsAll = 0x10, Level = 4
});
this.session = info.Create();
this.session.Start();
IObservable <EtwNativeEvent> stream = EtwObservable.FromSession(this.name);
this.subscription = stream.Subscribe(e => this.OnNext(e));
}
private static void CreateRealTimeTraceCollector()
{
RealTimeTraceCollectorInfo info = new RealTimeTraceCollectorInfo("MyRealTimeCollector");
// Microsoft-Windows-Kernel-Process
Guid providerId = new Guid("{22FB2CD6-0E7B-422B-A0C7-2FAD1FD0E716}");
info.Providers.Add(new ProviderInfo(providerId)
{
Level = 5
});
ISessionController controller = info.Create();
controller.Start();
Thread.Sleep(5000);
controller.Stop();
}
/// <inheritdoc />
protected sealed override async Task InitControllers(Action callBeforeRecurse, Task chatTask, DualReattachInformation reattachInfo, CancellationToken cancellationToken)
{
var serverToReattach = reattachInfo?.Alpha ?? reattachInfo?.Bravo;
var serverToKill = reattachInfo?.Bravo ?? reattachInfo?.Alpha;
// vice versa
if (serverToKill == serverToReattach)
{
serverToKill = null;
}
if (reattachInfo?.AlphaIsActive == false)
{
var temp = serverToReattach;
serverToReattach = serverToKill;
serverToKill = temp;
}
// don't need a new dmb if reattaching
var doesntNeedNewDmb = serverToReattach != null;
var dmbToUse = doesntNeedNewDmb ? null : DmbFactory.LockNextDmb(1);
// if this try catches something, both servers are killed
bool inactiveServerWasKilled = false;
try
{
// start the alpha server task, either by launch a new process or attaching to an existing one
// The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail
// The tasks pertaining to server startup times are in the ISessionControllers
Task <ISessionController> serverLaunchTask, inactiveReattachTask;
if (!doesntNeedNewDmb)
{
dmbToUse = await PrepServerForLaunch(dmbToUse, cancellationToken).ConfigureAwait(false);
serverLaunchTask = SessionControllerFactory.LaunchNew(
dmbToUse,
null,
ActiveLaunchParameters,
true,
true,
false,
cancellationToken);
}
else
{
serverLaunchTask = SessionControllerFactory.Reattach(serverToReattach, cancellationToken);
}
bool thereIsAnInactiveServerToKill = serverToKill != null;
if (thereIsAnInactiveServerToKill)
{
inactiveReattachTask = SessionControllerFactory.Reattach(serverToKill, cancellationToken);
}
else
{
inactiveReattachTask = Task.FromResult <ISessionController>(null);
}
// retrieve the session controller
Server = await serverLaunchTask.ConfigureAwait(false);
// failed reattaches will return null
Server?.SetHighPriority();
var inactiveServerController = await inactiveReattachTask.ConfigureAwait(false);
inactiveServerController?.Dispose();
inactiveServerWasKilled = inactiveServerController != null;
// possiblity of null servers due to failed reattaches
if (Server == null)
{
callBeforeRecurse();
await NotifyOfFailedReattach(thereIsAnInactiveServerToKill && !inactiveServerWasKilled, cancellationToken).ConfigureAwait(false);
return;
}
await CheckLaunchResult(Server, "Server", cancellationToken).ConfigureAwait(false);
Server.EnableCustomChatCommands();
}
catch
{
// kill the controllers
bool serverWasActive = Server != null;
DisposeAndNullControllers();
// server didn't get control of this dmb
if (dmbToUse != null && !serverWasActive)
{
dmbToUse.Dispose();
}
if (serverToKill != null && !inactiveServerWasKilled)
{
serverToKill.Dmb.Dispose();
}
throw;
}
}
/// <inheritdoc />
#pragma warning disable CA1502 // TODO: Decomplexify
protected override async Task MonitorLifetimes(CancellationToken cancellationToken)
{
Logger.LogTrace("Entered MonitorLifetimes");
// this function is responsible for calling HandlerMonitorWakeup when necessary and manitaining the MonitorState
var iteration = 1;
for (var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration)
{
// always start out with continue
monitorState.NextAction = MonitorAction.Continue;
// dump some info to the logs
Logger.LogDebug("Iteration {0} of monitor loop", iteration);
try
{
if (AlphaIsActive)
{
Logger.LogDebug("Alpha is the active server");
}
else
{
Logger.LogDebug("Bravo is the active server");
}
if (monitorState.RebootingInactiveServer)
{
Logger.LogDebug("Inactive server is rebooting");
}
// update the monitor state with the inactive/active servers
monitorState.ActiveServer = AlphaIsActive ? alphaServer : bravoServer;
monitorState.InactiveServer = AlphaIsActive ? bravoServer : alphaServer;
if (monitorState.ActiveServer.ClosePortOnReboot)
{
Logger.LogDebug("Active server will close port on reboot");
}
if (monitorState.InactiveServer.ClosePortOnReboot)
{
Logger.LogDebug("Inactive server will close port on reboot");
}
Logger.LogDebug("Active server Compile Job ID: {0}", monitorState.ActiveServer.Dmb.CompileJob.Id);
Logger.LogDebug("Inactive server Compile Job ID: {0}", monitorState.InactiveServer.Dmb.CompileJob.Id);
// load the activation tasks into local variables
Task activeServerLifetime = monitorState.ActiveServer.Lifetime;
Task inactiveServerLifetime = monitorState.InactiveServer.Lifetime;
var activeServerReboot = monitorState.ActiveServer.OnReboot;
var inactiveServerReboot = monitorState.InactiveServer.OnReboot;
Task inactiveServerStartup = monitorState.RebootingInactiveServer ? monitorState.InactiveServer.LaunchResult : null;
Task activeLaunchParametersChanged = ActiveParametersUpdated.Task;
var newDmbAvailable = DmbFactory.OnNewerDmb;
// cancel waiting if requested
var cancelTcs = new TaskCompletionSource <object>();
using (cancellationToken.Register(() => cancelTcs.SetCanceled()))
{
var toWaitOn = Task.WhenAny(activeServerLifetime, inactiveServerLifetime, activeServerReboot, inactiveServerReboot, newDmbAvailable, cancelTcs.Task, activeLaunchParametersChanged);
if (monitorState.RebootingInactiveServer)
{
toWaitOn = Task.WhenAny(toWaitOn, inactiveServerStartup);
}
// wait for something to happen
await toWaitOn.ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
}
var chatTask = Task.CompletedTask;
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
{
// always run HandleMonitorWakeup from the context of the semaphore lock
// multiple things may have happened, handle them one at a time
for (var moreActivationsToProcess = true; moreActivationsToProcess && (monitorState.NextAction == MonitorAction.Continue || monitorState.NextAction == MonitorAction.Skip);)
{
MonitorActivationReason activationReason = default; // this will always be assigned before being used
// process the tasks in this order and call HandlerMonitorWakup for each
bool CheckActivationReason(ref Task task, MonitorActivationReason testActivationReason)
{
var taskCompleted = task?.IsCompleted == true;
task = null;
if (monitorState.NextAction == MonitorAction.Skip)
{
monitorState.NextAction = MonitorAction.Continue;
}
else if (taskCompleted)
{
activationReason = testActivationReason;
return(true);
}
return(false);
}
if (CheckActivationReason(ref activeServerLifetime, MonitorActivationReason.ActiveServerCrashed) ||
CheckActivationReason(ref inactiveServerLifetime, MonitorActivationReason.InactiveServerCrashed) ||
CheckActivationReason(ref activeServerReboot, MonitorActivationReason.ActiveServerRebooted) ||
CheckActivationReason(ref inactiveServerReboot, MonitorActivationReason.InactiveServerRebooted) ||
CheckActivationReason(ref inactiveServerStartup, MonitorActivationReason.InactiveServerStartupComplete) ||
CheckActivationReason(ref newDmbAvailable, MonitorActivationReason.NewDmbAvailable) ||
CheckActivationReason(ref activeLaunchParametersChanged, MonitorActivationReason.ActiveLaunchParametersUpdated))
{
Logger.LogTrace("Monitor activation: {0}", activationReason);
await HandlerMonitorWakeup(activationReason, monitorState, cancellationToken).ConfigureAwait(false);
}
else
{
moreActivationsToProcess = false;
}
}
// writeback alphaServer and bravoServer from monitor state in case they changesd
alphaServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer;
bravoServer = !AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer;
}
// full reboot required
if (monitorState.NextAction == MonitorAction.Restart)
{
Logger.LogDebug("Next state action is to restart");
DisposeAndNullControllers();
chatTask = Chat.SendWatchdogMessage("Restarting entirely due to complications...", cancellationToken);
for (var retryAttempts = 1; monitorState.NextAction == MonitorAction.Restart; ++retryAttempts)
{
Exception launchException = null;
using (await SemaphoreSlimContext.Lock(Semaphore, cancellationToken).ConfigureAwait(false))
try
{
// use LaunchImplNoLock without announcements or restarting the monitor
await LaunchImplNoLock(false, false, null, cancellationToken).ConfigureAwait(false);
if (Running)
{
Logger.LogDebug("Relaunch successful, resetting monitor state...");
monitorState = new MonitorState(); // clean the slate and continue
}
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
launchException = e;
}
await chatTask.ConfigureAwait(false);
if (!Running)
{
if (launchException == null)
{
Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}", retryAttempts);
}
else
{
Logger.LogWarning("Failed to automatically restart the watchdog! Attempt: {0}, Exception: {1}", retryAttempts, launchException);
}
var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); // max of one hour, increasing by a power of 2 each time
chatTask = Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken);
await Task.WhenAll(AsyncDelayer.Delay(TimeSpan.FromSeconds(retryDelay), cancellationToken), chatTask).ConfigureAwait(false);
}
}
}
}
catch (OperationCanceledException)
{
Logger.LogDebug("Monitor cancelled");
break;
}
catch (Exception e)
{
// really, this should NEVER happen
Logger.LogError("Monitor crashed! Iteration: {0}, State: {1}, Exception: {2}", iteration, JsonConvert.SerializeObject(monitorState), e);
await Chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Monitor crashed, this should NEVER happen! Please report this, full details in logs! Restarting monitor... Error: {0}", e.Message), cancellationToken).ConfigureAwait(false);
}
}
Logger.LogTrace("Monitor exiting...");
}
#pragma warning restore CA1502
/// <inheritdoc />
#pragma warning disable CA1502 // TODO: Decomplexify
protected override async Task InitControllers(Action callBeforeRecurse, Task chatTask, WatchdogReattachInformation reattachInfo, CancellationToken cancellationToken)
{
Debug.Assert(alphaServer == null && bravoServer == null, "Entered LaunchNoLock with one or more of the servers not being null!");
// don't need a new dmb if reattaching
var doesntNeedNewDmb = reattachInfo?.Alpha != null && reattachInfo?.Bravo != null;
var dmbToUse = doesntNeedNewDmb ? null : DmbFactory.LockNextDmb(2);
// if this try catches something, both servers are killed
try
{
// start the alpha server task, either by launch a new process or attaching to an existing one
// The tasks returned are mainly for writing interop files to the directories among other things and should generally never fail
// The tasks pertaining to server startup times are in the ISessionControllers
Task <ISessionController> alphaServerTask;
if (!doesntNeedNewDmb)
{
alphaServerTask = SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, true, true, false, cancellationToken);
}
else
{
alphaServerTask = SessionControllerFactory.Reattach(reattachInfo.Alpha, cancellationToken);
}
// retrieve the session controller
var startTime = DateTimeOffset.Now;
alphaServer = await alphaServerTask.ConfigureAwait(false);
// failed reattaches will return null
alphaServer?.SetHighPriority();
// extra delay for total ordering
var now = DateTimeOffset.Now;
var delay = now - startTime;
// definitely not if reattaching though
if (reattachInfo == null && delay.TotalSeconds < AlphaBravoStartupSeperationInterval)
{
await AsyncDelayer.Delay(startTime.AddSeconds(AlphaBravoStartupSeperationInterval) - now, cancellationToken).ConfigureAwait(false);
}
// now bring bravo up
if (!doesntNeedNewDmb)
{
bravoServer = await SessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, false, false, false, cancellationToken).ConfigureAwait(false);
}
else
{
bravoServer = await SessionControllerFactory.Reattach(reattachInfo.Bravo, cancellationToken).ConfigureAwait(false);
}
// failed reattaches will return null
bravoServer?.SetHighPriority();
// possiblity of null servers due to failed reattaches
if (alphaServer == null || bravoServer == null)
{
await chatTask.ConfigureAwait(false);
var bothServersDead = alphaServer == null && bravoServer == null;
if (bothServersDead ||
(alphaServer == null && reattachInfo.AlphaIsActive) ||
(bravoServer == null && !reattachInfo.AlphaIsActive))
{
// we lost the active server, just restart entirely
DisposeAndNullControllers();
const string FailReattachMessage = "Unable to properly reattach to active server! Restarting...";
Logger.LogWarning(FailReattachMessage);
Logger.LogDebug(bothServersDead ? "Also could not reattach to inactive server!" : "Inactive server was reattached successfully!");
chatTask = Chat.SendWatchdogMessage(FailReattachMessage, cancellationToken);
callBeforeRecurse();
await LaunchImplNoLock(true, false, null, cancellationToken).ConfigureAwait(false);
await chatTask.ConfigureAwait(false);
return;
}
// we still have the active server but the other one is dead to us, hand it off to the monitor to restart
const string InactiveReattachFailureMessage = "Unable to reattach to inactive server. Leaving for monitor to reboot...";
chatTask = Chat.SendWatchdogMessage(InactiveReattachFailureMessage, cancellationToken);
Logger.LogWarning(InactiveReattachFailureMessage);
if (reattachInfo.AlphaIsActive)
{
bravoServer = SessionControllerFactory.CreateDeadSession(reattachInfo.Bravo.Dmb);
}
else
{
alphaServer = SessionControllerFactory.CreateDeadSession(reattachInfo.Alpha.Dmb);
}
}
var alphaLrt = CheckLaunchResult(alphaServer, "Alpha", cancellationToken);
var bravoLrt = CheckLaunchResult(bravoServer, "Bravo", cancellationToken);
// this task completes when both serers have finished booting
var allTask = Task.WhenAll(alphaLrt, bravoLrt);
await allTask.ConfigureAwait(false);
// both servers are now running, alpha is the active server(unless reattach), huzzah
alphaIsActive = reattachInfo?.AlphaIsActive ?? true;
var activeServer = AlphaIsActive ? alphaServer : bravoServer;
activeServer.EnableCustomChatCommands();
activeServer.ClosePortOnReboot = true;
}
catch
{
if (dmbToUse != null)
{
// we locked 2 dmbs
if (bravoServer == null)
{
// bravo didn't get control of his
dmbToUse.Dispose();
if (alphaServer == null)
{
dmbToUse.Dispose(); // alpha didn't get control of his
}
}
}
else if (doesntNeedNewDmb) // we have reattachInfo
{
if (bravoServer == null)
{
// bravo didn't get control of his
reattachInfo.Bravo?.Dmb.Dispose();
if (alphaServer == null)
{
reattachInfo.Alpha?.Dmb.Dispose(); // alpha didn't get control of his
}
}
}
// kill the controllers
DisposeAndNullControllers();
throw;
}
}
public GameModel(ISessionController <LogicGame> logicgame)
{
this.logic = logicgame.LoadOrCreate(LogicGameKey.Key);
this.logicgame = logicgame;
refreshOptionsList();
}
async Task <WatchdogLaunchResult> LaunchNoLock(bool startMonitor, bool announce, bool doReattach, CancellationToken cancellationToken)
{
logger.LogTrace("Begin LaunchNoLock");
using (var alphaStartCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
if (Running)
{
logger.LogTrace("Aborted due to already running!");
return(null);
}
Task chatTask;
//this is necessary, the monitor could be in it's sleep loop trying to restart
if (startMonitor && await StopMonitor().ConfigureAwait(false))
{
chatTask = chat.SendWatchdogMessage("Automatic retry sequence cancelled by manual launch. Restarting...", cancellationToken);
}
else if (announce)
{
chatTask = chat.SendWatchdogMessage("Starting...", cancellationToken);
}
else
{
chatTask = Task.CompletedTask;
}
//start both servers
LastLaunchParameters = ActiveLaunchParameters;
try
{
//good ole sanity
if (alphaServer != null || bravoServer != null)
{
throw new InvalidOperationException("Entered LaunchNoLock with one or more of the servers not being null!");
}
var reattachInfo = doReattach ? await reattachInfoHandler.Load(cancellationToken).ConfigureAwait(false) : null;
var doesntNeedNewDmb = doReattach && reattachInfo.Alpha != null && reattachInfo.Bravo != null;
var dmbToUse = doesntNeedNewDmb ? null : dmbFactory.LockNextDmb(2);
try
{
Task <ISessionController> alphaServerTask;
if (!doesntNeedNewDmb)
{
alphaServerTask = sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, true, true, false, alphaStartCts.Token);
}
else
{
alphaServerTask = sessionControllerFactory.Reattach(reattachInfo.Alpha, cancellationToken);
}
//wait until this boy officially starts so as not to confuse the servers as to who came first
var startTime = DateTimeOffset.Now;
alphaServer = await alphaServerTask.ConfigureAwait(false);
alphaServer.SetHighPriority();
//extra delay for total ordering
var now = DateTimeOffset.Now;
var delay = now - startTime;
if (delay.TotalSeconds < AlphaBravoStartupSeperationInterval)
{
await Task.Delay(startTime.AddSeconds(AlphaBravoStartupSeperationInterval) - now, cancellationToken).ConfigureAwait(false);
}
Task <ISessionController> bravoServerTask;
if (!doesntNeedNewDmb)
{
bravoServerTask = sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbToUse, null, false, false, false, cancellationToken);
}
else
{
bravoServerTask = sessionControllerFactory.Reattach(reattachInfo.Bravo, cancellationToken);
}
bravoServer = await bravoServerTask.ConfigureAwait(false);
bravoServer.SetHighPriority();
async Task <LaunchResult> CheckLaunch(ISessionController controller, string serverName)
{
var launch = await controller.LaunchResult.ConfigureAwait(false);
if (launch.ExitCode.HasValue)
{
//you killed us ray...
throw new JobException(String.Format(CultureInfo.InvariantCulture, "{1} server failed to start: {0}", launch.ToString(), serverName));
}
if (!launch.StartupTime.HasValue)
{
throw new JobException(String.Format(CultureInfo.InvariantCulture, "{1} server timed out on startup: {0}s", launch.ToString(), ActiveLaunchParameters.StartupTimeout.Value));
}
return(launch);
}
var alphaLrt = CheckLaunch(alphaServer, "Alpha");
var bravoLrt = CheckLaunch(bravoServer, "Bravo");
//now we have two booting servers, get them up and running
var allTask = Task.WhenAll(alphaLrt, bravoLrt);
//don't forget about the cancelationToken
var cancelTcs = new TaskCompletionSource <object>();
using (cancellationToken.Register(() => cancelTcs.SetCanceled()))
await Task.WhenAny(allTask, cancelTcs.Task).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
//both servers are now running, alpha is the active server, huzzah
AlphaIsActive = doReattach ? reattachInfo.AlphaIsActive : true;
LastLaunchResult = alphaLrt.Result;
logger.LogInformation("Launched servers successfully");
Running = true;
if (startMonitor)
{
await StopMonitor().ConfigureAwait(false);
monitorCts = new CancellationTokenSource();
monitorTask = MonitorLifetimes(monitorCts.Token);
}
return(new WatchdogLaunchResult
{
Alpha = alphaLrt.Result,
Bravo = bravoLrt.Result
});
}
catch
{
if (!doesntNeedNewDmb && (alphaServer == null && bravoServer == null))
{
dmbToUse.Dispose(); //guaranteed to not be null here
dmbToUse.Dispose(); //yes, dispose it twice. See the definition of IDmbFactory.LockNextDmb(), we called it with 2 locks
}
DisposeAndNullControllers();
throw;
}
}
catch (Exception e)
{
logger.LogWarning("Failed to start watchdog: {0}", e.ToString());
throw;
}
finally
{
try
{
await chatTask.ConfigureAwait(false);
}
catch (OperationCanceledException) { }
}
}
}
/// <summary>
/// The loop that watches the watchdog
/// </summary>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task MonitorLifetimes(CancellationToken cancellationToken)
{
logger.LogTrace("Entered MonitorLifetimes");
var iteration = 1;
for (var monitorState = new MonitorState(); monitorState.NextAction != MonitorAction.Exit; ++iteration)
{
monitorState.NextAction = MonitorAction.Continue;
logger.LogDebug("Iteration {0} of monitor loop", iteration);
try
{
if (AlphaIsActive)
{
logger.LogDebug("Alpha is the active server");
}
else
{
logger.LogDebug("Bravo is the active server");
}
if (monitorState.InactiveServerHasStagedDmb)
{
logger.LogDebug("Inactive server has staged .dmb");
}
if (monitorState.RebootingInactiveServer)
{
logger.LogDebug("Inactive server is rebooting");
}
monitorState.ActiveServer = AlphaIsActive ? alphaServer : bravoServer;
monitorState.InactiveServer = AlphaIsActive ? bravoServer : alphaServer;
if (monitorState.ActiveServer.ClosePortOnReboot)
{
logger.LogDebug("Active server will close port on reboot");
}
if (monitorState.InactiveServer.ClosePortOnReboot)
{
logger.LogDebug("Inactive server will close port on reboot");
}
var activeServerLifetime = monitorState.ActiveServer.Lifetime;
var inactiveServerLifetime = monitorState.InactiveServer.Lifetime;
var activeServerReboot = monitorState.ActiveServer.OnReboot;
var inactiveServerReboot = monitorState.InactiveServer.OnReboot;
var inactiveServerStartup = monitorState.RebootingInactiveServer ? monitorState.InactiveServer.LaunchResult : null;
var activeLaunchParametersChanged = activeParametersUpdated.Task;
var newDmbAvailable = dmbFactory.OnNewerDmb;
var cancelTcs = new TaskCompletionSource <object>();
using (cancellationToken.Register(() => cancelTcs.SetCanceled()))
{
var toWaitOn = Task.WhenAny(activeServerLifetime, inactiveServerLifetime, activeServerReboot, inactiveServerReboot, newDmbAvailable, cancelTcs.Task, activeLaunchParametersChanged);
if (monitorState.RebootingInactiveServer)
{
toWaitOn = Task.WhenAny(toWaitOn, inactiveServerStartup);
}
await toWaitOn.ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
}
var chatTask = Task.CompletedTask;
using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
{
MonitorActivationReason activationReason = default;
//multiple things may have happened, handle them one at a time
for (var moreActivationsToProcess = true; moreActivationsToProcess && monitorState.NextAction == MonitorAction.Continue;)
{
if (activeServerLifetime?.IsCompleted == true)
{
activationReason = MonitorActivationReason.ActiveServerCrashed;
activeServerLifetime = null;
}
else if (inactiveServerLifetime?.IsCompleted == true)
{
activationReason = MonitorActivationReason.InactiveServerCrashed;
inactiveServerLifetime = null;
}
else if (activeServerReboot?.IsCompleted == true)
{
activationReason = MonitorActivationReason.ActiveServerRebooted;
activeServerReboot = null;
}
else if (inactiveServerReboot?.IsCompleted == true)
{
activationReason = MonitorActivationReason.InactiveServerRebooted;
inactiveServerReboot = null;
}
else if (inactiveServerStartup?.IsCompleted == true)
{
activationReason = MonitorActivationReason.InactiveServerStartupComplete;
inactiveServerStartup = null;
}
else if (newDmbAvailable?.IsCompleted == true)
{
activationReason = MonitorActivationReason.NewDmbAvailable;
newDmbAvailable = null;
}
else if (activeLaunchParametersChanged?.IsCompleted == true)
{
activationReason = MonitorActivationReason.ActiveLaunchParametersUpdated;
activeLaunchParametersChanged = null;
}
else
{
moreActivationsToProcess = false;
}
if (moreActivationsToProcess)
{
await HandlerMonitorWakeup(activationReason, monitorState, cancellationToken).ConfigureAwait(false);
}
}
//writeback alphaServer and bravoServer
alphaServer = AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer;
bravoServer = !AlphaIsActive ? monitorState.ActiveServer : monitorState.InactiveServer;
}
//full reboot required
if (monitorState.NextAction == MonitorAction.Restart)
{
logger.LogDebug("Next state action is to restart");
DisposeAndNullControllers();
chatTask = chat.SendWatchdogMessage("Restarting entirely due to complications...", cancellationToken);
for (var retryAttempts = 1; monitorState.NextAction == MonitorAction.Restart; ++retryAttempts)
{
WatchdogLaunchResult result;
using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
{
result = await LaunchNoLock(false, false, false, cancellationToken).ConfigureAwait(false);
if (Running)
{
logger.LogDebug("Relaunch successful, resetting monitor state...");
monitorState = new MonitorState(); //clean the slate
}
}
await chatTask.ConfigureAwait(false);
if (!Running)
{
logger.LogWarning("Failed to automatically restart the watchdog! Alpha: {0}; Bravo: {1}", result.Alpha.ToString(), result.Bravo.ToString());
var retryDelay = Math.Min(Math.Pow(2, retryAttempts), 3600); //max of one hour
chatTask = chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Failed to restart watchdog (Attempt: {0}), retrying in {1} seconds...", retryAttempts, retryDelay), cancellationToken);
await Task.WhenAll(Task.Delay((int)retryDelay, cancellationToken), chatTask).ConfigureAwait(false);
}
}
}
}
catch (OperationCanceledException)
{
logger.LogDebug("Monitor cancelled");
break;
}
catch (Exception e)
{
logger.LogError("Monitor crashed! Iteration: {0}, State: {1}", iteration, JsonConvert.SerializeObject(monitorState));
await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Monitor crashed, this should NEVER happen! Please report this, full details in logs! Restarting monitor... Error: {0}", e.Message), cancellationToken).ConfigureAwait(false);
}
}
}
/// <summary>
/// Initialises a new instance of the <see cref="CurrentUser"/> class.
/// </summary>
/// <param name="log">Logging module</param>
/// <param name="sessionController">Session controller</param>
public CurrentUser(ILogger log, ISessionController sessionController)
{
this.log = log;
this.sessionController = sessionController;
}
/// <inheritdoc />
protected override void DisposeAndNullControllers()
{
Server?.Dispose();
Server = null;
Running = false;
}
public SessionDispatcher(ISessionController sessionController)
{
m_sessionController = sessionController;
}
public IndexModel(ISessionController <LogicGame> logicgame)
{
this.logic = logicgame.LoadOrCreate(LogicGameKey.Key);
this.logicgame = logicgame;
}
static string ExitWord(ISessionController controller) => controller.TerminationWasRequested ? "exited" : "crashed";
