/// <summary>
/// Run a quick DD instance to test the DMAPI is installed on the target code
/// </summary>
/// <param name="timeout">The timeout in seconds for validation</param>
/// <param name="securityLevel">The <see cref="DreamDaemonSecurity"/> level to use to validate the API</param>
/// <param name="job">The <see cref="Models.CompileJob"/> for the operation</param>
/// <param name="byondLock">The current <see cref="IByondExecutableLock"/></param>
/// <param name="portToUse">The port to use for API validation</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task{TResult}"/> resulting in <see langword="true"/> if the DMAPI was successfully validated, <see langword="false"/> otherwise</returns>
async Task <bool> VerifyApi(uint timeout, DreamDaemonSecurity securityLevel, Models.CompileJob job, IByondExecutableLock byondLock, ushort portToUse, CancellationToken cancellationToken)
{
logger.LogTrace("Verifying DMAPI...");
var launchParameters = new DreamDaemonLaunchParameters
{
AllowWebClient = false,
PrimaryPort = portToUse,
SecurityLevel = securityLevel, //all it needs to read the file and exit
StartupTimeout = timeout
};
var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName);
var provider = new TemporaryDmbProvider(ioManager.ResolvePath(dirA), String.Concat(job.DmeName, DmbExtension), job);
var timeoutAt = DateTimeOffset.Now.AddSeconds(timeout);
using (var controller = await sessionControllerFactory.LaunchNew(launchParameters, provider, byondLock, true, true, true, cancellationToken).ConfigureAwait(false))
{
var launchResult = await controller.LaunchResult.ConfigureAwait(false);
var now = DateTimeOffset.Now;
if (now < timeoutAt && launchResult.StartupTime.HasValue)
{
var timeoutTask = Task.Delay(timeoutAt - now, cancellationToken);
await Task.WhenAny(controller.Lifetime, timeoutTask).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
}
if (!controller.Lifetime.IsCompleted)
{
logger.LogDebug("API validation timed out!");
return(false);
}
var validated = controller.ApiValidated;
logger.LogTrace("API valid: {0}", validated);
return(validated);
}
}
/// <summary>
/// Run a quick DD instance to test the DMAPI is installed on the target code
/// </summary>
/// <param name="timeout">The timeout in seconds for validation</param>
/// <param name="securityLevel">The <see cref="DreamDaemonSecurity"/> level to use to validate the API</param>
/// <param name="job">The <see cref="Models.CompileJob"/> for the operation</param>
/// <param name="byondLock">The current <see cref="IByondExecutableLock"/></param>
/// <param name="portToUse">The port to use for API validation</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task VerifyApi(uint timeout, DreamDaemonSecurity securityLevel, Models.CompileJob job, IByondExecutableLock byondLock, ushort portToUse, CancellationToken cancellationToken)
{
logger.LogTrace("Verifying DMAPI...");
var launchParameters = new DreamDaemonLaunchParameters
{
AllowWebClient = false,
PrimaryPort = portToUse,
SecurityLevel = securityLevel,
StartupTimeout = timeout
};
var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName);
job.MinimumSecurityLevel = securityLevel; // needed for the TempDmbProvider
var timeoutAt = DateTimeOffset.Now.AddSeconds(timeout);
using (var provider = new TemporaryDmbProvider(ioManager.ResolvePath(dirA), String.Concat(job.DmeName, DmbExtension), job))
using (var controller = await sessionControllerFactory.LaunchNew(launchParameters, provider, byondLock, true, true, true, cancellationToken).ConfigureAwait(false))
{
var launchResult = await controller.LaunchResult.ConfigureAwait(false);
var now = DateTimeOffset.Now;
if (now < timeoutAt && launchResult.StartupTime.HasValue)
{
var timeoutTask = Task.Delay(timeoutAt - now, cancellationToken);
await Task.WhenAny(controller.Lifetime, timeoutTask).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
}
if (controller.Lifetime.IsCompleted)
{
var validationStatus = controller.ApiValidationStatus;
logger.LogTrace("API validation status: {0}", validationStatus);
switch (validationStatus)
{
case ApiValidationStatus.RequiresUltrasafe:
job.MinimumSecurityLevel = DreamDaemonSecurity.Ultrasafe;
return;
case ApiValidationStatus.RequiresSafe:
if (securityLevel == DreamDaemonSecurity.Ultrasafe)
{
throw new JobException("This game must be run with at least the 'Safe' DreamDaemon security level!");
}
job.MinimumSecurityLevel = DreamDaemonSecurity.Safe;
return;
case ApiValidationStatus.RequiresTrusted:
if (securityLevel != DreamDaemonSecurity.Trusted)
{
throw new JobException("This game must be run with at least the 'Trusted' DreamDaemon security level!");
}
job.MinimumSecurityLevel = DreamDaemonSecurity.Trusted;
return;
case ApiValidationStatus.NeverValidated:
break;
case ApiValidationStatus.BadValidationRequest:
throw new JobException("Recieved an unrecognized API validation request from DreamDaemon!");
case ApiValidationStatus.UnaskedValidationRequest:
default:
throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, "Session controller returned unexpected ApiValidationStatus: {0}", validationStatus));
}
}
throw new JobException("DMAPI validation timed out!");
}
}
/// <summary>
/// Handles the actions to take when the monitor has to "wake up"
/// </summary>
/// <param name="activationReason">The <see cref="MonitorActivationReason"/> that caused the invocation</param>
/// <param name="monitorState">The current <see cref="MonitorState"/>. Will be modified upon retrn</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task HandlerMonitorWakeup(MonitorActivationReason activationReason, MonitorState monitorState, CancellationToken cancellationToken)
{
logger.LogDebug("Monitor activation. Reason: {0}", activationReason);
//returns true if the inactive server can't be used immediately
bool FullRestartDeadInactive()
{
if (monitorState.RebootingInactiveServer || monitorState.InactiveServerCritFail)
{
logger.LogInformation("Inactive server is {0}! Restarting monitor...", monitorState.InactiveServerCritFail ? "critically failed" : "still rebooting");
monitorState.NextAction = MonitorAction.Restart; //will dispose server
return(true);
}
return(false);
};
//trys to set inactive server's port to the public port
//doesn't handle closing active server's port
async Task <bool> MakeInactiveActive()
{
logger.LogDebug("Setting inactive server to port {0}...", ActiveLaunchParameters.PrimaryPort.Value);
var result = await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.PrimaryPort.Value, cancellationToken).ConfigureAwait(false);
if (!result)
{
logger.LogWarning("Failed to activate inactive server! Restarting monitor...");
monitorState.NextAction = MonitorAction.Restart; //will dispose server
return(false);
}
//inactive server should always be using active launch parameters
LastLaunchParameters = ActiveLaunchParameters;
var tmp = monitorState.ActiveServer;
monitorState.ActiveServer = monitorState.InactiveServer;
monitorState.InactiveServer = tmp;
AlphaIsActive = !AlphaIsActive;
return(true);
}
// Tries to load inactive server with latest dmb, falling back to current dmb on failure. Requires a lock on <see cref="semaphore"/>
async Task <bool> RestartInactiveServer()
{
logger.LogInformation("Rebooting inactive server...");
var newDmb = dmbFactory.LockNextDmb(1);
bool usedMostRecentDmb;
try
{
monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, newDmb, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false);
usedMostRecentDmb = true;
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
logger.LogError("Error occurred while recreating server! Attempting backup strategy of running DMB of running server! Exception: {0}", e.ToString());
//ahh jeez, what do we do here?
//this is our fault, so it should never happen but
//idk maybe a database error while handling the newest dmb?
//either way try to start it using the active server's dmb as a backup
try
{
var dmbBackup = await dmbFactory.FromCompileJob(monitorState.ActiveServer.Dmb.CompileJob, cancellationToken).ConfigureAwait(false);
if (dmbBackup == null) //NANI!?
//just give up, if THAT compile job is failing then the ActiveServer is gonna crash soon too or already has
{
throw new JobException("Creating backup DMB provider failed!");
}
monitorState.InactiveServer = await sessionControllerFactory.LaunchNew(ActiveLaunchParameters, dmbBackup, null, false, !monitorState.ActiveServer.IsPrimary, false, cancellationToken).ConfigureAwait(false);
usedMostRecentDmb = false;
await chat.SendWatchdogMessage("Staging newest DMB on inactive server failed: {0} Falling back to previous dmb...", cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e2)
{
//fuuuuucckkk
logger.LogError("Backup strategy failed! Monitor will restart when active server reboots! Exception: {0}", e2.ToString());
monitorState.InactiveServerCritFail = true;
await chat.SendWatchdogMessage("Attempted reboot of inactive server failed. Watchdog will reset when active server fails or exits", cancellationToken).ConfigureAwait(false);
return(true); //we didn't use the old dmb
}
}
logger.LogInformation("Successfully relaunched inactive server!");
monitorState.RebootingInactiveServer = true;
return(usedMostRecentDmb);
}
async Task UpdateAndRestartInactiveServer(bool breakAfter)
{
//replace the notification tcs here so that the next loop will read a fresh one
activeParametersUpdated = new TaskCompletionSource <object>();
monitorState.InactiveServer.Dispose(); //kill or recycle it
monitorState.NextAction = breakAfter ? MonitorAction.Break : MonitorAction.Continue;
var usedLatestDmb = await RestartInactiveServer().ConfigureAwait(false);
if (monitorState.NextAction == (breakAfter ? MonitorAction.Break : MonitorAction.Continue))
{
monitorState.ActiveServer.ClosePortOnReboot = false;
if (monitorState.InactiveServerHasStagedDmb && !usedLatestDmb)
{
monitorState.InactiveServerHasStagedDmb = false; //don't try to load it again though
}
}
};
string ExitWord(ISessionController controller) => controller.TerminationWasRequested ? "exited" : "crashed";
//reason handling
switch (activationReason)
{
case MonitorActivationReason.ActiveServerCrashed:
if (monitorState.ActiveServer.RebootState == Components.Watchdog.RebootState.Shutdown)
{
await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Exiting due to graceful termination request...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false);
monitorState.NextAction = MonitorAction.Exit;
break;
}
if (FullRestartDeadInactive())
{
await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Inactive server unable to online!", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false);
break;
}
await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Active server {0}! Onlining inactive server...", ExitWord(monitorState.ActiveServer)), cancellationToken).ConfigureAwait(false);
if (!await MakeInactiveActive().ConfigureAwait(false))
{
break;
}
await UpdateAndRestartInactiveServer(true).ConfigureAwait(false);
break;
case MonitorActivationReason.InactiveServerCrashed:
await chat.SendWatchdogMessage(String.Format(CultureInfo.InvariantCulture, "Inactive server {0}! Rebooting...", ExitWord(monitorState.InactiveServer)), cancellationToken).ConfigureAwait(false);
await UpdateAndRestartInactiveServer(false).ConfigureAwait(false);
break;
case MonitorActivationReason.ActiveServerRebooted:
if (FullRestartDeadInactive())
{
break;
}
//what matters here is the RebootState
bool restartOnceSwapped = false;
var rebootState = monitorState.ActiveServer.RebootState;
monitorState.ActiveServer.ResetRebootState(); //the DMAPI has already done this internally
switch (rebootState)
{
case Components.Watchdog.RebootState.Normal:
break;
case Components.Watchdog.RebootState.Restart:
restartOnceSwapped = true;
break;
case Components.Watchdog.RebootState.Shutdown:
await chat.SendWatchdogMessage("Active server rebooted! Exiting due to graceful termination request...", cancellationToken).ConfigureAwait(false);
DisposeAndNullControllers();
monitorState.NextAction = MonitorAction.Exit;
return;
}
var sameCompileJob = monitorState.InactiveServer.Dmb.CompileJob.Id == monitorState.ActiveServer.Dmb.CompileJob.Id;
if (sameCompileJob && monitorState.InactiveServerHasStagedDmb)
{
//both servers up to date
monitorState.InactiveServerHasStagedDmb = false;
}
if (!sameCompileJob || ActiveLaunchParameters != LastLaunchParameters)
{
//need a new launch in ActiveServer
restartOnceSwapped = true;
}
if (restartOnceSwapped && !monitorState.ActiveServer.ClosePortOnReboot)
{
//we need to manually restart active server
//it won't listen to us right now so just kill it
monitorState.ActiveServer.Dispose();
}
if ((!restartOnceSwapped && !monitorState.ActiveServer.ClosePortOnReboot) || !await MakeInactiveActive().ConfigureAwait(false))
{
break;
}
monitorState.ActiveServer.ClosePortOnReboot = true;
if (!restartOnceSwapped)
{
monitorState.InactiveServer.ClosePortOnReboot = false;
//try to reopen inactive server on the private port so it's not pinging all the time
//failing that, just reboot it
restartOnceSwapped = !await monitorState.InactiveServer.SetPort(ActiveLaunchParameters.SecondaryPort.Value, cancellationToken).ConfigureAwait(false);
}
if (restartOnceSwapped) //for one reason or another
{
await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); //break because worse case, active server is still booting
}
else
{
monitorState.InactiveServer.ClosePortOnReboot = false;
monitorState.NextAction = MonitorAction.Break;
}
break;
case MonitorActivationReason.InactiveServerRebooted:
monitorState.RebootingInactiveServer = true;
monitorState.InactiveServer.ResetRebootState();
monitorState.ActiveServer.ClosePortOnReboot = false;
monitorState.NextAction = MonitorAction.Continue;
break;
case MonitorActivationReason.InactiveServerStartupComplete:
//eziest case of my life
monitorState.RebootingInactiveServer = false;
monitorState.ActiveServer.ClosePortOnReboot = true;
monitorState.NextAction = MonitorAction.Continue;
break;
case MonitorActivationReason.NewDmbAvailable:
monitorState.InactiveServerHasStagedDmb = true;
await UpdateAndRestartInactiveServer(true).ConfigureAwait(false); //next case does same thing
break;
case MonitorActivationReason.ActiveLaunchParametersUpdated:
await UpdateAndRestartInactiveServer(false).ConfigureAwait(false);
break;
}
}
/// <summary>
/// Run a quick DD instance to test the DMAPI is installed on the target code
/// </summary>
/// <param name="timeout">The timeout in seconds for validation</param>
/// <param name="securityLevel">The <see cref="DreamDaemonSecurity"/> level to use to validate the API</param>
/// <param name="job">The <see cref="CompileJob"/> for the operation</param>
/// <param name="byondLock">The current <see cref="IByondExecutableLock"/></param>
/// <param name="portToUse">The port to use for API validation</param>
/// <param name="requireValidate">If the API validation is required to complete the deployment.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task VerifyApi(
uint timeout,
DreamDaemonSecurity securityLevel,
Models.CompileJob job,
IByondExecutableLock byondLock,
ushort portToUse,
bool requireValidate,
CancellationToken cancellationToken)
{
logger.LogTrace("Verifying {0}DMAPI...", requireValidate ? "required " : String.Empty);
var launchParameters = new DreamDaemonLaunchParameters
{
AllowWebClient = false,
Port = portToUse,
SecurityLevel = securityLevel,
StartupTimeout = timeout,
TopicRequestTimeout = 0, // not used
HeartbeatSeconds = 0 // not used
};
job.MinimumSecurityLevel = securityLevel; // needed for the TempDmbProvider
ApiValidationStatus validationStatus;
using (var provider = new TemporaryDmbProvider(ioManager.ResolvePath(job.DirectoryName.ToString()), String.Concat(job.DmeName, DmbExtension), job))
await using (var controller = await sessionControllerFactory.LaunchNew(provider, byondLock, launchParameters, true, cancellationToken).ConfigureAwait(false))
{
var launchResult = await controller.LaunchResult.ConfigureAwait(false);
if (launchResult.StartupTime.HasValue)
{
await controller.Lifetime.WithToken(cancellationToken).ConfigureAwait(false);
}
if (!controller.Lifetime.IsCompleted)
{
await controller.DisposeAsync().ConfigureAwait(false);
}
validationStatus = controller.ApiValidationStatus;
if (requireValidate && validationStatus == ApiValidationStatus.NeverValidated)
{
throw new JobException(ErrorCode.DreamMakerNeverValidated);
}
logger.LogTrace("API validation status: {0}", validationStatus);
job.DMApiVersion = controller.DMApiVersion;
}
switch (validationStatus)
{
case ApiValidationStatus.RequiresUltrasafe:
job.MinimumSecurityLevel = DreamDaemonSecurity.Ultrasafe;
return;
case ApiValidationStatus.RequiresSafe:
job.MinimumSecurityLevel = DreamDaemonSecurity.Safe;
return;
case ApiValidationStatus.RequiresTrusted:
job.MinimumSecurityLevel = DreamDaemonSecurity.Trusted;
return;
case ApiValidationStatus.NeverValidated:
if (requireValidate)
{
throw new JobException(ErrorCode.DreamMakerNeverValidated);
}
job.MinimumSecurityLevel = DreamDaemonSecurity.Ultrasafe;
break;
case ApiValidationStatus.BadValidationRequest:
case ApiValidationStatus.Incompatible:
throw new JobException(ErrorCode.DreamMakerInvalidValidation);
case ApiValidationStatus.UnaskedValidationRequest:
default:
throw new InvalidOperationException(
$"Session controller returned unexpected ApiValidationStatus: {validationStatus}");
}
}