/// <summary>
/// Cleans up a failed compile <paramref name="job"/>.
/// </summary>
/// <param name="job">The running <see cref="CompileJob"/>.</param>
/// <param name="remoteDeploymentManager">The <see cref="IRemoteDeploymentManager"/> associated with the <paramref name="job"/>.</param>
/// <param name="exception">The <see cref="Exception"/> that was thrown.</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task CleanupFailedCompile(Models.CompileJob job, IRemoteDeploymentManager remoteDeploymentManager, Exception exception)
{
async Task CleanDir()
{
logger.LogTrace("Cleaning compile directory...");
var jobPath = job.DirectoryName.ToString();
try
{
// DCT: None available
await ioManager.DeleteDirectory(jobPath, default).ConfigureAwait(false);
}
catch (Exception e)
{
logger.LogWarning(e, "Error cleaning up compile directory {0}!", ioManager.ResolvePath(jobPath));
}
}
// DCT: None available
await Task.WhenAll(
CleanDir(),
remoteDeploymentManager.FailDeployment(
job,
FormatExceptionForUsers(exception),
default))
.ConfigureAwait(false);
}
/// <summary>
/// Adds server side includes to the .dme being compiled
/// </summary>
/// <param name="job">The <see cref="Models.CompileJob"/> for the operation</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task ModifyDme(Models.CompileJob job, CancellationToken cancellationToken)
{
var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName);
var dmeFileName = String.Join('.', job.DmeName, DmeExtension);
var dmePath = ioManager.ConcatPath(dirA, dmeFileName);
var dmeReadTask = ioManager.ReadAllBytes(dmePath, cancellationToken);
var dmeModificationsTask = configuration.CopyDMFilesTo(dmeFileName, ioManager.ResolvePath(dirA), cancellationToken);
var dmeBytes = await dmeReadTask.ConfigureAwait(false);
var dme = Encoding.UTF8.GetString(dmeBytes);
var dmeModifications = await dmeModificationsTask.ConfigureAwait(false);
if (dmeModifications == null || dmeModifications.TotalDmeOverwrite)
{
if (dmeModifications != null)
{
logger.LogDebug(".dme replacement configured!");
}
else
{
logger.LogTrace("No .dme modifications required.");
}
return;
}
if (dmeModifications.HeadIncludeLine != null)
{
logger.LogDebug("Head .dme include line: {0}", dmeModifications.HeadIncludeLine);
}
if (dmeModifications.TailIncludeLine != null)
{
logger.LogDebug("Tail .dme include line: {0}", dmeModifications.TailIncludeLine);
}
var dmeLines = new List <string>(dme.Split(new[] { Environment.NewLine }, StringSplitOptions.None));
for (var I = 0; I < dmeLines.Count; ++I)
{
var line = dmeLines[I];
if (line.Contains("BEGIN_INCLUDE", StringComparison.Ordinal) && dmeModifications.HeadIncludeLine != null)
{
dmeLines.Insert(I + 1, dmeModifications.HeadIncludeLine);
++I;
}
else if (line.Contains("END_INCLUDE", StringComparison.Ordinal) && dmeModifications.TailIncludeLine != null)
{
dmeLines.Insert(I, dmeModifications.TailIncludeLine);
break;
}
}
dmeBytes = Encoding.UTF8.GetBytes(String.Join(Environment.NewLine, dmeLines));
await ioManager.WriteAllBytes(dmePath, dmeBytes, cancellationToken).ConfigureAwait(false);
}
/// <summary>
/// Cleans up a failed compile <paramref name="job"/>
/// </summary>
/// <param name="job">The running <see cref="CompileJob"/></param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task CleanupFailedCompile(Models.CompileJob job)
{
logger.LogTrace("Cleaning compile directory...");
var jobPath = job.DirectoryName.ToString();
try
{
await ioManager.DeleteDirectory(jobPath, CancellationToken.None).ConfigureAwait(false);
}
catch (Exception e)
{
logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(jobPath), e);
}
}
/// <summary>
/// Compiles a .dme with DreamMaker
/// </summary>
/// <param name="dreamMakerPath">The path to the DreamMaker executable</param>
/// <param name="job">The <see cref="Models.CompileJob"/> for the operation</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task <int> RunDreamMaker(string dreamMakerPath, Models.CompileJob job, CancellationToken cancellationToken)
{
using (var dm = processExecutor.LaunchProcess(dreamMakerPath, ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)), String.Format(CultureInfo.InvariantCulture, "-clean {0}.{1}", job.DmeName, DmeExtension), true, true))
{
int exitCode;
using (cancellationToken.Register(() => dm.Terminate()))
exitCode = await dm.Lifetime.ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
logger.LogDebug("DreamMaker exit code: {0}", exitCode);
job.Output = dm.GetCombinedOutput();
logger.LogTrace("DreamMaker output: {0}{1}", Environment.NewLine, job.Output);
return(exitCode);
}
}
/// <summary>
/// Cleans up a failed compile <paramref name="job"/>
/// </summary>
/// <param name="job">The running <see cref="Models.CompileJob"/></param>
/// <param name="cancelled">If the <paramref name="job"/> was cancelled</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task CleanupFailedCompile(Models.CompileJob job, bool cancelled, CancellationToken cancellationToken)
{
logger.LogTrace("Cleaning compile directory...");
var chatTask = chat.SendUpdateMessage(cancelled ? "Deploy cancelled!" : "Deploy failed!", cancellationToken);
var jobPath = job.DirectoryName.ToString();
try
{
await ioManager.DeleteDirectory(jobPath, CancellationToken.None).ConfigureAwait(false);
}
catch (Exception e)
{
logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(jobPath), e);
}
await chatTask.ConfigureAwait(false);
}
/// <summary>
/// Run a quick DD instance to test the DMAPI is installed on the target code
/// </summary>
/// <param name="timeout">The timeout in seconds for validation</param>
/// <param name="securityLevel">The <see cref="DreamDaemonSecurity"/> level to use to validate the API</param>
/// <param name="job">The <see cref="Models.CompileJob"/> for the operation</param>
/// <param name="byondLock">The current <see cref="IByondExecutableLock"/></param>
/// <param name="portToUse">The port to use for API validation</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task{TResult}"/> resulting in <see langword="true"/> if the DMAPI was successfully validated, <see langword="false"/> otherwise</returns>
async Task <bool> VerifyApi(uint timeout, DreamDaemonSecurity securityLevel, Models.CompileJob job, IByondExecutableLock byondLock, ushort portToUse, CancellationToken cancellationToken)
{
logger.LogTrace("Verifying DMAPI...");
var launchParameters = new DreamDaemonLaunchParameters
{
AllowWebClient = false,
PrimaryPort = portToUse,
SecurityLevel = securityLevel, //all it needs to read the file and exit
StartupTimeout = timeout
};
var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName);
var provider = new TemporaryDmbProvider(ioManager.ResolvePath(dirA), String.Concat(job.DmeName, DmbExtension), job);
var timeoutAt = DateTimeOffset.Now.AddSeconds(timeout);
using (var controller = await sessionControllerFactory.LaunchNew(launchParameters, provider, byondLock, true, true, true, cancellationToken).ConfigureAwait(false))
{
var launchResult = await controller.LaunchResult.ConfigureAwait(false);
var now = DateTimeOffset.Now;
if (now < timeoutAt && launchResult.StartupTime.HasValue)
{
var timeoutTask = Task.Delay(timeoutAt - now, cancellationToken);
await Task.WhenAny(controller.Lifetime, timeoutTask).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
}
if (!controller.Lifetime.IsCompleted)
{
logger.LogDebug("API validation timed out!");
return(false);
}
var validated = controller.ApiValidated;
logger.LogTrace("API valid: {0}", validated);
return(validated);
}
}
/// <summary>
/// Compiles a .dme with DreamMaker
/// </summary>
/// <param name="dreamMakerPath">The path to the DreamMaker executable</param>
/// <param name="job">The <see cref="CompileJob"/> for the operation</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task <int> RunDreamMaker(string dreamMakerPath, Models.CompileJob job, CancellationToken cancellationToken)
{
using var dm = processExecutor.LaunchProcess(
dreamMakerPath,
ioManager.ResolvePath(
job.DirectoryName.ToString()),
$"-clean {job.DmeName}.{DmeExtension}",
true,
true,
true);
int exitCode;
using (cancellationToken.Register(() => dm.Terminate()))
exitCode = await dm.Lifetime.ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
logger.LogDebug("DreamMaker exit code: {0}", exitCode);
job.Output = await dm.GetCombinedOutput(cancellationToken).ConfigureAwait(false);
currentDreamMakerOutput = job.Output;
logger.LogDebug("DreamMaker output: {0}{1}", Environment.NewLine, job.Output);
return(exitCode);
}
/// <inheritdoc />
#pragma warning disable CA1506
public async Task DeploymentProcess(
Models.Job job,
IDatabaseContextFactory databaseContextFactory,
Action <int> progressReporter,
CancellationToken cancellationToken)
{
if (job == null)
{
throw new ArgumentNullException(nameof(job));
}
if (databaseContextFactory == null)
{
throw new ArgumentNullException(nameof(databaseContextFactory));
}
if (progressReporter == null)
{
throw new ArgumentNullException(nameof(progressReporter));
}
lock (deploymentLock)
{
if (deploying)
{
throw new JobException(ErrorCode.DreamMakerCompileJobInProgress);
}
deploying = true;
}
currentChatCallback = null;
currentDreamMakerOutput = null;
Models.CompileJob compileJob = null;
try
{
string repoOwner = null;
string repoName = null;
TimeSpan?averageSpan = null;
Models.RepositorySettings repositorySettings = null;
Models.DreamDaemonSettings ddSettings = null;
Models.DreamMakerSettings dreamMakerSettings = null;
IRepository repo = null;
IRemoteDeploymentManager remoteDeploymentManager = null;
Models.RevisionInformation revInfo = null;
await databaseContextFactory.UseContext(
async databaseContext =>
{
averageSpan = await CalculateExpectedDeploymentTime(databaseContext, cancellationToken).ConfigureAwait(false);
ddSettings = await databaseContext
.DreamDaemonSettings
.AsQueryable()
.Where(x => x.InstanceId == metadata.Id)
.Select(x => new Models.DreamDaemonSettings
{
StartupTimeout = x.StartupTimeout,
})
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
if (ddSettings == default)
{
throw new JobException(ErrorCode.InstanceMissingDreamDaemonSettings);
}
dreamMakerSettings = await databaseContext
.DreamMakerSettings
.AsQueryable()
.Where(x => x.InstanceId == metadata.Id)
.FirstAsync(cancellationToken)
.ConfigureAwait(false);
if (dreamMakerSettings == default)
{
throw new JobException(ErrorCode.InstanceMissingDreamMakerSettings);
}
repositorySettings = await databaseContext
.RepositorySettings
.AsQueryable()
.Where(x => x.InstanceId == metadata.Id)
.Select(x => new Models.RepositorySettings
{
AccessToken = x.AccessToken,
ShowTestMergeCommitters = x.ShowTestMergeCommitters,
PushTestMergeCommits = x.PushTestMergeCommits,
PostTestMergeComment = x.PostTestMergeComment,
})
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
if (repositorySettings == default)
{
throw new JobException(ErrorCode.InstanceMissingRepositorySettings);
}
repo = await repositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false);
try
{
if (repo == null)
{
throw new JobException(ErrorCode.RepoMissing);
}
remoteDeploymentManager = remoteDeploymentManagerFactory
.CreateRemoteDeploymentManager(metadata, repo.RemoteGitProvider.Value);
var repoSha = repo.Head;
repoOwner = repo.RemoteRepositoryOwner;
repoName = repo.RemoteRepositoryName;
revInfo = await databaseContext
.RevisionInformations
.AsQueryable()
.Where(x => x.CommitSha == repoSha && x.Instance.Id == metadata.Id)
.Include(x => x.ActiveTestMerges)
.ThenInclude(x => x.TestMerge)
.ThenInclude(x => x.MergedBy)
.FirstOrDefaultAsync(cancellationToken)
.ConfigureAwait(false);
if (revInfo == default)
{
revInfo = new Models.RevisionInformation
{
CommitSha = repoSha,
Timestamp = await repo.TimestampCommit(repoSha, cancellationToken).ConfigureAwait(false),
OriginCommitSha = repoSha,
Instance = new Models.Instance
{
Id = metadata.Id,
},
ActiveTestMerges = new List <RevInfoTestMerge>(),
};
logger.LogInformation(Repository.Repository.OriginTrackingErrorTemplate, repoSha);
databaseContext.Instances.Attach(revInfo.Instance);
await databaseContext.Save(cancellationToken).ConfigureAwait(false);
}
}
catch
{
repo?.Dispose();
throw;
}
})
.ConfigureAwait(false);
var likelyPushedTestMergeCommit =
repositorySettings.PushTestMergeCommits.Value &&
repositorySettings.AccessToken != null &&
repositorySettings.AccessUser != null;
using (repo)
compileJob = await Compile(
revInfo,
dreamMakerSettings,
ddSettings.StartupTimeout.Value,
repo,
remoteDeploymentManager,
progressReporter,
averageSpan,
likelyPushedTestMergeCommit,
cancellationToken)
.ConfigureAwait(false);
var activeCompileJob = compileJobConsumer.LatestCompileJob();
try
{
await databaseContextFactory.UseContext(
async databaseContext =>
{
var fullJob = compileJob.Job;
compileJob.Job = new Models.Job
{
Id = job.Id,
};
var fullRevInfo = compileJob.RevisionInformation;
compileJob.RevisionInformation = new Models.RevisionInformation
{
Id = revInfo.Id,
};
databaseContext.Jobs.Attach(compileJob.Job);
databaseContext.RevisionInformations.Attach(compileJob.RevisionInformation);
databaseContext.CompileJobs.Add(compileJob);
// The difficulty with compile jobs is they have a two part commit
await databaseContext.Save(cancellationToken).ConfigureAwait(false);
logger.LogTrace("Created CompileJob {0}", compileJob.Id);
try
{
await compileJobConsumer.LoadCompileJob(compileJob, cancellationToken).ConfigureAwait(false);
}
catch
{
// So we need to un-commit the compile job if the above throws
databaseContext.CompileJobs.Remove(compileJob);
// DCT: Cancellation token is for job, operation must run regardless
await databaseContext.Save(default).ConfigureAwait(false);
throw;
}
/// <inheritdoc />
public async Task <Models.CompileJob> Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action <int> progressReporter, TimeSpan?estimatedDuration, CancellationToken cancellationToken)
{
if (revisionInformation == null)
{
throw new ArgumentNullException(nameof(revisionInformation));
}
if (dreamMakerSettings == null)
{
throw new ArgumentNullException(nameof(dreamMakerSettings));
}
if (repository == null)
{
throw new ArgumentNullException(nameof(repository));
}
if (progressReporter == null)
{
throw new ArgumentNullException(nameof(progressReporter));
}
if (dreamMakerSettings.ApiValidationSecurityLevel == DreamDaemonSecurity.Ultrasafe)
{
throw new ArgumentOutOfRangeException(nameof(dreamMakerSettings), dreamMakerSettings, "Cannot compile with ultrasafe security!");
}
logger.LogTrace("Begin Compile");
lock (this)
{
if (compiling)
{
throw new JobException("There is already a compile job in progress!");
}
compiling = true;
}
using (var progressCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
var progressTask = estimatedDuration.HasValue ? ProgressTask(progressReporter, estimatedDuration.Value, cancellationToken) : Task.CompletedTask;
try
{
using (var byondLock = await byond.UseExecutables(null, cancellationToken).ConfigureAwait(false))
{
await SendDeploymentMessage(revisionInformation, byondLock, cancellationToken).ConfigureAwait(false);
var job = new Models.CompileJob
{
DirectoryName = Guid.NewGuid(),
DmeName = dreamMakerSettings.ProjectName,
RevisionInformation = revisionInformation,
ByondVersion = byondLock.Version.ToString()
};
await RunCompileJob(job, dreamMakerSettings, byondLock, repository, apiValidateTimeout, cancellationToken).ConfigureAwait(false);
return(job);
}
}
catch (OperationCanceledException)
{
await eventConsumer.HandleEvent(EventType.CompileCancelled, null, default).ConfigureAwait(false);
throw;
}
finally
{
compiling = false;
progressCts.Cancel();
await progressTask.ConfigureAwait(false);
}
}
}
/// <summary>
/// Executes and populate a given <paramref name="job"/>
/// </summary>
/// <param name="job">The <see cref="Models.CompileJob"/> to run and populate</param>
/// <param name="dreamMakerSettings">The <see cref="Api.Models.DreamMaker"/> settings to use</param>
/// <param name="byondLock">The <see cref="IByondExecutableLock"/> to use</param>
/// <param name="repository">The <see cref="IRepository"/> to use</param>
/// <param name="apiValidateTimeout">The timeout for validating the DMAPI</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task RunCompileJob(Models.CompileJob job, Api.Models.DreamMaker dreamMakerSettings, IByondExecutableLock byondLock, IRepository repository, uint apiValidateTimeout, CancellationToken cancellationToken)
{
var jobPath = job.DirectoryName.ToString();
logger.LogTrace("Compile output GUID: {0}", jobPath);
try
{
var dirA = ioManager.ConcatPath(jobPath, ADirectoryName);
var dirB = ioManager.ConcatPath(jobPath, BDirectoryName);
// copy the repository
logger.LogTrace("Copying repository to game directory...");
var resolvedADirectory = ioManager.ResolvePath(dirA);
var repoOrigin = repository.Origin;
using (repository)
await repository.CopyTo(resolvedADirectory, cancellationToken).ConfigureAwait(false);
// repository closed now
// run precompile scripts
await eventConsumer.HandleEvent(EventType.CompileStart, new List <string> {
resolvedADirectory, repoOrigin
}, cancellationToken).ConfigureAwait(false);
// determine the dme
if (job.DmeName == null)
{
logger.LogTrace("Searching for available .dmes...");
var foundPaths = await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false);
var foundPath = foundPaths.FirstOrDefault();
if (foundPath == default)
{
throw new JobException("Unable to find any .dme!");
}
var dmeWithExtension = ioManager.GetFileName(foundPath);
job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1);
}
else
{
var targetDme = ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension));
var targetDmeExists = await ioManager.FileExists(targetDme, cancellationToken).ConfigureAwait(false);
if (!targetDmeExists)
{
throw new JobException("Unable to locate specified .dme!");
}
}
logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName);
await ModifyDme(job, cancellationToken).ConfigureAwait(false);
// run compiler
var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false);
// verify api
try
{
if (exitCode != 0)
{
throw new JobException(String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}{2}", exitCode, Environment.NewLine, job.Output));
}
await VerifyApi(apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false);
}
catch (JobException)
{
// DD never validated or compile failed
await eventConsumer.HandleEvent(EventType.CompileFailure, new List <string> {
resolvedADirectory, exitCode == 0 ? "1" : "0"
}, cancellationToken).ConfigureAwait(false);
throw;
}
logger.LogTrace("Running post compile event...");
await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> {
resolvedADirectory
}, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Duplicating compiled game...");
// duplicate the dmb et al
await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Applying static game file symlinks...");
// symlink in the static data
var symATask = configuration.SymlinkStaticFilesTo(resolvedADirectory, cancellationToken);
var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken);
await Task.WhenAll(symATask, symBTask).ConfigureAwait(false);
await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deployment complete!{0}", watchdog.Running ? " Changes will be applied on next server reboot." : String.Empty), cancellationToken).ConfigureAwait(false);
logger.LogDebug("Compile complete!");
}
catch (Exception e)
{
await CleanupFailedCompile(job, e is OperationCanceledException, cancellationToken).ConfigureAwait(false);
throw;
}
}
/// <summary>
/// Run a quick DD instance to test the DMAPI is installed on the target code
/// </summary>
/// <param name="timeout">The timeout in seconds for validation</param>
/// <param name="securityLevel">The <see cref="DreamDaemonSecurity"/> level to use to validate the API</param>
/// <param name="job">The <see cref="Models.CompileJob"/> for the operation</param>
/// <param name="byondLock">The current <see cref="IByondExecutableLock"/></param>
/// <param name="portToUse">The port to use for API validation</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task VerifyApi(uint timeout, DreamDaemonSecurity securityLevel, Models.CompileJob job, IByondExecutableLock byondLock, ushort portToUse, CancellationToken cancellationToken)
{
logger.LogTrace("Verifying DMAPI...");
var launchParameters = new DreamDaemonLaunchParameters
{
AllowWebClient = false,
PrimaryPort = portToUse,
SecurityLevel = securityLevel,
StartupTimeout = timeout
};
var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName);
job.MinimumSecurityLevel = securityLevel; // needed for the TempDmbProvider
var timeoutAt = DateTimeOffset.Now.AddSeconds(timeout);
using (var provider = new TemporaryDmbProvider(ioManager.ResolvePath(dirA), String.Concat(job.DmeName, DmbExtension), job))
using (var controller = await sessionControllerFactory.LaunchNew(launchParameters, provider, byondLock, true, true, true, cancellationToken).ConfigureAwait(false))
{
var launchResult = await controller.LaunchResult.ConfigureAwait(false);
var now = DateTimeOffset.Now;
if (now < timeoutAt && launchResult.StartupTime.HasValue)
{
var timeoutTask = Task.Delay(timeoutAt - now, cancellationToken);
await Task.WhenAny(controller.Lifetime, timeoutTask).ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
}
if (controller.Lifetime.IsCompleted)
{
var validationStatus = controller.ApiValidationStatus;
logger.LogTrace("API validation status: {0}", validationStatus);
switch (validationStatus)
{
case ApiValidationStatus.RequiresUltrasafe:
job.MinimumSecurityLevel = DreamDaemonSecurity.Ultrasafe;
return;
case ApiValidationStatus.RequiresSafe:
if (securityLevel == DreamDaemonSecurity.Ultrasafe)
{
throw new JobException("This game must be run with at least the 'Safe' DreamDaemon security level!");
}
job.MinimumSecurityLevel = DreamDaemonSecurity.Safe;
return;
case ApiValidationStatus.RequiresTrusted:
if (securityLevel != DreamDaemonSecurity.Trusted)
{
throw new JobException("This game must be run with at least the 'Trusted' DreamDaemon security level!");
}
job.MinimumSecurityLevel = DreamDaemonSecurity.Trusted;
return;
case ApiValidationStatus.NeverValidated:
break;
case ApiValidationStatus.BadValidationRequest:
throw new JobException("Recieved an unrecognized API validation request from DreamDaemon!");
case ApiValidationStatus.UnaskedValidationRequest:
default:
throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, "Session controller returned unexpected ApiValidationStatus: {0}", validationStatus));
}
}
throw new JobException("DMAPI validation timed out!");
}
}
/// <inheritdoc />
public async Task <Models.CompileJob> Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action <int> progressReporter, TimeSpan?estimatedDuration, CancellationToken cancellationToken)
{
if (revisionInformation == null)
{
throw new ArgumentNullException(nameof(revisionInformation));
}
if (dreamMakerSettings == null)
{
throw new ArgumentNullException(nameof(dreamMakerSettings));
}
if (repository == null)
{
throw new ArgumentNullException(nameof(repository));
}
if (progressReporter == null)
{
throw new ArgumentNullException(nameof(progressReporter));
}
if (dreamMakerSettings.ApiValidationSecurityLevel == DreamDaemonSecurity.Ultrasafe)
{
throw new ArgumentOutOfRangeException(nameof(dreamMakerSettings), dreamMakerSettings, "Cannot compile with ultrasafe security!");
}
logger.LogTrace("Begin Compile");
var job = new Models.CompileJob
{
DirectoryName = Guid.NewGuid(),
DmeName = dreamMakerSettings.ProjectName,
RevisionInformation = revisionInformation
};
logger.LogTrace("Compile output GUID: {0}", job.DirectoryName);
lock (this)
{
if (compiling)
{
throw new JobException("There is already a compile job in progress!");
}
compiling = true;
}
using (var progressCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
async Task ProgressTask()
{
if (!estimatedDuration.HasValue)
{
return;
}
progressReporter(0);
var ct = progressCts.Token;
var sleepInterval = estimatedDuration.Value / 100;
try
{
for (var I = 0; I < 99; ++I)
{
await Task.Delay(sleepInterval, progressCts.Token).ConfigureAwait(false);
progressReporter(I + 1);
}
}
catch (OperationCanceledException) { }
}
var progressTask = ProgressTask();
try
{
var commitInsert = revisionInformation.CommitSha.Substring(0, 7);
string remoteCommitInsert;
if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha)
{
commitInsert = String.Format(CultureInfo.InvariantCulture, "^{0}", commitInsert);
remoteCommitInsert = String.Empty;
}
else
{
remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha.Substring(0, 7));
}
var testmergeInsert = revisionInformation.ActiveTestMerges.Count == 0 ? String.Empty : String.Format(CultureInfo.InvariantCulture, " (Test Merges: {0})",
String.Join(", ", revisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x =>
{
var result = String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.PullRequestRevision.Substring(0, 7));
if (x.Comment != null)
{
result += String.Format(CultureInfo.InvariantCulture, " ({0})", x.Comment);
}
return(result);
})));
using (var byondLock = await byond.UseExecutables(null, cancellationToken).ConfigureAwait(false))
{
await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deploying revision: {0}{1}{2} BYOND Version: {3}", commitInsert, testmergeInsert, remoteCommitInsert, byondLock.Version), cancellationToken).ConfigureAwait(false);
async Task CleanupFailedCompile(bool cancelled)
{
logger.LogTrace("Cleaning compile directory...");
var chatTask = chat.SendUpdateMessage(cancelled ? "Deploy cancelled!" : "Deploy failed!", cancellationToken);
try
{
await ioManager.DeleteDirectory(job.DirectoryName.ToString(), CancellationToken.None).ConfigureAwait(false);
}
catch (Exception e)
{
logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(job.DirectoryName.ToString()), e);
}
await chatTask.ConfigureAwait(false);
};
try
{
await ioManager.CreateDirectory(job.DirectoryName.ToString(), cancellationToken).ConfigureAwait(false);
var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName);
var dirB = ioManager.ConcatPath(job.DirectoryName.ToString(), BDirectoryName);
logger.LogTrace("Copying repository to game directory...");
//copy the repository
var fullDirA = ioManager.ResolvePath(dirA);
var repoOrigin = repository.Origin;
using (repository)
await repository.CopyTo(fullDirA, cancellationToken).ConfigureAwait(false);
//run precompile scripts
var resolvedGameDirectory = ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName));
await eventConsumer.HandleEvent(EventType.CompileStart, new List <string> {
resolvedGameDirectory, repoOrigin
}, cancellationToken).ConfigureAwait(false);
//determine the dme
if (job.DmeName == null)
{
logger.LogTrace("Searching for available .dmes...");
var path = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault();
if (path == default)
{
throw new JobException("Unable to find any .dme!");
}
var dmeWithExtension = ioManager.GetFileName(path);
job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1);
}
else if (!await ioManager.FileExists(ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension)), cancellationToken).ConfigureAwait(false))
{
throw new JobException("Unable to locate specified .dme!");
}
logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName);
await ModifyDme(job, cancellationToken).ConfigureAwait(false);
//run compiler, verify api
job.ByondVersion = byondLock.Version.ToString();
var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false);
try
{
if (exitCode != 0)
{
throw new JobException(String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}{2}", exitCode, Environment.NewLine, job.Output));
}
await VerifyApi(apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false);
}
catch (JobException)
{
//server never validated or compile failed
await eventConsumer.HandleEvent(EventType.CompileFailure, new List <string> {
resolvedGameDirectory, exitCode == 0 ? "1" : "0"
}, cancellationToken).ConfigureAwait(false);
throw;
}
logger.LogTrace("Running post compile event...");
await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> {
ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName))
}, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Duplicating compiled game...");
//duplicate the dmb et al
await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Applying static game file symlinks...");
//symlink in the static data
var symATask = configuration.SymlinkStaticFilesTo(fullDirA, cancellationToken);
var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken);
await Task.WhenAll(symATask, symBTask).ConfigureAwait(false);
await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deployment complete!{0}", watchdog.Running ? " Changes will be applied on next server reboot." : String.Empty), cancellationToken).ConfigureAwait(false);
logger.LogDebug("Compile complete!");
return(job);
}
catch (Exception e)
{
await CleanupFailedCompile(e is OperationCanceledException).ConfigureAwait(false);
throw;
}
}
}
catch (OperationCanceledException)
{
await eventConsumer.HandleEvent(EventType.CompileCancelled, null, default).ConfigureAwait(false);
throw;
}
finally
{
compiling = false;
progressCts.Cancel();
await progressTask.ConfigureAwait(false);
}
}
}
public async Task TestSuccessfulLaunchAndShutdown()
{
var mockChat = new Mock <IChat>();
mockChat.Setup(x => x.RegisterCommandHandler(It.IsNotNull <ICustomCommandHandler>())).Verifiable();
var mockSessionControllerFactory = new Mock <ISessionControllerFactory>();
var mockDmbFactory = new Mock <IDmbFactory>();
var mockLogger = new Mock <ILogger <ExperimentalWatchdog> >();
var mockReattachInfoHandler = new Mock <IReattachInfoHandler>();
var mockDatabaseContextFactory = new Mock <IDatabaseContextFactory>();
var mockByondTopicSender = new Mock <IByondTopicSender>();
var mockEventConsumer = new Mock <IEventConsumer>();
var mockJobManager = new Mock <IJobManager>();
var mockRestartRegistration = new Mock <IRestartRegistration>();
mockRestartRegistration.Setup(x => x.Dispose()).Verifiable();
var mockServerControl = new Mock <IServerControl>();
mockServerControl.Setup(x => x.RegisterForRestart(It.IsNotNull <IRestartHandler>())).Returns(mockRestartRegistration.Object).Verifiable();
var mockLaunchParameters = new DreamDaemonLaunchParameters();
var mockInstance = new Models.Instance();
var mockAsyncDelayer = new Mock <IAsyncDelayer>();
using (var wd = new ExperimentalWatchdog(mockChat.Object, mockSessionControllerFactory.Object, mockDmbFactory.Object, mockReattachInfoHandler.Object, mockDatabaseContextFactory.Object, mockByondTopicSender.Object, mockEventConsumer.Object, mockJobManager.Object, mockServerControl.Object, mockAsyncDelayer.Object, mockLogger.Object, mockLaunchParameters, mockInstance, default))
using (var cts = new CancellationTokenSource())
{
var mockCompileJob = new Models.CompileJob();
var mockDmbProvider = new Mock <IDmbProvider>();
mockDmbProvider.SetupGet(x => x.CompileJob).Returns(mockCompileJob).Verifiable();
var mDmbP = mockDmbProvider.Object;
var infiniteTask = new TaskCompletionSource <int>().Task;
mockDmbFactory.SetupGet(x => x.OnNewerDmb).Returns(infiniteTask);
mockDmbFactory.SetupGet(x => x.DmbAvailable).Returns(true).Verifiable();
mockDmbFactory.Setup(x => x.LockNextDmb(2)).Returns(mDmbP).Verifiable();
var sessionsToVerify = new List <Mock <ISessionController> >();
var cancellationToken = cts.Token;
mockSessionControllerFactory.Setup(x => x.LaunchNew(mockLaunchParameters, mDmbP, null, It.IsAny <bool>(), It.IsAny <bool>(), false, cancellationToken)).Returns(() =>
{
var mockSession = new Mock <ISessionController>();
mockSession.SetupGet(x => x.Lifetime).Returns(infiniteTask).Verifiable();
mockSession.SetupGet(x => x.OnReboot).Returns(infiniteTask).Verifiable();
mockSession.SetupGet(x => x.Dmb).Returns(mDmbP).Verifiable();
mockSession.SetupGet(x => x.LaunchResult).Returns(Task.FromResult(new LaunchResult
{
StartupTime = TimeSpan.FromSeconds(1)
})).Verifiable();
sessionsToVerify.Add(mockSession);
return(Task.FromResult(mockSession.Object));
}).Verifiable();
mockAsyncDelayer.Setup(x => x.Delay(It.IsAny <TimeSpan>(), cancellationToken)).Returns(Task.CompletedTask).Verifiable();
cts.CancelAfter(TimeSpan.FromSeconds(15));
try
{
await wd.Launch(cancellationToken).ConfigureAwait(false);
await wd.Terminate(false, cancellationToken).ConfigureAwait(false);
}
finally
{
cts.Cancel();
}
Assert.AreEqual(2, sessionsToVerify.Count);
foreach (var I in sessionsToVerify)
{
I.VerifyAll();
}
mockDmbProvider.VerifyAll();
}
mockSessionControllerFactory.VerifyAll();
mockDmbFactory.VerifyAll();
mockRestartRegistration.VerifyAll();
mockServerControl.VerifyAll();
mockChat.VerifyAll();
mockAsyncDelayer.VerifyAll();
}
/// <summary>
/// Executes and populate a given <paramref name="job"/>
/// </summary>
/// <param name="job">The <see cref="CompileJob"/> to run and populate</param>
/// <param name="dreamMakerSettings">The <see cref="Api.Models.Internal.DreamMakerSettings"/> to use</param>
/// <param name="byondLock">The <see cref="IByondExecutableLock"/> to use</param>
/// <param name="repository">The <see cref="IRepository"/> to use</param>
/// <param name="remoteDeploymentManager">The <see cref="IRemoteDeploymentManager"/> to use.</param>
/// <param name="apiValidateTimeout">The timeout for validating the DMAPI</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task RunCompileJob(
Models.CompileJob job,
Api.Models.Internal.DreamMakerSettings dreamMakerSettings,
IByondExecutableLock byondLock,
IRepository repository,
IRemoteDeploymentManager remoteDeploymentManager,
uint apiValidateTimeout,
CancellationToken cancellationToken)
{
var outputDirectory = job.DirectoryName.ToString();
logger.LogTrace("Compile output GUID: {0}", outputDirectory);
try
{
// copy the repository
logger.LogTrace("Copying repository to game directory...");
var resolvedOutputDirectory = ioManager.ResolvePath(outputDirectory);
var repoOrigin = repository.Origin;
using (repository)
await repository.CopyTo(resolvedOutputDirectory, cancellationToken).ConfigureAwait(false);
// repository closed now
// run precompile scripts
await eventConsumer.HandleEvent(
EventType.CompileStart,
new List <string>
{
resolvedOutputDirectory,
repoOrigin.ToString()
},
cancellationToken)
.ConfigureAwait(false);
// determine the dme
if (job.DmeName == null)
{
logger.LogTrace("Searching for available .dmes...");
var foundPaths = await ioManager.GetFilesWithExtension(resolvedOutputDirectory, DmeExtension, true, cancellationToken).ConfigureAwait(false);
var foundPath = foundPaths.FirstOrDefault();
if (foundPath == default)
{
throw new JobException(ErrorCode.DreamMakerNoDme);
}
job.DmeName = foundPath.Substring(
resolvedOutputDirectory.Length + 1,
foundPath.Length - resolvedOutputDirectory.Length - DmeExtension.Length - 2); // +1 for . in extension
}
else
{
var targetDme = ioManager.ConcatPath(outputDirectory, String.Join('.', job.DmeName, DmeExtension));
var targetDmeExists = await ioManager.FileExists(targetDme, cancellationToken).ConfigureAwait(false);
if (!targetDmeExists)
{
throw new JobException(ErrorCode.DreamMakerMissingDme);
}
}
logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName);
await ModifyDme(job, cancellationToken).ConfigureAwait(false);
// run compiler
var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false);
// verify api
try
{
if (exitCode != 0)
{
throw new JobException(
ErrorCode.DreamMakerExitCode,
new JobException($"Exit code: {exitCode}{Environment.NewLine}{Environment.NewLine}{job.Output}"));
}
await VerifyApi(
apiValidateTimeout,
dreamMakerSettings.ApiValidationSecurityLevel.Value,
job,
byondLock,
dreamMakerSettings.ApiValidationPort.Value,
dreamMakerSettings.RequireDMApiValidation.Value,
cancellationToken)
.ConfigureAwait(false);
}
catch (JobException)
{
// DD never validated or compile failed
await eventConsumer.HandleEvent(
EventType.CompileFailure,
new List <string>
{
resolvedOutputDirectory,
exitCode == 0 ? "1" : "0"
},
cancellationToken)
.ConfigureAwait(false);
throw;
}
await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> {
resolvedOutputDirectory
}, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Applying static game file symlinks...");
// symlink in the static data
await configuration.SymlinkStaticFilesTo(resolvedOutputDirectory, cancellationToken).ConfigureAwait(false);
logger.LogDebug("Compile complete!");
}
catch (Exception ex)
{
await CleanupFailedCompile(job, remoteDeploymentManager, ex).ConfigureAwait(false);
throw;
}
}
/// <summary>
/// Run a quick DD instance to test the DMAPI is installed on the target code
/// </summary>
/// <param name="timeout">The timeout in seconds for validation</param>
/// <param name="securityLevel">The <see cref="DreamDaemonSecurity"/> level to use to validate the API</param>
/// <param name="job">The <see cref="CompileJob"/> for the operation</param>
/// <param name="byondLock">The current <see cref="IByondExecutableLock"/></param>
/// <param name="portToUse">The port to use for API validation</param>
/// <param name="requireValidate">If the API validation is required to complete the deployment.</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task VerifyApi(
uint timeout,
DreamDaemonSecurity securityLevel,
Models.CompileJob job,
IByondExecutableLock byondLock,
ushort portToUse,
bool requireValidate,
CancellationToken cancellationToken)
{
logger.LogTrace("Verifying {0}DMAPI...", requireValidate ? "required " : String.Empty);
var launchParameters = new DreamDaemonLaunchParameters
{
AllowWebClient = false,
Port = portToUse,
SecurityLevel = securityLevel,
StartupTimeout = timeout,
TopicRequestTimeout = 0, // not used
HeartbeatSeconds = 0 // not used
};
job.MinimumSecurityLevel = securityLevel; // needed for the TempDmbProvider
ApiValidationStatus validationStatus;
using (var provider = new TemporaryDmbProvider(ioManager.ResolvePath(job.DirectoryName.ToString()), String.Concat(job.DmeName, DmbExtension), job))
await using (var controller = await sessionControllerFactory.LaunchNew(provider, byondLock, launchParameters, true, cancellationToken).ConfigureAwait(false))
{
var launchResult = await controller.LaunchResult.ConfigureAwait(false);
if (launchResult.StartupTime.HasValue)
{
await controller.Lifetime.WithToken(cancellationToken).ConfigureAwait(false);
}
if (!controller.Lifetime.IsCompleted)
{
await controller.DisposeAsync().ConfigureAwait(false);
}
validationStatus = controller.ApiValidationStatus;
if (requireValidate && validationStatus == ApiValidationStatus.NeverValidated)
{
throw new JobException(ErrorCode.DreamMakerNeverValidated);
}
logger.LogTrace("API validation status: {0}", validationStatus);
job.DMApiVersion = controller.DMApiVersion;
}
switch (validationStatus)
{
case ApiValidationStatus.RequiresUltrasafe:
job.MinimumSecurityLevel = DreamDaemonSecurity.Ultrasafe;
return;
case ApiValidationStatus.RequiresSafe:
job.MinimumSecurityLevel = DreamDaemonSecurity.Safe;
return;
case ApiValidationStatus.RequiresTrusted:
job.MinimumSecurityLevel = DreamDaemonSecurity.Trusted;
return;
case ApiValidationStatus.NeverValidated:
if (requireValidate)
{
throw new JobException(ErrorCode.DreamMakerNeverValidated);
}
job.MinimumSecurityLevel = DreamDaemonSecurity.Ultrasafe;
break;
case ApiValidationStatus.BadValidationRequest:
case ApiValidationStatus.Incompatible:
throw new JobException(ErrorCode.DreamMakerInvalidValidation);
case ApiValidationStatus.UnaskedValidationRequest:
default:
throw new InvalidOperationException(
$"Session controller returned unexpected ApiValidationStatus: {validationStatus}");
}
}
