#pragma warning restore CA1506
/// <inheritdoc />
#pragma warning disable CA1506 // TODO: Decomplexify
public async Task CleanUnusedCompileJobs(CompileJob exceptThisOne, CancellationToken cancellationToken)
{
List <long> jobIdsToSkip;
// don't clean locked directories
lock (this)
jobIdsToSkip = jobLockCounts.Select(x => x.Key).ToList();
List <string> jobUidsToNotErase = null;
// find the uids of locked directories
await databaseContextFactory.UseContext(async db =>
{
jobUidsToNotErase = await db.CompileJobs.Where(x => x.Job.Instance.Id == instance.Id && jobIdsToSkip.Contains(x.Id)).Select(x => x.DirectoryName.Value.ToString().ToUpperInvariant()).ToListAsync(cancellationToken).ConfigureAwait(false);
}).ConfigureAwait(false);
// add the other exemption
if (exceptThisOne != null)
{
jobUidsToNotErase.Add(exceptThisOne.DirectoryName.Value.ToString().ToUpperInvariant());
}
// cleanup
await ioManager.CreateDirectory(".", cancellationToken).ConfigureAwait(false);
var directories = await ioManager.GetDirectories(".", cancellationToken).ConfigureAwait(false);
int deleting = 0;
var tasks = directories.Select(async x =>
{
var nameOnly = ioManager.GetFileName(x);
if (jobUidsToNotErase.Contains(nameOnly.ToUpperInvariant()))
{
return;
}
try
{
++deleting;
await ioManager.DeleteDirectory(x, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
logger.LogWarning("Error deleting directory {0}! Exception: {1}", x, e);
}
}).ToList();
if (deleting > 0)
{
logger.LogDebug("Cleaning {0} unused game folders...", deleting);
await Task.WhenAll().ConfigureAwait(false);
}
}
/// <inheritdoc />
public async Task SymlinkStaticFilesTo(string destination, CancellationToken cancellationToken)
{
async Task SymlinkBase(bool files)
{
Task <IReadOnlyList <string> > task;
if (files)
{
task = ioManager.GetFiles(GameStaticFilesSubdirectory, cancellationToken);
}
else
{
task = ioManager.GetDirectories(GameStaticFilesSubdirectory, cancellationToken);
}
var entries = await task.ConfigureAwait(false);
await Task.WhenAll(task.Result.Select(async x =>
{
var destPath = ioManager.ConcatPath(destination, ioManager.GetFileName(x));
logger.LogTrace("Symlinking {0} to {1}...", x, destPath);
var fileExistsTask = ioManager.FileExists(destPath, cancellationToken);
if (await ioManager.DirectoryExists(destPath, cancellationToken).ConfigureAwait(false))
{
await ioManager.DeleteDirectory(destPath, cancellationToken).ConfigureAwait(false);
}
var fileExists = await fileExistsTask.ConfigureAwait(false);
if (fileExists)
{
await ioManager.DeleteFile(destPath, cancellationToken).ConfigureAwait(false);
}
await symlinkFactory.CreateSymbolicLink(ioManager.ResolvePath(x), ioManager.ResolvePath(destPath), cancellationToken).ConfigureAwait(false);
})).ConfigureAwait(false);
}
using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
{
await EnsureDirectories(cancellationToken).ConfigureAwait(false);
await Task.WhenAll(SymlinkBase(true), SymlinkBase(false)).ConfigureAwait(false);
}
}
public Task <IActionResult> ListLogs([FromQuery] int?page, [FromQuery] int?pageSize, CancellationToken cancellationToken)
=> Paginated(
async() =>
{
var path = fileLoggingConfiguration.GetFullLogDirectory(ioManager, assemblyInformationProvider, platformIdentifier);
try
{
var files = await ioManager.GetFiles(path, cancellationToken).ConfigureAwait(false);
var tasks = files.Select(
async file => new LogFileResponse
{
Name = ioManager.GetFileName(file),
LastModified = await ioManager
.GetLastModified(
ioManager.ConcatPath(path, file),
cancellationToken)
.ConfigureAwait(false),
})
.ToList();
await Task.WhenAll(tasks).ConfigureAwait(false);
return(new PaginatableResult <LogFileResponse>(
tasks
.AsQueryable()
.Select(x => x.Result)
.OrderByDescending(x => x.Name)));
}
catch (IOException ex)
{
return(new PaginatableResult <LogFileResponse>(
Conflict(new ErrorMessageResponse(ErrorCode.IOError)
{
AdditionalData = ex.ToString(),
})));
}
},
null,
page,
pageSize,
cancellationToken);
public async Task <IActionResult> ListLogs(CancellationToken cancellationToken)
{
var path = fileLoggingConfiguration.GetFullLogDirectory(ioManager, assemblyInformationProvider, platformIdentifier);
try
{
var files = await ioManager.GetFiles(path, cancellationToken).ConfigureAwait(false);
var tasks = files.Select(
async file => new LogFile
{
Name = ioManager.GetFileName(file),
LastModified = await ioManager.GetLastModified(
ioManager.ConcatPath(path, file),
cancellationToken)
.ConfigureAwait(false)
})
.ToList();
await Task.WhenAll(tasks).ConfigureAwait(false);
var result = tasks
.Select(x => x.Result)
.OrderByDescending(x => x.Name)
.ToList();
return(Ok(result));
}
catch (IOException ex)
{
return(Conflict(new ErrorMessage(ErrorCode.IOError)
{
AdditionalData = ex.ToString()
}));
}
}
/// <summary>
/// Executes and populate a given <paramref name="job"/>
/// </summary>
/// <param name="job">The <see cref="Models.CompileJob"/> to run and populate</param>
/// <param name="dreamMakerSettings">The <see cref="Api.Models.DreamMaker"/> settings to use</param>
/// <param name="byondLock">The <see cref="IByondExecutableLock"/> to use</param>
/// <param name="repository">The <see cref="IRepository"/> to use</param>
/// <param name="apiValidateTimeout">The timeout for validating the DMAPI</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task RunCompileJob(Models.CompileJob job, Api.Models.DreamMaker dreamMakerSettings, IByondExecutableLock byondLock, IRepository repository, uint apiValidateTimeout, CancellationToken cancellationToken)
{
var jobPath = job.DirectoryName.ToString();
logger.LogTrace("Compile output GUID: {0}", jobPath);
try
{
var dirA = ioManager.ConcatPath(jobPath, ADirectoryName);
var dirB = ioManager.ConcatPath(jobPath, BDirectoryName);
// copy the repository
logger.LogTrace("Copying repository to game directory...");
var resolvedADirectory = ioManager.ResolvePath(dirA);
var repoOrigin = repository.Origin;
using (repository)
await repository.CopyTo(resolvedADirectory, cancellationToken).ConfigureAwait(false);
// repository closed now
// run precompile scripts
await eventConsumer.HandleEvent(EventType.CompileStart, new List <string> {
resolvedADirectory, repoOrigin
}, cancellationToken).ConfigureAwait(false);
// determine the dme
if (job.DmeName == null)
{
logger.LogTrace("Searching for available .dmes...");
var foundPaths = await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false);
var foundPath = foundPaths.FirstOrDefault();
if (foundPath == default)
{
throw new JobException("Unable to find any .dme!");
}
var dmeWithExtension = ioManager.GetFileName(foundPath);
job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1);
}
else
{
var targetDme = ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension));
var targetDmeExists = await ioManager.FileExists(targetDme, cancellationToken).ConfigureAwait(false);
if (!targetDmeExists)
{
throw new JobException("Unable to locate specified .dme!");
}
}
logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName);
await ModifyDme(job, cancellationToken).ConfigureAwait(false);
// run compiler
var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false);
// verify api
try
{
if (exitCode != 0)
{
throw new JobException(String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}{2}", exitCode, Environment.NewLine, job.Output));
}
await VerifyApi(apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false);
}
catch (JobException)
{
// DD never validated or compile failed
await eventConsumer.HandleEvent(EventType.CompileFailure, new List <string> {
resolvedADirectory, exitCode == 0 ? "1" : "0"
}, cancellationToken).ConfigureAwait(false);
throw;
}
logger.LogTrace("Running post compile event...");
await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> {
resolvedADirectory
}, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Duplicating compiled game...");
// duplicate the dmb et al
await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Applying static game file symlinks...");
// symlink in the static data
var symATask = configuration.SymlinkStaticFilesTo(resolvedADirectory, cancellationToken);
var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken);
await Task.WhenAll(symATask, symBTask).ConfigureAwait(false);
await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deployment complete!{0}", watchdog.Running ? " Changes will be applied on next server reboot." : String.Empty), cancellationToken).ConfigureAwait(false);
logger.LogDebug("Compile complete!");
}
catch (Exception e)
{
await CleanupFailedCompile(job, e is OperationCanceledException, cancellationToken).ConfigureAwait(false);
throw;
}
}
/// <inheritdoc />
public async Task SymlinkStaticFilesTo(string destination, CancellationToken cancellationToken)
{
async Task <IReadOnlyList <string> > GetIgnoreFiles()
{
var ignoreFileBytes = await ioManager.ReadAllBytes(StaticIgnorePath(), cancellationToken).ConfigureAwait(false);
var ignoreFileText = Encoding.UTF8.GetString(ignoreFileBytes);
var results = new List <string> {
StaticIgnoreFile
};
//we don't want to lose trailing whitespace on linux
using (var reader = new StringReader(ignoreFileText))
{
cancellationToken.ThrowIfCancellationRequested();
var line = await reader.ReadLineAsync().ConfigureAwait(false);
if (!String.IsNullOrEmpty(line))
{
results.Add(line);
}
}
return(results);
};
IReadOnlyList <string> ignoreFiles;
async Task SymlinkBase(bool files)
{
Task <IReadOnlyList <string> > task;
if (files)
{
task = ioManager.GetFiles(GameStaticFilesSubdirectory, cancellationToken);
}
else
{
task = ioManager.GetDirectories(GameStaticFilesSubdirectory, cancellationToken);
}
var entries = await task.ConfigureAwait(false);
await Task.WhenAll(entries.Select(async x =>
{
var fileName = ioManager.GetFileName(x);
bool ignored;
if (platformIdentifier.IsWindows)
{
//need to normalize
ignored = ignoreFiles.Any(y => fileName.ToUpperInvariant() == y.ToUpperInvariant());
}
else
{
ignored = ignoreFiles.Any(y => fileName == y);
}
if (ignored)
{
logger.LogTrace("Ignoring static file {0}...", fileName);
return;
}
var destPath = ioManager.ConcatPath(destination, fileName);
logger.LogTrace("Symlinking {0} to {1}...", x, destPath);
var fileExistsTask = ioManager.FileExists(destPath, cancellationToken);
if (await ioManager.DirectoryExists(destPath, cancellationToken).ConfigureAwait(false))
{
await ioManager.DeleteDirectory(destPath, cancellationToken).ConfigureAwait(false);
}
var fileExists = await fileExistsTask.ConfigureAwait(false);
if (fileExists)
{
await ioManager.DeleteFile(destPath, cancellationToken).ConfigureAwait(false);
}
await symlinkFactory.CreateSymbolicLink(ioManager.ResolvePath(x), ioManager.ResolvePath(destPath), cancellationToken).ConfigureAwait(false);
})).ConfigureAwait(false);
}
using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
{
await EnsureDirectories(cancellationToken).ConfigureAwait(false);
ignoreFiles = await GetIgnoreFiles().ConfigureAwait(false);
await Task.WhenAll(SymlinkBase(true), SymlinkBase(false)).ConfigureAwait(false);
}
}
/// <inheritdoc />
public async Task <Models.CompileJob> Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action <int> progressReporter, TimeSpan?estimatedDuration, CancellationToken cancellationToken)
{
if (revisionInformation == null)
{
throw new ArgumentNullException(nameof(revisionInformation));
}
if (dreamMakerSettings == null)
{
throw new ArgumentNullException(nameof(dreamMakerSettings));
}
if (repository == null)
{
throw new ArgumentNullException(nameof(repository));
}
if (progressReporter == null)
{
throw new ArgumentNullException(nameof(progressReporter));
}
if (dreamMakerSettings.ApiValidationSecurityLevel == DreamDaemonSecurity.Ultrasafe)
{
throw new ArgumentOutOfRangeException(nameof(dreamMakerSettings), dreamMakerSettings, "Cannot compile with ultrasafe security!");
}
logger.LogTrace("Begin Compile");
var job = new Models.CompileJob
{
DirectoryName = Guid.NewGuid(),
DmeName = dreamMakerSettings.ProjectName,
RevisionInformation = revisionInformation
};
logger.LogTrace("Compile output GUID: {0}", job.DirectoryName);
lock (this)
{
if (compiling)
{
throw new JobException("There is already a compile job in progress!");
}
compiling = true;
}
using (var progressCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
async Task ProgressTask()
{
if (!estimatedDuration.HasValue)
{
return;
}
progressReporter(0);
var ct = progressCts.Token;
var sleepInterval = estimatedDuration.Value / 100;
try
{
for (var I = 0; I < 99; ++I)
{
await Task.Delay(sleepInterval, progressCts.Token).ConfigureAwait(false);
progressReporter(I + 1);
}
}
catch (OperationCanceledException) { }
}
var progressTask = ProgressTask();
try
{
var commitInsert = revisionInformation.CommitSha.Substring(0, 7);
string remoteCommitInsert;
if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha)
{
commitInsert = String.Format(CultureInfo.InvariantCulture, "^{0}", commitInsert);
remoteCommitInsert = String.Empty;
}
else
{
remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha.Substring(0, 7));
}
var testmergeInsert = revisionInformation.ActiveTestMerges.Count == 0 ? String.Empty : String.Format(CultureInfo.InvariantCulture, " (Test Merges: {0})",
String.Join(", ", revisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x =>
{
var result = String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.PullRequestRevision.Substring(0, 7));
if (x.Comment != null)
{
result += String.Format(CultureInfo.InvariantCulture, " ({0})", x.Comment);
}
return(result);
})));
using (var byondLock = await byond.UseExecutables(null, cancellationToken).ConfigureAwait(false))
{
await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deploying revision: {0}{1}{2} BYOND Version: {3}", commitInsert, testmergeInsert, remoteCommitInsert, byondLock.Version), cancellationToken).ConfigureAwait(false);
async Task CleanupFailedCompile(bool cancelled)
{
logger.LogTrace("Cleaning compile directory...");
var chatTask = chat.SendUpdateMessage(cancelled ? "Deploy cancelled!" : "Deploy failed!", cancellationToken);
try
{
await ioManager.DeleteDirectory(job.DirectoryName.ToString(), CancellationToken.None).ConfigureAwait(false);
}
catch (Exception e)
{
logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(job.DirectoryName.ToString()), e);
}
await chatTask.ConfigureAwait(false);
};
try
{
await ioManager.CreateDirectory(job.DirectoryName.ToString(), cancellationToken).ConfigureAwait(false);
var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName);
var dirB = ioManager.ConcatPath(job.DirectoryName.ToString(), BDirectoryName);
logger.LogTrace("Copying repository to game directory...");
//copy the repository
var fullDirA = ioManager.ResolvePath(dirA);
var repoOrigin = repository.Origin;
using (repository)
await repository.CopyTo(fullDirA, cancellationToken).ConfigureAwait(false);
//run precompile scripts
var resolvedGameDirectory = ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName));
await eventConsumer.HandleEvent(EventType.CompileStart, new List <string> {
resolvedGameDirectory, repoOrigin
}, cancellationToken).ConfigureAwait(false);
//determine the dme
if (job.DmeName == null)
{
logger.LogTrace("Searching for available .dmes...");
var path = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault();
if (path == default)
{
throw new JobException("Unable to find any .dme!");
}
var dmeWithExtension = ioManager.GetFileName(path);
job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1);
}
else if (!await ioManager.FileExists(ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension)), cancellationToken).ConfigureAwait(false))
{
throw new JobException("Unable to locate specified .dme!");
}
logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName);
await ModifyDme(job, cancellationToken).ConfigureAwait(false);
//run compiler, verify api
job.ByondVersion = byondLock.Version.ToString();
var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false);
try
{
if (exitCode != 0)
{
throw new JobException(String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}{2}", exitCode, Environment.NewLine, job.Output));
}
await VerifyApi(apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false);
}
catch (JobException)
{
//server never validated or compile failed
await eventConsumer.HandleEvent(EventType.CompileFailure, new List <string> {
resolvedGameDirectory, exitCode == 0 ? "1" : "0"
}, cancellationToken).ConfigureAwait(false);
throw;
}
logger.LogTrace("Running post compile event...");
await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> {
ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName))
}, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Duplicating compiled game...");
//duplicate the dmb et al
await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Applying static game file symlinks...");
//symlink in the static data
var symATask = configuration.SymlinkStaticFilesTo(fullDirA, cancellationToken);
var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken);
await Task.WhenAll(symATask, symBTask).ConfigureAwait(false);
await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deployment complete!{0}", watchdog.Running ? " Changes will be applied on next server reboot." : String.Empty), cancellationToken).ConfigureAwait(false);
logger.LogDebug("Compile complete!");
return(job);
}
catch (Exception e)
{
await CleanupFailedCompile(e is OperationCanceledException).ConfigureAwait(false);
throw;
}
}
}
catch (OperationCanceledException)
{
await eventConsumer.HandleEvent(EventType.CompileCancelled, null, default).ConfigureAwait(false);
throw;
}
finally
{
compiling = false;
progressCts.Cancel();
await progressTask.ConfigureAwait(false);
}
}
}
#pragma warning restore CA1506
/// <inheritdoc />
#pragma warning disable CA1506 // TODO: Decomplexify
public async Task CleanUnusedCompileJobs(CancellationToken cancellationToken)
{
List <long> jobIdsToSkip;
// don't clean locked directories
lock (jobLockCounts)
jobIdsToSkip = jobLockCounts.Select(x => x.Key).ToList();
List <string> jobUidsToNotErase = null;
// find the uids of locked directories
await databaseContextFactory.UseContext(async db =>
{
jobUidsToNotErase = (await db
.CompileJobs
.AsQueryable()
.Where(
x => x.Job.Instance.Id == metadata.Id &&
jobIdsToSkip.Contains(x.Id))
.Select(x => x.DirectoryName.Value)
.ToListAsync(cancellationToken)
.ConfigureAwait(false))
.Select(x => x.ToString())
.ToList();
}).ConfigureAwait(false);
jobUidsToNotErase.Add(SwappableDmbProvider.LiveGameDirectory);
logger.LogTrace("We will not clean the following directories: {0}", String.Join(", ", jobUidsToNotErase));
// cleanup
var gameDirectory = ioManager.ResolvePath();
await ioManager.CreateDirectory(gameDirectory, cancellationToken).ConfigureAwait(false);
var directories = await ioManager.GetDirectories(gameDirectory, cancellationToken).ConfigureAwait(false);
int deleting = 0;
var tasks = directories.Select(async x =>
{
var nameOnly = ioManager.GetFileName(x);
if (jobUidsToNotErase.Contains(nameOnly))
{
return;
}
logger.LogDebug("Cleaning unused game folder: {0}...", nameOnly);
try
{
++deleting;
await ioManager.DeleteDirectory(x, cancellationToken).ConfigureAwait(false);
}
catch (OperationCanceledException)
{
throw;
}
catch (Exception e)
{
logger.LogWarning(e, "Error deleting directory {0}!", x);
}
}).ToList();
if (deleting > 0)
{
await Task.WhenAll(tasks).ConfigureAwait(false);
}
}
