/// <inheritdoc />
public async Task <bool> HandleEvent(EventType eventType, IEnumerable <string> parameters, CancellationToken cancellationToken)
{
await EnsureDirectories(cancellationToken).ConfigureAwait(false);
if (!EventTypeScriptFileNameMap.TryGetValue(eventType, out var scriptName))
{
return(true);
}
//always execute in serial
using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false))
{
var files = await ioManager.GetFilesWithExtension(EventScriptsSubdirectory, SystemScriptFileExtension, cancellationToken).ConfigureAwait(false);
var resolvedScriptsDir = ioManager.ResolvePath(EventScriptsSubdirectory);
foreach (var I in files.Select(x => ioManager.GetFileName(x)).Where(x => x.StartsWith(scriptName, StringComparison.Ordinal)))
{
using (var script = processExecutor.LaunchProcess(ioManager.ConcatPath(resolvedScriptsDir, I), resolvedScriptsDir, String.Join(' ', parameters), noShellExecute: true))
using (cancellationToken.Register(() => script.Terminate()))
{
var exitCode = await script.Lifetime.ConfigureAwait(false);
cancellationToken.ThrowIfCancellationRequested();
if (exitCode != 0)
{
return(false);
}
}
}
}
return(true);
}
/// <summary>
/// Executes and populate a given <paramref name="job"/>
/// </summary>
/// <param name="job">The <see cref="Models.CompileJob"/> to run and populate</param>
/// <param name="dreamMakerSettings">The <see cref="Api.Models.DreamMaker"/> settings to use</param>
/// <param name="byondLock">The <see cref="IByondExecutableLock"/> to use</param>
/// <param name="repository">The <see cref="IRepository"/> to use</param>
/// <param name="apiValidateTimeout">The timeout for validating the DMAPI</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task RunCompileJob(Models.CompileJob job, Api.Models.DreamMaker dreamMakerSettings, IByondExecutableLock byondLock, IRepository repository, uint apiValidateTimeout, CancellationToken cancellationToken)
{
var jobPath = job.DirectoryName.ToString();
logger.LogTrace("Compile output GUID: {0}", jobPath);
try
{
var dirA = ioManager.ConcatPath(jobPath, ADirectoryName);
var dirB = ioManager.ConcatPath(jobPath, BDirectoryName);
// copy the repository
logger.LogTrace("Copying repository to game directory...");
var resolvedADirectory = ioManager.ResolvePath(dirA);
var repoOrigin = repository.Origin;
using (repository)
await repository.CopyTo(resolvedADirectory, cancellationToken).ConfigureAwait(false);
// repository closed now
// run precompile scripts
await eventConsumer.HandleEvent(EventType.CompileStart, new List <string> {
resolvedADirectory, repoOrigin
}, cancellationToken).ConfigureAwait(false);
// determine the dme
if (job.DmeName == null)
{
logger.LogTrace("Searching for available .dmes...");
var foundPaths = await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false);
var foundPath = foundPaths.FirstOrDefault();
if (foundPath == default)
{
throw new JobException("Unable to find any .dme!");
}
var dmeWithExtension = ioManager.GetFileName(foundPath);
job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1);
}
else
{
var targetDme = ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension));
var targetDmeExists = await ioManager.FileExists(targetDme, cancellationToken).ConfigureAwait(false);
if (!targetDmeExists)
{
throw new JobException("Unable to locate specified .dme!");
}
}
logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName);
await ModifyDme(job, cancellationToken).ConfigureAwait(false);
// run compiler
var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false);
// verify api
try
{
if (exitCode != 0)
{
throw new JobException(String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}{2}", exitCode, Environment.NewLine, job.Output));
}
await VerifyApi(apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false);
}
catch (JobException)
{
// DD never validated or compile failed
await eventConsumer.HandleEvent(EventType.CompileFailure, new List <string> {
resolvedADirectory, exitCode == 0 ? "1" : "0"
}, cancellationToken).ConfigureAwait(false);
throw;
}
logger.LogTrace("Running post compile event...");
await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> {
resolvedADirectory
}, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Duplicating compiled game...");
// duplicate the dmb et al
await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Applying static game file symlinks...");
// symlink in the static data
var symATask = configuration.SymlinkStaticFilesTo(resolvedADirectory, cancellationToken);
var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken);
await Task.WhenAll(symATask, symBTask).ConfigureAwait(false);
await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deployment complete!{0}", watchdog.Running ? " Changes will be applied on next server reboot." : String.Empty), cancellationToken).ConfigureAwait(false);
logger.LogDebug("Compile complete!");
}
catch (Exception e)
{
await CleanupFailedCompile(job, e is OperationCanceledException, cancellationToken).ConfigureAwait(false);
throw;
}
}
/// <inheritdoc />
public async Task <Models.CompileJob> Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action <int> progressReporter, TimeSpan?estimatedDuration, CancellationToken cancellationToken)
{
if (revisionInformation == null)
{
throw new ArgumentNullException(nameof(revisionInformation));
}
if (dreamMakerSettings == null)
{
throw new ArgumentNullException(nameof(dreamMakerSettings));
}
if (repository == null)
{
throw new ArgumentNullException(nameof(repository));
}
if (progressReporter == null)
{
throw new ArgumentNullException(nameof(progressReporter));
}
if (dreamMakerSettings.ApiValidationSecurityLevel == DreamDaemonSecurity.Ultrasafe)
{
throw new ArgumentOutOfRangeException(nameof(dreamMakerSettings), dreamMakerSettings, "Cannot compile with ultrasafe security!");
}
logger.LogTrace("Begin Compile");
var job = new Models.CompileJob
{
DirectoryName = Guid.NewGuid(),
DmeName = dreamMakerSettings.ProjectName,
RevisionInformation = revisionInformation
};
logger.LogTrace("Compile output GUID: {0}", job.DirectoryName);
lock (this)
{
if (compiling)
{
throw new JobException("There is already a compile job in progress!");
}
compiling = true;
}
using (var progressCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken))
{
async Task ProgressTask()
{
if (!estimatedDuration.HasValue)
{
return;
}
progressReporter(0);
var ct = progressCts.Token;
var sleepInterval = estimatedDuration.Value / 100;
try
{
for (var I = 0; I < 99; ++I)
{
await Task.Delay(sleepInterval, progressCts.Token).ConfigureAwait(false);
progressReporter(I + 1);
}
}
catch (OperationCanceledException) { }
}
var progressTask = ProgressTask();
try
{
var commitInsert = revisionInformation.CommitSha.Substring(0, 7);
string remoteCommitInsert;
if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha)
{
commitInsert = String.Format(CultureInfo.InvariantCulture, "^{0}", commitInsert);
remoteCommitInsert = String.Empty;
}
else
{
remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha.Substring(0, 7));
}
var testmergeInsert = revisionInformation.ActiveTestMerges.Count == 0 ? String.Empty : String.Format(CultureInfo.InvariantCulture, " (Test Merges: {0})",
String.Join(", ", revisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x =>
{
var result = String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.PullRequestRevision.Substring(0, 7));
if (x.Comment != null)
{
result += String.Format(CultureInfo.InvariantCulture, " ({0})", x.Comment);
}
return(result);
})));
using (var byondLock = await byond.UseExecutables(null, cancellationToken).ConfigureAwait(false))
{
await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deploying revision: {0}{1}{2} BYOND Version: {3}", commitInsert, testmergeInsert, remoteCommitInsert, byondLock.Version), cancellationToken).ConfigureAwait(false);
async Task CleanupFailedCompile(bool cancelled)
{
logger.LogTrace("Cleaning compile directory...");
var chatTask = chat.SendUpdateMessage(cancelled ? "Deploy cancelled!" : "Deploy failed!", cancellationToken);
try
{
await ioManager.DeleteDirectory(job.DirectoryName.ToString(), CancellationToken.None).ConfigureAwait(false);
}
catch (Exception e)
{
logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(job.DirectoryName.ToString()), e);
}
await chatTask.ConfigureAwait(false);
};
try
{
await ioManager.CreateDirectory(job.DirectoryName.ToString(), cancellationToken).ConfigureAwait(false);
var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName);
var dirB = ioManager.ConcatPath(job.DirectoryName.ToString(), BDirectoryName);
logger.LogTrace("Copying repository to game directory...");
//copy the repository
var fullDirA = ioManager.ResolvePath(dirA);
var repoOrigin = repository.Origin;
using (repository)
await repository.CopyTo(fullDirA, cancellationToken).ConfigureAwait(false);
//run precompile scripts
var resolvedGameDirectory = ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName));
await eventConsumer.HandleEvent(EventType.CompileStart, new List <string> {
resolvedGameDirectory, repoOrigin
}, cancellationToken).ConfigureAwait(false);
//determine the dme
if (job.DmeName == null)
{
logger.LogTrace("Searching for available .dmes...");
var path = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault();
if (path == default)
{
throw new JobException("Unable to find any .dme!");
}
var dmeWithExtension = ioManager.GetFileName(path);
job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1);
}
else if (!await ioManager.FileExists(ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension)), cancellationToken).ConfigureAwait(false))
{
throw new JobException("Unable to locate specified .dme!");
}
logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName);
await ModifyDme(job, cancellationToken).ConfigureAwait(false);
//run compiler, verify api
job.ByondVersion = byondLock.Version.ToString();
var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false);
try
{
if (exitCode != 0)
{
throw new JobException(String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}{2}", exitCode, Environment.NewLine, job.Output));
}
await VerifyApi(apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false);
}
catch (JobException)
{
//server never validated or compile failed
await eventConsumer.HandleEvent(EventType.CompileFailure, new List <string> {
resolvedGameDirectory, exitCode == 0 ? "1" : "0"
}, cancellationToken).ConfigureAwait(false);
throw;
}
logger.LogTrace("Running post compile event...");
await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> {
ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName))
}, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Duplicating compiled game...");
//duplicate the dmb et al
await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Applying static game file symlinks...");
//symlink in the static data
var symATask = configuration.SymlinkStaticFilesTo(fullDirA, cancellationToken);
var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken);
await Task.WhenAll(symATask, symBTask).ConfigureAwait(false);
await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deployment complete!{0}", watchdog.Running ? " Changes will be applied on next server reboot." : String.Empty), cancellationToken).ConfigureAwait(false);
logger.LogDebug("Compile complete!");
return(job);
}
catch (Exception e)
{
await CleanupFailedCompile(e is OperationCanceledException).ConfigureAwait(false);
throw;
}
}
}
catch (OperationCanceledException)
{
await eventConsumer.HandleEvent(EventType.CompileCancelled, null, default).ConfigureAwait(false);
throw;
}
finally
{
compiling = false;
progressCts.Cancel();
await progressTask.ConfigureAwait(false);
}
}
}
/// <inheritdoc />
#pragma warning disable CA1506 // TODO: Decomplexify
public async Task <ISessionController> LaunchNew(DreamDaemonLaunchParameters launchParameters, IDmbProvider dmbProvider, IByondExecutableLock currentByondLock, bool primaryPort, bool primaryDirectory, bool apiValidate, CancellationToken cancellationToken)
{
var portToUse = primaryPort ? launchParameters.PrimaryPort : launchParameters.SecondaryPort;
if (!portToUse.HasValue)
{
throw new InvalidOperationException("Given port is null!");
}
var accessIdentifier = cryptographySuite.GetSecureString();
const string JsonPostfix = "tgs.json";
var basePath = primaryDirectory ? dmbProvider.PrimaryDirectory : dmbProvider.SecondaryDirectory;
// delete all previous tgs json files
var files = await ioManager.GetFilesWithExtension(basePath, JsonPostfix, cancellationToken).ConfigureAwait(false);
await Task.WhenAll(files.Select(x => ioManager.DeleteFile(x, cancellationToken))).ConfigureAwait(false);
// i changed this back from guids, hopefully i don't regret that
string JsonFile(string name) => String.Format(CultureInfo.InvariantCulture, "{0}.{1}", name, JsonPostfix);
var securityLevelToUse = launchParameters.SecurityLevel.Value;
switch (dmbProvider.CompileJob.MinimumSecurityLevel)
{
case DreamDaemonSecurity.Ultrasafe:
break;
case DreamDaemonSecurity.Safe:
if (securityLevelToUse == DreamDaemonSecurity.Ultrasafe)
{
securityLevelToUse = DreamDaemonSecurity.Safe;
}
break;
case DreamDaemonSecurity.Trusted:
securityLevelToUse = DreamDaemonSecurity.Trusted;
break;
default:
throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, "Invalid DreamDaemonSecurity value: {0}", dmbProvider.CompileJob.MinimumSecurityLevel));
}
// setup interop files
var interopInfo = new JsonFile
{
AccessIdentifier = accessIdentifier,
ApiValidateOnly = apiValidate,
ChatChannelsJson = JsonFile("chat_channels"),
ChatCommandsJson = JsonFile("chat_commands"),
ServerCommandsJson = JsonFile("server_commands"),
InstanceName = instance.Name,
SecurityLevel = securityLevelToUse,
Revision = new Api.Models.Internal.RevisionInformation
{
CommitSha = dmbProvider.CompileJob.RevisionInformation.CommitSha,
OriginCommitSha = dmbProvider.CompileJob.RevisionInformation.OriginCommitSha
}
};
interopInfo.TestMerges.AddRange(dmbProvider.CompileJob.RevisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => new Interop.TestMerge(x, interopInfo.Revision)));
var interopJsonFile = JsonFile("interop");
var interopJson = JsonConvert.SerializeObject(interopInfo, new JsonSerializerSettings
{
ContractResolver = new CamelCasePropertyNamesContractResolver(),
ReferenceLoopHandling = ReferenceLoopHandling.Ignore
});
var chatJsonTrackingTask = chat.TrackJsons(basePath, interopInfo.ChatChannelsJson, interopInfo.ChatCommandsJson, cancellationToken);
await ioManager.WriteAllBytes(ioManager.ConcatPath(basePath, interopJsonFile), Encoding.UTF8.GetBytes(interopJson), cancellationToken).ConfigureAwait(false);
var chatJsonTrackingContext = await chatJsonTrackingTask.ConfigureAwait(false);
try
{
// get the byond lock
var byondLock = currentByondLock ?? await byond.UseExecutables(Version.Parse(dmbProvider.CompileJob.ByondVersion), cancellationToken).ConfigureAwait(false);
try
{
// create interop context
var context = new CommContext(ioManager, loggerFactory.CreateLogger <CommContext>(), basePath, interopInfo.ServerCommandsJson);
try
{
// set command line options
// more sanitization here cause it uses the same scheme
var parameters = String.Format(CultureInfo.InvariantCulture, "{2}={0}&{3}={1}", byondTopicSender.SanitizeString(application.Version.ToString()), byondTopicSender.SanitizeString(interopJsonFile), byondTopicSender.SanitizeString(Constants.DMParamHostVersion), byondTopicSender.SanitizeString(Constants.DMParamInfoJson));
var visibility = apiValidate ? "invisible" : "public";
// important to run on all ports to allow port changing
var arguments = String.Format(CultureInfo.InvariantCulture, "{0} -port {1} -ports 1-65535 {2}-close -{3} -{5} -public -params \"{4}\"",
dmbProvider.DmbName,
primaryPort ? launchParameters.PrimaryPort : launchParameters.SecondaryPort,
launchParameters.AllowWebClient.Value ? "-webclient " : String.Empty,
SecurityWord(securityLevelToUse),
parameters,
visibility);
// See https://github.com/tgstation/tgstation-server/issues/719
var noShellExecute = !platformIdentifier.IsWindows;
// launch dd
var process = processExecutor.LaunchProcess(byondLock.DreamDaemonPath, basePath, arguments, noShellExecute: noShellExecute);
try
{
networkPromptReaper.RegisterProcess(process);
// return the session controller for it
var result = new SessionController(new ReattachInformation
{
AccessIdentifier = accessIdentifier,
Dmb = dmbProvider,
IsPrimary = primaryDirectory,
Port = portToUse.Value,
ProcessId = process.Id,
ChatChannelsJson = interopInfo.ChatChannelsJson,
ChatCommandsJson = interopInfo.ChatCommandsJson,
ServerCommandsJson = interopInfo.ServerCommandsJson,
}, process, byondLock, byondTopicSender, chatJsonTrackingContext, context, chat, loggerFactory.CreateLogger <SessionController>(), launchParameters.SecurityLevel, launchParameters.StartupTimeout);
// writeback launch parameter's fixed security level
launchParameters.SecurityLevel = securityLevelToUse;
return(result);
}
catch
{
process.Dispose();
throw;
}
}
catch
{
context.Dispose();
throw;
}
}
catch
{
if (currentByondLock == null)
{
byondLock.Dispose();
}
throw;
}
}
catch
{
chatJsonTrackingContext.Dispose();
throw;
}
}
/// <summary>
/// Generate map diffs for a given <paramref name="pullRequest"/>
/// </summary>
/// <param name="pullRequest">The <see cref="PullRequest"/></param>
/// <param name="checkRunId">The <see cref="CheckRun.Id"/></param>
/// <param name="changedDmms">Paths to changed .dmm files</param>
/// <param name="scope">The <see cref="IServiceScope"/> for the operation</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task GenerateDiffs(PullRequest pullRequest, long checkRunId, IReadOnlyList <string> changedDmms, IServiceScope scope, CancellationToken cancellationToken)
{
using (logger.BeginScope("Generating {0} diffs for pull request #{1} in {2}/{3}", changedDmms.Count, pullRequest.Number, pullRequest.Base.Repository.Owner.Login, pullRequest.Base.Repository.Name))
{
const string OldMapExtension = ".old_map_diff_bot";
var gitHubManager = scope.ServiceProvider.GetRequiredService <IGitHubManager>();
var generatingCommentTask = Task.CompletedTask;
List <Task <RenderResult> > afterRenderings, beforeRenderings;
var workingDir = ioManager.ConcatPath(pullRequest.Base.Repository.Owner.Login, pullRequest.Base.Repository.Name, pullRequest.Number.ToString(CultureInfo.InvariantCulture));
logger.LogTrace("Setting workdir to {0}", workingDir);
IIOManager currentIOManager = new ResolvingIOManager(ioManager, workingDir);
string repoPath;
int lastProgress = -1;
Task lastProgressUpdate = Task.CompletedTask;
async Task OnCloneProgress(int progress)
{
lock (gitHubManager)
{
if (lastProgress >= progress)
{
return;
}
if (lastProgress == -1)
{
logger.LogInformation("Waiting on repository to finish cloning...");
}
lastProgress = progress;
}
await lastProgressUpdate.ConfigureAwait(false);
await gitHubManager.UpdateCheckRun(pullRequest.Base.Repository.Id, checkRunId, new CheckRunUpdate
{
Status = CheckStatus.InProgress,
Output = new NewCheckRunOutput(stringLocalizer["Cloning Repository"], stringLocalizer["Clone Progress: {0}%", progress]),
}, cancellationToken).ConfigureAwait(false);
};
Task CreateBlockedComment()
{
logger.LogInformation("Waiting for another diff generation on {0}/{1} to complete...", pullRequest.Base.Repository.Owner.Login, pullRequest.Base.Repository.Name);
return(gitHubManager.UpdateCheckRun(pullRequest.Base.Repository.Id, checkRunId, new CheckRunUpdate
{
Output = new NewCheckRunOutput(stringLocalizer["Waiting for Repository"], stringLocalizer["Waiting for another operation on this repository to complete..."]),
}, cancellationToken));
};
var progressBuilder = new StringBuilder();
void AddProgressLine(string line)
{
logger.LogTrace(line);
var ourTask = new TaskCompletionSource <object>();
Task toAwait;
async Task RunNext()
{
await toAwait.ConfigureAwait(false);
string ourLine;
lock (progressBuilder)
{
progressBuilder.AppendLine(String.Format(CultureInfo.InvariantCulture, "[{0}]: {1}", DateTimeOffset.Now.ToString("HH:mm:ss", CultureInfo.InvariantCulture), line));
ourLine = progressBuilder.ToString();
}
await gitHubManager.UpdateCheckRun(pullRequest.Base.Repository.Id, checkRunId, new CheckRunUpdate
{
Status = CheckStatus.InProgress,
Output = new NewCheckRunOutput(stringLocalizer["Generating Diffs"], stringLocalizer["Progress:"])
{
Text = String.Format(CultureInfo.InvariantCulture, "```{0}{1}{0}```", Environment.NewLine, ourLine)
},
}, cancellationToken).ConfigureAwait(false);
ourTask.SetResult(null);
};
lock (progressBuilder)
{
toAwait = generatingCommentTask;
generatingCommentTask = RunNext();
}
}
async Task DirectoryPrep(bool recreate)
{
AddProgressLine("Cleaning working directory...");
await currentIOManager.DeleteDirectory(".", cancellationToken).ConfigureAwait(false);
if (recreate)
{
await currentIOManager.CreateDirectory(".", cancellationToken).ConfigureAwait(false);
}
AddProgressLine("Working directory cleaned!");
};
logger.LogTrace("Locking repository...");
using (var repo = await repositoryManager.GetRepository(pullRequest.Base.Repository, OnCloneProgress, CreateBlockedComment, cancellationToken).ConfigureAwait(false))
{
logger.LogTrace("Repository ready");
AddProgressLine("Initializing...");
async Task <string> GetDmeToUse()
{
var customDme = await scope.ServiceProvider.GetRequiredService <IDatabaseContext>().InstallationRepositories.Where(x => x.Id == pullRequest.Base.Repository.Id).Select(x => x.TargetDme).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false);
if (customDme != null)
{
AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Using manually set dme: {0}", customDme));
return(customDme);
}
AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Looking for dme to use in {0}", repo.Path));
var availDmes = await ioManager.GetFilesWithExtension(repo.Path, "dme", cancellationToken).ConfigureAwait(false);
customDme = availDmes.First();
AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Selected {0} out of {1} possibilities", customDme, availDmes.Count));
return(customDme);
}
var dirPrepTask = DirectoryPrep(true);
//get the dme to use
var dmeToUseTask = GetDmeToUse();
var oldMapPaths = new List <string>()
{
Capacity = changedDmms.Count
};
try
{
//fetch base commit if necessary and check it out, fetch pull request
if (!await repo.ContainsCommit(pullRequest.Base.Sha, cancellationToken).ConfigureAwait(false))
{
AddProgressLine("Base commit not found, running fetch...");
await repo.Fetch(cancellationToken).ConfigureAwait(false);
}
AddProgressLine("Moving HEAD to pull request base...");
await repo.Checkout(pullRequest.Base.Sha, cancellationToken).ConfigureAwait(false);
//but since we don't need this right await don't await it yet
var pullRequestFetchTask = repo.FetchPullRequest(pullRequest.Number, cancellationToken);
try
{
//first copy all modified maps to the same location with the .old_map_diff_bot extension
async Task <string> CacheMap(string mapPath)
{
var originalPath = currentIOManager.ConcatPath(repoPath, mapPath);
if (await currentIOManager.FileExists(originalPath, cancellationToken).ConfigureAwait(false))
{
AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Creating old map cache of {0}", mapPath));
var oldMapPath = String.Format(CultureInfo.InvariantCulture, "{0}{1}", originalPath, OldMapExtension);
await currentIOManager.CopyFile(originalPath, oldMapPath, cancellationToken).ConfigureAwait(false);
return(oldMapPath);
}
return(null);
};
repoPath = repo.Path;
var tasks = changedDmms.Select(x => CacheMap(x)).ToList();
await Task.WhenAll(tasks).ConfigureAwait(false);
oldMapPaths.AddRange(tasks.Select(x => x.Result));
}
finally
{
logger.LogTrace("Waiting for pull request commits to be available...");
await pullRequestFetchTask.ConfigureAwait(false);
}
AddProgressLine("Creating and moving HEAD to pull request merge commit...");
//generate the merge commit ourselves since we can't get it from GitHub because itll return an outdated one
await repo.Merge(pullRequest.Head.Sha, cancellationToken).ConfigureAwait(false);
}
finally
{
logger.LogTrace("Waiting for configured project dme...");
await dmeToUseTask.ConfigureAwait(false);
}
//create empty array of map regions
var mapRegions = Enumerable.Repeat <MapRegion>(null, changedDmms.Count).ToList();
var dmeToUse = dmeToUseTask.Result;
var generator = generatorFactory.CreateGenerator(dmeToUse, new ResolvingIOManager(ioManager, repoPath));
var outputDirectory = currentIOManager.ResolvePath(".");
logger.LogTrace("Full workdir path: {0}", outputDirectory);
//Generate MapRegions for modified maps and render all new maps
async Task <RenderResult> DiffAndRenderNewMap(int I)
{
await dirPrepTask.ConfigureAwait(false);
var originalPath = currentIOManager.ConcatPath(repoPath, changedDmms[I]);
if (!await currentIOManager.FileExists(originalPath, cancellationToken).ConfigureAwait(false))
{
AddProgressLine(String.Format(CultureInfo.InvariantCulture, "No new map for path {0} exists, skipping region detection and after render", changedDmms[I]));
return(new RenderResult {
InputPath = changedDmms[I], ToolOutput = stringLocalizer["Map missing!"]
});
}
ToolResult result = null;
if (oldMapPaths[I] != null)
{
AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Getting diff region for {0}...", changedDmms[I]));
result = await generator.GetDifferences(oldMapPaths[I], originalPath, cancellationToken).ConfigureAwait(false);
var region = result.MapRegion;
AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Diff region for {0}: {1}", changedDmms[I], region));
if (region != null)
{
var xdiam = region.MaxX - region.MinX;
var ydiam = region.MaxY - region.MinY;
const int minDiffDimensions = 5 - 1;
if (xdiam < minDiffDimensions || ydiam < minDiffDimensions)
{
//need to expand
var fullResult = await generator.GetMapSize(originalPath, cancellationToken).ConfigureAwait(false);
var fullRegion = fullResult.MapRegion;
if (fullRegion == null)
{
//give up
region = null;
}
else
{
bool increaseMax = true;
if (xdiam < minDiffDimensions && ((fullRegion.MaxX - fullRegion.MinX) >= minDiffDimensions))
{
while ((region.MaxX - region.MinX) < minDiffDimensions)
{
if (increaseMax)
{
region.MaxX = (short)Math.Min(region.MaxX + 1, fullRegion.MaxX);
}
else
{
region.MinX = (short)Math.Max(region.MinX - 1, 1);
}
increaseMax = !increaseMax;
}
}
if (ydiam < minDiffDimensions && ((fullRegion.MaxY - fullRegion.MinY) >= minDiffDimensions))
{
while ((region.MaxY - region.MinY) < minDiffDimensions)
{
if (increaseMax)
{
region.MaxY = (short)Math.Min(region.MaxY + 1, fullRegion.MaxY);
}
else
{
region.MinY = (short)Math.Max(region.MinY - 1, 1);
}
increaseMax = !increaseMax;
}
}
}
AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Region for {0} expanded to {1}", changedDmms[I], region));
}
mapRegions[I] = region;
}
}
else
{
AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Skipping region detection for {0} due to old map not existing", changedDmms[I]));
}
AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Performing after rendering for {0}...", changedDmms[I]));
var renderResult = await generator.RenderMap(originalPath, mapRegions[I], outputDirectory, "after", cancellationToken).ConfigureAwait(false);
logger.LogTrace("After rendering for {0} complete! Result path: {1}, Output: {2}", changedDmms[I], renderResult.OutputPath, renderResult.ToolOutput);
AddProgressLine(String.Format(CultureInfo.InvariantCulture, "After rendering for {0} complete!", renderResult.OutputPath));
if (result != null)
{
renderResult.ToolOutput = String.Format(CultureInfo.InvariantCulture, "Differences task:{0}{1}{0}Render task:{0}{2}", Environment.NewLine, result.ToolOutput, renderResult.ToolOutput);
}
return(renderResult);
};
logger.LogTrace("Running iterations of DiffAndRenderNewMap...");
//finish up before we go back to the base branch
afterRenderings = Enumerable.Range(0, changedDmms.Count).Select(I => DiffAndRenderNewMap(I)).ToList();
try
{
await Task.WhenAll(afterRenderings).ConfigureAwait(false);
}
catch (Exception e)
{
logger.LogDebug(e, "After renderings produced exception!");
//at this point everything is done but some have failed
//we'll handle it later
}
AddProgressLine("Moving HEAD back to pull request base...");
await repo.Checkout(pullRequest.Base.Sha, cancellationToken).ConfigureAwait(false);
async Task <RenderResult> RenderOldMap(int i)
{
var oldPath = oldMapPaths[i];
if (oldMapPaths != null)
{
AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Performing before rendering for {0}...", changedDmms[i]));
var result = await generator.RenderMap(oldPath, mapRegions[i], outputDirectory, "before", cancellationToken).ConfigureAwait(false);
AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Before rendering for {0} complete!", changedDmms[i]));
return(result);
}
return(new RenderResult {
InputPath = changedDmms[i], ToolOutput = stringLocalizer["Map missing!"]
});
}
logger.LogTrace("Running iterations of RenderOldMap...");
//finish up rendering
beforeRenderings = Enumerable.Range(0, changedDmms.Count).Select(I => RenderOldMap(I)).ToList();
try
{
await Task.WhenAll(beforeRenderings).ConfigureAwait(false);
}
catch (Exception e)
{
logger.LogDebug(e, "Before renderings produced exception!");
//see above
}
AddProgressLine("Renderings complete, finalizing...");
//done with the repo at this point
logger.LogTrace("Renderings complete. Releasing reposiotory");
}
//collect results and errors
async Task <KeyValuePair <MapDiff, MapRegion> > GetResult(int i)
{
var beforeTask = beforeRenderings[i];
var afterTask = afterRenderings[i];
var result = new MapDiff
{
InstallationRepositoryId = pullRequest.Base.Repository.Id,
CheckRunId = checkRunId,
FileId = i,
};
RenderResult GetRenderingResult(Task <RenderResult> task)
{
if (task.Exception != null)
{
result.LogMessage = result.LogMessage == null?task.Exception.ToString() : String.Format(CultureInfo.InvariantCulture, "{0}{1}{2}", result.LogMessage, Environment.NewLine, task.Exception);
return(null);
}
return(task.Result);
};
var r1 = GetRenderingResult(beforeTask);
var r2 = GetRenderingResult(afterTask);
logger.LogTrace("Results for {0}: Before {1}, After {2}", changedDmms[i], r1?.OutputPath ?? "NONE", r2?.OutputPath ?? "NONE");
result.MapPath = changedDmms[i];
result.LogMessage = String.Format(CultureInfo.InvariantCulture, "Job {5}:{0}Path: {6}{0}Before:{0}Command Line: {1}{0}Output:{0}{2}{0}Logs:{0}{7}{0}After:{0}Command Line: {3}{0}Output:{0}{4}{0}Logs:{0}{8}{0}Exceptions:{0}{9}{0}", Environment.NewLine, r1?.CommandLine, r1?.OutputPath, r2?.CommandLine, r2?.OutputPath, i + 1, result.MapPath, r1?.ToolOutput, r2?.ToolOutput, result.LogMessage);
async Task <Image> ReadMapImage(string path)
{
if (path != null && await currentIOManager.FileExists(path, cancellationToken).ConfigureAwait(false))
{
var bytes = await currentIOManager.ReadAllBytes(path, cancellationToken).ConfigureAwait(false);
return(new Image {
Data = bytes
});
}
return(null);
}
var readBeforeTask = ReadMapImage(r1?.OutputPath);
result.AfterImage = await ReadMapImage(r2?.OutputPath).ConfigureAwait(false);
result.BeforeImage = await readBeforeTask.ConfigureAwait(false);
Image GenerateDifferenceImage(Image before, Image after)
{
if (before == null || result.AfterImage == null)
{
return(null);
}
using (var ms = new MemoryStream())
{
using (var diffI = new MagickImage())
{
using (var beforeI = new MagickImage(before.Data, new MagickReadSettings()))
using (var afterI = new MagickImage(after.Data, new MagickReadSettings()))
{
beforeI.Compare(afterI, ErrorMetric.Absolute, diffI);
}
diffI.Write(ms, MagickFormat.Png32);
}
return(new Image {
Data = ms.ToArray()
});
}
}
result.DifferenceImage = GenerateDifferenceImage(result.BeforeImage, result.AfterImage);
return(new KeyValuePair <MapDiff, MapRegion>(result, r2?.MapRegion));
}
logger.LogTrace("Collecting results...");
var results = Enumerable.Range(0, changedDmms.Count).Select(x => GetResult(x)).ToList();
await Task.WhenAll(results).ConfigureAwait(false);
await DirectoryPrep(false).ConfigureAwait(false);
logger.LogTrace("Waiting for GitHub check to finish updating...");
await generatingCommentTask.ConfigureAwait(false);
var dic = new Dictionary <MapDiff, MapRegion>();
foreach (var I in results.Select(x => x.Result))
{
dic.Add(I.Key, I.Value);
}
await HandleResults(pullRequest, checkRunId, dic, scope, cancellationToken).ConfigureAwait(false);
}
}
/// <summary>
/// Executes and populate a given <paramref name="job"/>
/// </summary>
/// <param name="job">The <see cref="CompileJob"/> to run and populate</param>
/// <param name="dreamMakerSettings">The <see cref="Api.Models.Internal.DreamMakerSettings"/> to use</param>
/// <param name="byondLock">The <see cref="IByondExecutableLock"/> to use</param>
/// <param name="repository">The <see cref="IRepository"/> to use</param>
/// <param name="remoteDeploymentManager">The <see cref="IRemoteDeploymentManager"/> to use.</param>
/// <param name="apiValidateTimeout">The timeout for validating the DMAPI</param>
/// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param>
/// <returns>A <see cref="Task"/> representing the running operation</returns>
async Task RunCompileJob(
Models.CompileJob job,
Api.Models.Internal.DreamMakerSettings dreamMakerSettings,
IByondExecutableLock byondLock,
IRepository repository,
IRemoteDeploymentManager remoteDeploymentManager,
uint apiValidateTimeout,
CancellationToken cancellationToken)
{
var outputDirectory = job.DirectoryName.ToString();
logger.LogTrace("Compile output GUID: {0}", outputDirectory);
try
{
// copy the repository
logger.LogTrace("Copying repository to game directory...");
var resolvedOutputDirectory = ioManager.ResolvePath(outputDirectory);
var repoOrigin = repository.Origin;
using (repository)
await repository.CopyTo(resolvedOutputDirectory, cancellationToken).ConfigureAwait(false);
// repository closed now
// run precompile scripts
await eventConsumer.HandleEvent(
EventType.CompileStart,
new List <string>
{
resolvedOutputDirectory,
repoOrigin.ToString()
},
cancellationToken)
.ConfigureAwait(false);
// determine the dme
if (job.DmeName == null)
{
logger.LogTrace("Searching for available .dmes...");
var foundPaths = await ioManager.GetFilesWithExtension(resolvedOutputDirectory, DmeExtension, true, cancellationToken).ConfigureAwait(false);
var foundPath = foundPaths.FirstOrDefault();
if (foundPath == default)
{
throw new JobException(ErrorCode.DreamMakerNoDme);
}
job.DmeName = foundPath.Substring(
resolvedOutputDirectory.Length + 1,
foundPath.Length - resolvedOutputDirectory.Length - DmeExtension.Length - 2); // +1 for . in extension
}
else
{
var targetDme = ioManager.ConcatPath(outputDirectory, String.Join('.', job.DmeName, DmeExtension));
var targetDmeExists = await ioManager.FileExists(targetDme, cancellationToken).ConfigureAwait(false);
if (!targetDmeExists)
{
throw new JobException(ErrorCode.DreamMakerMissingDme);
}
}
logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName);
await ModifyDme(job, cancellationToken).ConfigureAwait(false);
// run compiler
var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false);
// verify api
try
{
if (exitCode != 0)
{
throw new JobException(
ErrorCode.DreamMakerExitCode,
new JobException($"Exit code: {exitCode}{Environment.NewLine}{Environment.NewLine}{job.Output}"));
}
await VerifyApi(
apiValidateTimeout,
dreamMakerSettings.ApiValidationSecurityLevel.Value,
job,
byondLock,
dreamMakerSettings.ApiValidationPort.Value,
dreamMakerSettings.RequireDMApiValidation.Value,
cancellationToken)
.ConfigureAwait(false);
}
catch (JobException)
{
// DD never validated or compile failed
await eventConsumer.HandleEvent(
EventType.CompileFailure,
new List <string>
{
resolvedOutputDirectory,
exitCode == 0 ? "1" : "0"
},
cancellationToken)
.ConfigureAwait(false);
throw;
}
await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> {
resolvedOutputDirectory
}, cancellationToken).ConfigureAwait(false);
logger.LogTrace("Applying static game file symlinks...");
// symlink in the static data
await configuration.SymlinkStaticFilesTo(resolvedOutputDirectory, cancellationToken).ConfigureAwait(false);
logger.LogDebug("Compile complete!");
}
catch (Exception ex)
{
await CleanupFailedCompile(job, remoteDeploymentManager, ex).ConfigureAwait(false);
throw;
}
}