/// <inheritdoc /> public async Task <bool> HandleEvent(EventType eventType, IEnumerable <string> parameters, CancellationToken cancellationToken) { await EnsureDirectories(cancellationToken).ConfigureAwait(false); if (!EventTypeScriptFileNameMap.TryGetValue(eventType, out var scriptName)) { return(true); } //always execute in serial using (await SemaphoreSlimContext.Lock(semaphore, cancellationToken).ConfigureAwait(false)) { var files = await ioManager.GetFilesWithExtension(EventScriptsSubdirectory, SystemScriptFileExtension, cancellationToken).ConfigureAwait(false); var resolvedScriptsDir = ioManager.ResolvePath(EventScriptsSubdirectory); foreach (var I in files.Select(x => ioManager.GetFileName(x)).Where(x => x.StartsWith(scriptName, StringComparison.Ordinal))) { using (var script = processExecutor.LaunchProcess(ioManager.ConcatPath(resolvedScriptsDir, I), resolvedScriptsDir, String.Join(' ', parameters), noShellExecute: true)) using (cancellationToken.Register(() => script.Terminate())) { var exitCode = await script.Lifetime.ConfigureAwait(false); cancellationToken.ThrowIfCancellationRequested(); if (exitCode != 0) { return(false); } } } } return(true); }
/// <summary> /// Executes and populate a given <paramref name="job"/> /// </summary> /// <param name="job">The <see cref="Models.CompileJob"/> to run and populate</param> /// <param name="dreamMakerSettings">The <see cref="Api.Models.DreamMaker"/> settings to use</param> /// <param name="byondLock">The <see cref="IByondExecutableLock"/> to use</param> /// <param name="repository">The <see cref="IRepository"/> to use</param> /// <param name="apiValidateTimeout">The timeout for validating the DMAPI</param> /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param> /// <returns>A <see cref="Task"/> representing the running operation</returns> async Task RunCompileJob(Models.CompileJob job, Api.Models.DreamMaker dreamMakerSettings, IByondExecutableLock byondLock, IRepository repository, uint apiValidateTimeout, CancellationToken cancellationToken) { var jobPath = job.DirectoryName.ToString(); logger.LogTrace("Compile output GUID: {0}", jobPath); try { var dirA = ioManager.ConcatPath(jobPath, ADirectoryName); var dirB = ioManager.ConcatPath(jobPath, BDirectoryName); // copy the repository logger.LogTrace("Copying repository to game directory..."); var resolvedADirectory = ioManager.ResolvePath(dirA); var repoOrigin = repository.Origin; using (repository) await repository.CopyTo(resolvedADirectory, cancellationToken).ConfigureAwait(false); // repository closed now // run precompile scripts await eventConsumer.HandleEvent(EventType.CompileStart, new List <string> { resolvedADirectory, repoOrigin }, cancellationToken).ConfigureAwait(false); // determine the dme if (job.DmeName == null) { logger.LogTrace("Searching for available .dmes..."); var foundPaths = await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false); var foundPath = foundPaths.FirstOrDefault(); if (foundPath == default) { throw new JobException("Unable to find any .dme!"); } var dmeWithExtension = ioManager.GetFileName(foundPath); job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1); } else { var targetDme = ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension)); var targetDmeExists = await ioManager.FileExists(targetDme, cancellationToken).ConfigureAwait(false); if (!targetDmeExists) { throw new JobException("Unable to locate specified .dme!"); } } logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName); await ModifyDme(job, cancellationToken).ConfigureAwait(false); // run compiler var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false); // verify api try { if (exitCode != 0) { throw new JobException(String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}{2}", exitCode, Environment.NewLine, job.Output)); } await VerifyApi(apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false); } catch (JobException) { // DD never validated or compile failed await eventConsumer.HandleEvent(EventType.CompileFailure, new List <string> { resolvedADirectory, exitCode == 0 ? "1" : "0" }, cancellationToken).ConfigureAwait(false); throw; } logger.LogTrace("Running post compile event..."); await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> { resolvedADirectory }, cancellationToken).ConfigureAwait(false); logger.LogTrace("Duplicating compiled game..."); // duplicate the dmb et al await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false); logger.LogTrace("Applying static game file symlinks..."); // symlink in the static data var symATask = configuration.SymlinkStaticFilesTo(resolvedADirectory, cancellationToken); var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken); await Task.WhenAll(symATask, symBTask).ConfigureAwait(false); await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deployment complete!{0}", watchdog.Running ? " Changes will be applied on next server reboot." : String.Empty), cancellationToken).ConfigureAwait(false); logger.LogDebug("Compile complete!"); } catch (Exception e) { await CleanupFailedCompile(job, e is OperationCanceledException, cancellationToken).ConfigureAwait(false); throw; } }
/// <inheritdoc /> public async Task <Models.CompileJob> Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action <int> progressReporter, TimeSpan?estimatedDuration, CancellationToken cancellationToken) { if (revisionInformation == null) { throw new ArgumentNullException(nameof(revisionInformation)); } if (dreamMakerSettings == null) { throw new ArgumentNullException(nameof(dreamMakerSettings)); } if (repository == null) { throw new ArgumentNullException(nameof(repository)); } if (progressReporter == null) { throw new ArgumentNullException(nameof(progressReporter)); } if (dreamMakerSettings.ApiValidationSecurityLevel == DreamDaemonSecurity.Ultrasafe) { throw new ArgumentOutOfRangeException(nameof(dreamMakerSettings), dreamMakerSettings, "Cannot compile with ultrasafe security!"); } logger.LogTrace("Begin Compile"); var job = new Models.CompileJob { DirectoryName = Guid.NewGuid(), DmeName = dreamMakerSettings.ProjectName, RevisionInformation = revisionInformation }; logger.LogTrace("Compile output GUID: {0}", job.DirectoryName); lock (this) { if (compiling) { throw new JobException("There is already a compile job in progress!"); } compiling = true; } using (var progressCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { async Task ProgressTask() { if (!estimatedDuration.HasValue) { return; } progressReporter(0); var ct = progressCts.Token; var sleepInterval = estimatedDuration.Value / 100; try { for (var I = 0; I < 99; ++I) { await Task.Delay(sleepInterval, progressCts.Token).ConfigureAwait(false); progressReporter(I + 1); } } catch (OperationCanceledException) { } } var progressTask = ProgressTask(); try { var commitInsert = revisionInformation.CommitSha.Substring(0, 7); string remoteCommitInsert; if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha) { commitInsert = String.Format(CultureInfo.InvariantCulture, "^{0}", commitInsert); remoteCommitInsert = String.Empty; } else { remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha.Substring(0, 7)); } var testmergeInsert = revisionInformation.ActiveTestMerges.Count == 0 ? String.Empty : String.Format(CultureInfo.InvariantCulture, " (Test Merges: {0})", String.Join(", ", revisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => { var result = String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.PullRequestRevision.Substring(0, 7)); if (x.Comment != null) { result += String.Format(CultureInfo.InvariantCulture, " ({0})", x.Comment); } return(result); }))); using (var byondLock = await byond.UseExecutables(null, cancellationToken).ConfigureAwait(false)) { await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deploying revision: {0}{1}{2} BYOND Version: {3}", commitInsert, testmergeInsert, remoteCommitInsert, byondLock.Version), cancellationToken).ConfigureAwait(false); async Task CleanupFailedCompile(bool cancelled) { logger.LogTrace("Cleaning compile directory..."); var chatTask = chat.SendUpdateMessage(cancelled ? "Deploy cancelled!" : "Deploy failed!", cancellationToken); try { await ioManager.DeleteDirectory(job.DirectoryName.ToString(), CancellationToken.None).ConfigureAwait(false); } catch (Exception e) { logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(job.DirectoryName.ToString()), e); } await chatTask.ConfigureAwait(false); }; try { await ioManager.CreateDirectory(job.DirectoryName.ToString(), cancellationToken).ConfigureAwait(false); var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); var dirB = ioManager.ConcatPath(job.DirectoryName.ToString(), BDirectoryName); logger.LogTrace("Copying repository to game directory..."); //copy the repository var fullDirA = ioManager.ResolvePath(dirA); var repoOrigin = repository.Origin; using (repository) await repository.CopyTo(fullDirA, cancellationToken).ConfigureAwait(false); //run precompile scripts var resolvedGameDirectory = ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)); await eventConsumer.HandleEvent(EventType.CompileStart, new List <string> { resolvedGameDirectory, repoOrigin }, cancellationToken).ConfigureAwait(false); //determine the dme if (job.DmeName == null) { logger.LogTrace("Searching for available .dmes..."); var path = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault(); if (path == default) { throw new JobException("Unable to find any .dme!"); } var dmeWithExtension = ioManager.GetFileName(path); job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1); } else if (!await ioManager.FileExists(ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension)), cancellationToken).ConfigureAwait(false)) { throw new JobException("Unable to locate specified .dme!"); } logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName); await ModifyDme(job, cancellationToken).ConfigureAwait(false); //run compiler, verify api job.ByondVersion = byondLock.Version.ToString(); var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false); try { if (exitCode != 0) { throw new JobException(String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}{2}", exitCode, Environment.NewLine, job.Output)); } await VerifyApi(apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false); } catch (JobException) { //server never validated or compile failed await eventConsumer.HandleEvent(EventType.CompileFailure, new List <string> { resolvedGameDirectory, exitCode == 0 ? "1" : "0" }, cancellationToken).ConfigureAwait(false); throw; } logger.LogTrace("Running post compile event..."); await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> { ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)) }, cancellationToken).ConfigureAwait(false); logger.LogTrace("Duplicating compiled game..."); //duplicate the dmb et al await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false); logger.LogTrace("Applying static game file symlinks..."); //symlink in the static data var symATask = configuration.SymlinkStaticFilesTo(fullDirA, cancellationToken); var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken); await Task.WhenAll(symATask, symBTask).ConfigureAwait(false); await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deployment complete!{0}", watchdog.Running ? " Changes will be applied on next server reboot." : String.Empty), cancellationToken).ConfigureAwait(false); logger.LogDebug("Compile complete!"); return(job); } catch (Exception e) { await CleanupFailedCompile(e is OperationCanceledException).ConfigureAwait(false); throw; } } } catch (OperationCanceledException) { await eventConsumer.HandleEvent(EventType.CompileCancelled, null, default).ConfigureAwait(false); throw; } finally { compiling = false; progressCts.Cancel(); await progressTask.ConfigureAwait(false); } } }
/// <inheritdoc /> #pragma warning disable CA1506 // TODO: Decomplexify public async Task <ISessionController> LaunchNew(DreamDaemonLaunchParameters launchParameters, IDmbProvider dmbProvider, IByondExecutableLock currentByondLock, bool primaryPort, bool primaryDirectory, bool apiValidate, CancellationToken cancellationToken) { var portToUse = primaryPort ? launchParameters.PrimaryPort : launchParameters.SecondaryPort; if (!portToUse.HasValue) { throw new InvalidOperationException("Given port is null!"); } var accessIdentifier = cryptographySuite.GetSecureString(); const string JsonPostfix = "tgs.json"; var basePath = primaryDirectory ? dmbProvider.PrimaryDirectory : dmbProvider.SecondaryDirectory; // delete all previous tgs json files var files = await ioManager.GetFilesWithExtension(basePath, JsonPostfix, cancellationToken).ConfigureAwait(false); await Task.WhenAll(files.Select(x => ioManager.DeleteFile(x, cancellationToken))).ConfigureAwait(false); // i changed this back from guids, hopefully i don't regret that string JsonFile(string name) => String.Format(CultureInfo.InvariantCulture, "{0}.{1}", name, JsonPostfix); var securityLevelToUse = launchParameters.SecurityLevel.Value; switch (dmbProvider.CompileJob.MinimumSecurityLevel) { case DreamDaemonSecurity.Ultrasafe: break; case DreamDaemonSecurity.Safe: if (securityLevelToUse == DreamDaemonSecurity.Ultrasafe) { securityLevelToUse = DreamDaemonSecurity.Safe; } break; case DreamDaemonSecurity.Trusted: securityLevelToUse = DreamDaemonSecurity.Trusted; break; default: throw new InvalidOperationException(String.Format(CultureInfo.InvariantCulture, "Invalid DreamDaemonSecurity value: {0}", dmbProvider.CompileJob.MinimumSecurityLevel)); } // setup interop files var interopInfo = new JsonFile { AccessIdentifier = accessIdentifier, ApiValidateOnly = apiValidate, ChatChannelsJson = JsonFile("chat_channels"), ChatCommandsJson = JsonFile("chat_commands"), ServerCommandsJson = JsonFile("server_commands"), InstanceName = instance.Name, SecurityLevel = securityLevelToUse, Revision = new Api.Models.Internal.RevisionInformation { CommitSha = dmbProvider.CompileJob.RevisionInformation.CommitSha, OriginCommitSha = dmbProvider.CompileJob.RevisionInformation.OriginCommitSha } }; interopInfo.TestMerges.AddRange(dmbProvider.CompileJob.RevisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => new Interop.TestMerge(x, interopInfo.Revision))); var interopJsonFile = JsonFile("interop"); var interopJson = JsonConvert.SerializeObject(interopInfo, new JsonSerializerSettings { ContractResolver = new CamelCasePropertyNamesContractResolver(), ReferenceLoopHandling = ReferenceLoopHandling.Ignore }); var chatJsonTrackingTask = chat.TrackJsons(basePath, interopInfo.ChatChannelsJson, interopInfo.ChatCommandsJson, cancellationToken); await ioManager.WriteAllBytes(ioManager.ConcatPath(basePath, interopJsonFile), Encoding.UTF8.GetBytes(interopJson), cancellationToken).ConfigureAwait(false); var chatJsonTrackingContext = await chatJsonTrackingTask.ConfigureAwait(false); try { // get the byond lock var byondLock = currentByondLock ?? await byond.UseExecutables(Version.Parse(dmbProvider.CompileJob.ByondVersion), cancellationToken).ConfigureAwait(false); try { // create interop context var context = new CommContext(ioManager, loggerFactory.CreateLogger <CommContext>(), basePath, interopInfo.ServerCommandsJson); try { // set command line options // more sanitization here cause it uses the same scheme var parameters = String.Format(CultureInfo.InvariantCulture, "{2}={0}&{3}={1}", byondTopicSender.SanitizeString(application.Version.ToString()), byondTopicSender.SanitizeString(interopJsonFile), byondTopicSender.SanitizeString(Constants.DMParamHostVersion), byondTopicSender.SanitizeString(Constants.DMParamInfoJson)); var visibility = apiValidate ? "invisible" : "public"; // important to run on all ports to allow port changing var arguments = String.Format(CultureInfo.InvariantCulture, "{0} -port {1} -ports 1-65535 {2}-close -{3} -{5} -public -params \"{4}\"", dmbProvider.DmbName, primaryPort ? launchParameters.PrimaryPort : launchParameters.SecondaryPort, launchParameters.AllowWebClient.Value ? "-webclient " : String.Empty, SecurityWord(securityLevelToUse), parameters, visibility); // See https://github.com/tgstation/tgstation-server/issues/719 var noShellExecute = !platformIdentifier.IsWindows; // launch dd var process = processExecutor.LaunchProcess(byondLock.DreamDaemonPath, basePath, arguments, noShellExecute: noShellExecute); try { networkPromptReaper.RegisterProcess(process); // return the session controller for it var result = new SessionController(new ReattachInformation { AccessIdentifier = accessIdentifier, Dmb = dmbProvider, IsPrimary = primaryDirectory, Port = portToUse.Value, ProcessId = process.Id, ChatChannelsJson = interopInfo.ChatChannelsJson, ChatCommandsJson = interopInfo.ChatCommandsJson, ServerCommandsJson = interopInfo.ServerCommandsJson, }, process, byondLock, byondTopicSender, chatJsonTrackingContext, context, chat, loggerFactory.CreateLogger <SessionController>(), launchParameters.SecurityLevel, launchParameters.StartupTimeout); // writeback launch parameter's fixed security level launchParameters.SecurityLevel = securityLevelToUse; return(result); } catch { process.Dispose(); throw; } } catch { context.Dispose(); throw; } } catch { if (currentByondLock == null) { byondLock.Dispose(); } throw; } } catch { chatJsonTrackingContext.Dispose(); throw; } }
/// <summary> /// Generate map diffs for a given <paramref name="pullRequest"/> /// </summary> /// <param name="pullRequest">The <see cref="PullRequest"/></param> /// <param name="checkRunId">The <see cref="CheckRun.Id"/></param> /// <param name="changedDmms">Paths to changed .dmm files</param> /// <param name="scope">The <see cref="IServiceScope"/> for the operation</param> /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param> /// <returns>A <see cref="Task"/> representing the running operation</returns> async Task GenerateDiffs(PullRequest pullRequest, long checkRunId, IReadOnlyList <string> changedDmms, IServiceScope scope, CancellationToken cancellationToken) { using (logger.BeginScope("Generating {0} diffs for pull request #{1} in {2}/{3}", changedDmms.Count, pullRequest.Number, pullRequest.Base.Repository.Owner.Login, pullRequest.Base.Repository.Name)) { const string OldMapExtension = ".old_map_diff_bot"; var gitHubManager = scope.ServiceProvider.GetRequiredService <IGitHubManager>(); var generatingCommentTask = Task.CompletedTask; List <Task <RenderResult> > afterRenderings, beforeRenderings; var workingDir = ioManager.ConcatPath(pullRequest.Base.Repository.Owner.Login, pullRequest.Base.Repository.Name, pullRequest.Number.ToString(CultureInfo.InvariantCulture)); logger.LogTrace("Setting workdir to {0}", workingDir); IIOManager currentIOManager = new ResolvingIOManager(ioManager, workingDir); string repoPath; int lastProgress = -1; Task lastProgressUpdate = Task.CompletedTask; async Task OnCloneProgress(int progress) { lock (gitHubManager) { if (lastProgress >= progress) { return; } if (lastProgress == -1) { logger.LogInformation("Waiting on repository to finish cloning..."); } lastProgress = progress; } await lastProgressUpdate.ConfigureAwait(false); await gitHubManager.UpdateCheckRun(pullRequest.Base.Repository.Id, checkRunId, new CheckRunUpdate { Status = CheckStatus.InProgress, Output = new NewCheckRunOutput(stringLocalizer["Cloning Repository"], stringLocalizer["Clone Progress: {0}%", progress]), }, cancellationToken).ConfigureAwait(false); }; Task CreateBlockedComment() { logger.LogInformation("Waiting for another diff generation on {0}/{1} to complete...", pullRequest.Base.Repository.Owner.Login, pullRequest.Base.Repository.Name); return(gitHubManager.UpdateCheckRun(pullRequest.Base.Repository.Id, checkRunId, new CheckRunUpdate { Output = new NewCheckRunOutput(stringLocalizer["Waiting for Repository"], stringLocalizer["Waiting for another operation on this repository to complete..."]), }, cancellationToken)); }; var progressBuilder = new StringBuilder(); void AddProgressLine(string line) { logger.LogTrace(line); var ourTask = new TaskCompletionSource <object>(); Task toAwait; async Task RunNext() { await toAwait.ConfigureAwait(false); string ourLine; lock (progressBuilder) { progressBuilder.AppendLine(String.Format(CultureInfo.InvariantCulture, "[{0}]: {1}", DateTimeOffset.Now.ToString("HH:mm:ss", CultureInfo.InvariantCulture), line)); ourLine = progressBuilder.ToString(); } await gitHubManager.UpdateCheckRun(pullRequest.Base.Repository.Id, checkRunId, new CheckRunUpdate { Status = CheckStatus.InProgress, Output = new NewCheckRunOutput(stringLocalizer["Generating Diffs"], stringLocalizer["Progress:"]) { Text = String.Format(CultureInfo.InvariantCulture, "```{0}{1}{0}```", Environment.NewLine, ourLine) }, }, cancellationToken).ConfigureAwait(false); ourTask.SetResult(null); }; lock (progressBuilder) { toAwait = generatingCommentTask; generatingCommentTask = RunNext(); } } async Task DirectoryPrep(bool recreate) { AddProgressLine("Cleaning working directory..."); await currentIOManager.DeleteDirectory(".", cancellationToken).ConfigureAwait(false); if (recreate) { await currentIOManager.CreateDirectory(".", cancellationToken).ConfigureAwait(false); } AddProgressLine("Working directory cleaned!"); }; logger.LogTrace("Locking repository..."); using (var repo = await repositoryManager.GetRepository(pullRequest.Base.Repository, OnCloneProgress, CreateBlockedComment, cancellationToken).ConfigureAwait(false)) { logger.LogTrace("Repository ready"); AddProgressLine("Initializing..."); async Task <string> GetDmeToUse() { var customDme = await scope.ServiceProvider.GetRequiredService <IDatabaseContext>().InstallationRepositories.Where(x => x.Id == pullRequest.Base.Repository.Id).Select(x => x.TargetDme).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); if (customDme != null) { AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Using manually set dme: {0}", customDme)); return(customDme); } AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Looking for dme to use in {0}", repo.Path)); var availDmes = await ioManager.GetFilesWithExtension(repo.Path, "dme", cancellationToken).ConfigureAwait(false); customDme = availDmes.First(); AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Selected {0} out of {1} possibilities", customDme, availDmes.Count)); return(customDme); } var dirPrepTask = DirectoryPrep(true); //get the dme to use var dmeToUseTask = GetDmeToUse(); var oldMapPaths = new List <string>() { Capacity = changedDmms.Count }; try { //fetch base commit if necessary and check it out, fetch pull request if (!await repo.ContainsCommit(pullRequest.Base.Sha, cancellationToken).ConfigureAwait(false)) { AddProgressLine("Base commit not found, running fetch..."); await repo.Fetch(cancellationToken).ConfigureAwait(false); } AddProgressLine("Moving HEAD to pull request base..."); await repo.Checkout(pullRequest.Base.Sha, cancellationToken).ConfigureAwait(false); //but since we don't need this right await don't await it yet var pullRequestFetchTask = repo.FetchPullRequest(pullRequest.Number, cancellationToken); try { //first copy all modified maps to the same location with the .old_map_diff_bot extension async Task <string> CacheMap(string mapPath) { var originalPath = currentIOManager.ConcatPath(repoPath, mapPath); if (await currentIOManager.FileExists(originalPath, cancellationToken).ConfigureAwait(false)) { AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Creating old map cache of {0}", mapPath)); var oldMapPath = String.Format(CultureInfo.InvariantCulture, "{0}{1}", originalPath, OldMapExtension); await currentIOManager.CopyFile(originalPath, oldMapPath, cancellationToken).ConfigureAwait(false); return(oldMapPath); } return(null); }; repoPath = repo.Path; var tasks = changedDmms.Select(x => CacheMap(x)).ToList(); await Task.WhenAll(tasks).ConfigureAwait(false); oldMapPaths.AddRange(tasks.Select(x => x.Result)); } finally { logger.LogTrace("Waiting for pull request commits to be available..."); await pullRequestFetchTask.ConfigureAwait(false); } AddProgressLine("Creating and moving HEAD to pull request merge commit..."); //generate the merge commit ourselves since we can't get it from GitHub because itll return an outdated one await repo.Merge(pullRequest.Head.Sha, cancellationToken).ConfigureAwait(false); } finally { logger.LogTrace("Waiting for configured project dme..."); await dmeToUseTask.ConfigureAwait(false); } //create empty array of map regions var mapRegions = Enumerable.Repeat <MapRegion>(null, changedDmms.Count).ToList(); var dmeToUse = dmeToUseTask.Result; var generator = generatorFactory.CreateGenerator(dmeToUse, new ResolvingIOManager(ioManager, repoPath)); var outputDirectory = currentIOManager.ResolvePath("."); logger.LogTrace("Full workdir path: {0}", outputDirectory); //Generate MapRegions for modified maps and render all new maps async Task <RenderResult> DiffAndRenderNewMap(int I) { await dirPrepTask.ConfigureAwait(false); var originalPath = currentIOManager.ConcatPath(repoPath, changedDmms[I]); if (!await currentIOManager.FileExists(originalPath, cancellationToken).ConfigureAwait(false)) { AddProgressLine(String.Format(CultureInfo.InvariantCulture, "No new map for path {0} exists, skipping region detection and after render", changedDmms[I])); return(new RenderResult { InputPath = changedDmms[I], ToolOutput = stringLocalizer["Map missing!"] }); } ToolResult result = null; if (oldMapPaths[I] != null) { AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Getting diff region for {0}...", changedDmms[I])); result = await generator.GetDifferences(oldMapPaths[I], originalPath, cancellationToken).ConfigureAwait(false); var region = result.MapRegion; AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Diff region for {0}: {1}", changedDmms[I], region)); if (region != null) { var xdiam = region.MaxX - region.MinX; var ydiam = region.MaxY - region.MinY; const int minDiffDimensions = 5 - 1; if (xdiam < minDiffDimensions || ydiam < minDiffDimensions) { //need to expand var fullResult = await generator.GetMapSize(originalPath, cancellationToken).ConfigureAwait(false); var fullRegion = fullResult.MapRegion; if (fullRegion == null) { //give up region = null; } else { bool increaseMax = true; if (xdiam < minDiffDimensions && ((fullRegion.MaxX - fullRegion.MinX) >= minDiffDimensions)) { while ((region.MaxX - region.MinX) < minDiffDimensions) { if (increaseMax) { region.MaxX = (short)Math.Min(region.MaxX + 1, fullRegion.MaxX); } else { region.MinX = (short)Math.Max(region.MinX - 1, 1); } increaseMax = !increaseMax; } } if (ydiam < minDiffDimensions && ((fullRegion.MaxY - fullRegion.MinY) >= minDiffDimensions)) { while ((region.MaxY - region.MinY) < minDiffDimensions) { if (increaseMax) { region.MaxY = (short)Math.Min(region.MaxY + 1, fullRegion.MaxY); } else { region.MinY = (short)Math.Max(region.MinY - 1, 1); } increaseMax = !increaseMax; } } } AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Region for {0} expanded to {1}", changedDmms[I], region)); } mapRegions[I] = region; } } else { AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Skipping region detection for {0} due to old map not existing", changedDmms[I])); } AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Performing after rendering for {0}...", changedDmms[I])); var renderResult = await generator.RenderMap(originalPath, mapRegions[I], outputDirectory, "after", cancellationToken).ConfigureAwait(false); logger.LogTrace("After rendering for {0} complete! Result path: {1}, Output: {2}", changedDmms[I], renderResult.OutputPath, renderResult.ToolOutput); AddProgressLine(String.Format(CultureInfo.InvariantCulture, "After rendering for {0} complete!", renderResult.OutputPath)); if (result != null) { renderResult.ToolOutput = String.Format(CultureInfo.InvariantCulture, "Differences task:{0}{1}{0}Render task:{0}{2}", Environment.NewLine, result.ToolOutput, renderResult.ToolOutput); } return(renderResult); }; logger.LogTrace("Running iterations of DiffAndRenderNewMap..."); //finish up before we go back to the base branch afterRenderings = Enumerable.Range(0, changedDmms.Count).Select(I => DiffAndRenderNewMap(I)).ToList(); try { await Task.WhenAll(afterRenderings).ConfigureAwait(false); } catch (Exception e) { logger.LogDebug(e, "After renderings produced exception!"); //at this point everything is done but some have failed //we'll handle it later } AddProgressLine("Moving HEAD back to pull request base..."); await repo.Checkout(pullRequest.Base.Sha, cancellationToken).ConfigureAwait(false); async Task <RenderResult> RenderOldMap(int i) { var oldPath = oldMapPaths[i]; if (oldMapPaths != null) { AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Performing before rendering for {0}...", changedDmms[i])); var result = await generator.RenderMap(oldPath, mapRegions[i], outputDirectory, "before", cancellationToken).ConfigureAwait(false); AddProgressLine(String.Format(CultureInfo.InvariantCulture, "Before rendering for {0} complete!", changedDmms[i])); return(result); } return(new RenderResult { InputPath = changedDmms[i], ToolOutput = stringLocalizer["Map missing!"] }); } logger.LogTrace("Running iterations of RenderOldMap..."); //finish up rendering beforeRenderings = Enumerable.Range(0, changedDmms.Count).Select(I => RenderOldMap(I)).ToList(); try { await Task.WhenAll(beforeRenderings).ConfigureAwait(false); } catch (Exception e) { logger.LogDebug(e, "Before renderings produced exception!"); //see above } AddProgressLine("Renderings complete, finalizing..."); //done with the repo at this point logger.LogTrace("Renderings complete. Releasing reposiotory"); } //collect results and errors async Task <KeyValuePair <MapDiff, MapRegion> > GetResult(int i) { var beforeTask = beforeRenderings[i]; var afterTask = afterRenderings[i]; var result = new MapDiff { InstallationRepositoryId = pullRequest.Base.Repository.Id, CheckRunId = checkRunId, FileId = i, }; RenderResult GetRenderingResult(Task <RenderResult> task) { if (task.Exception != null) { result.LogMessage = result.LogMessage == null?task.Exception.ToString() : String.Format(CultureInfo.InvariantCulture, "{0}{1}{2}", result.LogMessage, Environment.NewLine, task.Exception); return(null); } return(task.Result); }; var r1 = GetRenderingResult(beforeTask); var r2 = GetRenderingResult(afterTask); logger.LogTrace("Results for {0}: Before {1}, After {2}", changedDmms[i], r1?.OutputPath ?? "NONE", r2?.OutputPath ?? "NONE"); result.MapPath = changedDmms[i]; result.LogMessage = String.Format(CultureInfo.InvariantCulture, "Job {5}:{0}Path: {6}{0}Before:{0}Command Line: {1}{0}Output:{0}{2}{0}Logs:{0}{7}{0}After:{0}Command Line: {3}{0}Output:{0}{4}{0}Logs:{0}{8}{0}Exceptions:{0}{9}{0}", Environment.NewLine, r1?.CommandLine, r1?.OutputPath, r2?.CommandLine, r2?.OutputPath, i + 1, result.MapPath, r1?.ToolOutput, r2?.ToolOutput, result.LogMessage); async Task <Image> ReadMapImage(string path) { if (path != null && await currentIOManager.FileExists(path, cancellationToken).ConfigureAwait(false)) { var bytes = await currentIOManager.ReadAllBytes(path, cancellationToken).ConfigureAwait(false); return(new Image { Data = bytes }); } return(null); } var readBeforeTask = ReadMapImage(r1?.OutputPath); result.AfterImage = await ReadMapImage(r2?.OutputPath).ConfigureAwait(false); result.BeforeImage = await readBeforeTask.ConfigureAwait(false); Image GenerateDifferenceImage(Image before, Image after) { if (before == null || result.AfterImage == null) { return(null); } using (var ms = new MemoryStream()) { using (var diffI = new MagickImage()) { using (var beforeI = new MagickImage(before.Data, new MagickReadSettings())) using (var afterI = new MagickImage(after.Data, new MagickReadSettings())) { beforeI.Compare(afterI, ErrorMetric.Absolute, diffI); } diffI.Write(ms, MagickFormat.Png32); } return(new Image { Data = ms.ToArray() }); } } result.DifferenceImage = GenerateDifferenceImage(result.BeforeImage, result.AfterImage); return(new KeyValuePair <MapDiff, MapRegion>(result, r2?.MapRegion)); } logger.LogTrace("Collecting results..."); var results = Enumerable.Range(0, changedDmms.Count).Select(x => GetResult(x)).ToList(); await Task.WhenAll(results).ConfigureAwait(false); await DirectoryPrep(false).ConfigureAwait(false); logger.LogTrace("Waiting for GitHub check to finish updating..."); await generatingCommentTask.ConfigureAwait(false); var dic = new Dictionary <MapDiff, MapRegion>(); foreach (var I in results.Select(x => x.Result)) { dic.Add(I.Key, I.Value); } await HandleResults(pullRequest, checkRunId, dic, scope, cancellationToken).ConfigureAwait(false); } }
/// <summary> /// Executes and populate a given <paramref name="job"/> /// </summary> /// <param name="job">The <see cref="CompileJob"/> to run and populate</param> /// <param name="dreamMakerSettings">The <see cref="Api.Models.Internal.DreamMakerSettings"/> to use</param> /// <param name="byondLock">The <see cref="IByondExecutableLock"/> to use</param> /// <param name="repository">The <see cref="IRepository"/> to use</param> /// <param name="remoteDeploymentManager">The <see cref="IRemoteDeploymentManager"/> to use.</param> /// <param name="apiValidateTimeout">The timeout for validating the DMAPI</param> /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param> /// <returns>A <see cref="Task"/> representing the running operation</returns> async Task RunCompileJob( Models.CompileJob job, Api.Models.Internal.DreamMakerSettings dreamMakerSettings, IByondExecutableLock byondLock, IRepository repository, IRemoteDeploymentManager remoteDeploymentManager, uint apiValidateTimeout, CancellationToken cancellationToken) { var outputDirectory = job.DirectoryName.ToString(); logger.LogTrace("Compile output GUID: {0}", outputDirectory); try { // copy the repository logger.LogTrace("Copying repository to game directory..."); var resolvedOutputDirectory = ioManager.ResolvePath(outputDirectory); var repoOrigin = repository.Origin; using (repository) await repository.CopyTo(resolvedOutputDirectory, cancellationToken).ConfigureAwait(false); // repository closed now // run precompile scripts await eventConsumer.HandleEvent( EventType.CompileStart, new List <string> { resolvedOutputDirectory, repoOrigin.ToString() }, cancellationToken) .ConfigureAwait(false); // determine the dme if (job.DmeName == null) { logger.LogTrace("Searching for available .dmes..."); var foundPaths = await ioManager.GetFilesWithExtension(resolvedOutputDirectory, DmeExtension, true, cancellationToken).ConfigureAwait(false); var foundPath = foundPaths.FirstOrDefault(); if (foundPath == default) { throw new JobException(ErrorCode.DreamMakerNoDme); } job.DmeName = foundPath.Substring( resolvedOutputDirectory.Length + 1, foundPath.Length - resolvedOutputDirectory.Length - DmeExtension.Length - 2); // +1 for . in extension } else { var targetDme = ioManager.ConcatPath(outputDirectory, String.Join('.', job.DmeName, DmeExtension)); var targetDmeExists = await ioManager.FileExists(targetDme, cancellationToken).ConfigureAwait(false); if (!targetDmeExists) { throw new JobException(ErrorCode.DreamMakerMissingDme); } } logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName); await ModifyDme(job, cancellationToken).ConfigureAwait(false); // run compiler var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false); // verify api try { if (exitCode != 0) { throw new JobException( ErrorCode.DreamMakerExitCode, new JobException($"Exit code: {exitCode}{Environment.NewLine}{Environment.NewLine}{job.Output}")); } await VerifyApi( apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, dreamMakerSettings.RequireDMApiValidation.Value, cancellationToken) .ConfigureAwait(false); } catch (JobException) { // DD never validated or compile failed await eventConsumer.HandleEvent( EventType.CompileFailure, new List <string> { resolvedOutputDirectory, exitCode == 0 ? "1" : "0" }, cancellationToken) .ConfigureAwait(false); throw; } await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> { resolvedOutputDirectory }, cancellationToken).ConfigureAwait(false); logger.LogTrace("Applying static game file symlinks..."); // symlink in the static data await configuration.SymlinkStaticFilesTo(resolvedOutputDirectory, cancellationToken).ConfigureAwait(false); logger.LogDebug("Compile complete!"); } catch (Exception ex) { await CleanupFailedCompile(job, remoteDeploymentManager, ex).ConfigureAwait(false); throw; } }