/// <summary> /// Send a message to <see cref="chat"/> about a deployment /// </summary> /// <param name="revisionInformation">The <see cref="Models.RevisionInformation"/> for the deployment</param> /// <param name="byondLock">The <see cref="IByondExecutableLock"/> for the deployment</param> /// <param name="cancellationToken">The <see cref="CancellationToken"/> for the operation</param> /// <returns>A <see cref="Task"/> representing the running operation</returns> async Task SendDeploymentMessage(Models.RevisionInformation revisionInformation, IByondExecutableLock byondLock, CancellationToken cancellationToken) { var commitInsert = revisionInformation.CommitSha.Substring(0, 7); string remoteCommitInsert; if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha) { commitInsert = String.Format(CultureInfo.InvariantCulture, "^{0}", commitInsert); remoteCommitInsert = String.Empty; } else { remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha.Substring(0, 7)); } var testmergeInsert = revisionInformation.ActiveTestMerges.Count == 0 ? String.Empty : String.Format(CultureInfo.InvariantCulture, " (Test Merges: {0})", String.Join(", ", revisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => { var result = String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.PullRequestRevision.Substring(0, 7)); if (x.Comment != null) { result += String.Format(CultureInfo.InvariantCulture, " ({0})", x.Comment); } return(result); }))); await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deploying revision: {0}{1}{2} BYOND Version: {3}", commitInsert, testmergeInsert, remoteCommitInsert, byondLock.Version), cancellationToken).ConfigureAwait(false); }
static async Task <bool> PopulateApi(Repository model, Components.Repository.IRepository repository, IDatabaseContext databaseContext, Models.Instance instance, string lastOriginCommitSha, Action <Models.RevisionInformation> revInfoSink, CancellationToken cancellationToken) { model.IsGitHub = repository.IsGitHubRepository; model.Origin = repository.Origin; model.Reference = repository.Reference; //rev info stuff Models.RevisionInformation revisionInfo = null; var needsDbUpdate = await LoadRevisionInformation(repository, databaseContext, instance, lastOriginCommitSha, x => revisionInfo = x, cancellationToken).ConfigureAwait(false); model.RevisionInformation = revisionInfo.ToApi(); revInfoSink?.Invoke(revisionInfo); return(needsDbUpdate); }
async Task <bool> LoadRevisionInformation(Components.Repository.IRepository repository, IDatabaseContext databaseContext, Models.Instance instance, string lastOriginCommitSha, Action <Models.RevisionInformation> revInfoSink, CancellationToken cancellationToken) { var repoSha = repository.Head; IQueryable <Models.RevisionInformation> ApplyQuery(IQueryable <Models.RevisionInformation> query) => query .Where(x => x.CommitSha == repoSha && x.Instance.Id == instance.Id) .Include(x => x.CompileJobs) .Include(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge).ThenInclude(x => x.MergedBy); var revisionInfo = await ApplyQuery(databaseContext.RevisionInformations).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); // If the DB doesn't have it, check the local set if (revisionInfo == default) { revisionInfo = databaseContext .RevisionInformations .Local .Where(x => x.CommitSha == repoSha && x.Instance.Id == instance.Id) .FirstOrDefault(); } var needsDbUpdate = revisionInfo == default; if (needsDbUpdate) { // needs insertion revisionInfo = new Models.RevisionInformation { Instance = instance, CommitSha = repoSha, CompileJobs = new List <Models.CompileJob>(), ActiveTestMerges = new List <RevInfoTestMerge>() // non null vals for api returns }; lock (databaseContext) // cleaner this way databaseContext.RevisionInformations.Add(revisionInfo); } revisionInfo.OriginCommitSha ??= lastOriginCommitSha; if (revisionInfo.OriginCommitSha == null) { revisionInfo.OriginCommitSha = repoSha; Logger.LogWarning(Components.Repository.Repository.OriginTrackingErrorTemplate, repoSha); } revInfoSink?.Invoke(revisionInfo); return(needsDbUpdate); }
async Task <bool> PopulateApi(Repository model, Components.Repository.IRepository repository, IDatabaseContext databaseContext, Models.Instance instance, CancellationToken cancellationToken) { model.RemoteGitProvider = repository.RemoteGitProvider; model.RemoteRepositoryOwner = repository.RemoteRepositoryOwner; model.RemoteRepositoryName = repository.RemoteRepositoryName; model.Origin = repository.Origin; model.Reference = repository.Reference; // rev info stuff Models.RevisionInformation revisionInfo = null; var needsDbUpdate = await LoadRevisionInformation(repository, databaseContext, instance, null, x => revisionInfo = x, cancellationToken).ConfigureAwait(false); model.RevisionInformation = revisionInfo.ToApi(); return(needsDbUpdate); }
/// <inheritdoc /> public override async Task <Func <string, string, Task> > SendUpdateMessage( Models.RevisionInformation revisionInformation, Version byondVersion, DateTimeOffset?estimatedCompletionTime, string gitHubOwner, string gitHubRepo, ulong channelId, bool localCommitPushed, CancellationToken cancellationToken) { bool gitHub = gitHubOwner != null && gitHubRepo != null; localCommitPushed |= revisionInformation.CommitSha == revisionInformation.OriginCommitSha; var fields = new List <EmbedFieldBuilder> { new EmbedFieldBuilder { Name = "BYOND Version", Value = $"{byondVersion.Major}.{byondVersion.Minor}{(byondVersion.Build > 0 ? $".{byondVersion.Build}" : String.Empty)}", IsInline = true }, new EmbedFieldBuilder { Name = "Local Commit", Value = localCommitPushed && gitHub ? $"[{revisionInformation.CommitSha.Substring(0, 7)}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{revisionInformation.CommitSha})" : revisionInformation.CommitSha.Substring(0, 7), IsInline = true }, new EmbedFieldBuilder { Name = "Branch Commit", Value = gitHub ? $"[{revisionInformation.OriginCommitSha.Substring(0, 7)}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{revisionInformation.OriginCommitSha})" : revisionInformation.OriginCommitSha.Substring(0, 7), IsInline = true } }; fields.AddRange((revisionInformation.ActiveTestMerges ?? Enumerable.Empty <RevInfoTestMerge>()) .Select(x => x.TestMerge) .Select(x => new EmbedFieldBuilder { Name = $"#{x.Number}", Value = $"[{x.TitleAtMerge}]({x.Url}) by _[@{x.Author}](https://github.com/{x.Author})_{Environment.NewLine}Commit: [{x.PullRequestRevision.Substring(0, 7)}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{x.PullRequestRevision}){(String.IsNullOrWhiteSpace(x.Comment) ? String.Empty : $"{Environment.NewLine}_**{x.Comment}**_")}" }));
/// <summary> /// Create a <see cref="List{T}"/> of <see cref="EmbedFieldBuilder"/>s for a discord update embed. /// </summary> /// <param name="revisionInformation">The <see cref="RevisionInformation"/> of the deployment.</param> /// <param name="byondVersion">The BYOND <see cref="Version"/> of the deployment.</param> /// <param name="gitHubOwner">The repository GitHub owner, if any.</param> /// <param name="gitHubRepo">The repository GitHub name, if any.</param> /// <param name="localCommitPushed"><see langword="true"/> if the local deployment commit was pushed to the remote repository.</param> /// <returns>A new <see cref="List{T}"/> of <see cref="EmbedFieldBuilder"/>s to use.</returns> static List <EmbedFieldBuilder> BuildUpdateEmbedFields( Models.RevisionInformation revisionInformation, Version byondVersion, string gitHubOwner, string gitHubRepo, bool localCommitPushed) { bool gitHub = gitHubOwner != null && gitHubRepo != null; var fields = new List <EmbedFieldBuilder> { new EmbedFieldBuilder { Name = "BYOND Version", Value = $"{byondVersion.Major}.{byondVersion.Minor}{(byondVersion.Build > 0 ? $".{byondVersion.Build}" : String.Empty)}", IsInline = true, }, new EmbedFieldBuilder { Name = "Local Commit", Value = localCommitPushed && gitHub ? $"[{revisionInformation.CommitSha.Substring(0, 7)}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{revisionInformation.CommitSha})" : revisionInformation.CommitSha.Substring(0, 7), IsInline = true, }, new EmbedFieldBuilder { Name = "Branch Commit", Value = gitHub ? $"[{revisionInformation.OriginCommitSha.Substring(0, 7)}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{revisionInformation.OriginCommitSha})" : revisionInformation.OriginCommitSha.Substring(0, 7), IsInline = true, }, }; fields.AddRange((revisionInformation.ActiveTestMerges ?? Enumerable.Empty <RevInfoTestMerge>()) .Select(x => x.TestMerge) .Select(x => new EmbedFieldBuilder { Name = $"#{x.Number}", Value = $"[{x.TitleAtMerge}]({x.Url}) by _[@{x.Author}](https://github.com/{x.Author})_{Environment.NewLine}Commit: [{x.TargetCommitSha.Substring(0, 7)}](https://github.com/{gitHubOwner}/{gitHubRepo}/commit/{x.TargetCommitSha}){(String.IsNullOrWhiteSpace(x.Comment) ? String.Empty : $"{Environment.NewLine}_**{x.Comment}**_")}", }));
static async Task <bool> LoadRevisionInformation(Components.Repository.IRepository repository, IDatabaseContext databaseContext, Models.Instance instance, string lastOriginCommitSha, Action <Models.RevisionInformation> revInfoSink, CancellationToken cancellationToken) { var repoSha = repository.Head; IQueryable <Models.RevisionInformation> queryTarget = databaseContext.RevisionInformations; var revisionInfo = await databaseContext.RevisionInformations.Where(x => x.CommitSha == repoSha && x.Instance.Id == instance.Id) .Include(x => x.CompileJobs) .Include(x => x.ActiveTestMerges).ThenInclude(x => x.TestMerge) //minimal info, they can query the rest if they're allowed .FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); //search every rev info because LOL SHA COLLISIONS if (revisionInfo == default) { revisionInfo = databaseContext.RevisionInformations.Local.Where(x => x.CommitSha == repoSha).FirstOrDefault(); } var needsDbUpdate = revisionInfo == default; if (needsDbUpdate) { //needs insertion revisionInfo = new Models.RevisionInformation { Instance = instance, CommitSha = repoSha, CompileJobs = new List <Models.CompileJob>(), ActiveTestMerges = new List <RevInfoTestMerge>() //non null vals for api returns }; lock (databaseContext) //cleaner this way databaseContext.RevisionInformations.Add(revisionInfo); } revisionInfo.OriginCommitSha = revisionInfo.OriginCommitSha ?? lastOriginCommitSha ?? repository.Head; revInfoSink?.Invoke(revisionInfo); return(needsDbUpdate); }
/// <inheritdoc /> #pragma warning disable CA1506 public async Task DeploymentProcess( Models.Job job, IDatabaseContextFactory databaseContextFactory, Action <int> progressReporter, CancellationToken cancellationToken) { if (job == null) { throw new ArgumentNullException(nameof(job)); } if (databaseContextFactory == null) { throw new ArgumentNullException(nameof(databaseContextFactory)); } if (progressReporter == null) { throw new ArgumentNullException(nameof(progressReporter)); } lock (deploymentLock) { if (deploying) { throw new JobException(ErrorCode.DreamMakerCompileJobInProgress); } deploying = true; } currentChatCallback = null; currentDreamMakerOutput = null; Models.CompileJob compileJob = null; try { string repoOwner = null; string repoName = null; TimeSpan?averageSpan = null; Models.RepositorySettings repositorySettings = null; Models.DreamDaemonSettings ddSettings = null; Models.DreamMakerSettings dreamMakerSettings = null; IRepository repo = null; IRemoteDeploymentManager remoteDeploymentManager = null; Models.RevisionInformation revInfo = null; await databaseContextFactory.UseContext( async databaseContext => { averageSpan = await CalculateExpectedDeploymentTime(databaseContext, cancellationToken).ConfigureAwait(false); ddSettings = await databaseContext .DreamDaemonSettings .AsQueryable() .Where(x => x.InstanceId == metadata.Id) .Select(x => new Models.DreamDaemonSettings { StartupTimeout = x.StartupTimeout, }) .FirstOrDefaultAsync(cancellationToken) .ConfigureAwait(false); if (ddSettings == default) { throw new JobException(ErrorCode.InstanceMissingDreamDaemonSettings); } dreamMakerSettings = await databaseContext .DreamMakerSettings .AsQueryable() .Where(x => x.InstanceId == metadata.Id) .FirstAsync(cancellationToken) .ConfigureAwait(false); if (dreamMakerSettings == default) { throw new JobException(ErrorCode.InstanceMissingDreamMakerSettings); } repositorySettings = await databaseContext .RepositorySettings .AsQueryable() .Where(x => x.InstanceId == metadata.Id) .Select(x => new Models.RepositorySettings { AccessToken = x.AccessToken, ShowTestMergeCommitters = x.ShowTestMergeCommitters, PushTestMergeCommits = x.PushTestMergeCommits, PostTestMergeComment = x.PostTestMergeComment, }) .FirstOrDefaultAsync(cancellationToken) .ConfigureAwait(false); if (repositorySettings == default) { throw new JobException(ErrorCode.InstanceMissingRepositorySettings); } repo = await repositoryManager.LoadRepository(cancellationToken).ConfigureAwait(false); try { if (repo == null) { throw new JobException(ErrorCode.RepoMissing); } remoteDeploymentManager = remoteDeploymentManagerFactory .CreateRemoteDeploymentManager(metadata, repo.RemoteGitProvider.Value); var repoSha = repo.Head; repoOwner = repo.RemoteRepositoryOwner; repoName = repo.RemoteRepositoryName; revInfo = await databaseContext .RevisionInformations .AsQueryable() .Where(x => x.CommitSha == repoSha && x.Instance.Id == metadata.Id) .Include(x => x.ActiveTestMerges) .ThenInclude(x => x.TestMerge) .ThenInclude(x => x.MergedBy) .FirstOrDefaultAsync(cancellationToken) .ConfigureAwait(false); if (revInfo == default) { revInfo = new Models.RevisionInformation { CommitSha = repoSha, Timestamp = await repo.TimestampCommit(repoSha, cancellationToken).ConfigureAwait(false), OriginCommitSha = repoSha, Instance = new Models.Instance { Id = metadata.Id, }, ActiveTestMerges = new List <RevInfoTestMerge>(), }; logger.LogInformation(Repository.Repository.OriginTrackingErrorTemplate, repoSha); databaseContext.Instances.Attach(revInfo.Instance); await databaseContext.Save(cancellationToken).ConfigureAwait(false); } } catch { repo?.Dispose(); throw; } }) .ConfigureAwait(false); var likelyPushedTestMergeCommit = repositorySettings.PushTestMergeCommits.Value && repositorySettings.AccessToken != null && repositorySettings.AccessUser != null; using (repo) compileJob = await Compile( revInfo, dreamMakerSettings, ddSettings.StartupTimeout.Value, repo, remoteDeploymentManager, progressReporter, averageSpan, likelyPushedTestMergeCommit, cancellationToken) .ConfigureAwait(false); var activeCompileJob = compileJobConsumer.LatestCompileJob(); try { await databaseContextFactory.UseContext( async databaseContext => { var fullJob = compileJob.Job; compileJob.Job = new Models.Job { Id = job.Id, }; var fullRevInfo = compileJob.RevisionInformation; compileJob.RevisionInformation = new Models.RevisionInformation { Id = revInfo.Id, }; databaseContext.Jobs.Attach(compileJob.Job); databaseContext.RevisionInformations.Attach(compileJob.RevisionInformation); databaseContext.CompileJobs.Add(compileJob); // The difficulty with compile jobs is they have a two part commit await databaseContext.Save(cancellationToken).ConfigureAwait(false); logger.LogTrace("Created CompileJob {0}", compileJob.Id); try { await compileJobConsumer.LoadCompileJob(compileJob, cancellationToken).ConfigureAwait(false); } catch { // So we need to un-commit the compile job if the above throws databaseContext.CompileJobs.Remove(compileJob); // DCT: Cancellation token is for job, operation must run regardless await databaseContext.Save(default).ConfigureAwait(false); throw; }
/// <inheritdoc /> public async Task <Models.CompileJob> Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action <int> progressReporter, TimeSpan?estimatedDuration, CancellationToken cancellationToken) { if (revisionInformation == null) { throw new ArgumentNullException(nameof(revisionInformation)); } if (dreamMakerSettings == null) { throw new ArgumentNullException(nameof(dreamMakerSettings)); } if (repository == null) { throw new ArgumentNullException(nameof(repository)); } if (progressReporter == null) { throw new ArgumentNullException(nameof(progressReporter)); } if (dreamMakerSettings.ApiValidationSecurityLevel == DreamDaemonSecurity.Ultrasafe) { throw new ArgumentOutOfRangeException(nameof(dreamMakerSettings), dreamMakerSettings, "Cannot compile with ultrasafe security!"); } logger.LogTrace("Begin Compile"); lock (this) { if (compiling) { throw new JobException("There is already a compile job in progress!"); } compiling = true; } using (var progressCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { var progressTask = estimatedDuration.HasValue ? ProgressTask(progressReporter, estimatedDuration.Value, cancellationToken) : Task.CompletedTask; try { using (var byondLock = await byond.UseExecutables(null, cancellationToken).ConfigureAwait(false)) { await SendDeploymentMessage(revisionInformation, byondLock, cancellationToken).ConfigureAwait(false); var job = new Models.CompileJob { DirectoryName = Guid.NewGuid(), DmeName = dreamMakerSettings.ProjectName, RevisionInformation = revisionInformation, ByondVersion = byondLock.Version.ToString() }; await RunCompileJob(job, dreamMakerSettings, byondLock, repository, apiValidateTimeout, cancellationToken).ConfigureAwait(false); return(job); } } catch (OperationCanceledException) { await eventConsumer.HandleEvent(EventType.CompileCancelled, null, default).ConfigureAwait(false); throw; } finally { compiling = false; progressCts.Cancel(); await progressTask.ConfigureAwait(false); } } }
/// <inheritdoc /> public async Task <Models.CompileJob> Compile(Models.RevisionInformation revisionInformation, Api.Models.DreamMaker dreamMakerSettings, uint apiValidateTimeout, IRepository repository, Action <int> progressReporter, TimeSpan?estimatedDuration, CancellationToken cancellationToken) { if (revisionInformation == null) { throw new ArgumentNullException(nameof(revisionInformation)); } if (dreamMakerSettings == null) { throw new ArgumentNullException(nameof(dreamMakerSettings)); } if (repository == null) { throw new ArgumentNullException(nameof(repository)); } if (progressReporter == null) { throw new ArgumentNullException(nameof(progressReporter)); } if (dreamMakerSettings.ApiValidationSecurityLevel == DreamDaemonSecurity.Ultrasafe) { throw new ArgumentOutOfRangeException(nameof(dreamMakerSettings), dreamMakerSettings, "Cannot compile with ultrasafe security!"); } logger.LogTrace("Begin Compile"); var job = new Models.CompileJob { DirectoryName = Guid.NewGuid(), DmeName = dreamMakerSettings.ProjectName, RevisionInformation = revisionInformation }; logger.LogTrace("Compile output GUID: {0}", job.DirectoryName); lock (this) { if (compiling) { throw new JobException("There is already a compile job in progress!"); } compiling = true; } using (var progressCts = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken)) { async Task ProgressTask() { if (!estimatedDuration.HasValue) { return; } progressReporter(0); var ct = progressCts.Token; var sleepInterval = estimatedDuration.Value / 100; try { for (var I = 0; I < 99; ++I) { await Task.Delay(sleepInterval, progressCts.Token).ConfigureAwait(false); progressReporter(I + 1); } } catch (OperationCanceledException) { } } var progressTask = ProgressTask(); try { var commitInsert = revisionInformation.CommitSha.Substring(0, 7); string remoteCommitInsert; if (revisionInformation.CommitSha == revisionInformation.OriginCommitSha) { commitInsert = String.Format(CultureInfo.InvariantCulture, "^{0}", commitInsert); remoteCommitInsert = String.Empty; } else { remoteCommitInsert = String.Format(CultureInfo.InvariantCulture, ". Remote commit: ^{0}", revisionInformation.OriginCommitSha.Substring(0, 7)); } var testmergeInsert = revisionInformation.ActiveTestMerges.Count == 0 ? String.Empty : String.Format(CultureInfo.InvariantCulture, " (Test Merges: {0})", String.Join(", ", revisionInformation.ActiveTestMerges.Select(x => x.TestMerge).Select(x => { var result = String.Format(CultureInfo.InvariantCulture, "#{0} at {1}", x.Number, x.PullRequestRevision.Substring(0, 7)); if (x.Comment != null) { result += String.Format(CultureInfo.InvariantCulture, " ({0})", x.Comment); } return(result); }))); using (var byondLock = await byond.UseExecutables(null, cancellationToken).ConfigureAwait(false)) { await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deploying revision: {0}{1}{2} BYOND Version: {3}", commitInsert, testmergeInsert, remoteCommitInsert, byondLock.Version), cancellationToken).ConfigureAwait(false); async Task CleanupFailedCompile(bool cancelled) { logger.LogTrace("Cleaning compile directory..."); var chatTask = chat.SendUpdateMessage(cancelled ? "Deploy cancelled!" : "Deploy failed!", cancellationToken); try { await ioManager.DeleteDirectory(job.DirectoryName.ToString(), CancellationToken.None).ConfigureAwait(false); } catch (Exception e) { logger.LogWarning("Error cleaning up compile directory {0}! Exception: {1}", ioManager.ResolvePath(job.DirectoryName.ToString()), e); } await chatTask.ConfigureAwait(false); }; try { await ioManager.CreateDirectory(job.DirectoryName.ToString(), cancellationToken).ConfigureAwait(false); var dirA = ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName); var dirB = ioManager.ConcatPath(job.DirectoryName.ToString(), BDirectoryName); logger.LogTrace("Copying repository to game directory..."); //copy the repository var fullDirA = ioManager.ResolvePath(dirA); var repoOrigin = repository.Origin; using (repository) await repository.CopyTo(fullDirA, cancellationToken).ConfigureAwait(false); //run precompile scripts var resolvedGameDirectory = ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)); await eventConsumer.HandleEvent(EventType.CompileStart, new List <string> { resolvedGameDirectory, repoOrigin }, cancellationToken).ConfigureAwait(false); //determine the dme if (job.DmeName == null) { logger.LogTrace("Searching for available .dmes..."); var path = (await ioManager.GetFilesWithExtension(dirA, DmeExtension, cancellationToken).ConfigureAwait(false)).FirstOrDefault(); if (path == default) { throw new JobException("Unable to find any .dme!"); } var dmeWithExtension = ioManager.GetFileName(path); job.DmeName = dmeWithExtension.Substring(0, dmeWithExtension.Length - DmeExtension.Length - 1); } else if (!await ioManager.FileExists(ioManager.ConcatPath(dirA, String.Join('.', job.DmeName, DmeExtension)), cancellationToken).ConfigureAwait(false)) { throw new JobException("Unable to locate specified .dme!"); } logger.LogDebug("Selected {0}.dme for compilation!", job.DmeName); await ModifyDme(job, cancellationToken).ConfigureAwait(false); //run compiler, verify api job.ByondVersion = byondLock.Version.ToString(); var exitCode = await RunDreamMaker(byondLock.DreamMakerPath, job, cancellationToken).ConfigureAwait(false); try { if (exitCode != 0) { throw new JobException(String.Format(CultureInfo.InvariantCulture, "DM exited with a non-zero code: {0}{1}{2}", exitCode, Environment.NewLine, job.Output)); } await VerifyApi(apiValidateTimeout, dreamMakerSettings.ApiValidationSecurityLevel.Value, job, byondLock, dreamMakerSettings.ApiValidationPort.Value, cancellationToken).ConfigureAwait(false); } catch (JobException) { //server never validated or compile failed await eventConsumer.HandleEvent(EventType.CompileFailure, new List <string> { resolvedGameDirectory, exitCode == 0 ? "1" : "0" }, cancellationToken).ConfigureAwait(false); throw; } logger.LogTrace("Running post compile event..."); await eventConsumer.HandleEvent(EventType.CompileComplete, new List <string> { ioManager.ResolvePath(ioManager.ConcatPath(job.DirectoryName.ToString(), ADirectoryName)) }, cancellationToken).ConfigureAwait(false); logger.LogTrace("Duplicating compiled game..."); //duplicate the dmb et al await ioManager.CopyDirectory(dirA, dirB, null, cancellationToken).ConfigureAwait(false); logger.LogTrace("Applying static game file symlinks..."); //symlink in the static data var symATask = configuration.SymlinkStaticFilesTo(fullDirA, cancellationToken); var symBTask = configuration.SymlinkStaticFilesTo(ioManager.ResolvePath(dirB), cancellationToken); await Task.WhenAll(symATask, symBTask).ConfigureAwait(false); await chat.SendUpdateMessage(String.Format(CultureInfo.InvariantCulture, "Deployment complete!{0}", watchdog.Running ? " Changes will be applied on next server reboot." : String.Empty), cancellationToken).ConfigureAwait(false); logger.LogDebug("Compile complete!"); return(job); } catch (Exception e) { await CleanupFailedCompile(e is OperationCanceledException).ConfigureAwait(false); throw; } } } catch (OperationCanceledException) { await eventConsumer.HandleEvent(EventType.CompileCancelled, null, default).ConfigureAwait(false); throw; } finally { compiling = false; progressCts.Cancel(); await progressTask.ConfigureAwait(false); } } }
#pragma warning disable CA1502 // TODO: Decomplexify #pragma warning disable CA1505 public async Task <IActionResult> Update([FromBody] Repository model, CancellationToken cancellationToken) { if (model == null) { throw new ArgumentNullException(nameof(model)); } if (model.AccessUser == null ^ model.AccessToken == null) { return(BadRequest(new ErrorMessage { Message = "Either both accessToken and accessUser must be present or neither!" })); } if (model.CheckoutSha != null && model.Reference != null) { return(BadRequest(new ErrorMessage { Message = "Only one of sha or reference may be specified!" })); } if (model.CheckoutSha != null && model.UpdateFromOrigin == true) { return(BadRequest(new ErrorMessage { Message = "Cannot update a reference when checking out a sha!" })); } if (model.Origin != null) { return(BadRequest(new ErrorMessage { Message = "origin cannot be modified without deleting the repository!" })); } if (model.NewTestMerges?.Any(x => !x.Number.HasValue) == true) { return(BadRequest(new ErrorMessage { Message = "All new test merges must provide a number!" })); } if (model.NewTestMerges?.Any(x => model.NewTestMerges.Any(y => x != y && x.Number == y.Number)) == true) { return(BadRequest(new ErrorMessage { Message = "Cannot test merge the same PR twice in one job!" })); } if (model.CommitterName?.Length == 0) { return(BadRequest(new ErrorMessage { Message = "Cannot set empty committer name!" })); } if (model.CommitterEmail?.Length == 0) { return(BadRequest(new ErrorMessage { Message = "Cannot set empty committer e=mail!" })); } var newTestMerges = model.NewTestMerges != null && model.NewTestMerges.Count > 0; var userRights = (RepositoryRights)AuthenticationContext.GetRight(RightsType.Repository); if (newTestMerges && !userRights.HasFlag(RepositoryRights.MergePullRequest)) { return(Forbid()); } var currentModel = await DatabaseContext.RepositorySettings.Where(x => x.InstanceId == Instance.Id).FirstOrDefaultAsync(cancellationToken).ConfigureAwait(false); if (currentModel == default) { return(StatusCode((int)HttpStatusCode.Gone)); } bool CheckModified <T>(Expression <Func <Api.Models.Internal.RepositorySettings, T> > expression, RepositoryRights requiredRight) { var memberSelectorExpression = (MemberExpression)expression.Body; var property = (PropertyInfo)memberSelectorExpression.Member; var newVal = property.GetValue(model); if (newVal == null) { return(false); } if (!userRights.HasFlag(requiredRight) && property.GetValue(currentModel) != newVal) { return(true); } property.SetValue(currentModel, newVal); return(false); } if (CheckModified(x => x.AccessToken, RepositoryRights.ChangeCredentials) || CheckModified(x => x.AccessUser, RepositoryRights.ChangeCredentials) || CheckModified(x => x.AutoUpdatesKeepTestMerges, RepositoryRights.ChangeAutoUpdateSettings) || CheckModified(x => x.AutoUpdatesSynchronize, RepositoryRights.ChangeAutoUpdateSettings) || CheckModified(x => x.CommitterEmail, RepositoryRights.ChangeCommitter) || CheckModified(x => x.CommitterName, RepositoryRights.ChangeCommitter) || CheckModified(x => x.PushTestMergeCommits, RepositoryRights.ChangeTestMergeCommits) || CheckModified(x => x.ShowTestMergeCommitters, RepositoryRights.ChangeTestMergeCommits) || CheckModified(x => x.PostTestMergeComment, RepositoryRights.ChangeTestMergeCommits) || (model.UpdateFromOrigin == true && !userRights.HasFlag(RepositoryRights.UpdateBranch))) { return(Forbid()); } if (currentModel.AccessToken?.Length == 0 && currentModel.AccessUser?.Length == 0) { // setting an empty string clears everything currentModel.AccessUser = null; currentModel.AccessToken = null; } var canRead = userRights.HasFlag(RepositoryRights.Read); var api = canRead ? currentModel.ToApi() : new Repository(); var repoManager = instanceManager.GetInstance(Instance).RepositoryManager; if (canRead) { if (repoManager.CloneInProgress) { return(Conflict(new ErrorMessage { Message = "A clone operation is in progress!" })); } if (repoManager.InUse) { return(Conflict(new ErrorMessage { Message = "The repo is busy!" })); } using (var repo = await repoManager.LoadRepository(cancellationToken).ConfigureAwait(false)) { if (repo == null) { return(Conflict(new ErrorMessage { Message = "Repository could not be loaded!" })); } await PopulateApi(api, repo, DatabaseContext, Instance, cancellationToken).ConfigureAwait(false); } } // this is just db stuf so stow it away await DatabaseContext.Save(cancellationToken).ConfigureAwait(false); // format the job description string description = null; if (model.UpdateFromOrigin == true) { if (model.Reference != null) { description = String.Format(CultureInfo.InvariantCulture, "Fetch and hard reset repository to origin/{0}", model.Reference); } else if (model.CheckoutSha != null) { description = String.Format(CultureInfo.InvariantCulture, "Fetch and checkout {0} in repository", model.CheckoutSha); } else { description = "Pull current repository reference"; } } else if (model.Reference != null || model.CheckoutSha != null) { description = String.Format(CultureInfo.InvariantCulture, "Checkout repository {0} {1}", model.Reference != null ? "reference" : "SHA", model.Reference ?? model.CheckoutSha); } if (newTestMerges) { description = String.Format(CultureInfo.InvariantCulture, "{0}est merge pull request(s) {1}{2}", description != null ? String.Format(CultureInfo.InvariantCulture, "{0} and t", description) : "T", String.Join(", ", model.NewTestMerges.Select(x => String.Format(CultureInfo.InvariantCulture, "#{0}{1}", x.Number, x.PullRequestRevision != null ? String.Format(CultureInfo.InvariantCulture, " at {0}", x.PullRequestRevision.Substring(0, 7)) : String.Empty))), description != null ? String.Empty : " in repository"); } if (description == null) { return(Json(api)); // no git changes } var job = new Models.Job { Description = description, StartedBy = AuthenticationContext.User, Instance = Instance, CancelRightsType = RightsType.Repository, CancelRight = (ulong)RepositoryRights.CancelPendingChanges, }; await jobManager.RegisterOperation(job, async (paramJob, databaseContext, progressReporter, ct) => { using (var repo = await repoManager.LoadRepository(ct).ConfigureAwait(false)) { if (repo == null) { throw new JobException("Repository could not be loaded!"); } var modelHasShaOrReference = model.CheckoutSha != null || model.Reference != null; var startReference = repo.Reference; var startSha = repo.Head; string postUpdateSha = null; if (newTestMerges && !repo.IsGitHubRepository) { throw new JobException("Cannot test merge on a non GitHub based repository!"); } var committerName = currentModel.ShowTestMergeCommitters.Value ? AuthenticationContext.User.Name : currentModel.CommitterName; var hardResettingToOriginReference = model.UpdateFromOrigin == true && model.Reference != null; var numSteps = (model.NewTestMerges?.Count ?? 0) + (model.UpdateFromOrigin == true ? 1 : 0) + (!modelHasShaOrReference ? 2 : (hardResettingToOriginReference ? 3 : 1)); var doneSteps = 0; Action <int> NextProgressReporter() { var tmpDoneSteps = doneSteps; ++doneSteps; return(progress => progressReporter((progress + (100 * tmpDoneSteps)) / numSteps)); } progressReporter(0); // get a base line for where we are Models.RevisionInformation lastRevisionInfo = null; var attachedInstance = new Models.Instance { Id = Instance.Id }; databaseContext.Instances.Attach(attachedInstance); await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false); // apply new rev info, tracking applied test merges async Task UpdateRevInfo() { var last = lastRevisionInfo; await LoadRevisionInformation(repo, databaseContext, attachedInstance, last.OriginCommitSha, x => lastRevisionInfo = x, ct).ConfigureAwait(false); lastRevisionInfo.ActiveTestMerges.AddRange(last.ActiveTestMerges); } try { // fetch/pull if (model.UpdateFromOrigin == true) { if (!repo.Tracking) { throw new JobException("Not on an updatable reference!"); } await repo.FetchOrigin(currentModel.AccessUser, currentModel.AccessToken, NextProgressReporter(), ct).ConfigureAwait(false); doneSteps = 1; if (!modelHasShaOrReference) { var fastForward = await repo.MergeOrigin(committerName, currentModel.CommitterEmail, NextProgressReporter(), ct).ConfigureAwait(false); if (!fastForward.HasValue) { throw new JobException("Merge conflict occurred during origin update!"); } await UpdateRevInfo().ConfigureAwait(false); if (fastForward.Value) { lastRevisionInfo.OriginCommitSha = repo.Head; await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), true, ct).ConfigureAwait(false); postUpdateSha = repo.Head; } else { NextProgressReporter()(100); } } } // checkout/hard reset if (modelHasShaOrReference) { if ((model.CheckoutSha != null && repo.Head.ToUpperInvariant().StartsWith(model.CheckoutSha.ToUpperInvariant(), StringComparison.Ordinal)) || (model.Reference != null && repo.Reference.ToUpperInvariant() != model.Reference.ToUpperInvariant())) { var committish = model.CheckoutSha ?? model.Reference; var isSha = await repo.IsSha(committish, cancellationToken).ConfigureAwait(false); if ((isSha && model.Reference != null) || (!isSha && model.CheckoutSha != null)) { throw new JobException("Attempted to checkout a SHA or reference that was actually the opposite!"); } await repo.CheckoutObject(committish, NextProgressReporter(), ct).ConfigureAwait(false); await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false); // we've either seen origin before or what we're checking out is on origin } else { NextProgressReporter()(100); } if (hardResettingToOriginReference) { if (!repo.Tracking) { throw new JobException("Checked out reference does not track a remote object!"); } await repo.ResetToOrigin(NextProgressReporter(), ct).ConfigureAwait(false); await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), true, ct).ConfigureAwait(false); await LoadRevisionInformation(repo, databaseContext, attachedInstance, null, x => lastRevisionInfo = x, ct).ConfigureAwait(false); // repo head is on origin so force this // will update the db if necessary lastRevisionInfo.OriginCommitSha = repo.Head; } } // test merging Dictionary <int, Octokit.PullRequest> prMap = null; if (newTestMerges) { // bit of sanitization foreach (var I in model.NewTestMerges.Where(x => String.IsNullOrWhiteSpace(x.PullRequestRevision))) { I.PullRequestRevision = null; } var gitHubClient = currentModel.AccessToken != null ? gitHubClientFactory.CreateClient(currentModel.AccessToken) : (String.IsNullOrEmpty(generalConfiguration.GitHubAccessToken) ? gitHubClientFactory.CreateClient() : gitHubClientFactory.CreateClient(generalConfiguration.GitHubAccessToken)); var repoOwner = repo.GitHubOwner; var repoName = repo.GitHubRepoName; // optimization: if we've already merged these exact same commits in this fashion before, just find the rev info for it and check it out Models.RevisionInformation revInfoWereLookingFor = null; bool needToApplyRemainingPrs = true; if (lastRevisionInfo.OriginCommitSha == lastRevisionInfo.CommitSha) { // In order for this to work though we need the shas of all the commits if (model.NewTestMerges.Any(x => x.PullRequestRevision == null)) { prMap = new Dictionary <int, Octokit.PullRequest>(); } bool cantSearch = false; foreach (var I in model.NewTestMerges) { if (I.PullRequestRevision != null) #pragma warning disable CA1308 // Normalize strings to uppercase { I.PullRequestRevision = I.PullRequestRevision?.ToLowerInvariant(); // ala libgit2 } #pragma warning restore CA1308 // Normalize strings to uppercase else { try { // retrieve the latest sha var pr = await gitHubClient.PullRequest.Get(repoOwner, repoName, I.Number.Value).ConfigureAwait(false); prMap.Add(I.Number.Value, pr); I.PullRequestRevision = pr.Head.Sha; } catch { cantSearch = true; break; } } } if (!cantSearch) { var dbPull = await databaseContext.RevisionInformations .Where(x => x.Instance.Id == Instance.Id && x.OriginCommitSha == lastRevisionInfo.OriginCommitSha && x.ActiveTestMerges.Count <= model.NewTestMerges.Count && x.ActiveTestMerges.Count > 0) .Include(x => x.ActiveTestMerges) .ThenInclude(x => x.TestMerge) .ToListAsync(cancellationToken).ConfigureAwait(false); // split here cause this bit has to be done locally revInfoWereLookingFor = dbPull .Where(x => x.ActiveTestMerges.Count == model.NewTestMerges.Count && x.ActiveTestMerges.Select(y => y.TestMerge) .All(y => model.NewTestMerges.Any(z => y.Number == z.Number && y.PullRequestRevision.StartsWith(z.PullRequestRevision, StringComparison.Ordinal) && (y.Comment?.Trim().ToUpperInvariant() == z.Comment?.Trim().ToUpperInvariant() || z.Comment == null)))) .FirstOrDefault(); if (revInfoWereLookingFor == null && model.NewTestMerges.Count > 1) { // okay try to add at least SOME prs we've seen before var search = model.NewTestMerges.ToList(); var appliedTestMergeIds = new List <long>(); Models.RevisionInformation lastGoodRevInfo = null; do { foreach (var I in search) { revInfoWereLookingFor = dbPull .Where(x => model.NewTestMerges.Any(z => x.PrimaryTestMerge.Number == z.Number && x.PrimaryTestMerge.PullRequestRevision.StartsWith(z.PullRequestRevision, StringComparison.Ordinal) && (x.PrimaryTestMerge.Comment?.Trim().ToUpperInvariant() == z.Comment?.Trim().ToUpperInvariant() || z.Comment == null)) && x.ActiveTestMerges.Select(y => y.TestMerge).All(y => appliedTestMergeIds.Contains(y.Id))) .FirstOrDefault(); if (revInfoWereLookingFor != null) { lastGoodRevInfo = revInfoWereLookingFor; appliedTestMergeIds.Add(revInfoWereLookingFor.PrimaryTestMerge.Id); search.Remove(I); break; } } }while (revInfoWereLookingFor != null && search.Count > 0); revInfoWereLookingFor = lastGoodRevInfo; needToApplyRemainingPrs = search.Count != 0; if (needToApplyRemainingPrs) { model.NewTestMerges = search; } } else if (revInfoWereLookingFor != null) { needToApplyRemainingPrs = false; } } } if (revInfoWereLookingFor != null) { // goteem await repo.ResetToSha(revInfoWereLookingFor.CommitSha, NextProgressReporter(), cancellationToken).ConfigureAwait(false); lastRevisionInfo = revInfoWereLookingFor; } if (needToApplyRemainingPrs) { // an invocation of LoadRevisionInformation could have already loaded this user var contextUser = databaseContext.Users.Local.Where(x => x.Id == AuthenticationContext.User.Id).FirstOrDefault(); if (contextUser == default) { contextUser = new Models.User { Id = AuthenticationContext.User.Id }; databaseContext.Users.Attach(contextUser); } else { Logger.LogTrace("Skipping attaching the user to the database context as it is already loaded!"); } foreach (var I in model.NewTestMerges) { Octokit.PullRequest pr = null; string errorMessage = null; if (lastRevisionInfo.ActiveTestMerges.Any(x => x.TestMerge.Number == I.Number.Value)) { throw new JobException("Cannot test merge the same PR twice in one HEAD!"); } try { // load from cache if possible if (prMap == null || !prMap.TryGetValue(I.Number.Value, out pr)) { pr = await gitHubClient.PullRequest.Get(repoOwner, repoName, I.Number.Value).ConfigureAwait(false); } } catch (Octokit.RateLimitExceededException) { // you look at your anonymous access and sigh errorMessage = "P.R.E. RATE LIMITED"; } catch (Octokit.AuthorizationException) { errorMessage = "P.R.E. BAD CREDENTIALS"; } catch (Octokit.NotFoundException) { // you look at your shithub and sigh errorMessage = "P.R.E. NOT FOUND"; } // we want to take the earliest truth possible to prevent RCEs, if this fails AddTestMerge will set it if (I.PullRequestRevision == null && pr != null) { I.PullRequestRevision = pr.Head.Sha; } var mergeResult = await repo.AddTestMerge(I, committerName, currentModel.CommitterEmail, currentModel.AccessUser, currentModel.AccessToken, NextProgressReporter(), ct).ConfigureAwait(false); if (!mergeResult.HasValue) { throw new JobException(String.Format(CultureInfo.InvariantCulture, "Merge of PR #{0} at {1} conflicted!", I.Number, I.PullRequestRevision.Substring(0, 7))); } ++doneSteps; var revInfoUpdateTask = UpdateRevInfo(); var tm = new Models.TestMerge { Author = pr?.User.Login ?? errorMessage, BodyAtMerge = pr?.Body ?? errorMessage ?? String.Empty, MergedAt = DateTimeOffset.Now, TitleAtMerge = pr?.Title ?? errorMessage ?? String.Empty, Comment = I.Comment, Number = I.Number, MergedBy = contextUser, PullRequestRevision = I.PullRequestRevision, Url = pr?.HtmlUrl ?? errorMessage }; await revInfoUpdateTask.ConfigureAwait(false); lastRevisionInfo.PrimaryTestMerge = tm; lastRevisionInfo.ActiveTestMerges.Add(new RevInfoTestMerge { TestMerge = tm }); } } } var currentHead = repo.Head; if (startSha != currentHead || (postUpdateSha != null && postUpdateSha != currentHead)) { await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), false, ct).ConfigureAwait(false); await UpdateRevInfo().ConfigureAwait(false); } await databaseContext.Save(ct).ConfigureAwait(false); } catch { doneSteps = 0; numSteps = 2; // the stuff didn't make it into the db, forget what we've done and abort await repo.CheckoutObject(startReference ?? startSha, NextProgressReporter(), default).ConfigureAwait(false); if (startReference != null && repo.Head != startSha) { await repo.ResetToSha(startSha, NextProgressReporter(), default).ConfigureAwait(false); } else { progressReporter(100); } throw; } } }, cancellationToken).ConfigureAwait(false); api.ActiveJob = job.ToApi(); return(Accepted(api)); }
/// <inheritdoc /> public Action <string, string> QueueDeploymentMessage( Models.RevisionInformation revisionInformation, Version byondVersion, DateTimeOffset?estimatedCompletionTime, string gitHubOwner, string gitHubRepo, bool localCommitPushed) { List <ulong> wdChannels; lock (mappedChannels) // so it doesn't change while we're using it wdChannels = mappedChannels.Where(x => x.Value.IsUpdatesChannel).Select(x => x.Key).ToList(); logger.LogTrace("Sending deployment message for RevisionInformation: {0}", revisionInformation.Id); var callbacks = new List <Func <string, string, Task> >(); var task = Task.WhenAll( wdChannels.Select( async x => { ChannelMapping channelMapping; lock (mappedChannels) if (!mappedChannels.TryGetValue(x, out channelMapping)) { return; } IProvider provider; lock (providers) if (!providers.TryGetValue(channelMapping.ProviderId, out provider)) { return; } try { var callback = await provider.SendUpdateMessage( revisionInformation, byondVersion, estimatedCompletionTime, gitHubOwner, gitHubRepo, channelMapping.ProviderChannelId, localCommitPushed, handlerCts.Token) .ConfigureAwait(false); callbacks.Add(callback); } catch (Exception ex) { logger.LogWarning( ex, "Error sending deploy message to provider {0}!", channelMapping.ProviderId); } })); AddMessageTask(task); return((errorMessage, dreamMakerOutput) => AddMessageTask( Task.WhenAll( callbacks.Select( x => x( errorMessage, dreamMakerOutput))))); }
#pragma warning disable CA1502, CA1505 // TODO: Decomplexify public async Task <IActionResult> Update([FromBody] Repository model, CancellationToken cancellationToken) { if (model == null) { throw new ArgumentNullException(nameof(model)); } if (model.AccessUser == null ^ model.AccessToken == null) { return(BadRequest(new ErrorMessage(ErrorCode.RepoMismatchUserAndAccessToken))); } if (model.CheckoutSha != null && model.Reference != null) { return(BadRequest(new ErrorMessage(ErrorCode.RepoMismatchShaAndReference))); } if (model.CheckoutSha != null && model.UpdateFromOrigin == true) { return(BadRequest(new ErrorMessage(ErrorCode.RepoMismatchShaAndUpdate))); } if (model.NewTestMerges?.Any(x => model.NewTestMerges.Any(y => x != y && x.Number == y.Number)) == true) { return(BadRequest(new ErrorMessage(ErrorCode.RepoDuplicateTestMerge))); } if (model.CommitterName?.Length == 0) { return(BadRequest(new ErrorMessage(ErrorCode.RepoWhitespaceCommitterName))); } if (model.CommitterEmail?.Length == 0) { return(BadRequest(new ErrorMessage(ErrorCode.RepoWhitespaceCommitterEmail))); } var newTestMerges = model.NewTestMerges != null && model.NewTestMerges.Count > 0; var userRights = (RepositoryRights)AuthenticationContext.GetRight(RightsType.Repository); if (newTestMerges && !userRights.HasFlag(RepositoryRights.MergePullRequest)) { return(Forbid()); } var currentModel = await DatabaseContext .RepositorySettings .AsQueryable() .Where(x => x.InstanceId == Instance.Id) .FirstOrDefaultAsync(cancellationToken) .ConfigureAwait(false); if (currentModel == default) { return(Gone()); } bool CheckModified <T>(Expression <Func <Api.Models.Internal.RepositorySettings, T> > expression, RepositoryRights requiredRight) { var memberSelectorExpression = (MemberExpression)expression.Body; var property = (PropertyInfo)memberSelectorExpression.Member; var newVal = property.GetValue(model); if (newVal == null) { return(false); } if (!userRights.HasFlag(requiredRight) && property.GetValue(currentModel) != newVal) { return(true); } property.SetValue(currentModel, newVal); return(false); } if (CheckModified(x => x.AccessToken, RepositoryRights.ChangeCredentials) || CheckModified(x => x.AccessUser, RepositoryRights.ChangeCredentials) || CheckModified(x => x.AutoUpdatesKeepTestMerges, RepositoryRights.ChangeAutoUpdateSettings) || CheckModified(x => x.AutoUpdatesSynchronize, RepositoryRights.ChangeAutoUpdateSettings) || CheckModified(x => x.CommitterEmail, RepositoryRights.ChangeCommitter) || CheckModified(x => x.CommitterName, RepositoryRights.ChangeCommitter) || CheckModified(x => x.PushTestMergeCommits, RepositoryRights.ChangeTestMergeCommits) || CheckModified(x => x.CreateGitHubDeployments, RepositoryRights.ChangeTestMergeCommits) || CheckModified(x => x.ShowTestMergeCommitters, RepositoryRights.ChangeTestMergeCommits) || CheckModified(x => x.PostTestMergeComment, RepositoryRights.ChangeTestMergeCommits) || (model.UpdateFromOrigin == true && !userRights.HasFlag(RepositoryRights.UpdateBranch))) { return(Forbid()); } if (model.AccessToken?.Length == 0 && model.AccessUser?.Length == 0) { // setting an empty string clears everything currentModel.AccessUser = null; currentModel.AccessToken = null; } var canRead = userRights.HasFlag(RepositoryRights.Read); var api = canRead ? currentModel.ToApi() : new Repository(); if (canRead) { var earlyOut = await WithComponentInstance( async instance => { var repoManager = instance.RepositoryManager; if (repoManager.CloneInProgress) { return(Conflict(new ErrorMessage(ErrorCode.RepoCloning))); } if (repoManager.InUse) { return(Conflict(new ErrorMessage(ErrorCode.RepoBusy))); } using var repo = await repoManager.LoadRepository(cancellationToken).ConfigureAwait(false); if (repo == null) { return(Conflict(new ErrorMessage(ErrorCode.RepoMissing))); } await PopulateApi(api, repo, DatabaseContext, Instance, cancellationToken).ConfigureAwait(false); if (model.Origin != null && model.Origin != repo.Origin) { return(BadRequest(new ErrorMessage(ErrorCode.RepoCantChangeOrigin))); } return(null); }) .ConfigureAwait(false); if (earlyOut != null) { return(earlyOut); } } // this is just db stuf so stow it away await DatabaseContext.Save(cancellationToken).ConfigureAwait(false); // format the job description string description = null; if (model.UpdateFromOrigin == true) { if (model.Reference != null) { description = String.Format(CultureInfo.InvariantCulture, "Fetch and hard reset repository to origin/{0}", model.Reference); } else if (model.CheckoutSha != null) { description = String.Format(CultureInfo.InvariantCulture, "Fetch and checkout {0} in repository", model.CheckoutSha); } else { description = "Pull current repository reference"; } } else if (model.Reference != null || model.CheckoutSha != null) { description = String.Format(CultureInfo.InvariantCulture, "Checkout repository {0} {1}", model.Reference != null ? "reference" : "SHA", model.Reference ?? model.CheckoutSha); } if (newTestMerges) { description = String.Format(CultureInfo.InvariantCulture, "{0}est merge(s) {1}{2}", description != null ? String.Format(CultureInfo.InvariantCulture, "{0} and t", description) : "T", String.Join(", ", model.NewTestMerges.Select(x => String.Format(CultureInfo.InvariantCulture, "#{0}{1}", x.Number, x.TargetCommitSha != null ? String.Format(CultureInfo.InvariantCulture, " at {0}", x.TargetCommitSha.Substring(0, 7)) : String.Empty))), description != null ? String.Empty : " in repository"); } if (description == null) { return(Json(api)); // no git changes } async Task <IActionResult> UpdateCallbackThatDesperatelyNeedsRefactoring( IInstanceCore instance, IDatabaseContextFactory databaseContextFactory, Action <int> progressReporter, CancellationToken ct) { var repoManager = instance.RepositoryManager; using var repo = await repoManager.LoadRepository(ct).ConfigureAwait(false); if (repo == null) { throw new JobException(ErrorCode.RepoMissing); } var modelHasShaOrReference = model.CheckoutSha != null || model.Reference != null; var startReference = repo.Reference; var startSha = repo.Head; string postUpdateSha = null; if (newTestMerges && repo.RemoteGitProvider == RemoteGitProvider.Unknown) { throw new JobException(ErrorCode.RepoUnsupportedTestMergeRemote); } var committerName = currentModel.ShowTestMergeCommitters.Value ? AuthenticationContext.User.Name : currentModel.CommitterName; var hardResettingToOriginReference = model.UpdateFromOrigin == true && model.Reference != null; var numSteps = (model.NewTestMerges?.Count ?? 0) + (model.UpdateFromOrigin == true ? 1 : 0) + (!modelHasShaOrReference ? 2 : (hardResettingToOriginReference ? 3 : 1)); var doneSteps = 0; Action <int> NextProgressReporter() { var tmpDoneSteps = doneSteps; ++doneSteps; return(progress => progressReporter((progress + (100 * tmpDoneSteps)) / numSteps)); } progressReporter(0); // get a base line for where we are Models.RevisionInformation lastRevisionInfo = null; var attachedInstance = new Models.Instance { Id = Instance.Id }; Task CallLoadRevInfo(Models.TestMerge testMergeToAdd = null, string lastOriginCommitSha = null) => databaseContextFactory .UseContext( async databaseContext => { databaseContext.Instances.Attach(attachedInstance); var previousRevInfo = lastRevisionInfo; var needsUpdate = await LoadRevisionInformation( repo, databaseContext, attachedInstance, lastOriginCommitSha, x => lastRevisionInfo = x, ct) .ConfigureAwait(false); if (testMergeToAdd != null) { // rev info may have already loaded the user var mergedBy = databaseContext.Users.Local.FirstOrDefault(x => x.Id == AuthenticationContext.User.Id); if (mergedBy == default) { mergedBy = new Models.User { Id = AuthenticationContext.User.Id }; databaseContext.Users.Attach(mergedBy); } testMergeToAdd.MergedBy = mergedBy; foreach (var activeTestMerge in previousRevInfo.ActiveTestMerges) { lastRevisionInfo.ActiveTestMerges.Add(activeTestMerge); } lastRevisionInfo.ActiveTestMerges.Add(new RevInfoTestMerge { TestMerge = testMergeToAdd }); lastRevisionInfo.PrimaryTestMerge = testMergeToAdd; needsUpdate = true; } if (needsUpdate) { await databaseContext.Save(cancellationToken).ConfigureAwait(false); } }); await CallLoadRevInfo().ConfigureAwait(false); // apply new rev info, tracking applied test merges Task UpdateRevInfo(Models.TestMerge testMergeToAdd = null) => CallLoadRevInfo(testMergeToAdd, lastRevisionInfo.OriginCommitSha); try { // fetch/pull if (model.UpdateFromOrigin == true) { if (!repo.Tracking) { throw new JobException(ErrorCode.RepoReferenceRequired); } await repo.FetchOrigin(currentModel.AccessUser, currentModel.AccessToken, NextProgressReporter(), ct).ConfigureAwait(false); doneSteps = 1; if (!modelHasShaOrReference) { var fastForward = await repo.MergeOrigin(committerName, currentModel.CommitterEmail, NextProgressReporter(), ct).ConfigureAwait(false); if (!fastForward.HasValue) { throw new JobException(ErrorCode.RepoMergeConflict); } lastRevisionInfo.OriginCommitSha = await repo.GetOriginSha(cancellationToken).ConfigureAwait(false); await UpdateRevInfo().ConfigureAwait(false); if (fastForward.Value) { await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), true, ct).ConfigureAwait(false); postUpdateSha = repo.Head; } else { NextProgressReporter()(100); } } } // checkout/hard reset if (modelHasShaOrReference) { var validCheckoutSha = model.CheckoutSha != null && !repo.Head.StartsWith(model.CheckoutSha, StringComparison.OrdinalIgnoreCase); var validCheckoutReference = model.Reference != null && !repo.Reference.Equals(model.Reference, StringComparison.OrdinalIgnoreCase); if (validCheckoutSha || validCheckoutReference) { var committish = model.CheckoutSha ?? model.Reference; var isSha = await repo.IsSha(committish, cancellationToken).ConfigureAwait(false); if ((isSha && model.Reference != null) || (!isSha && model.CheckoutSha != null)) { throw new JobException(ErrorCode.RepoSwappedShaOrReference); } await repo.CheckoutObject(committish, NextProgressReporter(), ct).ConfigureAwait(false); await CallLoadRevInfo().ConfigureAwait(false); // we've either seen origin before or what we're checking out is on origin } else { NextProgressReporter()(100); } if (hardResettingToOriginReference) { if (!repo.Tracking) { throw new JobException(ErrorCode.RepoReferenceNotTracking); } await repo.ResetToOrigin(NextProgressReporter(), ct).ConfigureAwait(false); await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), true, ct).ConfigureAwait(false); await CallLoadRevInfo().ConfigureAwait(false); // repo head is on origin so force this // will update the db if necessary lastRevisionInfo.OriginCommitSha = repo.Head; } } // test merging if (newTestMerges) { if (repo.RemoteGitProvider == RemoteGitProvider.Unknown) { throw new JobException(ErrorCode.RepoTestMergeInvalidRemote); } // bit of sanitization foreach (var I in model.NewTestMerges.Where(x => String.IsNullOrWhiteSpace(x.TargetCommitSha))) { I.TargetCommitSha = null; } var gitHubClient = currentModel.AccessToken != null ? gitHubClientFactory.CreateClient(currentModel.AccessToken) : gitHubClientFactory.CreateClient(); var repoOwner = repo.RemoteRepositoryOwner; var repoName = repo.RemoteRepositoryName; // optimization: if we've already merged these exact same commits in this fashion before, just find the rev info for it and check it out Models.RevisionInformation revInfoWereLookingFor = null; bool needToApplyRemainingPrs = true; if (lastRevisionInfo.OriginCommitSha == lastRevisionInfo.CommitSha) { bool cantSearch = false; foreach (var I in model.NewTestMerges) { if (I.TargetCommitSha != null) #pragma warning disable CA1308 // Normalize strings to uppercase { I.TargetCommitSha = I.TargetCommitSha?.ToLowerInvariant(); // ala libgit2 } #pragma warning restore CA1308 // Normalize strings to uppercase else { try { // retrieve the latest sha var pr = await repo.GetTestMerge(I, currentModel, ct).ConfigureAwait(false); // we want to take the earliest truth possible to prevent RCEs, if this fails AddTestMerge will set it I.TargetCommitSha = pr.TargetCommitSha; } catch { cantSearch = true; break; } } } if (!cantSearch) { List <Models.RevisionInformation> dbPull = null; await databaseContextFactory.UseContext( async databaseContext => dbPull = await databaseContext.RevisionInformations .AsQueryable() .Where(x => x.Instance.Id == Instance.Id && x.OriginCommitSha == lastRevisionInfo.OriginCommitSha && x.ActiveTestMerges.Count <= model.NewTestMerges.Count && x.ActiveTestMerges.Count > 0) .Include(x => x.ActiveTestMerges) .ThenInclude(x => x.TestMerge) .ToListAsync(cancellationToken) .ConfigureAwait(false)) .ConfigureAwait(false); // split here cause this bit has to be done locally revInfoWereLookingFor = dbPull .Where(x => x.ActiveTestMerges.Count == model.NewTestMerges.Count && x.ActiveTestMerges.Select(y => y.TestMerge) .All(y => model.NewTestMerges.Any(z => y.Number == z.Number && y.TargetCommitSha.StartsWith(z.TargetCommitSha, StringComparison.Ordinal) && (y.Comment?.Trim().ToUpperInvariant() == z.Comment?.Trim().ToUpperInvariant() || z.Comment == null)))) .FirstOrDefault(); if (revInfoWereLookingFor == default && model.NewTestMerges.Count > 1) { // okay try to add at least SOME prs we've seen before var search = model.NewTestMerges.ToList(); var appliedTestMergeIds = new List <long>(); Models.RevisionInformation lastGoodRevInfo = null; do { foreach (var I in search) { revInfoWereLookingFor = dbPull .Where(testRevInfo => { if (testRevInfo.PrimaryTestMerge == null) { return(false); } var testMergeMatch = model.NewTestMerges.Any(testTestMerge => { var numberMatch = testRevInfo.PrimaryTestMerge.Number == testTestMerge.Number; if (!numberMatch) { return(false); } var shaMatch = testRevInfo.PrimaryTestMerge.TargetCommitSha.StartsWith( testTestMerge.TargetCommitSha, StringComparison.Ordinal); if (!shaMatch) { return(false); } var commentMatch = testRevInfo.PrimaryTestMerge.Comment == testTestMerge.Comment; return(commentMatch); }); if (!testMergeMatch) { return(false); } var previousTestMergesMatch = testRevInfo .ActiveTestMerges .Select(previousRevInfoTestMerge => previousRevInfoTestMerge.TestMerge) .All(previousTestMerge => appliedTestMergeIds.Contains(previousTestMerge.Id)); return(previousTestMergesMatch); }) .FirstOrDefault(); if (revInfoWereLookingFor != null) { lastGoodRevInfo = revInfoWereLookingFor; appliedTestMergeIds.Add(revInfoWereLookingFor.PrimaryTestMerge.Id); search.Remove(I); break; } } }while (revInfoWereLookingFor != null && search.Count > 0); revInfoWereLookingFor = lastGoodRevInfo; needToApplyRemainingPrs = search.Count != 0; if (needToApplyRemainingPrs) { model.NewTestMerges = search; } } else if (revInfoWereLookingFor != null) { needToApplyRemainingPrs = false; } } } if (revInfoWereLookingFor != null) { // goteem Logger.LogDebug("Reusing existing SHA {0}...", revInfoWereLookingFor.CommitSha); await repo.ResetToSha(revInfoWereLookingFor.CommitSha, NextProgressReporter(), cancellationToken).ConfigureAwait(false); lastRevisionInfo = revInfoWereLookingFor; } if (needToApplyRemainingPrs) { foreach (var I in model.NewTestMerges) { if (lastRevisionInfo.ActiveTestMerges.Any(x => x.TestMerge.Number == I.Number)) { throw new JobException(ErrorCode.RepoDuplicateTestMerge); } var fullTestMergeTask = repo.GetTestMerge(I, currentModel, ct); var mergeResult = await repo.AddTestMerge( I, committerName, currentModel.CommitterEmail, currentModel.AccessUser, currentModel.AccessToken, NextProgressReporter(), ct).ConfigureAwait(false); if (mergeResult == null) { throw new JobException( ErrorCode.RepoTestMergeConflict, new JobException( $"Test Merge #{I.Number} at {I.TargetCommitSha.Substring(0, 7)} conflicted!")); } Models.TestMerge fullTestMerge; try { fullTestMerge = await fullTestMergeTask.ConfigureAwait(false); } catch (Exception ex) { Logger.LogWarning("Error retrieving metadata for test merge #{0}!", I.Number); fullTestMerge = new Models.TestMerge { Author = ex.Message, BodyAtMerge = ex.Message, MergedAt = DateTimeOffset.UtcNow, TitleAtMerge = ex.Message, Comment = I.Comment, Number = I.Number, Url = ex.Message }; } // Ensure we're getting the full sha from git itself fullTestMerge.TargetCommitSha = I.TargetCommitSha; // MergedBy will be set later ++doneSteps; await UpdateRevInfo(fullTestMerge).ConfigureAwait(false); } } } var currentHead = repo.Head; if (startSha != currentHead || (postUpdateSha != null && postUpdateSha != currentHead)) { await repo.Sychronize(currentModel.AccessUser, currentModel.AccessToken, currentModel.CommitterName, currentModel.CommitterEmail, NextProgressReporter(), false, ct).ConfigureAwait(false); await UpdateRevInfo().ConfigureAwait(false); } return(null); } catch { doneSteps = 0; numSteps = 2; // Forget what we've done and abort // DCTx2: Cancellation token is for job, operations should always run await repo.CheckoutObject(startReference ?? startSha, NextProgressReporter(), default).ConfigureAwait(false); if (startReference != null && repo.Head != startSha) { await repo.ResetToSha(startSha, NextProgressReporter(), default).ConfigureAwait(false); } else { progressReporter(100); } throw; } } var job = new Models.Job { Description = description, StartedBy = AuthenticationContext.User, Instance = Instance, CancelRightsType = RightsType.Repository, CancelRight = (ulong)RepositoryRights.CancelPendingChanges, }; // Time to access git, do it in a job await jobManager.RegisterOperation( job, (core, databaseContextFactory, paramJob, progressReporter, ct) => UpdateCallbackThatDesperatelyNeedsRefactoring( core, databaseContextFactory, progressReporter, ct), cancellationToken) .ConfigureAwait(false); api.ActiveJob = job.ToApi(); return(Accepted(api)); }