private async Task <bool> IsGroupJoinableAsync( BuildManifestLocation location, string commit, string buildId, IEnumerable <string> changedSemaphorePaths, JoinSemaphoreGroup joinGroup) { string[] remainingSemaphores = joinGroup .ParallelSemaphorePaths .Except(changedSemaphorePaths) .ToArray(); if (remainingSemaphores.Length == joinGroup.ParallelSemaphorePaths.Count()) { // No semaphores in this group are changing: it can't be joinable by this update. return(false); } // TODO: Avoid redundant fetches if multiple groups share a semaphore. https://github.com/dotnet/buildtools/issues/1910 bool[] remainingSemaphoreIsComplete = await Task.WhenAll( remainingSemaphores.Select( async path => { SemaphoreModel semaphore = await FetchSemaphoreAsync( location.GitHubProject, commit, location.GitHubBasePath, path); return(semaphore?.BuildId == buildId); })); return(remainingSemaphoreIsComplete.All(x => x)); }
private async Task <bool> PushUploadsAsync( BuildManifestLocation location, string message, string remoteCommit, IEnumerable <SupplementaryUploadRequest> uploads) { GitObject[] objects = uploads .Select(upload => new GitObject { Path = upload.GetAbsolutePath(location.GitHubBasePath), Mode = GitObject.ModeFile, Type = GitObject.TypeBlob, // Always upload files using LF to avoid bad dev scenarios with Git autocrlf. Content = upload.Contents.Replace("\r\n", "\n") }) .ToArray(); GitTree tree = await _github.PostTreeAsync( location.GitHubProject, remoteCommit, objects); GitCommit commit = await _github.PostCommitAsync( location.GitHubProject, message, tree.Sha, new[] { remoteCommit }); try { // Only fast-forward. Don't overwrite other changes: throw exception instead. await _github.PatchReferenceAsync( location.GitHubProject, location.GitHubRef, commit.Sha, force : false); } catch (NotFastForwardUpdateException e) { // Retry if there has been a commit since this update attempt started. Trace.TraceInformation($"Retrying: {e.Message}"); return(false); } return(true); }
public async Task PushNewBuildAsync( BuildManifestLocation location, OrchestratedBuildModel build, IEnumerable <SupplementaryUploadRequest> supplementaryUploads, string message) { await Retry.RunAsync(async attempt => { GitReference remoteRef = await _github.GetReferenceAsync( location.GitHubProject, location.GitHubRef); string remoteCommit = remoteRef.Object.Sha; Trace.TraceInformation($"Creating update on remote commit: {remoteCommit}"); IEnumerable <SupplementaryUploadRequest> uploads = supplementaryUploads.NullAsEmpty() .Concat(new[] { new SupplementaryUploadRequest { Path = BuildManifestXmlName, Contents = build.ToXml().ToString() }, new SupplementaryUploadRequest { Path = SemaphoreModel.BuildSemaphorePath, Contents = new SemaphoreModel { BuildId = build.Identity.BuildId }.ToFileContent() } }) .ToArray(); return(await PushUploadsAsync(location, message, remoteCommit, uploads)); }); }
public BuildManifestChange( BuildManifestLocation location, string commitMessage, string orchestratedBuildId, IEnumerable <string> semaphorePaths, Action <OrchestratedBuildModel> applyModelChanges) { if (location == null) { throw new ArgumentNullException(nameof(location)); } if (string.IsNullOrEmpty(commitMessage)) { throw new ArgumentException(nameof(commitMessage)); } if (string.IsNullOrEmpty(orchestratedBuildId)) { throw new ArgumentException(nameof(orchestratedBuildId)); } if (applyModelChanges == null) { throw new ArgumentNullException(nameof(applyModelChanges)); } if (semaphorePaths == null) { throw new ArgumentNullException(nameof(semaphorePaths)); } Location = location; CommitMessage = commitMessage; OrchestratedBuildId = orchestratedBuildId; SemaphorePaths = semaphorePaths; ApplyModelChanges = applyModelChanges; }
public async Task PushChangeAsync(BuildManifestChange change) { await Retry.RunAsync(async attempt => { BuildManifestLocation location = change.Location; // Get the current commit. Use this throughout to ensure a clean transaction. GitReference remoteRef = await _github.GetReferenceAsync( location.GitHubProject, location.GitHubRef); string remoteCommit = remoteRef.Object.Sha; Trace.TraceInformation($"Creating update on remote commit: {remoteCommit}"); XElement remoteModelXml = await FetchModelXmlAsync( location.GitHubProject, remoteCommit, location.GitHubBasePath); OrchestratedBuildModel remoteModel = OrchestratedBuildModel.Parse(remoteModelXml); // This is a subsequent publish step: make sure a new build hasn't happened already. if (change.OrchestratedBuildId != remoteModel.Identity.BuildId) { throw new ManifestChangeOutOfDateException( change.OrchestratedBuildId, remoteModel.Identity.BuildId); } OrchestratedBuildModel modifiedModel = OrchestratedBuildModel.Parse(remoteModelXml); change.ApplyModelChanges(modifiedModel); if (modifiedModel.Identity.BuildId != change.OrchestratedBuildId) { throw new ArgumentException( "Change action shouldn't modify BuildId. Changed from " + $"'{change.OrchestratedBuildId}' to '{modifiedModel.Identity.BuildId}'.", nameof(change)); } XElement modifiedModelXml = modifiedModel.ToXml(); string[] changedSemaphorePaths = change.SemaphorePaths.ToArray(); // Check if any join groups are completed by this change. var joinCompleteCheckTasks = change.JoinSemaphoreGroups.NullAsEmpty() .Select(async g => new { Group = g, Joinable = await IsGroupJoinableAsync( location, remoteCommit, change.OrchestratedBuildId, changedSemaphorePaths, g) }); var completeJoinedSemaphores = (await Task.WhenAll(joinCompleteCheckTasks)) .Where(g => g.Joinable) .Select(g => g.Group.JoinSemaphorePath) .ToArray(); IEnumerable <SupplementaryUploadRequest> semaphoreUploads = completeJoinedSemaphores .Concat(changedSemaphorePaths) .Select(p => new SupplementaryUploadRequest { Path = p, Contents = new SemaphoreModel { BuildId = change.OrchestratedBuildId }.ToFileContent() }); IEnumerable <SupplementaryUploadRequest> uploads = semaphoreUploads.Concat(change.SupplementaryUploads.NullAsEmpty()); if (!XNode.DeepEquals(modifiedModelXml, remoteModelXml)) { uploads = uploads.Concat(new[] { new SupplementaryUploadRequest { Path = BuildManifestXmlName, Contents = modifiedModelXml.ToString() } }); } return(await PushUploadsAsync( location, change.CommitMessage, remoteCommit, uploads)); }); }