internal static string GetDevDivPackagesDirPath(BuildVersion version) { // For example: "\\cpvsbuild\drops\Roslyn\Roslyn-Master-Signed-Release\20160315.3\DevDivPackages" return(Path.Combine(GetBuildDirectory(version), "DevDivPackages")); }
public static async Task PerformInsertionAsync( RoslynInsertionToolOptions options, ILogger log, CancellationToken cancellationToken) { Options = options; Log = log; Log.Info($"{Environment.NewLine}New Insertion Into {Options.VisualStudioBranchName} Started{Environment.NewLine}"); GitPullRequest pullRequest = null; var shouldRollBackGitChanges = false; var newPackageFiles = new List <string>(); var isInsertionCancelled = false; var noProgressOnFailedBuilds = false; try { // Verify that the arguments we were passed authenticate correctly Log.Trace($"Verifying given authentication for {Options.VSTSUri}"); try { ProjectCollection.Authenticate(); } catch (Exception ex) { Log.Error($"Could not authenticate with {Options.VSTSUri}"); Log.Error(ex); return; } Log.Trace($"Verification succeeded for {Options.VSTSUri}"); // ********************** Create dummy PR ***************************** if (Options.CreateDummyPr) { var dummyBranch = GetLatestAndCreateBranch(cancellationToken); try { CreateDummyCommit(cancellationToken); PushChanges(dummyBranch, cancellationToken); pullRequest = await CreatePullRequestAsync(dummyBranch.FriendlyName, $"DUMMY INSERTION FOR {Options.InsertionName}", "Not Specified", cancellationToken); } catch (Exception ex) { Log.Error($"Unable to create pull request for '{dummyBranch.FriendlyName}'"); Log.Error(ex); return; } if (pullRequest == null) { Log.Error($"Unable to create pull request for '{dummyBranch.FriendlyName}'"); } return; } // ********************** Get Last Insertion ***************************** cancellationToken.ThrowIfCancellationRequested(); BuildVersion buildVersion; Build buildToInsert; Build latestBuild = null; bool retainBuild = false; // Get the version from TFS build queue, e.g. Roslyn-Master-Signed-Release. // We assume all CoreXT packages we build (Roslyn and all dependencies we // insert) have the same version. if (string.IsNullOrEmpty(Options.SpecificBuild)) { buildToInsert = await GetLatestPassedBuildAsync(cancellationToken); buildVersion = BuildVersion.FromTfsBuildNumber(buildToInsert.BuildNumber, Options.BuildQueueName); // Get the latest build, whether passed or failed. If the buildToInsert has already been inserted but // there is a later failing build, then send an error latestBuild = await GetLatestBuildAsync(cancellationToken); } else { buildVersion = BuildVersion.FromString(Options.SpecificBuild); buildToInsert = await GetSpecificBuildAsync(buildVersion, cancellationToken); } string artifactsFolder = await GetBuildDirectoryAsync(buildToInsert, cancellationToken); Branch branch = null; cancellationToken.ThrowIfCancellationRequested(); if (Options.UpdateExistingPr != 0) { // ****************** Update existing PR *********************** pullRequest = await GetExistingPullRequestAsync(Options.UpdateExistingPr, cancellationToken); branch = SwitchToBranchAndUpdate(pullRequest.SourceRefName, Options.VisualStudioBranchName); } else { // ****************** Get Latest and Create Branch *********************** Log.Info($"Getting Latest From {Options.VisualStudioBranchName} and Creating New Branch"); branch = string.IsNullOrEmpty(Options.NewBranchName) ? null : GetLatestAndCreateBranch(cancellationToken); } shouldRollBackGitChanges = branch != null; if (Options.UpdateCoreXTLibraries) { // ************** Update paths to CoreFX libraries ************************ cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Update paths to CoreFX libraries"); if (!await TryUpdateFileAsync( artifactsFolder, Path.Combine("ProductData", "ContractAssemblies.props"), buildVersion, onlyCopyIfFileDoesNotExistAtDestination: false, cancellationToken: cancellationToken)) { return; } } var coreXT = CoreXT.Load(GetAbsolutePathForEnlistment()); if (Options.InsertCoreXTPackages) { // ************** Update Nuget Packages For Branch************************ cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Updating Nuget Packages"); bool success = false; (success, newPackageFiles) = UpdatePackages( buildVersion, coreXT, GetPackagesDirPath(artifactsFolder), cancellationToken); retainBuild |= success; // ************ Update .corext\Configs\default.config ******************** cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Updating CoreXT default.config file"); coreXT.SaveConfig(); } if (Options.UpdateCoreXTLibraries) { // ************** Update paths to CoreFX libraries ************************ cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Update paths to CoreFX libraries"); if (!await TryUpdateFileAsync( artifactsFolder, Path.Combine("ProductData", "ContractAssemblies.props"), buildVersion, onlyCopyIfFileDoesNotExistAtDestination: false, cancellationToken: cancellationToken)) { return; } } if (Options.UpdateCoreXTLibraries || Options.UpdateAssemblyVersions) { // ************** Update assembly versions ************************ cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Updating assembly versions"); UpdateAssemblyVersions(artifactsFolder); // if we got this far then we definitely need to retain this build retainBuild = true; } // *********** Update toolset ******************** if (Options.InsertToolset) { UpdateToolsetPackage(artifactsFolder, buildVersion, cancellationToken); retainBuild = true; } // *********** Update .corext\Configs\components.json ******************** if (Options.InsertWillowPackages) { cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Updating CoreXT components file"); var components = await GetLatestComponentsAsync(buildToInsert, cancellationToken); var shouldSave = false; foreach (var newComponent in components) { if (coreXT.TryGetComponentByName(newComponent.Name, out var oldComponent)) { coreXT.UpdateComponent(newComponent); shouldSave = true; } } if (shouldSave) { coreXT.SaveComponents(); retainBuild = true; } } // ************* Ensure the build is retained on the servers ************* if (Options.RetainInsertedBuild && retainBuild && !buildToInsert.KeepForever.GetValueOrDefault()) { Log.Info("Marking inserted build for retention."); buildToInsert.KeepForever = true; var buildClient = ProjectCollection.GetClient <BuildHttpClient>(); await buildClient.UpdateBuildAsync(buildToInsert, buildToInsert.Id); } // ********************* Verify Build Completes ************************** if (Options.PartitionsToBuild != null) { Log.Info($"Verifying build succeeds with changes"); foreach (var partition in Options.PartitionsToBuild) { Log.Info($"Starting build of {partition}"); if (!(await CanBuildPartitionAsync(partition, cancellationToken))) { Log.Error($"Build of partition {partition} failed"); return; } Log.Info($"Build of partition {partition} succeeded"); } } // ********************* Trigger a release ***************************** Log.Info($"Triggering a release for the build {buildToInsert.BuildNumber}"); var release = await CreateReleaseAsync(buildToInsert, cancellationToken); // The timeout for the below wait is primarily dependent on: // 1. The release task itself - Since its currently only triggering symbol archival, // it should not be very long but this should increase when more time intesive tasks are added to the release. // 2. The availability of machines to run the release on. This could be a problem at peak pool load // where getting a machine can take upto an hour or more. WaitForReleaseCompletion(release, TimeSpan.FromMinutes(10), cancellationToken); Log.Info($"Release succesfully triggered"); // ********************* Create pull request ***************************** if (branch != null) { cancellationToken.ThrowIfCancellationRequested(); var prDescription = $"Updating {Options.InsertionName} to {buildVersion}"; if (Options.UpdateExistingPr != 0 && pullRequest != null) { // update an existing pr try { branch = PushChanges(branch, buildVersion, cancellationToken, forcePush: true); pullRequest = await UpdatePullRequestDescriptionAsync(Options.UpdateExistingPr, prDescription, cancellationToken); shouldRollBackGitChanges = false; } catch (Exception ex) { Log.Error($"Unable to update pull request for '{branch.FriendlyName}'"); Log.Error(ex); return; } } else { // create a new PR Log.Info($"Create Pull Request"); try { branch = PushChanges(branch, buildVersion, cancellationToken); pullRequest = await CreatePullRequestAsync(branch.FriendlyName, prDescription, buildVersion.ToString(), cancellationToken); shouldRollBackGitChanges = false; } catch (EmptyCommitException ecx) { isInsertionCancelled = true; if (latestBuild != null && latestBuild.Result != BuildResult.Succeeded) { noProgressOnFailedBuilds = true; } Log.Warn($"Unable to create pull request for '{branch.FriendlyName}'"); Log.Warn(ecx); return; } catch (Exception ex) { Log.Error($"Unable to create pull request for '{branch.FriendlyName}'"); Log.Error(ex); return; } } if (pullRequest == null) { Log.Error($"Unable to create pull request for '{branch.FriendlyName}'"); return; } } // ********************* Create validation build ***************************** if (Options.QueueValidationBuild) { cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Create Validation Build"); if (pullRequest == null) { Log.Error("Unable to create a validation build: no pull request."); return; } try { await QueueBuildPolicy(pullRequest, "[Deprecated] ValBuild RPS"); } catch (Exception ex) { Log.Error($"Unable to create a deprecated validation build for '{pullRequest.SourceRefName}'"); Log.Error(ex); } try { await QueueBuildPolicy(pullRequest, "CloudBuild - Request RPS"); } catch (Exception ex) { Log.Error($"Unable to create a CloudBuild validation build for '{pullRequest.SourceRefName}'"); Log.Error(ex); } } } catch (Exception ex) { if (ex is OutdatedPackageException || ex is OperationCanceledException) { isInsertionCancelled = true; } Log.Error(ex); } finally { // ************************* Flush Log *********************************** Log.Factory.Flush(); // ********************* Rollback Git Changes **************************** if (shouldRollBackGitChanges) { try { Log.Info("Rolling back git changes"); var rollBackCommit = Enlistment.Branches[Options.VisualStudioBranchName].Commits.First(); Enlistment.Reset(ResetMode.Hard, rollBackCommit); Enlistment.RemoveUntrackedFiles(); } catch (Exception ex) { Log.Error(ex); } } // ********************* Send Status Mail ******************************** if (!string.IsNullOrEmpty(Options.EmailServerName) && !string.IsNullOrEmpty(Options.MailRecipient)) { try { SendMail(pullRequest, newPackageFiles, isInsertionCancelled, noProgressOnFailedBuilds); } catch (Exception ex) { Log.Error($"Unable to send mail, EmailServerName: '{Options.EmailServerName}', MailRecipient: '{Options.MailRecipient}'"); Log.Error(ex); } } Options = null; Log = null; } }
public static async Task <(bool success, int pullRequestId)> PerformInsertionAsync( RoslynInsertionToolOptions options, CancellationToken cancellationToken) { Options = options; Console.WriteLine($"{Environment.NewLine}New Insertion Into {Options.VisualStudioBranchName} Started{Environment.NewLine}"); var newPackageFiles = new List <string>(); try { Console.WriteLine($"Verifying given authentication for {Options.VSTSUri}"); try { ProjectCollection.Authenticate(); } catch (Exception ex) { LogError($"Could not authenticate with {Options.VSTSUri}"); LogError(ex); return(false, 0); } Console.WriteLine($"Verification succeeded for {Options.VSTSUri}"); // ********************** Create dummy PR ***************************** if (Options.CreateDummyPr) { GitPullRequest dummyPR; try { dummyPR = await CreatePlaceholderBranchAsync(cancellationToken); } catch (Exception ex) { LogError($"Unable to create placeholder PR for '{options.VisualStudioBranchName}'"); LogError(ex); return(false, 0); } if (dummyPR == null) { LogError($"Unable to create placeholder PR for '{options.VisualStudioBranchName}'"); return(false, 0); } return(true, dummyPR.PullRequestId); } // ********************** Get Last Insertion ***************************** cancellationToken.ThrowIfCancellationRequested(); BuildVersion buildVersion; Build buildToInsert; Build latestBuild = null; bool retainBuild = false; // Get the version from DevOps Pipelines queue, e.g. Roslyn-Master-Signed-Release. if (string.IsNullOrEmpty(Options.SpecificBuild)) { buildToInsert = await GetLatestPassedBuildAsync(cancellationToken); buildVersion = BuildVersion.FromTfsBuildNumber(buildToInsert.BuildNumber, Options.BuildQueueName); Console.WriteLine("Found build number " + buildVersion); // Get the latest build, whether passed or failed. If the buildToInsert has already been inserted but // there is a later failing build, then send an error latestBuild = await GetLatestBuildAsync(cancellationToken); } else { buildVersion = BuildVersion.FromString(Options.SpecificBuild); buildToInsert = await GetSpecificBuildAsync(buildVersion, cancellationToken); } var insertionArtifacts = await GetInsertionArtifactsAsync(buildToInsert, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); // *********** Look up existing PR ******************** var gitClient = ProjectCollection.GetClient <GitHttpClient>(); var branches = await gitClient.GetRefsAsync( VSRepoId, filter : $"heads/{Options.VisualStudioBranchName}", cancellationToken : cancellationToken); var baseBranch = branches.Single(b => b.Name == $"refs/heads/{Options.VisualStudioBranchName}"); var pullRequestId = Options.UpdateExistingPr; var useExistingPr = pullRequestId != 0; GitPullRequest pullRequest; string insertionBranchName; if (useExistingPr) { pullRequest = await gitClient.GetPullRequestByIdAsync(pullRequestId, cancellationToken : cancellationToken); insertionBranchName = pullRequest.SourceRefName.Substring("refs/heads/".Length); var refs = await gitClient.GetRefsAsync(VSRepoId, filter : $"heads/{insertionBranchName}", cancellationToken : cancellationToken); var insertionBranch = refs.Single(r => r.Name == $"refs/heads/{insertionBranchName}"); if (Options.OverwritePr) { // overwrite existing PR branch back to base before pushing new commit var updateToBase = new GitRefUpdate { OldObjectId = insertionBranch.ObjectId, NewObjectId = baseBranch.ObjectId, Name = $"refs/heads/{insertionBranchName}" }; await gitClient.UpdateRefsAsync(new[] { updateToBase }, VSRepoId, cancellationToken : cancellationToken); } else { // not overwriting PR, so the insertion branch is actually the base baseBranch = insertionBranch; } } else { pullRequest = null; insertionBranchName = GetNewBranchName(); } var allChanges = new List <GitChange>(); var coreXT = await CoreXT.Load(gitClient, baseBranch.ObjectId); if (Options.InsertCoreXTPackages) { // ************** Update Nuget Packages For Branch************************ cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating Nuget Packages"); bool success; (success, newPackageFiles) = UpdatePackages( buildVersion, coreXT, insertionArtifacts.GetPackagesDirectory(), cancellationToken); retainBuild |= success; // *********** Copy OptimizationInputs.props file *********************** foreach (var propsFile in insertionArtifacts.GetOptProfPropertyFiles()) { var targetFilePath = "src/Tests/config/runsettings/Official/OptProf/External/" + Path.GetFileName(propsFile); var version = new GitVersionDescriptor { VersionType = GitVersionType.Commit, Version = baseBranch.ObjectId }; var stream = await gitClient.GetItemContentAsync(VSRepoId, targetFilePath, download : true, versionDescriptor : version); var originalContent = new StreamReader(stream).ReadToEnd(); var newContent = File.ReadAllText(propsFile); if (GetChangeOpt(targetFilePath, originalContent, newContent) is GitChange change) { allChanges.Add(change); } } } if (Options.UpdateCoreXTLibraries || Options.UpdateAssemblyVersions) { // ************** Update assembly versions ************************ cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating assembly versions"); if (await UpdateAssemblyVersionsOpt(gitClient, baseBranch.ObjectId, insertionArtifacts) is GitChange assemblyVersionChange) { allChanges.Add(assemblyVersionChange); } // if we got this far then we definitely need to retain this build retainBuild = true; } // *********** Update toolset ******************** if (Options.InsertToolset) { UpdateToolsetPackage(coreXT, insertionArtifacts, buildVersion); retainBuild = true; } // ************ Update .corext\Configs\default.config ******************** cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating CoreXT default.config file"); if (coreXT.SaveConfigOpt() is GitChange configChange) { allChanges.Add(configChange); } // *********** Update .corext\Configs\components.json ******************** BuildVersion oldComponentVersion = default; if (Options.InsertWillowPackages) { cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating CoreXT components file"); var components = await GetLatestComponentsAsync(buildToInsert, cancellationToken); var shouldSave = false; foreach (var newComponent in components) { if (coreXT.TryGetComponentByName(newComponent.Name, out var oldComponent)) { if (oldComponent.BuildVersion != default) { oldComponentVersion = oldComponent.BuildVersion; } coreXT.UpdateComponent(newComponent); shouldSave = true; } } if (shouldSave) { var allComponentChanges = coreXT.SaveComponents(); allChanges.AddRange(allComponentChanges); retainBuild = true; } } // ************* Ensure the build is retained on the servers ************* if (Options.RetainInsertedBuild && retainBuild && !buildToInsert.KeepForever.GetValueOrDefault()) { Console.WriteLine("Marking inserted build for retention."); buildToInsert.KeepForever = true; var buildClient = ProjectCollection.GetClient <BuildHttpClient>(); await buildClient.UpdateBuildAsync(buildToInsert); } // ************* Bail out if there are no changes ************************ if (!allChanges.Any()) { LogWarning("No meaningful changes since the last insertion was merged. PR will not be created or updated."); return(true, 0); } // ********************* Create push ************************************* var insertionBranchUpdate = new GitRefUpdate { Name = $"refs/heads/{insertionBranchName}", OldObjectId = baseBranch.ObjectId }; var commit = new GitCommitRef { Comment = $"Updating {Options.InsertionName} to {buildVersion}", Changes = allChanges }; var push = new GitPush { RefUpdates = new[] { insertionBranchUpdate }, Commits = new[] { commit } }; await gitClient.CreatePushAsync(push, VSRepoId, cancellationToken : cancellationToken); // ********************* Create pull request ***************************** var oldBuild = await GetSpecificBuildAsync(oldComponentVersion, cancellationToken); var prDescriptionMarkdown = CreatePullRequestDescription(oldBuild, buildToInsert, useMarkdown: true); if (buildToInsert.Result == BuildResult.PartiallySucceeded) { prDescriptionMarkdown += Environment.NewLine + ":warning: The build being inserted has partially succeeded."; } if (!useExistingPr || Options.OverwritePr) { try { var nl = Environment.NewLine; if (oldBuild is null) { prDescriptionMarkdown += $"{nl}---{nl}Unable to find details for previous build ({oldComponentVersion}){nl}"; } else { var(changes, diffLink) = await GetChangesBetweenBuildsAsync(oldBuild, buildToInsert, cancellationToken); var diffDescription = changes.Any() ? $"[View Complete Diff of Changes]({diffLink})" : "No source changes since previous insertion"; prDescriptionMarkdown += nl + "---" + nl + diffDescription + nl; prDescriptionMarkdown = AppendChangesToDescription(prDescriptionMarkdown, oldBuild ?? buildToInsert, changes); } } catch (Exception e) { LogWarning("Failed to create diff links."); LogWarning(e.Message); } } if (useExistingPr) { try { if (Options.OverwritePr) { pullRequest = await OverwritePullRequestAsync(pullRequestId, prDescriptionMarkdown, buildVersion.ToString(), options.TitlePrefix, cancellationToken); } pullRequestId = pullRequest.PullRequestId; } catch (Exception ex) { LogError($"Unable to update pull request for '{pullRequest.SourceRefName}'"); LogError(ex); return(false, 0); } } else { // create a new PR Console.WriteLine($"Create Pull Request"); try { // If this insertion was queued for PR validation, for a dev branch, or for a feature branch, // then add the build queuer as a reviewer instead of mlinfraswat. var isPrValidation = !string.IsNullOrEmpty(GetBuildPRNumber(buildToInsert)); var isDevOrFeatureBranch = Options.BranchName.StartsWith("dev/") || Options.BranchName.StartsWith("features/"); var reviewerId = isPrValidation || isDevOrFeatureBranch ? buildToInsert.RequestedBy.Id : MLInfraSwatUserId.ToString(); pullRequest = await CreatePullRequestAsync(insertionBranchName, prDescriptionMarkdown, buildVersion.ToString(), options.TitlePrefix, reviewerId, cancellationToken); if (pullRequest == null) { LogError($"Unable to create pull request for '{insertionBranchName}'"); return(false, 0); } pullRequestId = pullRequest.PullRequestId; } catch (Exception ex) { LogError($"Unable to create pull request for '{insertionBranchName}'"); LogError(ex); return(false, 0); } } // ********************* Create validation build ***************************** if (Options.QueueValidationBuild) { cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Create Validation Build"); try { if (Options.CreateDraftPr) { // When creating Draft PRs no policies are automatically started. // If we do not queue a CloudBuild the Perf DDRITs request will // spin waiting for a build to test against until it timesout. await QueueBuildPolicy(pullRequest, "CloudBuild - PR"); } await QueueBuildPolicy(pullRequest, "Request Perf DDRITs"); } catch (Exception ex) { LogWarning($"Unable to create a CloudBuild validation build for '{insertionBranchName}'"); LogWarning(ex); } if (Options.CreateDraftPr) { // When creating Draft PRs no policies are automatically started. await TryQueueBuildPolicy(pullRequest, "Insertion Hash Check", insertionBranchName); await TryQueueBuildPolicy(pullRequest, "Insertion Sign Check", insertionBranchName); await TryQueueBuildPolicy(pullRequest, "Insertion Symbol Check", insertionBranchName); } } // ********************* Set PR to Auto-Complete ***************************** if (Options.SetAutoComplete) { cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Set PR to Auto-Complete"); try { var prDescriptionText = CreatePullRequestDescription(oldBuild, buildToInsert, useMarkdown: false); await SetAutoCompleteAsync(pullRequest, prDescriptionText, cancellationToken); } catch (Exception ex) { LogWarning($"Unable to Set PR to Auto-Complete for '{insertionBranchName}'"); LogWarning(ex); } } return(true, pullRequestId); } catch (Exception ex) { LogError(ex); return(false, 0); } finally { Options = null; } }
internal static string GetDevDivInsertionFilePath(BuildVersion version, string relativePath) { // For example: "\\cpvsbuild\drops\Roslyn\Roslyn-Master-Signed-Release\20160315.3\DevDivInsertionFiles\Roslyn\all.roslyn.locproj" return(Path.Combine(GetBuildDirectory(version), "DevDivInsertionFiles", relativePath)); }
private static void StageFiles(BuildVersion newRoslynVersion, CancellationToken cancellationToken) { var repositoryStatus = Enlistment.RetrieveStatus(); if (repositoryStatus.IsDirty) { cancellationToken.ThrowIfCancellationRequested(); var filesToStage = repositoryStatus .Where(item => item.State != FileStatus.Unaltered && item.State != FileStatus.Ignored) .Select(item => item.FilePath).ToList(); cancellationToken.ThrowIfCancellationRequested(); if (!isWhitespaceOnlyChange(Enlistment.Diff.Compare <Patch>(filesToStage))) { Console.WriteLine($"Staging {filesToStage.Count()} file(s)"); var watch = Stopwatch.StartNew(); Enlistment.Stage(filesToStage, GetStageOptions()); Console.WriteLine($"Staging took {watch.Elapsed.TotalSeconds} seconds"); } else { Console.WriteLine("Only whitespace changes found"); } } bool isWhitespaceOnlyChange(Patch p) { var before = new StringBuilder(); var after = new StringBuilder(); foreach (var change in p) { if (change.Status != ChangeKind.Modified) { return(false); } using (var reader = new StringReader(change.Patch)) { string line; while ((line = reader.ReadLine()) != null) { if (!line.StartsWith("---") && !line.StartsWith("+++")) { if (line.StartsWith("+")) { after.Append(line.Substring(1).Trim()); continue; } else if (line.StartsWith("-")) { before.Append(line.Substring(1).Trim()); continue; } } after.AppendLine(line); before.AppendLine(line); } } if (after.ToString() != before.ToString()) { return(false); } after.Clear(); before.Clear(); } return(true); } }
/// <returns>A tuple containing (success, pullRequestId).</returns> public static async Task <(bool, int)> PerformInsertionAsync( RoslynInsertionToolOptions options, CancellationToken cancellationToken) { Options = options; Console.WriteLine($"{Environment.NewLine}New Insertion Into {Options.VisualStudioBranchName} Started{Environment.NewLine}"); GitPullRequest pullRequest = null; var shouldRollBackGitChanges = false; var newPackageFiles = new List <string>(); try { // Verify that the arguments we were passed authenticate correctly Console.WriteLine($"Verifying given authentication for {Options.VSTSUri}"); try { ProjectCollection.Authenticate(); } catch (Exception ex) { Console.WriteLine($"Could not authenticate with {Options.VSTSUri}"); Console.WriteLine(ex); return(false, 0); } Console.WriteLine($"Verification succeeded for {Options.VSTSUri}"); // ********************** Create dummy PR ***************************** if (Options.CreateDummyPr) { var dummyBranch = CreateBranch(cancellationToken); try { CreateDummyCommit(cancellationToken); PushChanges(dummyBranch, cancellationToken); pullRequest = await CreatePullRequestAsync(dummyBranch.FriendlyName, $"DUMMY INSERTION FOR {Options.InsertionName}", "Not Specified", options.TitlePrefix, cancellationToken); } catch (Exception ex) { Console.WriteLine($"Unable to create pull request for '{dummyBranch.FriendlyName}'"); Console.WriteLine(ex); return(false, 0); } if (pullRequest == null) { Console.WriteLine($"Unable to create pull request for '{dummyBranch.FriendlyName}'"); return(false, 0); } return(true, pullRequest.PullRequestId); } // ********************** Get Last Insertion ***************************** cancellationToken.ThrowIfCancellationRequested(); BuildVersion buildVersion; Build buildToInsert; Build latestBuild = null; bool retainBuild = false; // Get the version from DevOps Pipelines queue, e.g. Roslyn-Master-Signed-Release. if (string.IsNullOrEmpty(Options.SpecificBuild)) { buildToInsert = await GetLatestPassedBuildAsync(cancellationToken); buildVersion = BuildVersion.FromTfsBuildNumber(buildToInsert.BuildNumber, Options.BuildQueueName); Console.WriteLine("Found build number " + buildVersion); // Get the latest build, whether passed or failed. If the buildToInsert has already been inserted but // there is a later failing build, then send an error latestBuild = await GetLatestBuildAsync(cancellationToken); } else { buildVersion = BuildVersion.FromString(Options.SpecificBuild); buildToInsert = await GetSpecificBuildAsync(buildVersion, cancellationToken); } var insertionArtifacts = await GetInsertionArtifactsAsync(buildToInsert, cancellationToken); Branch branch = null; cancellationToken.ThrowIfCancellationRequested(); var useExistingPr = Options.UpdateExistingPr != 0; if (useExistingPr) { // ****************** Update existing PR *********************** pullRequest = await GetExistingPullRequestAsync(Options.UpdateExistingPr, cancellationToken); branch = SwitchToBranchAndUpdate(pullRequest.SourceRefName, Options.VisualStudioBranchName, overwriteExistingChanges: Options.OverwritePr); } else { // ****************** Create Branch *********************** Console.WriteLine("Creating New Branch"); branch = string.IsNullOrEmpty(Options.NewBranchName) ? null : CreateBranch(cancellationToken); } shouldRollBackGitChanges = branch != null; var coreXT = CoreXT.Load(GetAbsolutePathForEnlistment()); if (Options.InsertCoreXTPackages) { // ************** Update Nuget Packages For Branch************************ cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating Nuget Packages"); bool success = false; (success, newPackageFiles) = UpdatePackages( buildVersion, coreXT, insertionArtifacts.GetPackagesDirectory(), cancellationToken); retainBuild |= success; // ************ Update .corext\Configs\default.config ******************** cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating CoreXT default.config file"); coreXT.SaveConfig(); } if (Options.UpdateCoreXTLibraries || Options.UpdateAssemblyVersions) { // ************** Update assembly versions ************************ cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating assembly versions"); UpdateAssemblyVersions(insertionArtifacts); // if we got this far then we definitely need to retain this build retainBuild = true; } // *********** Update toolset ******************** if (Options.InsertToolset) { UpdateToolsetPackage(insertionArtifacts, buildVersion, cancellationToken); retainBuild = true; } // *********** Update .corext\Configs\components.json ******************** if (Options.InsertWillowPackages) { cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating CoreXT components file"); var components = await GetLatestComponentsAsync(buildToInsert, cancellationToken); var shouldSave = false; foreach (var newComponent in components) { if (coreXT.TryGetComponentByName(newComponent.Name, out var oldComponent)) { coreXT.UpdateComponent(newComponent); shouldSave = true; } } if (shouldSave) { coreXT.SaveComponents(); retainBuild = true; } } // ************* Ensure the build is retained on the servers ************* if (Options.RetainInsertedBuild && retainBuild && !buildToInsert.KeepForever.GetValueOrDefault()) { Console.WriteLine("Marking inserted build for retention."); buildToInsert.KeepForever = true; var buildClient = ProjectCollection.GetClient <BuildHttpClient>(); await buildClient.UpdateBuildAsync(buildToInsert, buildToInsert.Id); } // ********************* Verify Build Completes ************************** if (Options.PartitionsToBuild != null) { Console.WriteLine($"Verifying build succeeds with changes"); foreach (var partition in Options.PartitionsToBuild) { Console.WriteLine($"Starting build of {partition}"); if (!(await CanBuildPartitionAsync(partition, cancellationToken))) { Console.WriteLine($"Build of partition {partition} failed"); return(false, 0); } Console.WriteLine($"Build of partition {partition} succeeded"); } } // ********************* Trigger a release ***************************** Console.WriteLine($"Triggering a release for the build {buildToInsert.BuildNumber}"); var release = await CreateReleaseAsync(buildToInsert, cancellationToken); // The timeout for the below wait is primarily dependent on: // 1. The release task itself - Since its currently only triggering symbol archival, // it should not be very long but this should increase when more time intesive tasks are added to the release. // 2. The availability of machines to run the release on. This could be a problem at peak pool load // where getting a machine can take upto an hour or more. WaitForReleaseCompletion(release, TimeSpan.FromMinutes(10), cancellationToken); Console.WriteLine($"Release succesfully triggered"); // ********************* Create pull request ***************************** var pullRequestId = 0; if (branch != null) { cancellationToken.ThrowIfCancellationRequested(); var prDescription = $"Updating {Options.InsertionName} to {buildVersion}"; if (useExistingPr && pullRequest != null) { // update an existing pr try { branch = PushChanges(branch, buildVersion, cancellationToken, forcePush: true); if (Options.OverwritePr) { pullRequest = await UpdatePullRequestDescriptionAsync(Options.UpdateExistingPr, prDescription, cancellationToken); } shouldRollBackGitChanges = false; pullRequestId = pullRequest.PullRequestId; } catch (Exception ex) { Console.WriteLine($"Unable to update pull request for '{branch.FriendlyName}'"); Console.WriteLine(ex); return(false, 0); } } else { // create a new PR Console.WriteLine($"Create Pull Request"); try { branch = PushChanges(branch, buildVersion, cancellationToken); pullRequest = await CreatePullRequestAsync(branch.FriendlyName, prDescription, buildVersion.ToString(), options.TitlePrefix, cancellationToken); shouldRollBackGitChanges = false; pullRequestId = pullRequest.PullRequestId; } catch (EmptyCommitException ecx) { Console.WriteLine($"Unable to create pull request for '{branch.FriendlyName}'"); Console.WriteLine(ecx); return(false, 0); } catch (Exception ex) { Console.WriteLine($"Unable to create pull request for '{branch.FriendlyName}'"); Console.WriteLine(ex); return(false, 0); } } if (pullRequest == null) { Console.WriteLine($"Unable to create pull request for '{branch.FriendlyName}'"); return(false, 0); } } // ********************* Create validation build ***************************** if (Options.QueueValidationBuild) { cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Create Validation Build"); if (pullRequest == null) { Console.WriteLine("Unable to create a validation build: no pull request."); return(false, 0); } try { await QueueBuildPolicy(pullRequest, "CloudBuild - Request RPS"); } catch (Exception ex) { Console.WriteLine($"Unable to create a CloudBuild validation build for '{pullRequest.SourceRefName}'"); Console.WriteLine(ex); } } return(true, pullRequestId); } catch (Exception ex) { Console.WriteLine(ex); return(false, 0); } finally { // ********************* Rollback Git Changes **************************** if (shouldRollBackGitChanges) { try { Console.WriteLine("Rolling back git changes"); var rollBackCommit = Enlistment.Branches[Options.VisualStudioBranchName].Commits.First(); Enlistment.Reset(ResetMode.Hard, rollBackCommit); Enlistment.RemoveUntrackedFiles(); } catch (Exception ex) { Console.WriteLine(ex); } } Options = null; } }
private static Branch PushChanges(Branch branch, BuildVersion newRoslynVersion, CancellationToken cancellationToken, bool forcePush = false) { StageFiles(newRoslynVersion, cancellationToken); CommitStagedChanges(newRoslynVersion, cancellationToken); return(PushChanges(branch, cancellationToken, forcePush: forcePush)); }
public static async Task <(bool success, int pullRequestId)> PerformInsertionAsync( RoslynInsertionToolOptions options, CancellationToken cancellationToken) { Options = options; Console.WriteLine($"{Environment.NewLine}New Insertion Into {Options.VisualStudioBranchName} Started{Environment.NewLine}"); GitPullRequest pullRequest = null; var shouldRollBackGitChanges = false; var newPackageFiles = new List <string>(); try { // Verify that the arguments we were passed authenticate correctly Console.WriteLine($"Verifying given authentication for {Options.VSTSUri}"); try { ProjectCollection.Authenticate(); } catch (Exception ex) { Console.WriteLine($"Could not authenticate with {Options.VSTSUri}"); Console.WriteLine(ex); return(false, 0); } Console.WriteLine($"Verification succeeded for {Options.VSTSUri}"); // ********************** Create dummy PR ***************************** if (Options.CreateDummyPr) { try { pullRequest = await CreatePlaceholderBranchAsync(cancellationToken); } catch (Exception ex) { Console.WriteLine($"Unable to create placeholder PR for '{options.VisualStudioBranchName}'"); Console.WriteLine(ex); return(false, 0); } if (pullRequest == null) { Console.WriteLine($"Unable to create placeholder PR for '{options.VisualStudioBranchName}'"); return(false, 0); } return(true, pullRequest.PullRequestId); } // ********************** Get Last Insertion ***************************** cancellationToken.ThrowIfCancellationRequested(); BuildVersion buildVersion; Build buildToInsert; Build latestBuild = null; bool retainBuild = false; // Get the version from DevOps Pipelines queue, e.g. Roslyn-Master-Signed-Release. if (string.IsNullOrEmpty(Options.SpecificBuild)) { buildToInsert = await GetLatestPassedBuildAsync(cancellationToken); buildVersion = BuildVersion.FromTfsBuildNumber(buildToInsert.BuildNumber, Options.BuildQueueName); Console.WriteLine("Found build number " + buildVersion); // Get the latest build, whether passed or failed. If the buildToInsert has already been inserted but // there is a later failing build, then send an error latestBuild = await GetLatestBuildAsync(cancellationToken); } else { buildVersion = BuildVersion.FromString(Options.SpecificBuild); buildToInsert = await GetSpecificBuildAsync(buildVersion, cancellationToken); } string commitSHA = buildToInsert.SourceVersion.Substring(0, 7); string lastCommitUrl = string.Empty; if (buildToInsert.Links.Links.ContainsKey("sourceVersionDisplayUri")) { // Get a link to the commit the build was built from. var sourceLink = (ReferenceLink)buildToInsert.Links.Links["sourceVersionDisplayUri"]; lastCommitUrl = sourceLink.Href; } var insertionArtifacts = await GetInsertionArtifactsAsync(buildToInsert, cancellationToken); Branch branch = null; cancellationToken.ThrowIfCancellationRequested(); var useExistingPr = Options.UpdateExistingPr != 0; if (useExistingPr) { // ****************** Update existing PR *********************** pullRequest = await GetExistingPullRequestAsync(Options.UpdateExistingPr, cancellationToken); branch = SwitchToBranchAndUpdate(pullRequest.SourceRefName, Options.VisualStudioBranchName, overwriteExistingChanges: Options.OverwritePr); } else { // ****************** Create Branch *********************** Console.WriteLine("Creating New Branch"); branch = string.IsNullOrEmpty(Options.NewBranchName) ? null : CreateBranch(cancellationToken); } shouldRollBackGitChanges = branch != null; var enlistmentRoot = GetAbsolutePathForEnlistment(); var coreXT = CoreXT.Load(enlistmentRoot); if (Options.InsertCoreXTPackages) { // ************** Update Nuget Packages For Branch************************ cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating Nuget Packages"); bool success = false; (success, newPackageFiles) = UpdatePackages( buildVersion, coreXT, insertionArtifacts.GetPackagesDirectory(), cancellationToken); retainBuild |= success; // ************ Update .corext\Configs\default.config ******************** cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating CoreXT default.config file"); coreXT.SaveConfig(); // *********** Copy OptimizationInputs.props file *********************** foreach (var propsFile in insertionArtifacts.GetOptProfPropertyFiles()) { var targetDirectory = Path.Combine(enlistmentRoot, @"src\Tests\config\runsettings\Official\OptProf\External"); var targetFilePath = Path.Combine(targetDirectory, Path.GetFileName(propsFile)); Console.WriteLine($"Updating {targetFilePath}"); Directory.CreateDirectory(targetDirectory); File.Copy(propsFile, targetFilePath, overwrite: true); } } if (Options.UpdateCoreXTLibraries || Options.UpdateAssemblyVersions) { // ************** Update assembly versions ************************ cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating assembly versions"); UpdateAssemblyVersions(insertionArtifacts); // if we got this far then we definitely need to retain this build retainBuild = true; } // *********** Update toolset ******************** if (Options.InsertToolset) { UpdateToolsetPackage(insertionArtifacts, buildVersion, cancellationToken); retainBuild = true; } // *********** Update .corext\Configs\components.json ******************** BuildVersion oldComponentVersion = default; if (Options.InsertWillowPackages) { cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating CoreXT components file"); var components = await GetLatestComponentsAsync(buildToInsert, cancellationToken); var shouldSave = false; foreach (var newComponent in components) { if (coreXT.TryGetComponentByName(newComponent.Name, out var oldComponent)) { if (oldComponent.BuildVersion != default) { oldComponentVersion = oldComponent.BuildVersion; } coreXT.UpdateComponent(newComponent); shouldSave = true; } } if (shouldSave) { coreXT.SaveComponents(); retainBuild = true; } } // ************* Ensure the build is retained on the servers ************* if (Options.RetainInsertedBuild && retainBuild && !buildToInsert.KeepForever.GetValueOrDefault()) { Console.WriteLine("Marking inserted build for retention."); buildToInsert.KeepForever = true; var buildClient = ProjectCollection.GetClient <BuildHttpClient>(); await buildClient.UpdateBuildAsync(buildToInsert, buildToInsert.Id); } // ********************* Verify Build Completes ************************** if (Options.PartitionsToBuild != null) { Console.WriteLine($"Verifying build succeeds with changes"); foreach (var partition in Options.PartitionsToBuild) { Console.WriteLine($"Starting build of {partition}"); if (!(await CanBuildPartitionAsync(partition, cancellationToken))) { Console.WriteLine($"Build of partition {partition} failed"); return(false, 0); } Console.WriteLine($"Build of partition {partition} succeeded"); } } // ********************* Create pull request ***************************** var pullRequestId = 0; if (branch != null) { cancellationToken.ThrowIfCancellationRequested(); var prDescription = $"Updating {Options.InsertionName} to {buildVersion} ([{commitSHA}]({lastCommitUrl}))"; if (useExistingPr && pullRequest != null) { // update an existing pr try { branch = PushChanges(branch, buildVersion, cancellationToken, forcePush: true); if (Options.OverwritePr) { pullRequest = await UpdatePullRequestDescriptionAsync(Options.UpdateExistingPr, prDescription, cancellationToken); } shouldRollBackGitChanges = false; pullRequestId = pullRequest.PullRequestId; } catch (Exception ex) { Console.WriteLine($"Unable to update pull request for '{branch.FriendlyName}'"); Console.WriteLine(ex); return(false, 0); } } else { // create a new PR Console.WriteLine($"Create Pull Request"); try { try { var oldBuild = await GetSpecificBuildAsync(oldComponentVersion, cancellationToken); var(changes, diffLink) = await GetChangesBetweenBuildsAsync(oldBuild ?? buildToInsert, buildToInsert, cancellationToken); prDescription = AppendDiffToDescription(prDescription, diffLink); prDescription = AppendChangesToDescription(prDescription, oldBuild ?? buildToInsert, changes); } catch (Exception e) { Console.WriteLine("##vso[task.logissue type=warning] Failed to create diff links."); Console.WriteLine($"##vso[task.logissue type=warning] {e.Message}"); } branch = PushChanges(branch, buildVersion, cancellationToken); pullRequest = await CreatePullRequestAsync(branch.FriendlyName, prDescription, buildVersion.ToString(), options.TitlePrefix, cancellationToken); shouldRollBackGitChanges = false; pullRequestId = pullRequest.PullRequestId; } catch (EmptyCommitException ecx) { Console.WriteLine($"Unable to create pull request for '{branch.FriendlyName}'"); Console.WriteLine(ecx); return(false, 0); } catch (Exception ex) { Console.WriteLine($"Unable to create pull request for '{branch.FriendlyName}'"); Console.WriteLine(ex); return(false, 0); } } if (pullRequest == null) { Console.WriteLine($"Unable to create pull request for '{branch.FriendlyName}'"); return(false, 0); } } // ********************* Create validation build ***************************** if (Options.QueueValidationBuild) { cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Create Validation Build"); if (pullRequest == null) { Console.WriteLine("Unable to create a validation build: no pull request."); return(false, 0); } try { await QueueBuildPolicy(pullRequest, "Request Perf DDRITs"); } catch (Exception ex) { Console.WriteLine($"Unable to create a CloudBuild validation build for '{pullRequest.SourceRefName}'"); Console.WriteLine(ex); } } return(true, pullRequestId); } catch (RepositoryNotFoundException ex) { Console.Error.WriteLine(ex.Message); Console.Error.WriteLine(@"Please ensure a VS enlistment exists at the given path, or pass the `/enlistmentpath=C:\path\to\VS` argument on the command line."); return(false, 0); } catch (Exception ex) { Console.WriteLine(ex); return(false, 0); } finally { // ********************* Rollback Git Changes **************************** if (shouldRollBackGitChanges) { try { Console.WriteLine("Rolling back git changes"); var rollBackCommit = Enlistment.Branches[Options.VisualStudioBranchName].Commits.First(); Enlistment.Reset(ResetMode.Hard, rollBackCommit); Enlistment.RemoveUntrackedFiles(); } catch (Exception ex) { Console.WriteLine(ex); } } Options = null; } }
public static async Task <(bool success, int pullRequestId)> PerformInsertionAsync( RoslynInsertionToolOptions options, CancellationToken cancellationToken) { Options = options; Console.WriteLine($"{Environment.NewLine}New Insertion Into {Options.VisualStudioBranchName} Started{Environment.NewLine}"); var newPackageFiles = new List <string>(); try { Console.WriteLine($"Verifying given authentication for {Options.VisualStudioRepoAzdoUri}"); try { VisualStudioRepoConnection.Authenticate(); } catch (Exception ex) { LogError($"Could not authenticate with {Options.VisualStudioRepoAzdoUri}"); LogError(ex); return(false, 0); } Console.WriteLine($"Verification succeeded for {Options.VisualStudioRepoAzdoUri}"); if (ComponentBuildConnection != VisualStudioRepoConnection) { Console.WriteLine($"Verifying given authentication for {Options.ComponentBuildAzdoUri}"); try { ComponentBuildConnection.Authenticate(); } catch (Exception ex) { LogError($"Could not authenticate with {Options.ComponentBuildAzdoUri}"); LogError(ex); return(false, 0); } Console.WriteLine($"Verification succeeded for {Options.ComponentBuildAzdoUri}"); } // ********************** Create dummy PR ***************************** if (Options.CreateDummyPr) { GitPullRequest dummyPR; try { dummyPR = await CreatePlaceholderVSBranchAsync(cancellationToken); } catch (Exception ex) { LogError($"Unable to create placeholder PR for '{options.VisualStudioBranchName}'"); LogError(ex); return(false, 0); } if (dummyPR == null) { LogError($"Unable to create placeholder PR for '{options.VisualStudioBranchName}'"); return(false, 0); } return(true, dummyPR.PullRequestId); } // ********************** Get Last Insertion ***************************** cancellationToken.ThrowIfCancellationRequested(); BuildVersion buildVersion; Build buildToInsert; Build latestBuild = null; bool retainBuild = false; // Get the version from DevOps Pipelines queue, e.g. Roslyn-Main-Signed-Release. if (string.IsNullOrEmpty(Options.SpecificBuild)) { buildToInsert = await GetLatestPassedComponentBuildAsync(cancellationToken); buildVersion = BuildVersion.FromTfsBuildNumber(buildToInsert.BuildNumber, Options.ComponentBuildQueueName); Console.WriteLine("Found " + buildToInsert.Definition.Name + " build number " + buildVersion); // Get the latest build, whether passed or failed. If the buildToInsert has already been inserted but // there is a later failing build, then send an error latestBuild = await GetLatestComponentBuildAsync(cancellationToken); } else { buildVersion = BuildVersion.FromString(Options.SpecificBuild); buildToInsert = await GetSpecificComponentBuildAsync(buildVersion, cancellationToken); } var insertionArtifacts = await GetInsertionArtifactsAsync(buildToInsert, cancellationToken); cancellationToken.ThrowIfCancellationRequested(); // *********** Look up existing PR ******************** var gitClient = VisualStudioRepoConnection.GetClient <GitHttpClient>(); var branches = await gitClient.GetRefsAsync( VSRepoId, filter : $"heads/{Options.VisualStudioBranchName}", cancellationToken : cancellationToken); var baseBranch = branches.Single(b => b.Name == $"refs/heads/{Options.VisualStudioBranchName}"); var pullRequestId = Options.UpdateExistingPr; var useExistingPr = pullRequestId != 0; GitPullRequest pullRequest; string insertionBranchName; if (useExistingPr) { pullRequest = await gitClient.GetPullRequestByIdAsync(pullRequestId, cancellationToken : cancellationToken); insertionBranchName = pullRequest.SourceRefName.Substring("refs/heads/".Length); var refs = await gitClient.GetRefsAsync(VSRepoId, filter : $"heads/{insertionBranchName}", cancellationToken : cancellationToken); var insertionBranch = refs.Single(r => r.Name == $"refs/heads/{insertionBranchName}"); if (Options.OverwritePr) { // overwrite existing PR branch back to base before pushing new commit var updateToBase = new GitRefUpdate { OldObjectId = insertionBranch.ObjectId, NewObjectId = baseBranch.ObjectId, Name = $"refs/heads/{insertionBranchName}" }; var results = await gitClient.UpdateRefsAsync(new[] { updateToBase }, VSRepoId, cancellationToken : cancellationToken); foreach (var result in results) { if (!result.Success) { LogError("Failed to overwrite PR: " + result.CustomMessage); } } } else { // not overwriting PR, so the insertion branch is actually the base baseBranch = insertionBranch; } } else { pullRequest = null; insertionBranchName = GetNewBranchName(); } var allChanges = new List <GitChange>(); var coreXT = await CoreXT.Load(gitClient, baseBranch.ObjectId); if (Options.InsertCoreXTPackages) { // ************** Update Nuget Packages For Branch************************ cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating Nuget Packages"); bool success; (success, newPackageFiles) = UpdatePackages( buildVersion, coreXT, insertionArtifacts.GetPackagesDirectory(), Options.SkipCoreXTPackages, cancellationToken); retainBuild |= success; // *********** Copy OptimizationInputs.props file *********************** foreach (var propsFile in insertionArtifacts.GetOptProfPropertyFiles()) { var propsFilename = Path.GetFileName(propsFile); if (propsFilename == "dotnet-roslyn.props") { // Since the propsFilename is based on repo name, during Roslyn's transition from inserting // from GH dotnet/roslyn builds to inserting from dnceng dotnet-roslyn builds, this will // ensure that we look for the proper props filename. propsFilename = "dotnet.roslyn.props"; } var targetFilePath = $"src/Tests/config/runsettings/Official/OptProf/External/{propsFilename}"; var version = new GitVersionDescriptor { VersionType = GitVersionType.Commit, Version = baseBranch.ObjectId }; var stream = await gitClient.GetItemContentAsync(VSRepoId, targetFilePath, download : true, versionDescriptor : version); var originalContent = new StreamReader(stream).ReadToEnd(); var newContent = File.ReadAllText(propsFile); if (GetChangeOpt(targetFilePath, originalContent, newContent) is GitChange change) { allChanges.Add(change); } } } if (Options.UpdateCoreXTLibraries || Options.UpdateAssemblyVersions) { // ************** Update assembly versions ************************ cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating assembly versions"); if (await UpdateAssemblyVersionsOpt(gitClient, baseBranch.ObjectId, insertionArtifacts) is GitChange assemblyVersionChange) { allChanges.Add(assemblyVersionChange); } // if we got this far then we definitely need to retain this build retainBuild = true; } // *********** Update toolset ******************** if (Options.InsertToolset) { UpdateToolsetPackage(coreXT, insertionArtifacts, buildVersion); retainBuild = true; } // ************ Update .corext\Configs\default.config ******************** cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating CoreXT default.config and props files under src/ConfigData/Packages"); foreach (var configChange in coreXT.SaveConfigs()) { if (configChange is not null) { allChanges.Add(configChange); } } // *********** Update .corext\Configs\components.json ******************** BuildVersion oldComponentVersion = default; if (Options.InsertWillowPackages) { cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Updating CoreXT components file"); var components = GetLatestBuildComponents(buildToInsert, insertionArtifacts, cancellationToken); var shouldSave = false; foreach (var newComponent in components) { if (coreXT.TryGetComponentByName(newComponent.Name, out var oldComponent)) { if (oldComponent.BuildVersion != default) { oldComponentVersion = oldComponent.BuildVersion; } coreXT.UpdateComponent(newComponent); shouldSave = true; } } if (shouldSave) { var allComponentChanges = coreXT.SaveComponents(); allChanges.AddRange(allComponentChanges); retainBuild = true; } } // ************* Ensure the build is retained on the servers ************* if (Options.RetainInsertedBuild && retainBuild && !buildToInsert.KeepForever.GetValueOrDefault()) { await RetainComponentBuild(buildToInsert); } // ************* Bail out if there are no changes ************************ if (!allChanges.Any() && options.CherryPick.IsDefaultOrEmpty) { LogWarning("No meaningful changes since the last insertion was merged. PR will not be created or updated."); return(true, 0); } // ********************* Create push ************************************* var currentCommit = baseBranch.ObjectId; if (allChanges.Any()) { var insertionBranchUpdate = new GitRefUpdate { Name = $"refs/heads/{insertionBranchName}", OldObjectId = baseBranch.ObjectId }; var commit = new GitCommitRef { Comment = $"Updating {Options.InsertionName} to {buildVersion}", Changes = allChanges }; var push = new GitPush { RefUpdates = new[] { insertionBranchUpdate }, Commits = new[] { commit } }; push = await gitClient.CreatePushAsync(push, VSRepoId, cancellationToken : cancellationToken); currentCommit = push.Commits.Single().CommitId; } // ********************* Cherry-pick VS commits ***************************** var cherryPickCommits = Options.CherryPick; if (!cherryPickCommits.IsDefaultOrEmpty) { Console.WriteLine("Cherry-picking the following VS commits:"); foreach (var cherryPickCommit in cherryPickCommits) { var gc = await gitClient.GetCommitAsync(cherryPickCommit, VSRepoId, cancellationToken : cancellationToken); Console.WriteLine("- " + gc.RemoteUrl); } var commitRefs = cherryPickCommits.Select(id => new GitCommitRef() { CommitId = id }).ToArray(); var cherryPickBranchName = $"{insertionBranchName}-cherry-pick-{DateTime.Now:yyyyMMddHHmmss}"; var cherryPickArgs = new GitAsyncRefOperationParameters() { Source = new GitAsyncRefOperationSource() { CommitList = commitRefs }, OntoRefName = $"refs/heads/{insertionBranchName}", GeneratedRefName = $"refs/heads/{cherryPickBranchName}" }; // Cherry-pick VS commits into insertion branch. var cherryPick = await gitClient.CreateCherryPickAsync(cherryPickArgs, Options.VisualStudioRepoProjectName, VSRepoId, cancellationToken : cancellationToken); while (cherryPick.Status < GitAsyncOperationStatus.Completed) { Console.WriteLine($"Cherry-pick progress: {cherryPick.DetailedStatus?.Progress ?? 0:P}"); await Task.Delay(5000); cherryPick = await gitClient.GetCherryPickAsync(options.VisualStudioRepoProjectName, cherryPick.CherryPickId, VSRepoId, cancellationToken : cancellationToken); } Console.WriteLine($"Cherry-pick status: {cherryPick.Status}"); if (cherryPick.Status == GitAsyncOperationStatus.Completed) { var cherryPickBranch = await gitClient.GetBranchAsync(VSRepoId, cherryPickBranchName, cancellationToken : cancellationToken); var addCherryPickedCommits = new GitRefUpdate { OldObjectId = currentCommit, NewObjectId = cherryPickBranch.Commit.CommitId, Name = $"refs/heads/{insertionBranchName}" }; var results = await gitClient.UpdateRefsAsync(new[] { addCherryPickedCommits }, VSRepoId, cancellationToken : cancellationToken); foreach (var result in results) { if (!result.Success) { LogError("Failed to reset ref to cherry-pick branch: " + result.CustomMessage); } } } else { LogError("Cherry-picking failed: " + cherryPick.DetailedStatus.FailureMessage); } } // ********************* Create pull request ***************************** var oldBuild = await GetSpecificComponentBuildAsync(oldComponentVersion, cancellationToken); var prDescriptionMarkdown = CreatePullRequestDescription(oldBuild, buildToInsert, useMarkdown: true); if (buildToInsert.Result == BuildResult.PartiallySucceeded) { prDescriptionMarkdown += Environment.NewLine + ":warning: The build being inserted has partially succeeded."; } if (!useExistingPr || Options.OverwritePr) { try { var nl = Environment.NewLine; if (oldBuild is null) { prDescriptionMarkdown += $"{nl}---{nl}Unable to find details for previous build ({oldComponentVersion}){nl}"; } else { var(changes, diffLink) = await GetChangesBetweenBuildsAsync(oldBuild, buildToInsert, cancellationToken); var diffDescription = changes.Any() ? $"[View Complete Diff of Changes]({diffLink})" : "No source changes since previous insertion"; prDescriptionMarkdown += nl + "---" + nl + diffDescription + nl; prDescriptionMarkdown = AppendChangesToDescription(prDescriptionMarkdown, oldBuild ?? buildToInsert, changes); } } catch (Exception e) { LogWarning("Failed to create diff links."); LogWarning(e.Message); } } if (useExistingPr) { try { if (Options.OverwritePr) { pullRequest = await OverwritePullRequestAsync(pullRequestId, prDescriptionMarkdown, buildVersion.ToString(), cancellationToken); } pullRequestId = pullRequest.PullRequestId; } catch (Exception ex) { LogError($"Unable to update pull request for '{pullRequest.SourceRefName}'"); LogError(ex); return(false, 0); } } else { // create a new PR Console.WriteLine($"Create Pull Request"); try { // If this insertion was queued for PR validation, for a dev branch, for a feature branch, // or if no default reviewer is specified, then add the build queuer as a reviewer. var isPrValidation = !string.IsNullOrEmpty(GetBuildPRNumber(buildToInsert)); var isDevOrFeatureBranch = Options.ComponentBranchName.StartsWith("dev/") || Options.ComponentBranchName.StartsWith("features/"); bool hasReviewer = !string.IsNullOrEmpty(Options.ReviewerGUID); // Easiest way to get the reviewer GUIDs is to create a PR search in AzDo // You'll get something like https://dev.azure.com/devdiv/DevDiv/_git/VS/pullrequests?_a=active&createdBy=GUID-here var reviewerId = (isPrValidation || isDevOrFeatureBranch) || !hasReviewer ? buildToInsert.RequestedBy.Id : Options.ReviewerGUID; pullRequest = await CreateVSPullRequestAsync(insertionBranchName, prDescriptionMarkdown, buildVersion.ToString(), reviewerId, cancellationToken); if (pullRequest == null) { LogError($"Unable to create pull request for '{insertionBranchName}'"); return(false, 0); } pullRequestId = pullRequest.PullRequestId; } catch (Exception ex) { LogError($"Unable to create pull request for '{insertionBranchName}'"); LogError(ex); return(false, 0); } } // ********************* Create validation build ***************************** if (Options.QueueValidationBuild) { cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Create Validation Build"); try { if (Options.CreateDraftPr) { // When creating Draft PRs no policies are automatically started. // If we do not queue a CloudBuild the Perf DDRITs request will // spin waiting for a build to test against until it timesout. await QueueVSBuildPolicy(pullRequest, "CloudBuild - PR"); } await QueueVSBuildPolicy(pullRequest, "Request Perf DDRITs"); } catch (Exception ex) { LogWarning($"Unable to create a CloudBuild validation build for '{insertionBranchName}'"); LogWarning(ex); } if (Options.CreateDraftPr) { // When creating Draft PRs no policies are automatically started. await TryQueueVSBuildPolicy(pullRequest, "Insertion Hash Check", insertionBranchName); await TryQueueVSBuildPolicy(pullRequest, "Insertion Sign Check", insertionBranchName); await TryQueueVSBuildPolicy(pullRequest, "Insertion Symbol Check", insertionBranchName); } } // ********************* Set PR to Auto-Complete ***************************** if (Options.SetAutoComplete) { cancellationToken.ThrowIfCancellationRequested(); Console.WriteLine($"Set PR to Auto-Complete"); try { var prDescriptionText = CreatePullRequestDescription(oldBuild, buildToInsert, useMarkdown: false); await SetAutoCompleteAsync(pullRequest, prDescriptionText, cancellationToken); } catch (Exception ex) { LogWarning($"Unable to Set PR to Auto-Complete for '{insertionBranchName}'"); LogWarning(ex); } } return(true, pullRequestId); } catch (Exception ex) { LogError(ex); return(false, 0); } finally { Options = null; } }
public static async Task PerformInsertionAsync( RoslynInsertionToolOptions options, ILogger log, CancellationToken cancellationToken) { Options = options; Log = log; File.Delete(LogFilePath); Log.Info($"{Environment.NewLine}New Insertion Into {Options.VisualStudioBranchName} Started{Environment.NewLine}"); GitPullRequest pullRequest = null; var shouldRollBackGitChanges = false; var newPackageFiles = new List <string>(); var isInsertionCancelled = false; try { // Verify that the arguments we were passed authenticate correctly Log.Trace($"Verifying given authentication for {Options.VSTSUri}"); try { ProjectCollection.Authenticate(); } catch (Exception ex) { Log.Error($"Could not authenticate with {Options.VSTSUri}"); Log.Error(ex); return; } Log.Trace($"Verification succeeded for {Options.VSTSUri}"); // ********************** Get Last Insertion ***************************** cancellationToken.ThrowIfCancellationRequested(); BuildVersion roslynBuildVersion; Build newestBuild; bool retainBuild = false; // Get the version from TFS build queue, e.g. Roslyn-Master-Signed-Release. // We assume all CoreXT packages we build (Roslyn and all dependencies we // insert) have the same version. if (string.IsNullOrEmpty(Options.SpecificBuild)) { newestBuild = await GetLatestBuildAsync(cancellationToken); roslynBuildVersion = BuildVersion.FromTfsBuildNumber(newestBuild.BuildNumber, Options.RoslynBuildQueueName); } else { roslynBuildVersion = BuildVersion.FromString(Options.SpecificBuild); newestBuild = await GetSpecificBuildAsync(roslynBuildVersion, cancellationToken); } // ****************** Get Latest and Create Branch *********************** cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Getting Latest From {Options.VisualStudioBranchName} and Creating New Branch"); var branch = string.IsNullOrEmpty(Options.NewBranchName) ? null : GetLatestAndCreateBranch(cancellationToken); shouldRollBackGitChanges = branch != null; var coreXT = CoreXT.Load(GetAbsolutePathForEnlistment()); // ************** Update Nuget Packages For Branch************************ if (Options.InsertCoreXTPackages) { cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Updating Nuget Packages"); retainBuild |= UpdatePackages( newPackageFiles, roslynBuildVersion, coreXT, GetDevDivPackagesDirPath(roslynBuildVersion), cancellationToken); // ************ Update .corext\Configs\default.config ******************** cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Updating CoreXT config file"); coreXT.SaveConfig(); // ************** Update paths to CoreFX libraries ************************ cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Update paths to CoreFX libraries"); if (!await TryUpdateFileAsync( Path.Combine("ProductData", "ContractAssemblies.props"), roslynBuildVersion, onlyCopyIfFileDoesNotExistAtDestination: false, cancellationToken: cancellationToken)) { return; } // ************** Update assembly versions ************************ cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Updating assembly versions"); UpdateAssemblyVersions(roslynBuildVersion); // if we got this far then we definitely need to retain this build retainBuild = true; } // *********** Update toolset ******************** if (Options.InsertToolset) { UpdateToolsetPackage(roslynBuildVersion, cancellationToken); retainBuild = true; } // *********** Update .corext\Configs\components.json ******************** if (Options.InsertWillowPackages) { cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Updating CoreXT components file"); var components = await GetLatestComponentsAsync(newestBuild, cancellationToken); var shouldSave = false; foreach (var newComponent in components) { if (coreXT.TryGetComponentByName(newComponent.Name, out var oldComponent)) { coreXT.UpdateComponent(newComponent); shouldSave = true; } } if (shouldSave) { coreXT.SaveComponents(); retainBuild = true; } } // ************* Ensure the build is retained on the servers ************* if (Options.RetainInsertedBuild && retainBuild && !newestBuild.KeepForever.GetValueOrDefault()) { Log.Info("Marking inserted build for retention."); newestBuild.KeepForever = true; var buildClient = ProjectCollection.GetClient <BuildHttpClient>(); await buildClient.UpdateBuildAsync(newestBuild, newestBuild.Id); } // ********************* Verify Build Completes ************************** if (Options.PartitionsToBuild != null) { Log.Info($"Verifying build succeeds with changes"); foreach (var partition in Options.PartitionsToBuild) { Log.Info($"Starting build of {partition}"); if (!(await CanBuildPartitionAsync(partition, cancellationToken))) { Log.Error($"Build of partition {partition} failed"); return; } Log.Info($"Build of partition {partition} succeeded"); } } // ********************* Create pull request ***************************** if (branch != null) { cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Create Pull Request"); try { PushChanges(branch, roslynBuildVersion, cancellationToken); pullRequest = await CreatePullRequestAsync(branch.FriendlyName, $"Updating {Options.InsertionName} to {roslynBuildVersion}", cancellationToken); shouldRollBackGitChanges = false; } catch (EmptyCommitException ecx) { isInsertionCancelled = true; Log.Warn($"Unable to create pull request for '{branch.FriendlyName}'"); Log.Warn(ecx); return; } catch (Exception ex) { Log.Error($"Unable to create pull request for '{branch.FriendlyName}'"); Log.Error(ex); return; } if (pullRequest == null) { Log.Error($"Unable to create pull request for '{branch.FriendlyName}'"); return; } } // ********************* Create validation build ***************************** if (Options.QueueValidationBuild) { cancellationToken.ThrowIfCancellationRequested(); Log.Info($"Create Validation Build"); if (pullRequest == null) { Log.Error("Unable to create a validation build: no pull request."); return; } string buildUrl; int buildId; try { var build = await QueueValidationBuildAsync(pullRequest.SourceRefName); buildId = build.Id; var buildWebLink = (ReferenceLink)build.Links.Links["web"]; buildUrl = buildWebLink.Href; Log.Info($"Created build {buildUrl}"); } catch (Exception ex) { Log.Error($"Unable to create a validation build for '{pullRequest.SourceRefName}'"); Log.Error(ex); return; } try { string commentContent = $"Validation build: [{buildId}]({buildUrl})"; var commentThread = await CreateGitPullRequestCommentThread(pullRequest.PullRequestId, commentContent); Log.Info($"Added comment '{commentContent} to the pull request'"); } catch (Exception ex) { Log.Error($"Unable to add comment to PR about validation build"); Log.Error(ex); return; } } } catch (Exception ex) { if (ex is OutdatedPackageException || ex is OperationCanceledException) { isInsertionCancelled = true; } Log.Error(ex); } finally { // ************************* Flush Log *********************************** Log.Factory.Flush(); // ********************* Rollback Git Changes **************************** if (shouldRollBackGitChanges) { try { Log.Info("Rolling back git changes"); var rollBackCommit = Enlistment.Branches[Options.VisualStudioBranchName].Commits.First(); Enlistment.Reset(ResetMode.Hard, rollBackCommit); Enlistment.RemoveUntrackedFiles(); } catch (Exception ex) { Log.Error(ex); } } // ********************* Send Status Mail ******************************** if (!string.IsNullOrEmpty(Options.EmailServerName) && !string.IsNullOrEmpty(Options.MailRecipient)) { try { SendMail(pullRequest, newPackageFiles, isInsertionCancelled); } catch (Exception ex) { Log.Error($"Unable to send mail, EmailServerName: '{Options.EmailServerName}', MailRecipient: '{Options.MailRecipient}'"); Log.Error(ex); } } Options = null; Log = null; } }
private static void PushChanges(Branch branch, BuildVersion newRoslynVersion, CancellationToken cancellationToken) { StageFiles(newRoslynVersion, cancellationToken); CommitStagedChanges(newRoslynVersion, cancellationToken); PushChanges(branch, cancellationToken); }