internal static string GetDevDivPackagesDirPath(BuildVersion version)
{
// For example: "\\cpvsbuild\drops\Roslyn\Roslyn-Master-Signed-Release\20160315.3\DevDivPackages"
return(Path.Combine(GetBuildDirectory(version), "DevDivPackages"));
}
public static async Task PerformInsertionAsync(
RoslynInsertionToolOptions options,
ILogger log,
CancellationToken cancellationToken)
{
Options = options;
Log = log;
Log.Info($"{Environment.NewLine}New Insertion Into {Options.VisualStudioBranchName} Started{Environment.NewLine}");
GitPullRequest pullRequest = null;
var shouldRollBackGitChanges = false;
var newPackageFiles = new List <string>();
var isInsertionCancelled = false;
var noProgressOnFailedBuilds = false;
try
{
// Verify that the arguments we were passed authenticate correctly
Log.Trace($"Verifying given authentication for {Options.VSTSUri}");
try
{
ProjectCollection.Authenticate();
}
catch (Exception ex)
{
Log.Error($"Could not authenticate with {Options.VSTSUri}");
Log.Error(ex);
return;
}
Log.Trace($"Verification succeeded for {Options.VSTSUri}");
// ********************** Create dummy PR *****************************
if (Options.CreateDummyPr)
{
var dummyBranch = GetLatestAndCreateBranch(cancellationToken);
try
{
CreateDummyCommit(cancellationToken);
PushChanges(dummyBranch, cancellationToken);
pullRequest = await CreatePullRequestAsync(dummyBranch.FriendlyName, $"DUMMY INSERTION FOR {Options.InsertionName}", "Not Specified", cancellationToken);
}
catch (Exception ex)
{
Log.Error($"Unable to create pull request for '{dummyBranch.FriendlyName}'");
Log.Error(ex);
return;
}
if (pullRequest == null)
{
Log.Error($"Unable to create pull request for '{dummyBranch.FriendlyName}'");
}
return;
}
// ********************** Get Last Insertion *****************************
cancellationToken.ThrowIfCancellationRequested();
BuildVersion buildVersion;
Build buildToInsert;
Build latestBuild = null;
bool retainBuild = false;
// Get the version from TFS build queue, e.g. Roslyn-Master-Signed-Release.
// We assume all CoreXT packages we build (Roslyn and all dependencies we
// insert) have the same version.
if (string.IsNullOrEmpty(Options.SpecificBuild))
{
buildToInsert = await GetLatestPassedBuildAsync(cancellationToken);
buildVersion = BuildVersion.FromTfsBuildNumber(buildToInsert.BuildNumber, Options.BuildQueueName);
// Get the latest build, whether passed or failed. If the buildToInsert has already been inserted but
// there is a later failing build, then send an error
latestBuild = await GetLatestBuildAsync(cancellationToken);
}
else
{
buildVersion = BuildVersion.FromString(Options.SpecificBuild);
buildToInsert = await GetSpecificBuildAsync(buildVersion, cancellationToken);
}
string artifactsFolder = await GetBuildDirectoryAsync(buildToInsert, cancellationToken);
Branch branch = null;
cancellationToken.ThrowIfCancellationRequested();
if (Options.UpdateExistingPr != 0)
{
// ****************** Update existing PR ***********************
pullRequest = await GetExistingPullRequestAsync(Options.UpdateExistingPr, cancellationToken);
branch = SwitchToBranchAndUpdate(pullRequest.SourceRefName, Options.VisualStudioBranchName);
}
else
{
// ****************** Get Latest and Create Branch ***********************
Log.Info($"Getting Latest From {Options.VisualStudioBranchName} and Creating New Branch");
branch = string.IsNullOrEmpty(Options.NewBranchName)
? null
: GetLatestAndCreateBranch(cancellationToken);
}
shouldRollBackGitChanges = branch != null;
if (Options.UpdateCoreXTLibraries)
{
// ************** Update paths to CoreFX libraries ************************
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Update paths to CoreFX libraries");
if (!await TryUpdateFileAsync(
artifactsFolder,
Path.Combine("ProductData", "ContractAssemblies.props"),
buildVersion,
onlyCopyIfFileDoesNotExistAtDestination: false,
cancellationToken: cancellationToken))
{
return;
}
}
var coreXT = CoreXT.Load(GetAbsolutePathForEnlistment());
if (Options.InsertCoreXTPackages)
{
// ************** Update Nuget Packages For Branch************************
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Updating Nuget Packages");
bool success = false;
(success, newPackageFiles) = UpdatePackages(
buildVersion,
coreXT,
GetPackagesDirPath(artifactsFolder),
cancellationToken);
retainBuild |= success;
// ************ Update .corext\Configs\default.config ********************
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Updating CoreXT default.config file");
coreXT.SaveConfig();
}
if (Options.UpdateCoreXTLibraries)
{
// ************** Update paths to CoreFX libraries ************************
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Update paths to CoreFX libraries");
if (!await TryUpdateFileAsync(
artifactsFolder,
Path.Combine("ProductData", "ContractAssemblies.props"),
buildVersion,
onlyCopyIfFileDoesNotExistAtDestination: false,
cancellationToken: cancellationToken))
{
return;
}
}
if (Options.UpdateCoreXTLibraries || Options.UpdateAssemblyVersions)
{
// ************** Update assembly versions ************************
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Updating assembly versions");
UpdateAssemblyVersions(artifactsFolder);
// if we got this far then we definitely need to retain this build
retainBuild = true;
}
// *********** Update toolset ********************
if (Options.InsertToolset)
{
UpdateToolsetPackage(artifactsFolder, buildVersion, cancellationToken);
retainBuild = true;
}
// *********** Update .corext\Configs\components.json ********************
if (Options.InsertWillowPackages)
{
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Updating CoreXT components file");
var components = await GetLatestComponentsAsync(buildToInsert, cancellationToken);
var shouldSave = false;
foreach (var newComponent in components)
{
if (coreXT.TryGetComponentByName(newComponent.Name, out var oldComponent))
{
coreXT.UpdateComponent(newComponent);
shouldSave = true;
}
}
if (shouldSave)
{
coreXT.SaveComponents();
retainBuild = true;
}
}
// ************* Ensure the build is retained on the servers *************
if (Options.RetainInsertedBuild && retainBuild && !buildToInsert.KeepForever.GetValueOrDefault())
{
Log.Info("Marking inserted build for retention.");
buildToInsert.KeepForever = true;
var buildClient = ProjectCollection.GetClient <BuildHttpClient>();
await buildClient.UpdateBuildAsync(buildToInsert, buildToInsert.Id);
}
// ********************* Verify Build Completes **************************
if (Options.PartitionsToBuild != null)
{
Log.Info($"Verifying build succeeds with changes");
foreach (var partition in Options.PartitionsToBuild)
{
Log.Info($"Starting build of {partition}");
if (!(await CanBuildPartitionAsync(partition, cancellationToken)))
{
Log.Error($"Build of partition {partition} failed");
return;
}
Log.Info($"Build of partition {partition} succeeded");
}
}
// ********************* Trigger a release *****************************
Log.Info($"Triggering a release for the build {buildToInsert.BuildNumber}");
var release = await CreateReleaseAsync(buildToInsert, cancellationToken);
// The timeout for the below wait is primarily dependent on:
// 1. The release task itself - Since its currently only triggering symbol archival,
// it should not be very long but this should increase when more time intesive tasks are added to the release.
// 2. The availability of machines to run the release on. This could be a problem at peak pool load
// where getting a machine can take upto an hour or more.
WaitForReleaseCompletion(release, TimeSpan.FromMinutes(10), cancellationToken);
Log.Info($"Release succesfully triggered");
// ********************* Create pull request *****************************
if (branch != null)
{
cancellationToken.ThrowIfCancellationRequested();
var prDescription = $"Updating {Options.InsertionName} to {buildVersion}";
if (Options.UpdateExistingPr != 0 && pullRequest != null)
{
// update an existing pr
try
{
branch = PushChanges(branch, buildVersion, cancellationToken, forcePush: true);
pullRequest = await UpdatePullRequestDescriptionAsync(Options.UpdateExistingPr, prDescription, cancellationToken);
shouldRollBackGitChanges = false;
}
catch (Exception ex)
{
Log.Error($"Unable to update pull request for '{branch.FriendlyName}'");
Log.Error(ex);
return;
}
}
else
{
// create a new PR
Log.Info($"Create Pull Request");
try
{
branch = PushChanges(branch, buildVersion, cancellationToken);
pullRequest = await CreatePullRequestAsync(branch.FriendlyName, prDescription, buildVersion.ToString(), cancellationToken);
shouldRollBackGitChanges = false;
}
catch (EmptyCommitException ecx)
{
isInsertionCancelled = true;
if (latestBuild != null && latestBuild.Result != BuildResult.Succeeded)
{
noProgressOnFailedBuilds = true;
}
Log.Warn($"Unable to create pull request for '{branch.FriendlyName}'");
Log.Warn(ecx);
return;
}
catch (Exception ex)
{
Log.Error($"Unable to create pull request for '{branch.FriendlyName}'");
Log.Error(ex);
return;
}
}
if (pullRequest == null)
{
Log.Error($"Unable to create pull request for '{branch.FriendlyName}'");
return;
}
}
// ********************* Create validation build *****************************
if (Options.QueueValidationBuild)
{
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Create Validation Build");
if (pullRequest == null)
{
Log.Error("Unable to create a validation build: no pull request.");
return;
}
try
{
await QueueBuildPolicy(pullRequest, "[Deprecated] ValBuild RPS");
}
catch (Exception ex)
{
Log.Error($"Unable to create a deprecated validation build for '{pullRequest.SourceRefName}'");
Log.Error(ex);
}
try
{
await QueueBuildPolicy(pullRequest, "CloudBuild - Request RPS");
}
catch (Exception ex)
{
Log.Error($"Unable to create a CloudBuild validation build for '{pullRequest.SourceRefName}'");
Log.Error(ex);
}
}
}
catch (Exception ex)
{
if (ex is OutdatedPackageException || ex is OperationCanceledException)
{
isInsertionCancelled = true;
}
Log.Error(ex);
}
finally
{
// ************************* Flush Log ***********************************
Log.Factory.Flush();
// ********************* Rollback Git Changes ****************************
if (shouldRollBackGitChanges)
{
try
{
Log.Info("Rolling back git changes");
var rollBackCommit = Enlistment.Branches[Options.VisualStudioBranchName].Commits.First();
Enlistment.Reset(ResetMode.Hard, rollBackCommit);
Enlistment.RemoveUntrackedFiles();
}
catch (Exception ex)
{
Log.Error(ex);
}
}
// ********************* Send Status Mail ********************************
if (!string.IsNullOrEmpty(Options.EmailServerName) &&
!string.IsNullOrEmpty(Options.MailRecipient))
{
try
{
SendMail(pullRequest, newPackageFiles, isInsertionCancelled, noProgressOnFailedBuilds);
}
catch (Exception ex)
{
Log.Error($"Unable to send mail, EmailServerName: '{Options.EmailServerName}', MailRecipient: '{Options.MailRecipient}'");
Log.Error(ex);
}
}
Options = null;
Log = null;
}
}
public static async Task <(bool success, int pullRequestId)> PerformInsertionAsync(
RoslynInsertionToolOptions options,
CancellationToken cancellationToken)
{
Options = options;
Console.WriteLine($"{Environment.NewLine}New Insertion Into {Options.VisualStudioBranchName} Started{Environment.NewLine}");
var newPackageFiles = new List <string>();
try
{
Console.WriteLine($"Verifying given authentication for {Options.VSTSUri}");
try
{
ProjectCollection.Authenticate();
}
catch (Exception ex)
{
LogError($"Could not authenticate with {Options.VSTSUri}");
LogError(ex);
return(false, 0);
}
Console.WriteLine($"Verification succeeded for {Options.VSTSUri}");
// ********************** Create dummy PR *****************************
if (Options.CreateDummyPr)
{
GitPullRequest dummyPR;
try
{
dummyPR = await CreatePlaceholderBranchAsync(cancellationToken);
}
catch (Exception ex)
{
LogError($"Unable to create placeholder PR for '{options.VisualStudioBranchName}'");
LogError(ex);
return(false, 0);
}
if (dummyPR == null)
{
LogError($"Unable to create placeholder PR for '{options.VisualStudioBranchName}'");
return(false, 0);
}
return(true, dummyPR.PullRequestId);
}
// ********************** Get Last Insertion *****************************
cancellationToken.ThrowIfCancellationRequested();
BuildVersion buildVersion;
Build buildToInsert;
Build latestBuild = null;
bool retainBuild = false;
// Get the version from DevOps Pipelines queue, e.g. Roslyn-Master-Signed-Release.
if (string.IsNullOrEmpty(Options.SpecificBuild))
{
buildToInsert = await GetLatestPassedBuildAsync(cancellationToken);
buildVersion = BuildVersion.FromTfsBuildNumber(buildToInsert.BuildNumber, Options.BuildQueueName);
Console.WriteLine("Found build number " + buildVersion);
// Get the latest build, whether passed or failed. If the buildToInsert has already been inserted but
// there is a later failing build, then send an error
latestBuild = await GetLatestBuildAsync(cancellationToken);
}
else
{
buildVersion = BuildVersion.FromString(Options.SpecificBuild);
buildToInsert = await GetSpecificBuildAsync(buildVersion, cancellationToken);
}
var insertionArtifacts = await GetInsertionArtifactsAsync(buildToInsert, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
// *********** Look up existing PR ********************
var gitClient = ProjectCollection.GetClient <GitHttpClient>();
var branches = await gitClient.GetRefsAsync(
VSRepoId,
filter : $"heads/{Options.VisualStudioBranchName}",
cancellationToken : cancellationToken);
var baseBranch = branches.Single(b => b.Name == $"refs/heads/{Options.VisualStudioBranchName}");
var pullRequestId = Options.UpdateExistingPr;
var useExistingPr = pullRequestId != 0;
GitPullRequest pullRequest;
string insertionBranchName;
if (useExistingPr)
{
pullRequest = await gitClient.GetPullRequestByIdAsync(pullRequestId, cancellationToken : cancellationToken);
insertionBranchName = pullRequest.SourceRefName.Substring("refs/heads/".Length);
var refs = await gitClient.GetRefsAsync(VSRepoId, filter : $"heads/{insertionBranchName}", cancellationToken : cancellationToken);
var insertionBranch = refs.Single(r => r.Name == $"refs/heads/{insertionBranchName}");
if (Options.OverwritePr)
{
// overwrite existing PR branch back to base before pushing new commit
var updateToBase = new GitRefUpdate
{
OldObjectId = insertionBranch.ObjectId,
NewObjectId = baseBranch.ObjectId,
Name = $"refs/heads/{insertionBranchName}"
};
await gitClient.UpdateRefsAsync(new[] { updateToBase }, VSRepoId, cancellationToken : cancellationToken);
}
else
{
// not overwriting PR, so the insertion branch is actually the base
baseBranch = insertionBranch;
}
}
else
{
pullRequest = null;
insertionBranchName = GetNewBranchName();
}
var allChanges = new List <GitChange>();
var coreXT = await CoreXT.Load(gitClient, baseBranch.ObjectId);
if (Options.InsertCoreXTPackages)
{
// ************** Update Nuget Packages For Branch************************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating Nuget Packages");
bool success;
(success, newPackageFiles) = UpdatePackages(
buildVersion,
coreXT,
insertionArtifacts.GetPackagesDirectory(),
cancellationToken);
retainBuild |= success;
// *********** Copy OptimizationInputs.props file ***********************
foreach (var propsFile in insertionArtifacts.GetOptProfPropertyFiles())
{
var targetFilePath = "src/Tests/config/runsettings/Official/OptProf/External/" + Path.GetFileName(propsFile);
var version = new GitVersionDescriptor {
VersionType = GitVersionType.Commit, Version = baseBranch.ObjectId
};
var stream = await gitClient.GetItemContentAsync(VSRepoId, targetFilePath, download : true, versionDescriptor : version);
var originalContent = new StreamReader(stream).ReadToEnd();
var newContent = File.ReadAllText(propsFile);
if (GetChangeOpt(targetFilePath, originalContent, newContent) is GitChange change)
{
allChanges.Add(change);
}
}
}
if (Options.UpdateCoreXTLibraries || Options.UpdateAssemblyVersions)
{
// ************** Update assembly versions ************************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating assembly versions");
if (await UpdateAssemblyVersionsOpt(gitClient, baseBranch.ObjectId, insertionArtifacts) is GitChange assemblyVersionChange)
{
allChanges.Add(assemblyVersionChange);
}
// if we got this far then we definitely need to retain this build
retainBuild = true;
}
// *********** Update toolset ********************
if (Options.InsertToolset)
{
UpdateToolsetPackage(coreXT, insertionArtifacts, buildVersion);
retainBuild = true;
}
// ************ Update .corext\Configs\default.config ********************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating CoreXT default.config file");
if (coreXT.SaveConfigOpt() is GitChange configChange)
{
allChanges.Add(configChange);
}
// *********** Update .corext\Configs\components.json ********************
BuildVersion oldComponentVersion = default;
if (Options.InsertWillowPackages)
{
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating CoreXT components file");
var components = await GetLatestComponentsAsync(buildToInsert, cancellationToken);
var shouldSave = false;
foreach (var newComponent in components)
{
if (coreXT.TryGetComponentByName(newComponent.Name, out var oldComponent))
{
if (oldComponent.BuildVersion != default)
{
oldComponentVersion = oldComponent.BuildVersion;
}
coreXT.UpdateComponent(newComponent);
shouldSave = true;
}
}
if (shouldSave)
{
var allComponentChanges = coreXT.SaveComponents();
allChanges.AddRange(allComponentChanges);
retainBuild = true;
}
}
// ************* Ensure the build is retained on the servers *************
if (Options.RetainInsertedBuild && retainBuild && !buildToInsert.KeepForever.GetValueOrDefault())
{
Console.WriteLine("Marking inserted build for retention.");
buildToInsert.KeepForever = true;
var buildClient = ProjectCollection.GetClient <BuildHttpClient>();
await buildClient.UpdateBuildAsync(buildToInsert);
}
// ************* Bail out if there are no changes ************************
if (!allChanges.Any())
{
LogWarning("No meaningful changes since the last insertion was merged. PR will not be created or updated.");
return(true, 0);
}
// ********************* Create push *************************************
var insertionBranchUpdate = new GitRefUpdate
{
Name = $"refs/heads/{insertionBranchName}",
OldObjectId = baseBranch.ObjectId
};
var commit = new GitCommitRef
{
Comment = $"Updating {Options.InsertionName} to {buildVersion}",
Changes = allChanges
};
var push = new GitPush
{
RefUpdates = new[] { insertionBranchUpdate },
Commits = new[] { commit }
};
await gitClient.CreatePushAsync(push, VSRepoId, cancellationToken : cancellationToken);
// ********************* Create pull request *****************************
var oldBuild = await GetSpecificBuildAsync(oldComponentVersion, cancellationToken);
var prDescriptionMarkdown = CreatePullRequestDescription(oldBuild, buildToInsert, useMarkdown: true);
if (buildToInsert.Result == BuildResult.PartiallySucceeded)
{
prDescriptionMarkdown += Environment.NewLine + ":warning: The build being inserted has partially succeeded.";
}
if (!useExistingPr || Options.OverwritePr)
{
try
{
var nl = Environment.NewLine;
if (oldBuild is null)
{
prDescriptionMarkdown += $"{nl}---{nl}Unable to find details for previous build ({oldComponentVersion}){nl}";
}
else
{
var(changes, diffLink) = await GetChangesBetweenBuildsAsync(oldBuild, buildToInsert, cancellationToken);
var diffDescription = changes.Any()
? $"[View Complete Diff of Changes]({diffLink})"
: "No source changes since previous insertion";
prDescriptionMarkdown += nl + "---" + nl + diffDescription + nl;
prDescriptionMarkdown = AppendChangesToDescription(prDescriptionMarkdown, oldBuild ?? buildToInsert, changes);
}
}
catch (Exception e)
{
LogWarning("Failed to create diff links.");
LogWarning(e.Message);
}
}
if (useExistingPr)
{
try
{
if (Options.OverwritePr)
{
pullRequest = await OverwritePullRequestAsync(pullRequestId, prDescriptionMarkdown, buildVersion.ToString(), options.TitlePrefix, cancellationToken);
}
pullRequestId = pullRequest.PullRequestId;
}
catch (Exception ex)
{
LogError($"Unable to update pull request for '{pullRequest.SourceRefName}'");
LogError(ex);
return(false, 0);
}
}
else
{
// create a new PR
Console.WriteLine($"Create Pull Request");
try
{
// If this insertion was queued for PR validation, for a dev branch, or for a feature branch,
// then add the build queuer as a reviewer instead of mlinfraswat.
var isPrValidation = !string.IsNullOrEmpty(GetBuildPRNumber(buildToInsert));
var isDevOrFeatureBranch = Options.BranchName.StartsWith("dev/") || Options.BranchName.StartsWith("features/");
var reviewerId = isPrValidation || isDevOrFeatureBranch
? buildToInsert.RequestedBy.Id
: MLInfraSwatUserId.ToString();
pullRequest = await CreatePullRequestAsync(insertionBranchName, prDescriptionMarkdown, buildVersion.ToString(), options.TitlePrefix, reviewerId, cancellationToken);
if (pullRequest == null)
{
LogError($"Unable to create pull request for '{insertionBranchName}'");
return(false, 0);
}
pullRequestId = pullRequest.PullRequestId;
}
catch (Exception ex)
{
LogError($"Unable to create pull request for '{insertionBranchName}'");
LogError(ex);
return(false, 0);
}
}
// ********************* Create validation build *****************************
if (Options.QueueValidationBuild)
{
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Create Validation Build");
try
{
if (Options.CreateDraftPr)
{
// When creating Draft PRs no policies are automatically started.
// If we do not queue a CloudBuild the Perf DDRITs request will
// spin waiting for a build to test against until it timesout.
await QueueBuildPolicy(pullRequest, "CloudBuild - PR");
}
await QueueBuildPolicy(pullRequest, "Request Perf DDRITs");
}
catch (Exception ex)
{
LogWarning($"Unable to create a CloudBuild validation build for '{insertionBranchName}'");
LogWarning(ex);
}
if (Options.CreateDraftPr)
{
// When creating Draft PRs no policies are automatically started.
await TryQueueBuildPolicy(pullRequest, "Insertion Hash Check", insertionBranchName);
await TryQueueBuildPolicy(pullRequest, "Insertion Sign Check", insertionBranchName);
await TryQueueBuildPolicy(pullRequest, "Insertion Symbol Check", insertionBranchName);
}
}
// ********************* Set PR to Auto-Complete *****************************
if (Options.SetAutoComplete)
{
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Set PR to Auto-Complete");
try
{
var prDescriptionText = CreatePullRequestDescription(oldBuild, buildToInsert, useMarkdown: false);
await SetAutoCompleteAsync(pullRequest, prDescriptionText, cancellationToken);
}
catch (Exception ex)
{
LogWarning($"Unable to Set PR to Auto-Complete for '{insertionBranchName}'");
LogWarning(ex);
}
}
return(true, pullRequestId);
}
catch (Exception ex)
{
LogError(ex);
return(false, 0);
}
finally
{
Options = null;
}
}
internal static string GetDevDivInsertionFilePath(BuildVersion version, string relativePath)
{
// For example: "\\cpvsbuild\drops\Roslyn\Roslyn-Master-Signed-Release\20160315.3\DevDivInsertionFiles\Roslyn\all.roslyn.locproj"
return(Path.Combine(GetBuildDirectory(version), "DevDivInsertionFiles", relativePath));
}
private static void StageFiles(BuildVersion newRoslynVersion, CancellationToken cancellationToken)
{
var repositoryStatus = Enlistment.RetrieveStatus();
if (repositoryStatus.IsDirty)
{
cancellationToken.ThrowIfCancellationRequested();
var filesToStage = repositoryStatus
.Where(item => item.State != FileStatus.Unaltered && item.State != FileStatus.Ignored)
.Select(item => item.FilePath).ToList();
cancellationToken.ThrowIfCancellationRequested();
if (!isWhitespaceOnlyChange(Enlistment.Diff.Compare <Patch>(filesToStage)))
{
Console.WriteLine($"Staging {filesToStage.Count()} file(s)");
var watch = Stopwatch.StartNew();
Enlistment.Stage(filesToStage, GetStageOptions());
Console.WriteLine($"Staging took {watch.Elapsed.TotalSeconds} seconds");
}
else
{
Console.WriteLine("Only whitespace changes found");
}
}
bool isWhitespaceOnlyChange(Patch p)
{
var before = new StringBuilder();
var after = new StringBuilder();
foreach (var change in p)
{
if (change.Status != ChangeKind.Modified)
{
return(false);
}
using (var reader = new StringReader(change.Patch))
{
string line;
while ((line = reader.ReadLine()) != null)
{
if (!line.StartsWith("---") && !line.StartsWith("+++"))
{
if (line.StartsWith("+"))
{
after.Append(line.Substring(1).Trim());
continue;
}
else if (line.StartsWith("-"))
{
before.Append(line.Substring(1).Trim());
continue;
}
}
after.AppendLine(line);
before.AppendLine(line);
}
}
if (after.ToString() != before.ToString())
{
return(false);
}
after.Clear();
before.Clear();
}
return(true);
}
}
/// <returns>A tuple containing (success, pullRequestId).</returns>
public static async Task <(bool, int)> PerformInsertionAsync(
RoslynInsertionToolOptions options,
CancellationToken cancellationToken)
{
Options = options;
Console.WriteLine($"{Environment.NewLine}New Insertion Into {Options.VisualStudioBranchName} Started{Environment.NewLine}");
GitPullRequest pullRequest = null;
var shouldRollBackGitChanges = false;
var newPackageFiles = new List <string>();
try
{
// Verify that the arguments we were passed authenticate correctly
Console.WriteLine($"Verifying given authentication for {Options.VSTSUri}");
try
{
ProjectCollection.Authenticate();
}
catch (Exception ex)
{
Console.WriteLine($"Could not authenticate with {Options.VSTSUri}");
Console.WriteLine(ex);
return(false, 0);
}
Console.WriteLine($"Verification succeeded for {Options.VSTSUri}");
// ********************** Create dummy PR *****************************
if (Options.CreateDummyPr)
{
var dummyBranch = CreateBranch(cancellationToken);
try
{
CreateDummyCommit(cancellationToken);
PushChanges(dummyBranch, cancellationToken);
pullRequest = await CreatePullRequestAsync(dummyBranch.FriendlyName, $"DUMMY INSERTION FOR {Options.InsertionName}", "Not Specified", options.TitlePrefix, cancellationToken);
}
catch (Exception ex)
{
Console.WriteLine($"Unable to create pull request for '{dummyBranch.FriendlyName}'");
Console.WriteLine(ex);
return(false, 0);
}
if (pullRequest == null)
{
Console.WriteLine($"Unable to create pull request for '{dummyBranch.FriendlyName}'");
return(false, 0);
}
return(true, pullRequest.PullRequestId);
}
// ********************** Get Last Insertion *****************************
cancellationToken.ThrowIfCancellationRequested();
BuildVersion buildVersion;
Build buildToInsert;
Build latestBuild = null;
bool retainBuild = false;
// Get the version from DevOps Pipelines queue, e.g. Roslyn-Master-Signed-Release.
if (string.IsNullOrEmpty(Options.SpecificBuild))
{
buildToInsert = await GetLatestPassedBuildAsync(cancellationToken);
buildVersion = BuildVersion.FromTfsBuildNumber(buildToInsert.BuildNumber, Options.BuildQueueName);
Console.WriteLine("Found build number " + buildVersion);
// Get the latest build, whether passed or failed. If the buildToInsert has already been inserted but
// there is a later failing build, then send an error
latestBuild = await GetLatestBuildAsync(cancellationToken);
}
else
{
buildVersion = BuildVersion.FromString(Options.SpecificBuild);
buildToInsert = await GetSpecificBuildAsync(buildVersion, cancellationToken);
}
var insertionArtifacts = await GetInsertionArtifactsAsync(buildToInsert, cancellationToken);
Branch branch = null;
cancellationToken.ThrowIfCancellationRequested();
var useExistingPr = Options.UpdateExistingPr != 0;
if (useExistingPr)
{
// ****************** Update existing PR ***********************
pullRequest = await GetExistingPullRequestAsync(Options.UpdateExistingPr, cancellationToken);
branch = SwitchToBranchAndUpdate(pullRequest.SourceRefName, Options.VisualStudioBranchName, overwriteExistingChanges: Options.OverwritePr);
}
else
{
// ****************** Create Branch ***********************
Console.WriteLine("Creating New Branch");
branch = string.IsNullOrEmpty(Options.NewBranchName)
? null
: CreateBranch(cancellationToken);
}
shouldRollBackGitChanges = branch != null;
var coreXT = CoreXT.Load(GetAbsolutePathForEnlistment());
if (Options.InsertCoreXTPackages)
{
// ************** Update Nuget Packages For Branch************************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating Nuget Packages");
bool success = false;
(success, newPackageFiles) = UpdatePackages(
buildVersion,
coreXT,
insertionArtifacts.GetPackagesDirectory(),
cancellationToken);
retainBuild |= success;
// ************ Update .corext\Configs\default.config ********************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating CoreXT default.config file");
coreXT.SaveConfig();
}
if (Options.UpdateCoreXTLibraries || Options.UpdateAssemblyVersions)
{
// ************** Update assembly versions ************************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating assembly versions");
UpdateAssemblyVersions(insertionArtifacts);
// if we got this far then we definitely need to retain this build
retainBuild = true;
}
// *********** Update toolset ********************
if (Options.InsertToolset)
{
UpdateToolsetPackage(insertionArtifacts, buildVersion, cancellationToken);
retainBuild = true;
}
// *********** Update .corext\Configs\components.json ********************
if (Options.InsertWillowPackages)
{
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating CoreXT components file");
var components = await GetLatestComponentsAsync(buildToInsert, cancellationToken);
var shouldSave = false;
foreach (var newComponent in components)
{
if (coreXT.TryGetComponentByName(newComponent.Name, out var oldComponent))
{
coreXT.UpdateComponent(newComponent);
shouldSave = true;
}
}
if (shouldSave)
{
coreXT.SaveComponents();
retainBuild = true;
}
}
// ************* Ensure the build is retained on the servers *************
if (Options.RetainInsertedBuild && retainBuild && !buildToInsert.KeepForever.GetValueOrDefault())
{
Console.WriteLine("Marking inserted build for retention.");
buildToInsert.KeepForever = true;
var buildClient = ProjectCollection.GetClient <BuildHttpClient>();
await buildClient.UpdateBuildAsync(buildToInsert, buildToInsert.Id);
}
// ********************* Verify Build Completes **************************
if (Options.PartitionsToBuild != null)
{
Console.WriteLine($"Verifying build succeeds with changes");
foreach (var partition in Options.PartitionsToBuild)
{
Console.WriteLine($"Starting build of {partition}");
if (!(await CanBuildPartitionAsync(partition, cancellationToken)))
{
Console.WriteLine($"Build of partition {partition} failed");
return(false, 0);
}
Console.WriteLine($"Build of partition {partition} succeeded");
}
}
// ********************* Trigger a release *****************************
Console.WriteLine($"Triggering a release for the build {buildToInsert.BuildNumber}");
var release = await CreateReleaseAsync(buildToInsert, cancellationToken);
// The timeout for the below wait is primarily dependent on:
// 1. The release task itself - Since its currently only triggering symbol archival,
// it should not be very long but this should increase when more time intesive tasks are added to the release.
// 2. The availability of machines to run the release on. This could be a problem at peak pool load
// where getting a machine can take upto an hour or more.
WaitForReleaseCompletion(release, TimeSpan.FromMinutes(10), cancellationToken);
Console.WriteLine($"Release succesfully triggered");
// ********************* Create pull request *****************************
var pullRequestId = 0;
if (branch != null)
{
cancellationToken.ThrowIfCancellationRequested();
var prDescription = $"Updating {Options.InsertionName} to {buildVersion}";
if (useExistingPr && pullRequest != null)
{
// update an existing pr
try
{
branch = PushChanges(branch, buildVersion, cancellationToken, forcePush: true);
if (Options.OverwritePr)
{
pullRequest = await UpdatePullRequestDescriptionAsync(Options.UpdateExistingPr, prDescription, cancellationToken);
}
shouldRollBackGitChanges = false;
pullRequestId = pullRequest.PullRequestId;
}
catch (Exception ex)
{
Console.WriteLine($"Unable to update pull request for '{branch.FriendlyName}'");
Console.WriteLine(ex);
return(false, 0);
}
}
else
{
// create a new PR
Console.WriteLine($"Create Pull Request");
try
{
branch = PushChanges(branch, buildVersion, cancellationToken);
pullRequest = await CreatePullRequestAsync(branch.FriendlyName, prDescription, buildVersion.ToString(), options.TitlePrefix, cancellationToken);
shouldRollBackGitChanges = false;
pullRequestId = pullRequest.PullRequestId;
}
catch (EmptyCommitException ecx)
{
Console.WriteLine($"Unable to create pull request for '{branch.FriendlyName}'");
Console.WriteLine(ecx);
return(false, 0);
}
catch (Exception ex)
{
Console.WriteLine($"Unable to create pull request for '{branch.FriendlyName}'");
Console.WriteLine(ex);
return(false, 0);
}
}
if (pullRequest == null)
{
Console.WriteLine($"Unable to create pull request for '{branch.FriendlyName}'");
return(false, 0);
}
}
// ********************* Create validation build *****************************
if (Options.QueueValidationBuild)
{
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Create Validation Build");
if (pullRequest == null)
{
Console.WriteLine("Unable to create a validation build: no pull request.");
return(false, 0);
}
try
{
await QueueBuildPolicy(pullRequest, "CloudBuild - Request RPS");
}
catch (Exception ex)
{
Console.WriteLine($"Unable to create a CloudBuild validation build for '{pullRequest.SourceRefName}'");
Console.WriteLine(ex);
}
}
return(true, pullRequestId);
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(false, 0);
}
finally
{
// ********************* Rollback Git Changes ****************************
if (shouldRollBackGitChanges)
{
try
{
Console.WriteLine("Rolling back git changes");
var rollBackCommit = Enlistment.Branches[Options.VisualStudioBranchName].Commits.First();
Enlistment.Reset(ResetMode.Hard, rollBackCommit);
Enlistment.RemoveUntrackedFiles();
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
Options = null;
}
}
private static Branch PushChanges(Branch branch, BuildVersion newRoslynVersion, CancellationToken cancellationToken, bool forcePush = false)
{
StageFiles(newRoslynVersion, cancellationToken);
CommitStagedChanges(newRoslynVersion, cancellationToken);
return(PushChanges(branch, cancellationToken, forcePush: forcePush));
}
public static async Task <(bool success, int pullRequestId)> PerformInsertionAsync(
RoslynInsertionToolOptions options,
CancellationToken cancellationToken)
{
Options = options;
Console.WriteLine($"{Environment.NewLine}New Insertion Into {Options.VisualStudioBranchName} Started{Environment.NewLine}");
GitPullRequest pullRequest = null;
var shouldRollBackGitChanges = false;
var newPackageFiles = new List <string>();
try
{
// Verify that the arguments we were passed authenticate correctly
Console.WriteLine($"Verifying given authentication for {Options.VSTSUri}");
try
{
ProjectCollection.Authenticate();
}
catch (Exception ex)
{
Console.WriteLine($"Could not authenticate with {Options.VSTSUri}");
Console.WriteLine(ex);
return(false, 0);
}
Console.WriteLine($"Verification succeeded for {Options.VSTSUri}");
// ********************** Create dummy PR *****************************
if (Options.CreateDummyPr)
{
try
{
pullRequest = await CreatePlaceholderBranchAsync(cancellationToken);
}
catch (Exception ex)
{
Console.WriteLine($"Unable to create placeholder PR for '{options.VisualStudioBranchName}'");
Console.WriteLine(ex);
return(false, 0);
}
if (pullRequest == null)
{
Console.WriteLine($"Unable to create placeholder PR for '{options.VisualStudioBranchName}'");
return(false, 0);
}
return(true, pullRequest.PullRequestId);
}
// ********************** Get Last Insertion *****************************
cancellationToken.ThrowIfCancellationRequested();
BuildVersion buildVersion;
Build buildToInsert;
Build latestBuild = null;
bool retainBuild = false;
// Get the version from DevOps Pipelines queue, e.g. Roslyn-Master-Signed-Release.
if (string.IsNullOrEmpty(Options.SpecificBuild))
{
buildToInsert = await GetLatestPassedBuildAsync(cancellationToken);
buildVersion = BuildVersion.FromTfsBuildNumber(buildToInsert.BuildNumber, Options.BuildQueueName);
Console.WriteLine("Found build number " + buildVersion);
// Get the latest build, whether passed or failed. If the buildToInsert has already been inserted but
// there is a later failing build, then send an error
latestBuild = await GetLatestBuildAsync(cancellationToken);
}
else
{
buildVersion = BuildVersion.FromString(Options.SpecificBuild);
buildToInsert = await GetSpecificBuildAsync(buildVersion, cancellationToken);
}
string commitSHA = buildToInsert.SourceVersion.Substring(0, 7);
string lastCommitUrl = string.Empty;
if (buildToInsert.Links.Links.ContainsKey("sourceVersionDisplayUri"))
{
// Get a link to the commit the build was built from.
var sourceLink = (ReferenceLink)buildToInsert.Links.Links["sourceVersionDisplayUri"];
lastCommitUrl = sourceLink.Href;
}
var insertionArtifacts = await GetInsertionArtifactsAsync(buildToInsert, cancellationToken);
Branch branch = null;
cancellationToken.ThrowIfCancellationRequested();
var useExistingPr = Options.UpdateExistingPr != 0;
if (useExistingPr)
{
// ****************** Update existing PR ***********************
pullRequest = await GetExistingPullRequestAsync(Options.UpdateExistingPr, cancellationToken);
branch = SwitchToBranchAndUpdate(pullRequest.SourceRefName, Options.VisualStudioBranchName, overwriteExistingChanges: Options.OverwritePr);
}
else
{
// ****************** Create Branch ***********************
Console.WriteLine("Creating New Branch");
branch = string.IsNullOrEmpty(Options.NewBranchName)
? null
: CreateBranch(cancellationToken);
}
shouldRollBackGitChanges = branch != null;
var enlistmentRoot = GetAbsolutePathForEnlistment();
var coreXT = CoreXT.Load(enlistmentRoot);
if (Options.InsertCoreXTPackages)
{
// ************** Update Nuget Packages For Branch************************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating Nuget Packages");
bool success = false;
(success, newPackageFiles) = UpdatePackages(
buildVersion,
coreXT,
insertionArtifacts.GetPackagesDirectory(),
cancellationToken);
retainBuild |= success;
// ************ Update .corext\Configs\default.config ********************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating CoreXT default.config file");
coreXT.SaveConfig();
// *********** Copy OptimizationInputs.props file ***********************
foreach (var propsFile in insertionArtifacts.GetOptProfPropertyFiles())
{
var targetDirectory = Path.Combine(enlistmentRoot, @"src\Tests\config\runsettings\Official\OptProf\External");
var targetFilePath = Path.Combine(targetDirectory, Path.GetFileName(propsFile));
Console.WriteLine($"Updating {targetFilePath}");
Directory.CreateDirectory(targetDirectory);
File.Copy(propsFile, targetFilePath, overwrite: true);
}
}
if (Options.UpdateCoreXTLibraries || Options.UpdateAssemblyVersions)
{
// ************** Update assembly versions ************************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating assembly versions");
UpdateAssemblyVersions(insertionArtifacts);
// if we got this far then we definitely need to retain this build
retainBuild = true;
}
// *********** Update toolset ********************
if (Options.InsertToolset)
{
UpdateToolsetPackage(insertionArtifacts, buildVersion, cancellationToken);
retainBuild = true;
}
// *********** Update .corext\Configs\components.json ********************
BuildVersion oldComponentVersion = default;
if (Options.InsertWillowPackages)
{
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating CoreXT components file");
var components = await GetLatestComponentsAsync(buildToInsert, cancellationToken);
var shouldSave = false;
foreach (var newComponent in components)
{
if (coreXT.TryGetComponentByName(newComponent.Name, out var oldComponent))
{
if (oldComponent.BuildVersion != default)
{
oldComponentVersion = oldComponent.BuildVersion;
}
coreXT.UpdateComponent(newComponent);
shouldSave = true;
}
}
if (shouldSave)
{
coreXT.SaveComponents();
retainBuild = true;
}
}
// ************* Ensure the build is retained on the servers *************
if (Options.RetainInsertedBuild && retainBuild && !buildToInsert.KeepForever.GetValueOrDefault())
{
Console.WriteLine("Marking inserted build for retention.");
buildToInsert.KeepForever = true;
var buildClient = ProjectCollection.GetClient <BuildHttpClient>();
await buildClient.UpdateBuildAsync(buildToInsert, buildToInsert.Id);
}
// ********************* Verify Build Completes **************************
if (Options.PartitionsToBuild != null)
{
Console.WriteLine($"Verifying build succeeds with changes");
foreach (var partition in Options.PartitionsToBuild)
{
Console.WriteLine($"Starting build of {partition}");
if (!(await CanBuildPartitionAsync(partition, cancellationToken)))
{
Console.WriteLine($"Build of partition {partition} failed");
return(false, 0);
}
Console.WriteLine($"Build of partition {partition} succeeded");
}
}
// ********************* Create pull request *****************************
var pullRequestId = 0;
if (branch != null)
{
cancellationToken.ThrowIfCancellationRequested();
var prDescription = $"Updating {Options.InsertionName} to {buildVersion} ([{commitSHA}]({lastCommitUrl}))";
if (useExistingPr && pullRequest != null)
{
// update an existing pr
try
{
branch = PushChanges(branch, buildVersion, cancellationToken, forcePush: true);
if (Options.OverwritePr)
{
pullRequest = await UpdatePullRequestDescriptionAsync(Options.UpdateExistingPr, prDescription, cancellationToken);
}
shouldRollBackGitChanges = false;
pullRequestId = pullRequest.PullRequestId;
}
catch (Exception ex)
{
Console.WriteLine($"Unable to update pull request for '{branch.FriendlyName}'");
Console.WriteLine(ex);
return(false, 0);
}
}
else
{
// create a new PR
Console.WriteLine($"Create Pull Request");
try
{
try
{
var oldBuild = await GetSpecificBuildAsync(oldComponentVersion, cancellationToken);
var(changes, diffLink) = await GetChangesBetweenBuildsAsync(oldBuild ?? buildToInsert, buildToInsert, cancellationToken);
prDescription = AppendDiffToDescription(prDescription, diffLink);
prDescription = AppendChangesToDescription(prDescription, oldBuild ?? buildToInsert, changes);
}
catch (Exception e)
{
Console.WriteLine("##vso[task.logissue type=warning] Failed to create diff links.");
Console.WriteLine($"##vso[task.logissue type=warning] {e.Message}");
}
branch = PushChanges(branch, buildVersion, cancellationToken);
pullRequest = await CreatePullRequestAsync(branch.FriendlyName, prDescription, buildVersion.ToString(), options.TitlePrefix, cancellationToken);
shouldRollBackGitChanges = false;
pullRequestId = pullRequest.PullRequestId;
}
catch (EmptyCommitException ecx)
{
Console.WriteLine($"Unable to create pull request for '{branch.FriendlyName}'");
Console.WriteLine(ecx);
return(false, 0);
}
catch (Exception ex)
{
Console.WriteLine($"Unable to create pull request for '{branch.FriendlyName}'");
Console.WriteLine(ex);
return(false, 0);
}
}
if (pullRequest == null)
{
Console.WriteLine($"Unable to create pull request for '{branch.FriendlyName}'");
return(false, 0);
}
}
// ********************* Create validation build *****************************
if (Options.QueueValidationBuild)
{
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Create Validation Build");
if (pullRequest == null)
{
Console.WriteLine("Unable to create a validation build: no pull request.");
return(false, 0);
}
try
{
await QueueBuildPolicy(pullRequest, "Request Perf DDRITs");
}
catch (Exception ex)
{
Console.WriteLine($"Unable to create a CloudBuild validation build for '{pullRequest.SourceRefName}'");
Console.WriteLine(ex);
}
}
return(true, pullRequestId);
}
catch (RepositoryNotFoundException ex)
{
Console.Error.WriteLine(ex.Message);
Console.Error.WriteLine(@"Please ensure a VS enlistment exists at the given path, or pass the `/enlistmentpath=C:\path\to\VS` argument on the command line.");
return(false, 0);
}
catch (Exception ex)
{
Console.WriteLine(ex);
return(false, 0);
}
finally
{
// ********************* Rollback Git Changes ****************************
if (shouldRollBackGitChanges)
{
try
{
Console.WriteLine("Rolling back git changes");
var rollBackCommit = Enlistment.Branches[Options.VisualStudioBranchName].Commits.First();
Enlistment.Reset(ResetMode.Hard, rollBackCommit);
Enlistment.RemoveUntrackedFiles();
}
catch (Exception ex)
{
Console.WriteLine(ex);
}
}
Options = null;
}
}
public static async Task <(bool success, int pullRequestId)> PerformInsertionAsync(
RoslynInsertionToolOptions options,
CancellationToken cancellationToken)
{
Options = options;
Console.WriteLine($"{Environment.NewLine}New Insertion Into {Options.VisualStudioBranchName} Started{Environment.NewLine}");
var newPackageFiles = new List <string>();
try
{
Console.WriteLine($"Verifying given authentication for {Options.VisualStudioRepoAzdoUri}");
try
{
VisualStudioRepoConnection.Authenticate();
}
catch (Exception ex)
{
LogError($"Could not authenticate with {Options.VisualStudioRepoAzdoUri}");
LogError(ex);
return(false, 0);
}
Console.WriteLine($"Verification succeeded for {Options.VisualStudioRepoAzdoUri}");
if (ComponentBuildConnection != VisualStudioRepoConnection)
{
Console.WriteLine($"Verifying given authentication for {Options.ComponentBuildAzdoUri}");
try
{
ComponentBuildConnection.Authenticate();
}
catch (Exception ex)
{
LogError($"Could not authenticate with {Options.ComponentBuildAzdoUri}");
LogError(ex);
return(false, 0);
}
Console.WriteLine($"Verification succeeded for {Options.ComponentBuildAzdoUri}");
}
// ********************** Create dummy PR *****************************
if (Options.CreateDummyPr)
{
GitPullRequest dummyPR;
try
{
dummyPR = await CreatePlaceholderVSBranchAsync(cancellationToken);
}
catch (Exception ex)
{
LogError($"Unable to create placeholder PR for '{options.VisualStudioBranchName}'");
LogError(ex);
return(false, 0);
}
if (dummyPR == null)
{
LogError($"Unable to create placeholder PR for '{options.VisualStudioBranchName}'");
return(false, 0);
}
return(true, dummyPR.PullRequestId);
}
// ********************** Get Last Insertion *****************************
cancellationToken.ThrowIfCancellationRequested();
BuildVersion buildVersion;
Build buildToInsert;
Build latestBuild = null;
bool retainBuild = false;
// Get the version from DevOps Pipelines queue, e.g. Roslyn-Main-Signed-Release.
if (string.IsNullOrEmpty(Options.SpecificBuild))
{
buildToInsert = await GetLatestPassedComponentBuildAsync(cancellationToken);
buildVersion = BuildVersion.FromTfsBuildNumber(buildToInsert.BuildNumber, Options.ComponentBuildQueueName);
Console.WriteLine("Found " + buildToInsert.Definition.Name + " build number " + buildVersion);
// Get the latest build, whether passed or failed. If the buildToInsert has already been inserted but
// there is a later failing build, then send an error
latestBuild = await GetLatestComponentBuildAsync(cancellationToken);
}
else
{
buildVersion = BuildVersion.FromString(Options.SpecificBuild);
buildToInsert = await GetSpecificComponentBuildAsync(buildVersion, cancellationToken);
}
var insertionArtifacts = await GetInsertionArtifactsAsync(buildToInsert, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
// *********** Look up existing PR ********************
var gitClient = VisualStudioRepoConnection.GetClient <GitHttpClient>();
var branches = await gitClient.GetRefsAsync(
VSRepoId,
filter : $"heads/{Options.VisualStudioBranchName}",
cancellationToken : cancellationToken);
var baseBranch = branches.Single(b => b.Name == $"refs/heads/{Options.VisualStudioBranchName}");
var pullRequestId = Options.UpdateExistingPr;
var useExistingPr = pullRequestId != 0;
GitPullRequest pullRequest;
string insertionBranchName;
if (useExistingPr)
{
pullRequest = await gitClient.GetPullRequestByIdAsync(pullRequestId, cancellationToken : cancellationToken);
insertionBranchName = pullRequest.SourceRefName.Substring("refs/heads/".Length);
var refs = await gitClient.GetRefsAsync(VSRepoId, filter : $"heads/{insertionBranchName}", cancellationToken : cancellationToken);
var insertionBranch = refs.Single(r => r.Name == $"refs/heads/{insertionBranchName}");
if (Options.OverwritePr)
{
// overwrite existing PR branch back to base before pushing new commit
var updateToBase = new GitRefUpdate
{
OldObjectId = insertionBranch.ObjectId,
NewObjectId = baseBranch.ObjectId,
Name = $"refs/heads/{insertionBranchName}"
};
var results = await gitClient.UpdateRefsAsync(new[] { updateToBase }, VSRepoId, cancellationToken : cancellationToken);
foreach (var result in results)
{
if (!result.Success)
{
LogError("Failed to overwrite PR: " + result.CustomMessage);
}
}
}
else
{
// not overwriting PR, so the insertion branch is actually the base
baseBranch = insertionBranch;
}
}
else
{
pullRequest = null;
insertionBranchName = GetNewBranchName();
}
var allChanges = new List <GitChange>();
var coreXT = await CoreXT.Load(gitClient, baseBranch.ObjectId);
if (Options.InsertCoreXTPackages)
{
// ************** Update Nuget Packages For Branch************************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating Nuget Packages");
bool success;
(success, newPackageFiles) = UpdatePackages(
buildVersion,
coreXT,
insertionArtifacts.GetPackagesDirectory(),
Options.SkipCoreXTPackages,
cancellationToken);
retainBuild |= success;
// *********** Copy OptimizationInputs.props file ***********************
foreach (var propsFile in insertionArtifacts.GetOptProfPropertyFiles())
{
var propsFilename = Path.GetFileName(propsFile);
if (propsFilename == "dotnet-roslyn.props")
{
// Since the propsFilename is based on repo name, during Roslyn's transition from inserting
// from GH dotnet/roslyn builds to inserting from dnceng dotnet-roslyn builds, this will
// ensure that we look for the proper props filename.
propsFilename = "dotnet.roslyn.props";
}
var targetFilePath = $"src/Tests/config/runsettings/Official/OptProf/External/{propsFilename}";
var version = new GitVersionDescriptor {
VersionType = GitVersionType.Commit, Version = baseBranch.ObjectId
};
var stream = await gitClient.GetItemContentAsync(VSRepoId, targetFilePath, download : true, versionDescriptor : version);
var originalContent = new StreamReader(stream).ReadToEnd();
var newContent = File.ReadAllText(propsFile);
if (GetChangeOpt(targetFilePath, originalContent, newContent) is GitChange change)
{
allChanges.Add(change);
}
}
}
if (Options.UpdateCoreXTLibraries || Options.UpdateAssemblyVersions)
{
// ************** Update assembly versions ************************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating assembly versions");
if (await UpdateAssemblyVersionsOpt(gitClient, baseBranch.ObjectId, insertionArtifacts) is GitChange assemblyVersionChange)
{
allChanges.Add(assemblyVersionChange);
}
// if we got this far then we definitely need to retain this build
retainBuild = true;
}
// *********** Update toolset ********************
if (Options.InsertToolset)
{
UpdateToolsetPackage(coreXT, insertionArtifacts, buildVersion);
retainBuild = true;
}
// ************ Update .corext\Configs\default.config ********************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating CoreXT default.config and props files under src/ConfigData/Packages");
foreach (var configChange in coreXT.SaveConfigs())
{
if (configChange is not null)
{
allChanges.Add(configChange);
}
}
// *********** Update .corext\Configs\components.json ********************
BuildVersion oldComponentVersion = default;
if (Options.InsertWillowPackages)
{
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating CoreXT components file");
var components = GetLatestBuildComponents(buildToInsert, insertionArtifacts, cancellationToken);
var shouldSave = false;
foreach (var newComponent in components)
{
if (coreXT.TryGetComponentByName(newComponent.Name, out var oldComponent))
{
if (oldComponent.BuildVersion != default)
{
oldComponentVersion = oldComponent.BuildVersion;
}
coreXT.UpdateComponent(newComponent);
shouldSave = true;
}
}
if (shouldSave)
{
var allComponentChanges = coreXT.SaveComponents();
allChanges.AddRange(allComponentChanges);
retainBuild = true;
}
}
// ************* Ensure the build is retained on the servers *************
if (Options.RetainInsertedBuild && retainBuild && !buildToInsert.KeepForever.GetValueOrDefault())
{
await RetainComponentBuild(buildToInsert);
}
// ************* Bail out if there are no changes ************************
if (!allChanges.Any() && options.CherryPick.IsDefaultOrEmpty)
{
LogWarning("No meaningful changes since the last insertion was merged. PR will not be created or updated.");
return(true, 0);
}
// ********************* Create push *************************************
var currentCommit = baseBranch.ObjectId;
if (allChanges.Any())
{
var insertionBranchUpdate = new GitRefUpdate
{
Name = $"refs/heads/{insertionBranchName}",
OldObjectId = baseBranch.ObjectId
};
var commit = new GitCommitRef
{
Comment = $"Updating {Options.InsertionName} to {buildVersion}",
Changes = allChanges
};
var push = new GitPush
{
RefUpdates = new[] { insertionBranchUpdate },
Commits = new[] { commit }
};
push = await gitClient.CreatePushAsync(push, VSRepoId, cancellationToken : cancellationToken);
currentCommit = push.Commits.Single().CommitId;
}
// ********************* Cherry-pick VS commits *****************************
var cherryPickCommits = Options.CherryPick;
if (!cherryPickCommits.IsDefaultOrEmpty)
{
Console.WriteLine("Cherry-picking the following VS commits:");
foreach (var cherryPickCommit in cherryPickCommits)
{
var gc = await gitClient.GetCommitAsync(cherryPickCommit, VSRepoId, cancellationToken : cancellationToken);
Console.WriteLine("- " + gc.RemoteUrl);
}
var commitRefs = cherryPickCommits.Select(id => new GitCommitRef()
{
CommitId = id
}).ToArray();
var cherryPickBranchName = $"{insertionBranchName}-cherry-pick-{DateTime.Now:yyyyMMddHHmmss}";
var cherryPickArgs = new GitAsyncRefOperationParameters()
{
Source = new GitAsyncRefOperationSource()
{
CommitList = commitRefs
},
OntoRefName = $"refs/heads/{insertionBranchName}",
GeneratedRefName = $"refs/heads/{cherryPickBranchName}"
};
// Cherry-pick VS commits into insertion branch.
var cherryPick = await gitClient.CreateCherryPickAsync(cherryPickArgs, Options.VisualStudioRepoProjectName, VSRepoId, cancellationToken : cancellationToken);
while (cherryPick.Status < GitAsyncOperationStatus.Completed)
{
Console.WriteLine($"Cherry-pick progress: {cherryPick.DetailedStatus?.Progress ?? 0:P}");
await Task.Delay(5000);
cherryPick = await gitClient.GetCherryPickAsync(options.VisualStudioRepoProjectName, cherryPick.CherryPickId, VSRepoId, cancellationToken : cancellationToken);
}
Console.WriteLine($"Cherry-pick status: {cherryPick.Status}");
if (cherryPick.Status == GitAsyncOperationStatus.Completed)
{
var cherryPickBranch = await gitClient.GetBranchAsync(VSRepoId, cherryPickBranchName, cancellationToken : cancellationToken);
var addCherryPickedCommits = new GitRefUpdate
{
OldObjectId = currentCommit,
NewObjectId = cherryPickBranch.Commit.CommitId,
Name = $"refs/heads/{insertionBranchName}"
};
var results = await gitClient.UpdateRefsAsync(new[] { addCherryPickedCommits }, VSRepoId, cancellationToken : cancellationToken);
foreach (var result in results)
{
if (!result.Success)
{
LogError("Failed to reset ref to cherry-pick branch: " + result.CustomMessage);
}
}
}
else
{
LogError("Cherry-picking failed: " + cherryPick.DetailedStatus.FailureMessage);
}
}
// ********************* Create pull request *****************************
var oldBuild = await GetSpecificComponentBuildAsync(oldComponentVersion, cancellationToken);
var prDescriptionMarkdown = CreatePullRequestDescription(oldBuild, buildToInsert, useMarkdown: true);
if (buildToInsert.Result == BuildResult.PartiallySucceeded)
{
prDescriptionMarkdown += Environment.NewLine + ":warning: The build being inserted has partially succeeded.";
}
if (!useExistingPr || Options.OverwritePr)
{
try
{
var nl = Environment.NewLine;
if (oldBuild is null)
{
prDescriptionMarkdown += $"{nl}---{nl}Unable to find details for previous build ({oldComponentVersion}){nl}";
}
else
{
var(changes, diffLink) = await GetChangesBetweenBuildsAsync(oldBuild, buildToInsert, cancellationToken);
var diffDescription = changes.Any()
? $"[View Complete Diff of Changes]({diffLink})"
: "No source changes since previous insertion";
prDescriptionMarkdown += nl + "---" + nl + diffDescription + nl;
prDescriptionMarkdown = AppendChangesToDescription(prDescriptionMarkdown, oldBuild ?? buildToInsert, changes);
}
}
catch (Exception e)
{
LogWarning("Failed to create diff links.");
LogWarning(e.Message);
}
}
if (useExistingPr)
{
try
{
if (Options.OverwritePr)
{
pullRequest = await OverwritePullRequestAsync(pullRequestId, prDescriptionMarkdown, buildVersion.ToString(), cancellationToken);
}
pullRequestId = pullRequest.PullRequestId;
}
catch (Exception ex)
{
LogError($"Unable to update pull request for '{pullRequest.SourceRefName}'");
LogError(ex);
return(false, 0);
}
}
else
{
// create a new PR
Console.WriteLine($"Create Pull Request");
try
{
// If this insertion was queued for PR validation, for a dev branch, for a feature branch,
// or if no default reviewer is specified, then add the build queuer as a reviewer.
var isPrValidation = !string.IsNullOrEmpty(GetBuildPRNumber(buildToInsert));
var isDevOrFeatureBranch = Options.ComponentBranchName.StartsWith("dev/") || Options.ComponentBranchName.StartsWith("features/");
bool hasReviewer = !string.IsNullOrEmpty(Options.ReviewerGUID);
// Easiest way to get the reviewer GUIDs is to create a PR search in AzDo
// You'll get something like https://dev.azure.com/devdiv/DevDiv/_git/VS/pullrequests?_a=active&createdBy=GUID-here
var reviewerId = (isPrValidation || isDevOrFeatureBranch) || !hasReviewer
? buildToInsert.RequestedBy.Id
: Options.ReviewerGUID;
pullRequest = await CreateVSPullRequestAsync(insertionBranchName, prDescriptionMarkdown, buildVersion.ToString(), reviewerId, cancellationToken);
if (pullRequest == null)
{
LogError($"Unable to create pull request for '{insertionBranchName}'");
return(false, 0);
}
pullRequestId = pullRequest.PullRequestId;
}
catch (Exception ex)
{
LogError($"Unable to create pull request for '{insertionBranchName}'");
LogError(ex);
return(false, 0);
}
}
// ********************* Create validation build *****************************
if (Options.QueueValidationBuild)
{
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Create Validation Build");
try
{
if (Options.CreateDraftPr)
{
// When creating Draft PRs no policies are automatically started.
// If we do not queue a CloudBuild the Perf DDRITs request will
// spin waiting for a build to test against until it timesout.
await QueueVSBuildPolicy(pullRequest, "CloudBuild - PR");
}
await QueueVSBuildPolicy(pullRequest, "Request Perf DDRITs");
}
catch (Exception ex)
{
LogWarning($"Unable to create a CloudBuild validation build for '{insertionBranchName}'");
LogWarning(ex);
}
if (Options.CreateDraftPr)
{
// When creating Draft PRs no policies are automatically started.
await TryQueueVSBuildPolicy(pullRequest, "Insertion Hash Check", insertionBranchName);
await TryQueueVSBuildPolicy(pullRequest, "Insertion Sign Check", insertionBranchName);
await TryQueueVSBuildPolicy(pullRequest, "Insertion Symbol Check", insertionBranchName);
}
}
// ********************* Set PR to Auto-Complete *****************************
if (Options.SetAutoComplete)
{
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Set PR to Auto-Complete");
try
{
var prDescriptionText = CreatePullRequestDescription(oldBuild, buildToInsert, useMarkdown: false);
await SetAutoCompleteAsync(pullRequest, prDescriptionText, cancellationToken);
}
catch (Exception ex)
{
LogWarning($"Unable to Set PR to Auto-Complete for '{insertionBranchName}'");
LogWarning(ex);
}
}
return(true, pullRequestId);
}
catch (Exception ex)
{
LogError(ex);
return(false, 0);
}
finally
{
Options = null;
}
}
public static async Task PerformInsertionAsync(
RoslynInsertionToolOptions options,
ILogger log,
CancellationToken cancellationToken)
{
Options = options;
Log = log;
File.Delete(LogFilePath);
Log.Info($"{Environment.NewLine}New Insertion Into {Options.VisualStudioBranchName} Started{Environment.NewLine}");
GitPullRequest pullRequest = null;
var shouldRollBackGitChanges = false;
var newPackageFiles = new List <string>();
var isInsertionCancelled = false;
try
{
// Verify that the arguments we were passed authenticate correctly
Log.Trace($"Verifying given authentication for {Options.VSTSUri}");
try
{
ProjectCollection.Authenticate();
}
catch (Exception ex)
{
Log.Error($"Could not authenticate with {Options.VSTSUri}");
Log.Error(ex);
return;
}
Log.Trace($"Verification succeeded for {Options.VSTSUri}");
// ********************** Get Last Insertion *****************************
cancellationToken.ThrowIfCancellationRequested();
BuildVersion roslynBuildVersion;
Build newestBuild;
bool retainBuild = false;
// Get the version from TFS build queue, e.g. Roslyn-Master-Signed-Release.
// We assume all CoreXT packages we build (Roslyn and all dependencies we
// insert) have the same version.
if (string.IsNullOrEmpty(Options.SpecificBuild))
{
newestBuild = await GetLatestBuildAsync(cancellationToken);
roslynBuildVersion = BuildVersion.FromTfsBuildNumber(newestBuild.BuildNumber, Options.RoslynBuildQueueName);
}
else
{
roslynBuildVersion = BuildVersion.FromString(Options.SpecificBuild);
newestBuild = await GetSpecificBuildAsync(roslynBuildVersion, cancellationToken);
}
// ****************** Get Latest and Create Branch ***********************
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Getting Latest From {Options.VisualStudioBranchName} and Creating New Branch");
var branch = string.IsNullOrEmpty(Options.NewBranchName)
? null
: GetLatestAndCreateBranch(cancellationToken);
shouldRollBackGitChanges = branch != null;
var coreXT = CoreXT.Load(GetAbsolutePathForEnlistment());
// ************** Update Nuget Packages For Branch************************
if (Options.InsertCoreXTPackages)
{
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Updating Nuget Packages");
retainBuild |= UpdatePackages(
newPackageFiles,
roslynBuildVersion,
coreXT,
GetDevDivPackagesDirPath(roslynBuildVersion),
cancellationToken);
// ************ Update .corext\Configs\default.config ********************
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Updating CoreXT config file");
coreXT.SaveConfig();
// ************** Update paths to CoreFX libraries ************************
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Update paths to CoreFX libraries");
if (!await TryUpdateFileAsync(
Path.Combine("ProductData", "ContractAssemblies.props"),
roslynBuildVersion,
onlyCopyIfFileDoesNotExistAtDestination: false,
cancellationToken: cancellationToken))
{
return;
}
// ************** Update assembly versions ************************
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Updating assembly versions");
UpdateAssemblyVersions(roslynBuildVersion);
// if we got this far then we definitely need to retain this build
retainBuild = true;
}
// *********** Update toolset ********************
if (Options.InsertToolset)
{
UpdateToolsetPackage(roslynBuildVersion, cancellationToken);
retainBuild = true;
}
// *********** Update .corext\Configs\components.json ********************
if (Options.InsertWillowPackages)
{
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Updating CoreXT components file");
var components = await GetLatestComponentsAsync(newestBuild, cancellationToken);
var shouldSave = false;
foreach (var newComponent in components)
{
if (coreXT.TryGetComponentByName(newComponent.Name, out var oldComponent))
{
coreXT.UpdateComponent(newComponent);
shouldSave = true;
}
}
if (shouldSave)
{
coreXT.SaveComponents();
retainBuild = true;
}
}
// ************* Ensure the build is retained on the servers *************
if (Options.RetainInsertedBuild && retainBuild && !newestBuild.KeepForever.GetValueOrDefault())
{
Log.Info("Marking inserted build for retention.");
newestBuild.KeepForever = true;
var buildClient = ProjectCollection.GetClient <BuildHttpClient>();
await buildClient.UpdateBuildAsync(newestBuild, newestBuild.Id);
}
// ********************* Verify Build Completes **************************
if (Options.PartitionsToBuild != null)
{
Log.Info($"Verifying build succeeds with changes");
foreach (var partition in Options.PartitionsToBuild)
{
Log.Info($"Starting build of {partition}");
if (!(await CanBuildPartitionAsync(partition, cancellationToken)))
{
Log.Error($"Build of partition {partition} failed");
return;
}
Log.Info($"Build of partition {partition} succeeded");
}
}
// ********************* Create pull request *****************************
if (branch != null)
{
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Create Pull Request");
try
{
PushChanges(branch, roslynBuildVersion, cancellationToken);
pullRequest = await CreatePullRequestAsync(branch.FriendlyName, $"Updating {Options.InsertionName} to {roslynBuildVersion}", cancellationToken);
shouldRollBackGitChanges = false;
}
catch (EmptyCommitException ecx)
{
isInsertionCancelled = true;
Log.Warn($"Unable to create pull request for '{branch.FriendlyName}'");
Log.Warn(ecx);
return;
}
catch (Exception ex)
{
Log.Error($"Unable to create pull request for '{branch.FriendlyName}'");
Log.Error(ex);
return;
}
if (pullRequest == null)
{
Log.Error($"Unable to create pull request for '{branch.FriendlyName}'");
return;
}
}
// ********************* Create validation build *****************************
if (Options.QueueValidationBuild)
{
cancellationToken.ThrowIfCancellationRequested();
Log.Info($"Create Validation Build");
if (pullRequest == null)
{
Log.Error("Unable to create a validation build: no pull request.");
return;
}
string buildUrl;
int buildId;
try
{
var build = await QueueValidationBuildAsync(pullRequest.SourceRefName);
buildId = build.Id;
var buildWebLink = (ReferenceLink)build.Links.Links["web"];
buildUrl = buildWebLink.Href;
Log.Info($"Created build {buildUrl}");
}
catch (Exception ex)
{
Log.Error($"Unable to create a validation build for '{pullRequest.SourceRefName}'");
Log.Error(ex);
return;
}
try
{
string commentContent = $"Validation build: [{buildId}]({buildUrl})";
var commentThread = await CreateGitPullRequestCommentThread(pullRequest.PullRequestId, commentContent);
Log.Info($"Added comment '{commentContent} to the pull request'");
}
catch (Exception ex)
{
Log.Error($"Unable to add comment to PR about validation build");
Log.Error(ex);
return;
}
}
}
catch (Exception ex)
{
if (ex is OutdatedPackageException || ex is OperationCanceledException)
{
isInsertionCancelled = true;
}
Log.Error(ex);
}
finally
{
// ************************* Flush Log ***********************************
Log.Factory.Flush();
// ********************* Rollback Git Changes ****************************
if (shouldRollBackGitChanges)
{
try
{
Log.Info("Rolling back git changes");
var rollBackCommit = Enlistment.Branches[Options.VisualStudioBranchName].Commits.First();
Enlistment.Reset(ResetMode.Hard, rollBackCommit);
Enlistment.RemoveUntrackedFiles();
}
catch (Exception ex)
{
Log.Error(ex);
}
}
// ********************* Send Status Mail ********************************
if (!string.IsNullOrEmpty(Options.EmailServerName) &&
!string.IsNullOrEmpty(Options.MailRecipient))
{
try
{
SendMail(pullRequest, newPackageFiles, isInsertionCancelled);
}
catch (Exception ex)
{
Log.Error($"Unable to send mail, EmailServerName: '{Options.EmailServerName}', MailRecipient: '{Options.MailRecipient}'");
Log.Error(ex);
}
}
Options = null;
Log = null;
}
}
private static void PushChanges(Branch branch, BuildVersion newRoslynVersion, CancellationToken cancellationToken)
{
StageFiles(newRoslynVersion, cancellationToken);
CommitStagedChanges(newRoslynVersion, cancellationToken);
PushChanges(branch, cancellationToken);
}
