private static async Task <GitPullRequest> CreatePlaceholderVSBranchAsync(CancellationToken cancellationToken)
{
cancellationToken.ThrowIfCancellationRequested();
var gitClient = VisualStudioRepoConnection.GetClient <GitHttpClient>();
var repository = await gitClient.GetRepositoryAsync(project : Options.VisualStudioRepoProjectName, repositoryId : "VS", cancellationToken : cancellationToken);
var refs = await gitClient.GetRefsAsync(
repository.Id,
filter : $"heads/{Options.VisualStudioBranchName}",
cancellationToken : cancellationToken);
GitRef sourceBranch = refs.Single(r => r.Name == $"refs/heads/{Options.VisualStudioBranchName}");
var branchName = GetNewBranchName();
_ = await gitClient.CreatePushAsync(new GitPush()
{
RefUpdates = new[] {
new GitRefUpdate()
{
Name = $"refs/heads/{branchName}",
OldObjectId = sourceBranch.ObjectId,
}
},
Commits = new[] {
new GitCommitRef()
{
Comment = $"PLACEHOLDER INSERTION FOR {Options.InsertionName}",
Changes = new GitChange[]
{
new GitChange()
{
ChangeType = VersionControlChangeType.Delete,
Item = new GitItem()
{
Path = "/Init.ps1"
}
},
}
},
},
}, repository.Id, cancellationToken : cancellationToken);
return(await CreateVSPullRequestAsync(
branchName,
$"PLACEHOLDER INSERTION FOR {Options.InsertionName}",
"Not Specified",
reviewerId : Options.ReviewerGUID,
cancellationToken));
}
public static async Task <(bool success, int pullRequestId)> PerformInsertionAsync(
RoslynInsertionToolOptions options,
CancellationToken cancellationToken)
{
Options = options;
Console.WriteLine($"{Environment.NewLine}New Insertion Into {Options.VisualStudioBranchName} Started{Environment.NewLine}");
var newPackageFiles = new List <string>();
try
{
Console.WriteLine($"Verifying given authentication for {Options.VisualStudioRepoAzdoUri}");
try
{
VisualStudioRepoConnection.Authenticate();
}
catch (Exception ex)
{
LogError($"Could not authenticate with {Options.VisualStudioRepoAzdoUri}");
LogError(ex);
return(false, 0);
}
Console.WriteLine($"Verification succeeded for {Options.VisualStudioRepoAzdoUri}");
if (ComponentBuildConnection != VisualStudioRepoConnection)
{
Console.WriteLine($"Verifying given authentication for {Options.ComponentBuildAzdoUri}");
try
{
ComponentBuildConnection.Authenticate();
}
catch (Exception ex)
{
LogError($"Could not authenticate with {Options.ComponentBuildAzdoUri}");
LogError(ex);
return(false, 0);
}
Console.WriteLine($"Verification succeeded for {Options.ComponentBuildAzdoUri}");
}
// ********************** Create dummy PR *****************************
if (Options.CreateDummyPr)
{
GitPullRequest dummyPR;
try
{
dummyPR = await CreatePlaceholderVSBranchAsync(cancellationToken);
}
catch (Exception ex)
{
LogError($"Unable to create placeholder PR for '{options.VisualStudioBranchName}'");
LogError(ex);
return(false, 0);
}
if (dummyPR == null)
{
LogError($"Unable to create placeholder PR for '{options.VisualStudioBranchName}'");
return(false, 0);
}
return(true, dummyPR.PullRequestId);
}
// ********************** Get Last Insertion *****************************
cancellationToken.ThrowIfCancellationRequested();
BuildVersion buildVersion;
Build buildToInsert;
Build latestBuild = null;
bool retainBuild = false;
// Get the version from DevOps Pipelines queue, e.g. Roslyn-Main-Signed-Release.
if (string.IsNullOrEmpty(Options.SpecificBuild))
{
buildToInsert = await GetLatestPassedComponentBuildAsync(cancellationToken);
buildVersion = BuildVersion.FromTfsBuildNumber(buildToInsert.BuildNumber, Options.ComponentBuildQueueName);
Console.WriteLine("Found " + buildToInsert.Definition.Name + " build number " + buildVersion);
// Get the latest build, whether passed or failed. If the buildToInsert has already been inserted but
// there is a later failing build, then send an error
latestBuild = await GetLatestComponentBuildAsync(cancellationToken);
}
else
{
buildVersion = BuildVersion.FromString(Options.SpecificBuild);
buildToInsert = await GetSpecificComponentBuildAsync(buildVersion, cancellationToken);
}
var insertionArtifacts = await GetInsertionArtifactsAsync(buildToInsert, cancellationToken);
cancellationToken.ThrowIfCancellationRequested();
// *********** Look up existing PR ********************
var gitClient = VisualStudioRepoConnection.GetClient <GitHttpClient>();
var branches = await gitClient.GetRefsAsync(
VSRepoId,
filter : $"heads/{Options.VisualStudioBranchName}",
cancellationToken : cancellationToken);
var baseBranch = branches.Single(b => b.Name == $"refs/heads/{Options.VisualStudioBranchName}");
var pullRequestId = Options.UpdateExistingPr;
var useExistingPr = pullRequestId != 0;
GitPullRequest pullRequest;
string insertionBranchName;
if (useExistingPr)
{
pullRequest = await gitClient.GetPullRequestByIdAsync(pullRequestId, cancellationToken : cancellationToken);
insertionBranchName = pullRequest.SourceRefName.Substring("refs/heads/".Length);
var refs = await gitClient.GetRefsAsync(VSRepoId, filter : $"heads/{insertionBranchName}", cancellationToken : cancellationToken);
var insertionBranch = refs.Single(r => r.Name == $"refs/heads/{insertionBranchName}");
if (Options.OverwritePr)
{
// overwrite existing PR branch back to base before pushing new commit
var updateToBase = new GitRefUpdate
{
OldObjectId = insertionBranch.ObjectId,
NewObjectId = baseBranch.ObjectId,
Name = $"refs/heads/{insertionBranchName}"
};
var results = await gitClient.UpdateRefsAsync(new[] { updateToBase }, VSRepoId, cancellationToken : cancellationToken);
foreach (var result in results)
{
if (!result.Success)
{
LogError("Failed to overwrite PR: " + result.CustomMessage);
}
}
}
else
{
// not overwriting PR, so the insertion branch is actually the base
baseBranch = insertionBranch;
}
}
else
{
pullRequest = null;
insertionBranchName = GetNewBranchName();
}
var allChanges = new List <GitChange>();
var coreXT = await CoreXT.Load(gitClient, baseBranch.ObjectId);
if (Options.InsertCoreXTPackages)
{
// ************** Update Nuget Packages For Branch************************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating Nuget Packages");
bool success;
(success, newPackageFiles) = UpdatePackages(
buildVersion,
coreXT,
insertionArtifacts.GetPackagesDirectory(),
Options.SkipCoreXTPackages,
cancellationToken);
retainBuild |= success;
// *********** Copy OptimizationInputs.props file ***********************
foreach (var propsFile in insertionArtifacts.GetOptProfPropertyFiles())
{
var propsFilename = Path.GetFileName(propsFile);
if (propsFilename == "dotnet-roslyn.props")
{
// Since the propsFilename is based on repo name, during Roslyn's transition from inserting
// from GH dotnet/roslyn builds to inserting from dnceng dotnet-roslyn builds, this will
// ensure that we look for the proper props filename.
propsFilename = "dotnet.roslyn.props";
}
var targetFilePath = $"src/Tests/config/runsettings/Official/OptProf/External/{propsFilename}";
var version = new GitVersionDescriptor {
VersionType = GitVersionType.Commit, Version = baseBranch.ObjectId
};
var stream = await gitClient.GetItemContentAsync(VSRepoId, targetFilePath, download : true, versionDescriptor : version);
var originalContent = new StreamReader(stream).ReadToEnd();
var newContent = File.ReadAllText(propsFile);
if (GetChangeOpt(targetFilePath, originalContent, newContent) is GitChange change)
{
allChanges.Add(change);
}
}
}
if (Options.UpdateCoreXTLibraries || Options.UpdateAssemblyVersions)
{
// ************** Update assembly versions ************************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating assembly versions");
if (await UpdateAssemblyVersionsOpt(gitClient, baseBranch.ObjectId, insertionArtifacts) is GitChange assemblyVersionChange)
{
allChanges.Add(assemblyVersionChange);
}
// if we got this far then we definitely need to retain this build
retainBuild = true;
}
// *********** Update toolset ********************
if (Options.InsertToolset)
{
UpdateToolsetPackage(coreXT, insertionArtifacts, buildVersion);
retainBuild = true;
}
// ************ Update .corext\Configs\default.config ********************
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating CoreXT default.config and props files under src/ConfigData/Packages");
foreach (var configChange in coreXT.SaveConfigs())
{
if (configChange is not null)
{
allChanges.Add(configChange);
}
}
// *********** Update .corext\Configs\components.json ********************
BuildVersion oldComponentVersion = default;
if (Options.InsertWillowPackages)
{
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Updating CoreXT components file");
var components = GetLatestBuildComponents(buildToInsert, insertionArtifacts, cancellationToken);
var shouldSave = false;
foreach (var newComponent in components)
{
if (coreXT.TryGetComponentByName(newComponent.Name, out var oldComponent))
{
if (oldComponent.BuildVersion != default)
{
oldComponentVersion = oldComponent.BuildVersion;
}
coreXT.UpdateComponent(newComponent);
shouldSave = true;
}
}
if (shouldSave)
{
var allComponentChanges = coreXT.SaveComponents();
allChanges.AddRange(allComponentChanges);
retainBuild = true;
}
}
// ************* Ensure the build is retained on the servers *************
if (Options.RetainInsertedBuild && retainBuild && !buildToInsert.KeepForever.GetValueOrDefault())
{
await RetainComponentBuild(buildToInsert);
}
// ************* Bail out if there are no changes ************************
if (!allChanges.Any() && options.CherryPick.IsDefaultOrEmpty)
{
LogWarning("No meaningful changes since the last insertion was merged. PR will not be created or updated.");
return(true, 0);
}
// ********************* Create push *************************************
var currentCommit = baseBranch.ObjectId;
if (allChanges.Any())
{
var insertionBranchUpdate = new GitRefUpdate
{
Name = $"refs/heads/{insertionBranchName}",
OldObjectId = baseBranch.ObjectId
};
var commit = new GitCommitRef
{
Comment = $"Updating {Options.InsertionName} to {buildVersion}",
Changes = allChanges
};
var push = new GitPush
{
RefUpdates = new[] { insertionBranchUpdate },
Commits = new[] { commit }
};
push = await gitClient.CreatePushAsync(push, VSRepoId, cancellationToken : cancellationToken);
currentCommit = push.Commits.Single().CommitId;
}
// ********************* Cherry-pick VS commits *****************************
var cherryPickCommits = Options.CherryPick;
if (!cherryPickCommits.IsDefaultOrEmpty)
{
Console.WriteLine("Cherry-picking the following VS commits:");
foreach (var cherryPickCommit in cherryPickCommits)
{
var gc = await gitClient.GetCommitAsync(cherryPickCommit, VSRepoId, cancellationToken : cancellationToken);
Console.WriteLine("- " + gc.RemoteUrl);
}
var commitRefs = cherryPickCommits.Select(id => new GitCommitRef()
{
CommitId = id
}).ToArray();
var cherryPickBranchName = $"{insertionBranchName}-cherry-pick-{DateTime.Now:yyyyMMddHHmmss}";
var cherryPickArgs = new GitAsyncRefOperationParameters()
{
Source = new GitAsyncRefOperationSource()
{
CommitList = commitRefs
},
OntoRefName = $"refs/heads/{insertionBranchName}",
GeneratedRefName = $"refs/heads/{cherryPickBranchName}"
};
// Cherry-pick VS commits into insertion branch.
var cherryPick = await gitClient.CreateCherryPickAsync(cherryPickArgs, Options.VisualStudioRepoProjectName, VSRepoId, cancellationToken : cancellationToken);
while (cherryPick.Status < GitAsyncOperationStatus.Completed)
{
Console.WriteLine($"Cherry-pick progress: {cherryPick.DetailedStatus?.Progress ?? 0:P}");
await Task.Delay(5000);
cherryPick = await gitClient.GetCherryPickAsync(options.VisualStudioRepoProjectName, cherryPick.CherryPickId, VSRepoId, cancellationToken : cancellationToken);
}
Console.WriteLine($"Cherry-pick status: {cherryPick.Status}");
if (cherryPick.Status == GitAsyncOperationStatus.Completed)
{
var cherryPickBranch = await gitClient.GetBranchAsync(VSRepoId, cherryPickBranchName, cancellationToken : cancellationToken);
var addCherryPickedCommits = new GitRefUpdate
{
OldObjectId = currentCommit,
NewObjectId = cherryPickBranch.Commit.CommitId,
Name = $"refs/heads/{insertionBranchName}"
};
var results = await gitClient.UpdateRefsAsync(new[] { addCherryPickedCommits }, VSRepoId, cancellationToken : cancellationToken);
foreach (var result in results)
{
if (!result.Success)
{
LogError("Failed to reset ref to cherry-pick branch: " + result.CustomMessage);
}
}
}
else
{
LogError("Cherry-picking failed: " + cherryPick.DetailedStatus.FailureMessage);
}
}
// ********************* Create pull request *****************************
var oldBuild = await GetSpecificComponentBuildAsync(oldComponentVersion, cancellationToken);
var prDescriptionMarkdown = CreatePullRequestDescription(oldBuild, buildToInsert, useMarkdown: true);
if (buildToInsert.Result == BuildResult.PartiallySucceeded)
{
prDescriptionMarkdown += Environment.NewLine + ":warning: The build being inserted has partially succeeded.";
}
if (!useExistingPr || Options.OverwritePr)
{
try
{
var nl = Environment.NewLine;
if (oldBuild is null)
{
prDescriptionMarkdown += $"{nl}---{nl}Unable to find details for previous build ({oldComponentVersion}){nl}";
}
else
{
var(changes, diffLink) = await GetChangesBetweenBuildsAsync(oldBuild, buildToInsert, cancellationToken);
var diffDescription = changes.Any()
? $"[View Complete Diff of Changes]({diffLink})"
: "No source changes since previous insertion";
prDescriptionMarkdown += nl + "---" + nl + diffDescription + nl;
prDescriptionMarkdown = AppendChangesToDescription(prDescriptionMarkdown, oldBuild ?? buildToInsert, changes);
}
}
catch (Exception e)
{
LogWarning("Failed to create diff links.");
LogWarning(e.Message);
}
}
if (useExistingPr)
{
try
{
if (Options.OverwritePr)
{
pullRequest = await OverwritePullRequestAsync(pullRequestId, prDescriptionMarkdown, buildVersion.ToString(), cancellationToken);
}
pullRequestId = pullRequest.PullRequestId;
}
catch (Exception ex)
{
LogError($"Unable to update pull request for '{pullRequest.SourceRefName}'");
LogError(ex);
return(false, 0);
}
}
else
{
// create a new PR
Console.WriteLine($"Create Pull Request");
try
{
// If this insertion was queued for PR validation, for a dev branch, for a feature branch,
// or if no default reviewer is specified, then add the build queuer as a reviewer.
var isPrValidation = !string.IsNullOrEmpty(GetBuildPRNumber(buildToInsert));
var isDevOrFeatureBranch = Options.ComponentBranchName.StartsWith("dev/") || Options.ComponentBranchName.StartsWith("features/");
bool hasReviewer = !string.IsNullOrEmpty(Options.ReviewerGUID);
// Easiest way to get the reviewer GUIDs is to create a PR search in AzDo
// You'll get something like https://dev.azure.com/devdiv/DevDiv/_git/VS/pullrequests?_a=active&createdBy=GUID-here
var reviewerId = (isPrValidation || isDevOrFeatureBranch) || !hasReviewer
? buildToInsert.RequestedBy.Id
: Options.ReviewerGUID;
pullRequest = await CreateVSPullRequestAsync(insertionBranchName, prDescriptionMarkdown, buildVersion.ToString(), reviewerId, cancellationToken);
if (pullRequest == null)
{
LogError($"Unable to create pull request for '{insertionBranchName}'");
return(false, 0);
}
pullRequestId = pullRequest.PullRequestId;
}
catch (Exception ex)
{
LogError($"Unable to create pull request for '{insertionBranchName}'");
LogError(ex);
return(false, 0);
}
}
// ********************* Create validation build *****************************
if (Options.QueueValidationBuild)
{
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Create Validation Build");
try
{
if (Options.CreateDraftPr)
{
// When creating Draft PRs no policies are automatically started.
// If we do not queue a CloudBuild the Perf DDRITs request will
// spin waiting for a build to test against until it timesout.
await QueueVSBuildPolicy(pullRequest, "CloudBuild - PR");
}
await QueueVSBuildPolicy(pullRequest, "Request Perf DDRITs");
}
catch (Exception ex)
{
LogWarning($"Unable to create a CloudBuild validation build for '{insertionBranchName}'");
LogWarning(ex);
}
if (Options.CreateDraftPr)
{
// When creating Draft PRs no policies are automatically started.
await TryQueueVSBuildPolicy(pullRequest, "Insertion Hash Check", insertionBranchName);
await TryQueueVSBuildPolicy(pullRequest, "Insertion Sign Check", insertionBranchName);
await TryQueueVSBuildPolicy(pullRequest, "Insertion Symbol Check", insertionBranchName);
}
}
// ********************* Set PR to Auto-Complete *****************************
if (Options.SetAutoComplete)
{
cancellationToken.ThrowIfCancellationRequested();
Console.WriteLine($"Set PR to Auto-Complete");
try
{
var prDescriptionText = CreatePullRequestDescription(oldBuild, buildToInsert, useMarkdown: false);
await SetAutoCompleteAsync(pullRequest, prDescriptionText, cancellationToken);
}
catch (Exception ex)
{
LogWarning($"Unable to Set PR to Auto-Complete for '{insertionBranchName}'");
LogWarning(ex);
}
}
return(true, pullRequestId);
}
catch (Exception ex)
{
LogError(ex);
return(false, 0);
}
finally
{
Options = null;
}
}