public void TestMergeBuildManifests()
{
var orchestratedModel = new OrchestratedBuildModel(new BuildIdentity("Orchestrated", "123"))
{
Endpoints = new List <EndpointModel>
{
EndpointModel.CreateOrchestratedBlobFeed("http://example.org")
}
};
orchestratedModel.AddParticipantBuild(CreatePackageOnlyBuildManifestModel());
orchestratedModel.AddParticipantBuild(BuildModel.Parse(XElement.Parse(ExampleBuildString)));
XElement modelXml = orchestratedModel.ToXml();
XElement xml = XElement.Parse(@"
<OrchestratedBuild Name=""Orchestrated"" BuildId=""123"">
<Endpoint Id=""Orchestrated"" Type=""BlobFeed"" Url=""http://example.org"">
<Package Id=""Foo"" Version=""1.2.3-example"" />
<Package Id=""runtime.rhel.6-x64.Microsoft.Private.CoreFx.NETCoreApp"" Version=""4.5.0-preview1-25929-04"" Category=""noship"" />
<Package Id=""System.Memory"" Version=""4.5.0-preview1-25927-01"" />
<Blob Id=""symbols/inner/blank-dir-nonshipping"" NonShipping=""false"" />
<Blob Id=""symbols/runtime.rhel.6-x64.Microsoft.Private.CoreFx.NETCoreApp.4.5.0-preview1-25929-04.symbols.nupkg"" />
<Blob Id=""symbols/System.ValueTuple.4.5.0-preview1-25929-04.symbols.nupkg"" NonShipping=""true"" />
</Endpoint>
<Build Name=""SimpleBuildManifest"" BuildId=""123"" />
<Build Name=""corefx"" BuildId=""20171129-04"" Branch=""master"" Commit=""defb6d52047cc3d6b5f5d0853b0afdb1512dfbf4"" />
</OrchestratedBuild>");
Assert.True(XNode.DeepEquals(xml, modelXml));
}
public override bool Execute()
{
// Leave out attributes if they would just have empty string values.
if (ManifestBranch == string.Empty)
{
ManifestBranch = null;
}
if (ManifestCommit == string.Empty)
{
ManifestCommit = null;
}
var identity = new BuildIdentity
{
Name = ManifestName,
BuildId = ManifestBuildId,
Branch = ManifestBranch,
Commit = ManifestCommit,
IsStable = IsStable,
VersionStamp = VersionStamp
};
var orchestratedBuild = new OrchestratedBuildModel(identity)
{
Endpoints = new List <EndpointModel>
{
EndpointModel.CreateOrchestratedBlobFeed(BlobFeedUrl)
}
};
foreach (ITaskItem buildManifestFile in BuildManifestFiles)
{
string contents = System.IO.File.ReadAllText(buildManifestFile.ItemSpec);
BuildModel build = BuildModel.Parse(XElement.Parse(contents));
foreach (PackageArtifactModel package in build.Artifacts.Packages)
{
package.OriginBuildName = build.Identity.Name;
}
orchestratedBuild.AddParticipantBuild(build);
}
System.IO.File.WriteAllText(File, orchestratedBuild.ToXml().ToString());
return(!Log.HasLoggedErrors);
}
public async Task TestPushConflictingChangeAsync()
{
var mockGitHub = new Mock <IGitHubClient>(MockBehavior.Strict);
var client = new BuildManifestClient(mockGitHub.Object);
var proj = new GitHubProject("versions", "dotnet");
string @ref = "heads/master";
string basePath = "build-info/dotnet/product/cli/master";
string message = "Test change manifest commit";
string addSemaphorePath = "add-identity.semaphore";
var fakeExistingBuild = new OrchestratedBuildModel(new BuildIdentity {
Name = "orch", BuildId = "123"
});
var fakeNewExistingBuild = new OrchestratedBuildModel(new BuildIdentity {
Name = "orch", BuildId = "456"
});
string fakeCommitHash = "fakeCommitHash";
mockGitHub
.Setup(c => c.GetReferenceAsync(proj, @ref))
.ReturnsAsync(() => new GitReference
{
Object = new GitReferenceObject {
Sha = fakeCommitHash
}
});
mockGitHub
.Setup(c => c.GetGitHubFileContentsAsync(It.IsAny <string>(), proj, fakeCommitHash))
.ReturnsAsync(() => fakeNewExistingBuild.ToXml().ToString());
var pushClient = client.PushChangeAsync(
new BuildManifestChange(
new BuildManifestLocation(proj, @ref, basePath),
message,
fakeExistingBuild.Identity.BuildId,
new[] { addSemaphorePath },
_ => { }
));
Func <Task> act = async() => { await pushClient; };
await act.Should().ThrowAsync <ManifestChangeOutOfDateException>();
mockGitHub.VerifyAll();
}
public async Task PushNewBuildAsync(
BuildManifestLocation location,
OrchestratedBuildModel build,
IEnumerable <SupplementaryUploadRequest> supplementaryUploads,
string message)
{
await Retry.RunAsync(async attempt =>
{
GitReference remoteRef = await _github.GetReferenceAsync(
location.GitHubProject,
location.GitHubRef);
string remoteCommit = remoteRef.Object.Sha;
Trace.TraceInformation($"Creating update on remote commit: {remoteCommit}");
IEnumerable <SupplementaryUploadRequest> uploads = supplementaryUploads.NullAsEmpty()
.Concat(new[]
{
new SupplementaryUploadRequest
{
Path = BuildManifestXmlName,
Contents = build.ToXml().ToString()
},
new SupplementaryUploadRequest
{
Path = SemaphoreModel.BuildSemaphorePath,
Contents = new SemaphoreModel
{
BuildId = build.Identity.BuildId
}.ToFileContent()
}
})
.ToArray();
return(await PushUploadsAsync(location, message, remoteCommit, uploads));
});
}
public override bool Execute()
{
// Leave out attributes if they would just have empty string values.
if (ManifestBranch == string.Empty)
{
ManifestBranch = null;
}
if (ManifestCommit == string.Empty)
{
ManifestCommit = null;
}
var orchestratedBuild = new OrchestratedBuildModel(new BuildIdentity(
ManifestName,
ManifestBuildId,
ManifestBranch,
ManifestCommit))
{
Endpoints = new List <EndpointModel>
{
EndpointModel.CreateOrchestratedBlobFeed(BlobFeedUrl)
}
};
foreach (ITaskItem buildManifestFile in BuildManifestFiles)
{
string contents = System.IO.File.ReadAllText(buildManifestFile.ItemSpec);
BuildModel build = BuildModel.Parse(XElement.Parse(contents));
orchestratedBuild.AddParticipantBuild(build);
}
System.IO.File.WriteAllText(File, orchestratedBuild.ToXml().ToString());
return(!Log.HasLoggedErrors);
}
public async Task TestPushNewBuildAsync()
{
var mockGitHub = new Mock <IGitHubClient>(MockBehavior.Strict);
var client = new BuildManifestClient(mockGitHub.Object);
var build = new OrchestratedBuildModel(new BuildIdentity {
Name = "orch", BuildId = "123"
});
var proj = new GitHubProject("versions", "dotnet");
string @ref = "heads/master";
string basePath = "build-info/dotnet/product/cli/master";
string message = "Test build upload commit";
string fakeCommitHash = "fakeCommitHash";
string fakeTreeHash = "fakeTreeHash";
string fakeNewCommitHash = "fakeNewCommitHash";
mockGitHub
.Setup(c => c.GetReferenceAsync(proj, @ref))
.ReturnsAsync(() => new GitReference
{
Object = new GitReferenceObject {
Sha = fakeCommitHash
}
});
mockGitHub
.Setup(c => c.PostTreeAsync(
proj,
fakeCommitHash,
It.Is <GitObject[]>(
objects =>
objects.Length == 2 &&
objects[0].Path == $"{basePath}/{BuildManifestClient.BuildManifestXmlName}" &&
objects[0].Content == build.ToXml().ToString() &&
objects[1].Path == $"{basePath}/{SemaphoreModel.BuildSemaphorePath}" &&
objects[1].Content == build.Identity.BuildId + "\n")))
.ReturnsAsync(() => new GitTree {
Sha = fakeTreeHash
});
mockGitHub
.Setup(c => c.PostCommitAsync(
proj,
message,
fakeTreeHash,
It.Is <string[]>(parents => parents.Single() == fakeCommitHash)))
.ReturnsAsync(() => new GitCommit {
Sha = fakeNewCommitHash
});
mockGitHub
.Setup(c => c.PatchReferenceAsync(proj, @ref, fakeNewCommitHash, false))
.ReturnsAsync(() => null);
await client.PushNewBuildAsync(
new BuildManifestLocation(proj, @ref, basePath),
build,
null,
message);
mockGitHub.VerifyAll();
}
public async Task PushChangeAsync(BuildManifestChange change)
{
await Retry.RunAsync(async attempt =>
{
BuildManifestLocation location = change.Location;
// Get the current commit. Use this throughout to ensure a clean transaction.
GitReference remoteRef = await _github.GetReferenceAsync(
location.GitHubProject,
location.GitHubRef);
string remoteCommit = remoteRef.Object.Sha;
Trace.TraceInformation($"Creating update on remote commit: {remoteCommit}");
XElement remoteModelXml = await FetchModelXmlAsync(
location.GitHubProject,
remoteCommit,
location.GitHubBasePath);
OrchestratedBuildModel remoteModel = OrchestratedBuildModel.Parse(remoteModelXml);
// This is a subsequent publish step: make sure a new build hasn't happened already.
if (change.OrchestratedBuildId != remoteModel.Identity.BuildId)
{
throw new ManifestChangeOutOfDateException(
change.OrchestratedBuildId,
remoteModel.Identity.BuildId);
}
OrchestratedBuildModel modifiedModel = OrchestratedBuildModel.Parse(remoteModelXml);
change.ApplyModelChanges(modifiedModel);
if (modifiedModel.Identity.BuildId != change.OrchestratedBuildId)
{
throw new ArgumentException(
"Change action shouldn't modify BuildId. Changed from " +
$"'{change.OrchestratedBuildId}' to '{modifiedModel.Identity.BuildId}'.",
nameof(change));
}
XElement modifiedModelXml = modifiedModel.ToXml();
string[] changedSemaphorePaths = change.SemaphorePaths.ToArray();
// Check if any join groups are completed by this change.
var joinCompleteCheckTasks = change.JoinSemaphoreGroups.NullAsEmpty()
.Select(async g => new
{
Group = g,
Joinable = await IsGroupJoinableAsync(
location,
remoteCommit,
change.OrchestratedBuildId,
changedSemaphorePaths,
g)
});
var completeJoinedSemaphores = (await Task.WhenAll(joinCompleteCheckTasks))
.Where(g => g.Joinable)
.Select(g => g.Group.JoinSemaphorePath)
.ToArray();
IEnumerable <SupplementaryUploadRequest> semaphoreUploads = completeJoinedSemaphores
.Concat(changedSemaphorePaths)
.Select(p => new SupplementaryUploadRequest
{
Path = p,
Contents = new SemaphoreModel
{
BuildId = change.OrchestratedBuildId
}.ToFileContent()
});
IEnumerable <SupplementaryUploadRequest> uploads =
semaphoreUploads.Concat(change.SupplementaryUploads.NullAsEmpty());
if (!XNode.DeepEquals(modifiedModelXml, remoteModelXml))
{
uploads = uploads.Concat(new[]
{
new SupplementaryUploadRequest
{
Path = BuildManifestXmlName,
Contents = modifiedModelXml.ToString()
}
});
}
return(await PushUploadsAsync(
location,
change.CommitMessage,
remoteCommit,
uploads));
});
}
public async Task TestPushChangeSemaphoreAsync()
{
var mockGitHub = new Mock <IGitHubClient>(MockBehavior.Strict);
var client = new BuildManifestClient(mockGitHub.Object);
var proj = new GitHubProject("versions", "dotnet");
string @ref = "heads/master";
string basePath = "build-info/dotnet/product/cli/master";
string message = "Test change manifest commit";
string addSemaphorePath = "add-identity.semaphore";
var fakeExistingBuild = new OrchestratedBuildModel(new BuildIdentity("orch", "123"));
string fakeExistingBuildString = fakeExistingBuild.ToXml().ToString();
string fakeCommitHash = "fakeCommitHash";
string fakeTreeHash = "fakeTreeHash";
string fakeNewCommitHash = "fakeNewCommitHash";
mockGitHub
.Setup(c => c.GetReferenceAsync(proj, @ref))
.ReturnsAsync(() => new GitReference
{
Object = new GitReferenceObject {
Sha = fakeCommitHash
}
});
mockGitHub
.Setup(c => c.GetGitHubFileContentsAsync(
$"{basePath}/{BuildManifestClient.BuildManifestXmlName}",
proj,
fakeCommitHash))
.ReturnsAsync(() => fakeExistingBuildString);
mockGitHub
.Setup(c => c.PostTreeAsync(
proj,
fakeCommitHash,
It.Is <GitObject[]>(
objects =>
objects.Length == 1 &&
objects[0].Path == $"{basePath}/{addSemaphorePath}" &&
objects[0].Content == fakeExistingBuild.Identity.BuildId + "\n")))
.ReturnsAsync(() => new GitTree {
Sha = fakeTreeHash
});
mockGitHub
.Setup(c => c.PostCommitAsync(
proj,
message,
fakeTreeHash,
It.Is <string[]>(parents => parents.Single() == fakeCommitHash)))
.ReturnsAsync(() => new GitCommit {
Sha = fakeNewCommitHash
});
mockGitHub
.Setup(c => c.PatchReferenceAsync(proj, @ref, fakeNewCommitHash, false))
.ReturnsAsync(() => null);
await client.PushChangeAsync(
proj,
@ref,
basePath,
fakeExistingBuild.Identity.BuildId,
_ => { },
new[] { addSemaphorePath },
null,
message);
mockGitHub.VerifyAll();
}
public async Task PushChangeAsync(
GitHubProject project,
string @ref,
string basePath,
string orchestratedBuildId,
Action <OrchestratedBuildModel> changeModel,
IEnumerable <string> semaphorePaths,
IEnumerable <SupplementaryUploadRequest> supplementaryUploads,
string message)
{
await Retry.RunAsync(async attempt =>
{
// Get the current commit. Use this throughout to ensure a clean transaction.
string remoteCommit = (await _github.GetReferenceAsync(project, @ref)).Object.Sha;
Trace.TraceInformation($"Creating update on remote commit: {remoteCommit}");
// This is a subsequent publish step: check to make sure the build id matches.
XElement remoteModelXml = await FetchModelXmlAsync(project, remoteCommit, basePath);
OrchestratedBuildModel remoteModel = OrchestratedBuildModel.Parse(remoteModelXml);
if (orchestratedBuildId != remoteModel.Identity.BuildId)
{
throw new ManifestChangeOutOfDateException(
orchestratedBuildId,
remoteModel.Identity.BuildId);
}
OrchestratedBuildModel modifiedModel = OrchestratedBuildModel.Parse(remoteModelXml);
changeModel(modifiedModel);
if (modifiedModel.Identity.BuildId != orchestratedBuildId)
{
throw new ArgumentException(
"Change action shouldn't modify BuildId. Changed from " +
$"'{orchestratedBuildId}' to '{modifiedModel.Identity.BuildId}'.",
nameof(changeModel));
}
XElement modifiedModelXml = modifiedModel.ToXml();
IEnumerable <SupplementaryUploadRequest> uploads = semaphorePaths.NullAsEmpty()
.Select(p => new SupplementaryUploadRequest
{
Path = p,
Contents = new SemaphoreModel
{
BuildId = orchestratedBuildId
}.ToFileContent()
})
.Concat(supplementaryUploads.NullAsEmpty())
.ToArray();
if (!XNode.DeepEquals(modifiedModelXml, remoteModelXml))
{
uploads = uploads.Concat(new[]
{
new SupplementaryUploadRequest
{
Path = BuildManifestXmlName,
Contents = modifiedModelXml.ToString()
}
});
}
return(await PushUploadsAsync(project, @ref, basePath, message, remoteCommit, uploads));
});
}