public OrchestratedBuildDependencyInfo(
string simpleName,
OrchestratedBuildModel model)
{
SimpleName = simpleName;
OrchestratedBuildModel = model;
}
public override bool Execute()
{
string contents = System.IO.File.ReadAllText(ManifestFile);
var model = OrchestratedBuildModel.Parse(XElement.Parse(contents));
if (string.IsNullOrEmpty(CommitMessage))
{
CommitMessage = $"{model.Identity} orchestrated build manifest";
}
var gitHubAuth = new GitHubAuth(GitHubAuthToken, GitHubUser, GitHubEmail);
using (var gitHubClient = new GitHubClient(gitHubAuth))
{
var client = new BuildManifestClient(gitHubClient);
var location = new BuildManifestLocation(
new GitHubProject(VersionsRepo, VersionsRepoOwner),
$"heads/{VersionsRepoBranch}",
VersionsRepoPath);
var pushTask = client.PushNewBuildAsync(
location,
model,
CreateUploadRequests(SupplementaryFiles),
CommitMessage);
pushTask.Wait();
}
return(!Log.HasLoggedErrors);
}
public async Task <OrchestratedBuildModel> FetchManifestAsync(
GitHubProject project,
string @ref,
string basePath)
{
return(OrchestratedBuildModel.Parse(await FetchModelXmlAsync(project, @ref, basePath)));
}
private static async Task <IEnumerable <IDependencyInfo> > LoadBuildInfoXml()
{
Trace.TraceInformation($"Retrieving build info from '{Options.BuildInfoUrl}'");
XDocument buildInfoXml;
if (File.Exists(Options.BuildInfoUrl.LocalPath))
{
buildInfoXml = XDocument.Load(Options.BuildInfoUrl.LocalPath);
}
else
{
using (HttpClient client = new HttpClient())
using (Stream stream = await client.GetStreamAsync(Options.BuildInfoUrl))
{
buildInfoXml = XDocument.Load(stream);
}
}
OrchestratedBuildModel buildInfo = OrchestratedBuildModel.Parse(buildInfoXml.Root);
return(new[]
{
CreateDependencyBuildInfo(SdkBuildInfoName, buildInfo.Builds),
CreateDependencyBuildInfo(RuntimeBuildInfoName, buildInfo.Builds),
CreateDependencyBuildInfo(AspNetCoreBuildInfoName, buildInfo.Builds),
});
}
private static void ApplyUpdate(OrchestratedBuildModel manifest, ITaskItem update)
{
string type = update.GetMetadata(nameof(UpdateType));
UpdateType updateType;
if (!Enum.TryParse(type, true, out updateType))
{
throw new ArgumentException(
$"UpdateType '{type}' on update '{update.ItemSpec}' is not valid.");
}
switch (updateType)
{
case UpdateType.AddOrMergeEndpoint:
var xml = XElement.Parse(update.GetMetadata(XmlMetadataName));
EndpointModel endpoint = EndpointModel.Parse(xml);
EndpointModel existingEndpoint = manifest.Endpoints
.FirstOrDefault(e => SameAttributes(endpoint.Attributes, e.Attributes));
if (existingEndpoint == null)
{
manifest.Endpoints.Add(endpoint);
}
else
{
existingEndpoint.Artifacts.Add(endpoint.Artifacts);
}
break;
}
}
public void TestMergeBuildManifests()
{
var orchestratedModel = new OrchestratedBuildModel(new BuildIdentity("Orchestrated", "123"))
{
Endpoints = new List <EndpointModel>
{
EndpointModel.CreateOrchestratedBlobFeed("http://example.org")
}
};
orchestratedModel.AddParticipantBuild(CreatePackageOnlyBuildManifestModel());
orchestratedModel.AddParticipantBuild(BuildModel.Parse(XElement.Parse(ExampleBuildString)));
XElement modelXml = orchestratedModel.ToXml();
XElement xml = XElement.Parse(@"
<OrchestratedBuild Name=""Orchestrated"" BuildId=""123"">
<Endpoint Id=""Orchestrated"" Type=""BlobFeed"" Url=""http://example.org"">
<Package Id=""Foo"" Version=""1.2.3-example"" />
<Package Id=""runtime.rhel.6-x64.Microsoft.Private.CoreFx.NETCoreApp"" Version=""4.5.0-preview1-25929-04"" Category=""noship"" />
<Package Id=""System.Memory"" Version=""4.5.0-preview1-25927-01"" />
<Blob Id=""symbols/inner/blank-dir-nonshipping"" NonShipping=""false"" />
<Blob Id=""symbols/runtime.rhel.6-x64.Microsoft.Private.CoreFx.NETCoreApp.4.5.0-preview1-25929-04.symbols.nupkg"" />
<Blob Id=""symbols/System.ValueTuple.4.5.0-preview1-25929-04.symbols.nupkg"" NonShipping=""true"" />
</Endpoint>
<Build Name=""SimpleBuildManifest"" BuildId=""123"" />
<Build Name=""corefx"" BuildId=""20171129-04"" Branch=""master"" Commit=""defb6d52047cc3d6b5f5d0853b0afdb1512dfbf4"" />
</OrchestratedBuild>");
Assert.True(XNode.DeepEquals(xml, modelXml));
}
public void TestExampleOrchestratedBuildManifestRoundtrip()
{
XElement xml = XElement.Parse(ExampleOrchestratedBuildString);
var model = OrchestratedBuildModel.Parse(xml);
XElement modelXml = model.ToXml();
XNode.DeepEquals(xml, modelXml).Should().BeTrue("Model failed to output the parsed XML.");
}
public async Task TestPushConflictAsync()
{
var mockGitHub = new Mock <IGitHubClient>(MockBehavior.Strict);
var client = new BuildManifestClient(mockGitHub.Object);
var build = new OrchestratedBuildModel(new BuildIdentity {
Name = "orch", BuildId = "123"
});
var proj = new GitHubProject("versions", "dotnet");
string @ref = "heads/master";
string basePath = "build-info/dotnet/product/cli/master";
string message = "Test build upload commit";
string fakeCommitHash = "fakeCommitHash";
string fakeTreeHash = "fakeTreeHash";
string fakeNewCommitHash = "fakeNewCommitHash";
mockGitHub
.Setup(c => c.GetReferenceAsync(proj, @ref))
.ReturnsAsync(() => new GitReference
{
Object = new GitReferenceObject {
Sha = fakeCommitHash
}
});
mockGitHub
.Setup(c => c.PostTreeAsync(proj, fakeCommitHash, It.IsAny <GitObject[]>()))
.ReturnsAsync(() => new GitTree {
Sha = fakeTreeHash
});
mockGitHub
.Setup(c => c.PostCommitAsync(proj, message, fakeTreeHash, It.IsAny <string[]>()))
.ReturnsAsync(() => new GitCommit {
Sha = fakeNewCommitHash
});
mockGitHub
.Setup(c => c.PatchReferenceAsync(proj, @ref, fakeNewCommitHash, false))
.Callback(() =>
{
// Once the exception is hit, let the next patch call work.
mockGitHub
.Setup(c => c.PatchReferenceAsync(proj, @ref, fakeNewCommitHash, false))
.ReturnsAsync(() => null);
})
.ThrowsAsync(new NotFastForwardUpdateException("Testing non-fast-forward update."));
await client.PushNewBuildAsync(
new BuildManifestLocation(proj, @ref, basePath),
build,
null,
message);
mockGitHub.VerifyAll();
}
public OrchestratedBuildDependencyInfo(
string simpleName,
string basePath,
OrchestratedBuildModel model)
{
SimpleName = simpleName;
BasePath = basePath;
OrchestratedBuildModel = model;
}
protected IEnumerable <IDependencyInfo> CreateDependencyInfos(
bool remote,
string versionsCommit)
{
foreach (ITaskItem info in DependencyInfo ?? Enumerable.Empty <ITaskItem>())
{
IDependencyInfo dependencyInfo;
string type = info.GetMetadata("DependencyType");
switch (type)
{
case "Build":
SetVersionsCommitOverride(info, versionsCommit);
dependencyInfo = CreateBuildInfoDependency(info, BuildInfoCacheDir);
break;
case "Submodule":
dependencyInfo = SubmoduleDependencyInfo.Create(
GetRequiredMetadata(info, "Repository"),
GetRequiredMetadata(info, "Ref"),
GetRequiredMetadata(info, "Path"),
remote);
break;
case "Orchestrated build":
SetVersionsCommitOverride(info, versionsCommit);
dependencyInfo = OrchestratedBuildDependencyInfo.CreateAsync(
info.ItemSpec,
new GitHubProject(
GetRequiredMetadata(info, "VersionsRepo"),
GetRequiredMetadata(info, "VersionsRepoOwner")),
GetRequiredMetadata(info, CurrentRefMetadataName),
GetRequiredMetadata(info, "BasePath"),
new BuildManifestClient(GitHubClient)).Result;
break;
case "Orchestrated build file":
dependencyInfo = new OrchestratedBuildDependencyInfo(
info.ItemSpec,
OrchestratedBuildModel.Parse(
XElement.Parse(
File.ReadAllText(
GetRequiredMetadata(info, "Path")))));
break;
default:
throw new NotSupportedException(
$"Unsupported DependencyInfo '{info.ItemSpec}': DependencyType '{type}'.");
}
DependencyInfoConfigItems[dependencyInfo] = info;
yield return(dependencyInfo);
}
}
public async Task <OrchestratedBuildModel> FetchManifestAsync(
GitHubProject project,
string @ref,
string basePath)
{
XElement contents = await FetchModelXmlAsync(project, @ref, basePath);
if (contents == null)
{
return(null);
}
return(OrchestratedBuildModel.Parse(contents));
}
public override bool Execute()
{
// Leave out attributes if they would just have empty string values.
if (ManifestBranch == string.Empty)
{
ManifestBranch = null;
}
if (ManifestCommit == string.Empty)
{
ManifestCommit = null;
}
var identity = new BuildIdentity
{
Name = ManifestName,
BuildId = ManifestBuildId,
Branch = ManifestBranch,
Commit = ManifestCommit,
IsStable = IsStable,
VersionStamp = VersionStamp
};
var orchestratedBuild = new OrchestratedBuildModel(identity)
{
Endpoints = new List <EndpointModel>
{
EndpointModel.CreateOrchestratedBlobFeed(BlobFeedUrl)
}
};
foreach (ITaskItem buildManifestFile in BuildManifestFiles)
{
string contents = System.IO.File.ReadAllText(buildManifestFile.ItemSpec);
BuildModel build = BuildModel.Parse(XElement.Parse(contents));
foreach (PackageArtifactModel package in build.Artifacts.Packages)
{
package.OriginBuildName = build.Identity.Name;
}
orchestratedBuild.AddParticipantBuild(build);
}
System.IO.File.WriteAllText(File, orchestratedBuild.ToXml().ToString());
return(!Log.HasLoggedErrors);
}
public override bool Execute()
{
GitHubAuth gitHubAuth = null;
if (!string.IsNullOrEmpty(GitHubAuthToken))
{
gitHubAuth = new GitHubAuth(GitHubAuthToken, GitHubUser, GitHubEmail);
}
using (var gitHubClient = new GitHubClient(gitHubAuth))
{
var client = new BuildManifestClient(gitHubClient);
OrchestratedBuildModel manifest = client.FetchManifestAsync(
new GitHubProject(VersionsRepo, VersionsRepoOwner),
VersionsRepoRef,
VersionsRepoPath)
.Result;
OrchestratedBuild = CreateItem(manifest.Identity);
EndpointModel[] orchestratedFeeds = manifest.Endpoints
.Where(e => e.IsOrchestratedBlobFeed)
.ToArray();
if (orchestratedFeeds.Length != 1)
{
throw new Exception(
"Invalid manifest. Expected 1 orchestrated blob feed, " +
$"found {orchestratedFeeds.Length}.");
}
EndpointModel feed = orchestratedFeeds[0];
IEnumerable <ITaskItem> packageItems = feed.Artifacts.Packages.Select(CreateItem);
IEnumerable <ITaskItem> blobItems = feed.Artifacts.Blobs.Select(CreateItem);
OrchestratedBlobFeed = new[] { new TaskItem("Endpoint", feed.Attributes) };
OrchestratedBlobFeedArtifacts = packageItems.Concat(blobItems).ToArray();
IEnumerable <ITaskItem> buildItems = manifest.Builds.Select(CreateItem);
OrchestratedBuildConstituents = buildItems.ToArray();
}
return(!Log.HasLoggedErrors);
}
public async Task TestPushConflictingChangeAsync()
{
var mockGitHub = new Mock <IGitHubClient>(MockBehavior.Strict);
var client = new BuildManifestClient(mockGitHub.Object);
var proj = new GitHubProject("versions", "dotnet");
string @ref = "heads/master";
string basePath = "build-info/dotnet/product/cli/master";
string message = "Test change manifest commit";
string addSemaphorePath = "add-identity.semaphore";
var fakeExistingBuild = new OrchestratedBuildModel(new BuildIdentity {
Name = "orch", BuildId = "123"
});
var fakeNewExistingBuild = new OrchestratedBuildModel(new BuildIdentity {
Name = "orch", BuildId = "456"
});
string fakeCommitHash = "fakeCommitHash";
mockGitHub
.Setup(c => c.GetReferenceAsync(proj, @ref))
.ReturnsAsync(() => new GitReference
{
Object = new GitReferenceObject {
Sha = fakeCommitHash
}
});
mockGitHub
.Setup(c => c.GetGitHubFileContentsAsync(It.IsAny <string>(), proj, fakeCommitHash))
.ReturnsAsync(() => fakeNewExistingBuild.ToXml().ToString());
var pushClient = client.PushChangeAsync(
new BuildManifestChange(
new BuildManifestLocation(proj, @ref, basePath),
message,
fakeExistingBuild.Identity.BuildId,
new[] { addSemaphorePath },
_ => { }
));
Func <Task> act = async() => { await pushClient; };
await act.Should().ThrowAsync <ManifestChangeOutOfDateException>();
mockGitHub.VerifyAll();
}
private static async Task <IEnumerable <IDependencyInfo> > GetBuildInfoAsync()
{
Trace.TraceInformation($"Retrieving build info from '{Options.BuildInfoUrl}'");
using (HttpClient client = new HttpClient())
using (Stream stream = await client.GetStreamAsync(Options.BuildInfoUrl))
{
XDocument buildInfoXml = XDocument.Load(stream);
OrchestratedBuildModel buildInfo = OrchestratedBuildModel.Parse(buildInfoXml.Root);
BuildIdentity sdkBuild = buildInfo.Builds
.First(build => string.Equals(build.Name, "cli", StringComparison.OrdinalIgnoreCase));
BuildIdentity coreSetupBuild = buildInfo.Builds
.First(build => string.Equals(build.Name, "core-setup", StringComparison.OrdinalIgnoreCase));
return(new[]
{
CreateDependencyBuildInfo(SdkBuildInfoName, sdkBuild.ProductVersion),
CreateDependencyBuildInfo(RuntimeBuildInfoName, coreSetupBuild.ProductVersion),
});
};
}
public static async Task <OrchestratedBuildDependencyInfo> CreateAsync(
string simpleName,
GitHubProject project,
string @ref,
string basePath,
BuildManifestClient client)
{
OrchestratedBuildModel model = await client.FetchManifestAsync(
project,
@ref,
basePath);
if (model == null)
{
throw new ArgumentException(
$"Found no manifest for '{simpleName}' at " +
$"'{project.Segments}' '{basePath}' ref '{@ref}'");
}
return(new OrchestratedBuildDependencyInfo(simpleName, basePath, model));
}
public async Task PushNewBuildAsync(
BuildManifestLocation location,
OrchestratedBuildModel build,
IEnumerable <SupplementaryUploadRequest> supplementaryUploads,
string message)
{
await Retry.RunAsync(async attempt =>
{
GitReference remoteRef = await _github.GetReferenceAsync(
location.GitHubProject,
location.GitHubRef);
string remoteCommit = remoteRef.Object.Sha;
Trace.TraceInformation($"Creating update on remote commit: {remoteCommit}");
IEnumerable <SupplementaryUploadRequest> uploads = supplementaryUploads.NullAsEmpty()
.Concat(new[]
{
new SupplementaryUploadRequest
{
Path = BuildManifestXmlName,
Contents = build.ToXml().ToString()
},
new SupplementaryUploadRequest
{
Path = SemaphoreModel.BuildSemaphorePath,
Contents = new SemaphoreModel
{
BuildId = build.Identity.BuildId
}.ToFileContent()
}
})
.ToArray();
return(await PushUploadsAsync(location, message, remoteCommit, uploads));
});
}
private async Task <IDependencyInfo> GetBuildInfoAsync()
{
Trace.TraceInformation($"Retrieving build info from '{BuildInfoUrl}'");
using (var client = new HttpClient())
using (var stream = await client.GetStreamAsync(BuildInfoUrl))
{
var buildInfoXml = XDocument.Load(stream);
var buildInfo = OrchestratedBuildModel.Parse(buildInfoXml.Root);
var aspnetBuild = buildInfo.Builds
.First(build => string.Equals(build.Name, "aspnet", StringComparison.OrdinalIgnoreCase));
return(new BuildDependencyInfo(
new BuildInfo()
{
Name = RuntimeBuildInfo,
LatestReleaseVersion = aspnetBuild.ProductVersion,
LatestPackages = new Dictionary <string, string>()
},
false,
Enumerable.Empty <string>()));
}
}
public override bool Execute()
{
// Leave out attributes if they would just have empty string values.
if (ManifestBranch == string.Empty)
{
ManifestBranch = null;
}
if (ManifestCommit == string.Empty)
{
ManifestCommit = null;
}
var orchestratedBuild = new OrchestratedBuildModel(new BuildIdentity(
ManifestName,
ManifestBuildId,
ManifestBranch,
ManifestCommit))
{
Endpoints = new List <EndpointModel>
{
EndpointModel.CreateOrchestratedBlobFeed(BlobFeedUrl)
}
};
foreach (ITaskItem buildManifestFile in BuildManifestFiles)
{
string contents = System.IO.File.ReadAllText(buildManifestFile.ItemSpec);
BuildModel build = BuildModel.Parse(XElement.Parse(contents));
orchestratedBuild.AddParticipantBuild(build);
}
System.IO.File.WriteAllText(File, orchestratedBuild.ToXml().ToString());
return(!Log.HasLoggedErrors);
}
public static OrchestratedBuild Load(string xmlText, string branch)
{
var model = OrchestratedBuildModel.Parse(XDocument.Parse(xmlText).Root);
var orchestratedBuild = new OrchestratedBuild
{
BuildNumber = model.Identity.BuildId,
Branch = branch,
OrchestratedBuildId = $"{branch}/{model.Identity.BuildId}",
Name = model.Identity.Name,
IsStable = string.IsNullOrEmpty(model.Identity.IsStable) ? false : bool.Parse(model.Identity.IsStable),
VersionStamp = model.Identity.VersionStamp
};
var buildsIndex = new Dictionary <string, Build>();
foreach (var buildModel in model.Builds.Where(b => b.Name != "anonymous"))
{
var build = new Build
{
OrchestratedBuildId = orchestratedBuild.OrchestratedBuildId,
Name = buildModel.Name,
BuildNumber = buildModel.BuildId,
BuildId = $"{orchestratedBuild.OrchestratedBuildId}/builds/{buildModel.Name}/{buildModel.BuildId}",
Branch = buildModel.Branch,
ProductVersion = buildModel.ProductVersion,
Commit = buildModel.Commit,
};
buildsIndex[build.Name] = build;
orchestratedBuild.Builds.Add(build);
}
var endpointIndex = new Dictionary <string, Endpoint>();
var packageIndex = new Dictionary <string, Package>();
var blobIndex = new Dictionary <string, Blob>();
foreach (var endpoint in model.Endpoints)
{
var baseUrl = endpoint.Url;
if (baseUrl.EndsWith("/index.json"))
{
baseUrl = baseUrl.Substring(0, baseUrl.Length - 11);
}
if (endpoint.IsOrchestratedBlobFeed)
{
baseUrl += "/assets";
}
var ep = new Endpoint
{
EndpointId = $"{orchestratedBuild.OrchestratedBuildId}/endpoints/{endpoint.Id}",
Id = endpoint.Id,
Type = endpoint.Type,
Url = endpoint.Url
};
endpointIndex[ep.EndpointId] = ep;
orchestratedBuild.Endpoints.Add(ep);
foreach (var artifact in endpoint.Artifacts.Packages)
{
if (packageIndex.TryGetValue(artifact.Id, out var existingRef))
{
existingRef.Endpoints.Add(new EndpointRef()
{
EndpointRefId = $"{orchestratedBuild.OrchestratedBuildId}/endpoints/{ep.EndpointId}/packages/{artifact.Id}",
EndpointId = ep.EndpointId,
ArtifactUrl = $"{baseUrl}/flatcontainer/{artifact.Id}/{artifact.Version}/{artifact.Id}.{artifact.Version}.nupkg",
});
}
else
{
var packageRef = new Package
{
PackageId = $"{orchestratedBuild.OrchestratedBuildId}/packages/{artifact.Id}",
Id = artifact.Id,
Version = artifact.Version,
NonShipping = bool.Parse(artifact.Attributes.TryGetValue("NonShipping", bool.TrueString))
};
if (!string.IsNullOrEmpty(artifact.OriginBuildName) && buildsIndex.TryGetValue(artifact.OriginBuildName, out var build))
{
packageRef.OriginBuildId = build.BuildId;
}
packageIndex[artifact.Id] = packageRef;
orchestratedBuild.Packages.Add(packageRef);
}
}
foreach (var artifact in endpoint.Artifacts.Blobs)
{
if (blobIndex.TryGetValue(artifact.Id, out var existingRef))
{
existingRef.Endpoints.Add(new EndpointRef()
{
EndpointRefId = $"{orchestratedBuild.OrchestratedBuildId}/endpoints/{ep.EndpointId}/blobs/{artifact.Id}",
EndpointId = ep.EndpointId,
ArtifactUrl = $"{baseUrl}/assets/{artifact.Id}",
});
}
else
{
var blobRef = new Blob
{
BlobId = $"{orchestratedBuild.OrchestratedBuildId}/blobs/{artifact.Id}",
Id = artifact.Id,
Type = artifact.Attributes.TryGetValue("Type"),
ShipInstaller = artifact.Attributes.TryGetValue("ShipInstaller")
};
blobIndex[artifact.Id] = blobRef;
orchestratedBuild.Blobs.Add(blobRef);
}
}
}
return(orchestratedBuild);
}
public async Task PushChangeAsync(
GitHubProject project,
string @ref,
string basePath,
string orchestratedBuildId,
Action <OrchestratedBuildModel> changeModel,
IEnumerable <string> semaphorePaths,
IEnumerable <SupplementaryUploadRequest> supplementaryUploads,
string message)
{
await Retry.RunAsync(async attempt =>
{
// Get the current commit. Use this throughout to ensure a clean transaction.
string remoteCommit = (await _github.GetReferenceAsync(project, @ref)).Object.Sha;
Trace.TraceInformation($"Creating update on remote commit: {remoteCommit}");
// This is a subsequent publish step: check to make sure the build id matches.
XElement remoteModelXml = await FetchModelXmlAsync(project, remoteCommit, basePath);
OrchestratedBuildModel remoteModel = OrchestratedBuildModel.Parse(remoteModelXml);
if (orchestratedBuildId != remoteModel.Identity.BuildId)
{
throw new ManifestChangeOutOfDateException(
orchestratedBuildId,
remoteModel.Identity.BuildId);
}
OrchestratedBuildModel modifiedModel = OrchestratedBuildModel.Parse(remoteModelXml);
changeModel(modifiedModel);
if (modifiedModel.Identity.BuildId != orchestratedBuildId)
{
throw new ArgumentException(
"Change action shouldn't modify BuildId. Changed from " +
$"'{orchestratedBuildId}' to '{modifiedModel.Identity.BuildId}'.",
nameof(changeModel));
}
XElement modifiedModelXml = modifiedModel.ToXml();
IEnumerable <SupplementaryUploadRequest> uploads = semaphorePaths.NullAsEmpty()
.Select(p => new SupplementaryUploadRequest
{
Path = p,
Contents = new SemaphoreModel
{
BuildId = orchestratedBuildId
}.ToFileContent()
})
.Concat(supplementaryUploads.NullAsEmpty())
.ToArray();
if (!XNode.DeepEquals(modifiedModelXml, remoteModelXml))
{
uploads = uploads.Concat(new[]
{
new SupplementaryUploadRequest
{
Path = BuildManifestXmlName,
Contents = modifiedModelXml.ToString()
}
});
}
return(await PushUploadsAsync(project, @ref, basePath, message, remoteCommit, uploads));
});
}
public override bool Execute()
{
string contents = System.IO.File.ReadAllText(ManifestFile);
OrchestratedBuildModel model = OrchestratedBuildModel.Parse(XElement.Parse(contents));
EndpointModel blobFeed = model.Endpoints.First(e => e.IsOrchestratedBlobFeed);
string feedAssetsRoot = blobFeed.Url.Replace("/index.json", "/assets");
string sdkProductVersion = model.Builds
.FirstOrDefault(b => b.Name == "cli")
?.ProductVersion;
string runtimeProductVersion = model.Builds
.FirstOrDefault(b => b.Name == "core-setup")
?.ProductVersion;
string aspnetProductVersion = model.Builds
.FirstOrDefault(b => b.Name == "aspnet")
?.ProductVersion;
var builder = new StringBuilder();
builder.Append("## Product build: ");
builder.AppendLine(model.Identity.ToString());
if (!string.IsNullOrEmpty(SdkTableTemplateFile) && sdkProductVersion != null)
{
builder.AppendLine();
builder.AppendLine(FillTemplate(
SdkTableTemplateFile,
feedAssetsRoot,
sdkProductVersion));
}
if (!string.IsNullOrEmpty(DotNetRuntimeTableTemplateFile) && runtimeProductVersion != null)
{
builder.AppendLine();
builder.AppendLine(FillTemplate(
DotNetRuntimeTableTemplateFile,
feedAssetsRoot,
runtimeProductVersion));
}
if (!string.IsNullOrEmpty(AspNetCoreRuntimeTableTemplateFile) && aspnetProductVersion != null)
{
builder.AppendLine();
builder.AppendLine(FillTemplate(
AspNetCoreRuntimeTableTemplateFile,
feedAssetsRoot,
aspnetProductVersion));
}
builder.AppendLine();
builder.AppendLine("### Built Repositories");
foreach (BuildIdentity build in model.Builds
.Where(b => b.Name != "anonymous")
.OrderBy(b => b.Name))
{
builder.Append(" * ");
builder.AppendLine(build.ToString());
}
System.IO.File.WriteAllText(File, builder.ToString());
return(!Log.HasLoggedErrors);
}
public async Task TestPushChangeSemaphoreAsync()
{
var mockGitHub = new Mock <IGitHubClient>(MockBehavior.Strict);
var client = new BuildManifestClient(mockGitHub.Object);
var proj = new GitHubProject("versions", "dotnet");
string @ref = "heads/master";
string basePath = "build-info/dotnet/product/cli/master";
string message = "Test change manifest commit";
string addSemaphorePath = "add-identity.semaphore";
var fakeExistingBuild = new OrchestratedBuildModel(new BuildIdentity("orch", "123"));
string fakeExistingBuildString = fakeExistingBuild.ToXml().ToString();
string fakeCommitHash = "fakeCommitHash";
string fakeTreeHash = "fakeTreeHash";
string fakeNewCommitHash = "fakeNewCommitHash";
mockGitHub
.Setup(c => c.GetReferenceAsync(proj, @ref))
.ReturnsAsync(() => new GitReference
{
Object = new GitReferenceObject {
Sha = fakeCommitHash
}
});
mockGitHub
.Setup(c => c.GetGitHubFileContentsAsync(
$"{basePath}/{BuildManifestClient.BuildManifestXmlName}",
proj,
fakeCommitHash))
.ReturnsAsync(() => fakeExistingBuildString);
mockGitHub
.Setup(c => c.PostTreeAsync(
proj,
fakeCommitHash,
It.Is <GitObject[]>(
objects =>
objects.Length == 1 &&
objects[0].Path == $"{basePath}/{addSemaphorePath}" &&
objects[0].Content == fakeExistingBuild.Identity.BuildId + "\n")))
.ReturnsAsync(() => new GitTree {
Sha = fakeTreeHash
});
mockGitHub
.Setup(c => c.PostCommitAsync(
proj,
message,
fakeTreeHash,
It.Is <string[]>(parents => parents.Single() == fakeCommitHash)))
.ReturnsAsync(() => new GitCommit {
Sha = fakeNewCommitHash
});
mockGitHub
.Setup(c => c.PatchReferenceAsync(proj, @ref, fakeNewCommitHash, false))
.ReturnsAsync(() => null);
await client.PushChangeAsync(
proj,
@ref,
basePath,
fakeExistingBuild.Identity.BuildId,
_ => { },
new[] { addSemaphorePath },
null,
message);
mockGitHub.VerifyAll();
}
public async Task TestPushNewBuildAsync()
{
var mockGitHub = new Mock <IGitHubClient>(MockBehavior.Strict);
var client = new BuildManifestClient(mockGitHub.Object);
var build = new OrchestratedBuildModel(new BuildIdentity {
Name = "orch", BuildId = "123"
});
var proj = new GitHubProject("versions", "dotnet");
string @ref = "heads/master";
string basePath = "build-info/dotnet/product/cli/master";
string message = "Test build upload commit";
string fakeCommitHash = "fakeCommitHash";
string fakeTreeHash = "fakeTreeHash";
string fakeNewCommitHash = "fakeNewCommitHash";
mockGitHub
.Setup(c => c.GetReferenceAsync(proj, @ref))
.ReturnsAsync(() => new GitReference
{
Object = new GitReferenceObject {
Sha = fakeCommitHash
}
});
mockGitHub
.Setup(c => c.PostTreeAsync(
proj,
fakeCommitHash,
It.Is <GitObject[]>(
objects =>
objects.Length == 2 &&
objects[0].Path == $"{basePath}/{BuildManifestClient.BuildManifestXmlName}" &&
objects[0].Content == build.ToXml().ToString() &&
objects[1].Path == $"{basePath}/{SemaphoreModel.BuildSemaphorePath}" &&
objects[1].Content == build.Identity.BuildId + "\n")))
.ReturnsAsync(() => new GitTree {
Sha = fakeTreeHash
});
mockGitHub
.Setup(c => c.PostCommitAsync(
proj,
message,
fakeTreeHash,
It.Is <string[]>(parents => parents.Single() == fakeCommitHash)))
.ReturnsAsync(() => new GitCommit {
Sha = fakeNewCommitHash
});
mockGitHub
.Setup(c => c.PatchReferenceAsync(proj, @ref, fakeNewCommitHash, false))
.ReturnsAsync(() => null);
await client.PushNewBuildAsync(
new BuildManifestLocation(proj, @ref, basePath),
build,
null,
message);
mockGitHub.VerifyAll();
}
public async Task PushChangeAsync(BuildManifestChange change)
{
await Retry.RunAsync(async attempt =>
{
BuildManifestLocation location = change.Location;
// Get the current commit. Use this throughout to ensure a clean transaction.
GitReference remoteRef = await _github.GetReferenceAsync(
location.GitHubProject,
location.GitHubRef);
string remoteCommit = remoteRef.Object.Sha;
Trace.TraceInformation($"Creating update on remote commit: {remoteCommit}");
XElement remoteModelXml = await FetchModelXmlAsync(
location.GitHubProject,
remoteCommit,
location.GitHubBasePath);
OrchestratedBuildModel remoteModel = OrchestratedBuildModel.Parse(remoteModelXml);
// This is a subsequent publish step: make sure a new build hasn't happened already.
if (change.OrchestratedBuildId != remoteModel.Identity.BuildId)
{
throw new ManifestChangeOutOfDateException(
change.OrchestratedBuildId,
remoteModel.Identity.BuildId);
}
OrchestratedBuildModel modifiedModel = OrchestratedBuildModel.Parse(remoteModelXml);
change.ApplyModelChanges(modifiedModel);
if (modifiedModel.Identity.BuildId != change.OrchestratedBuildId)
{
throw new ArgumentException(
"Change action shouldn't modify BuildId. Changed from " +
$"'{change.OrchestratedBuildId}' to '{modifiedModel.Identity.BuildId}'.",
nameof(change));
}
XElement modifiedModelXml = modifiedModel.ToXml();
string[] changedSemaphorePaths = change.SemaphorePaths.ToArray();
// Check if any join groups are completed by this change.
var joinCompleteCheckTasks = change.JoinSemaphoreGroups.NullAsEmpty()
.Select(async g => new
{
Group = g,
Joinable = await IsGroupJoinableAsync(
location,
remoteCommit,
change.OrchestratedBuildId,
changedSemaphorePaths,
g)
});
var completeJoinedSemaphores = (await Task.WhenAll(joinCompleteCheckTasks))
.Where(g => g.Joinable)
.Select(g => g.Group.JoinSemaphorePath)
.ToArray();
IEnumerable <SupplementaryUploadRequest> semaphoreUploads = completeJoinedSemaphores
.Concat(changedSemaphorePaths)
.Select(p => new SupplementaryUploadRequest
{
Path = p,
Contents = new SemaphoreModel
{
BuildId = change.OrchestratedBuildId
}.ToFileContent()
});
IEnumerable <SupplementaryUploadRequest> uploads =
semaphoreUploads.Concat(change.SupplementaryUploads.NullAsEmpty());
if (!XNode.DeepEquals(modifiedModelXml, remoteModelXml))
{
uploads = uploads.Concat(new[]
{
new SupplementaryUploadRequest
{
Path = BuildManifestXmlName,
Contents = modifiedModelXml.ToString()
}
});
}
return(await PushUploadsAsync(
location,
change.CommitMessage,
remoteCommit,
uploads));
});
}