public async Task <OrchestratedBuildModel> FetchManifestAsync(
GitHubProject project,
string @ref,
string basePath)
{
return(OrchestratedBuildModel.Parse(await FetchModelXmlAsync(project, @ref, basePath)));
}
private static async Task <IEnumerable <IDependencyInfo> > LoadBuildInfoXml()
{
Trace.TraceInformation($"Retrieving build info from '{Options.BuildInfoUrl}'");
XDocument buildInfoXml;
if (File.Exists(Options.BuildInfoUrl.LocalPath))
{
buildInfoXml = XDocument.Load(Options.BuildInfoUrl.LocalPath);
}
else
{
using (HttpClient client = new HttpClient())
using (Stream stream = await client.GetStreamAsync(Options.BuildInfoUrl))
{
buildInfoXml = XDocument.Load(stream);
}
}
OrchestratedBuildModel buildInfo = OrchestratedBuildModel.Parse(buildInfoXml.Root);
return(new[]
{
CreateDependencyBuildInfo(SdkBuildInfoName, buildInfo.Builds),
CreateDependencyBuildInfo(RuntimeBuildInfoName, buildInfo.Builds),
CreateDependencyBuildInfo(AspNetCoreBuildInfoName, buildInfo.Builds),
});
}
public override bool Execute()
{
string contents = System.IO.File.ReadAllText(ManifestFile);
var model = OrchestratedBuildModel.Parse(XElement.Parse(contents));
if (string.IsNullOrEmpty(CommitMessage))
{
CommitMessage = $"{model.Identity} orchestrated build manifest";
}
var gitHubAuth = new GitHubAuth(GitHubAuthToken, GitHubUser, GitHubEmail);
using (var gitHubClient = new GitHubClient(gitHubAuth))
{
var client = new BuildManifestClient(gitHubClient);
var location = new BuildManifestLocation(
new GitHubProject(VersionsRepo, VersionsRepoOwner),
$"heads/{VersionsRepoBranch}",
VersionsRepoPath);
var pushTask = client.PushNewBuildAsync(
location,
model,
CreateUploadRequests(SupplementaryFiles),
CommitMessage);
pushTask.Wait();
}
return(!Log.HasLoggedErrors);
}
public void TestExampleOrchestratedBuildManifestRoundtrip()
{
XElement xml = XElement.Parse(ExampleOrchestratedBuildString);
var model = OrchestratedBuildModel.Parse(xml);
XElement modelXml = model.ToXml();
XNode.DeepEquals(xml, modelXml).Should().BeTrue("Model failed to output the parsed XML.");
}
protected IEnumerable <IDependencyInfo> CreateDependencyInfos(
bool remote,
string versionsCommit)
{
foreach (ITaskItem info in DependencyInfo ?? Enumerable.Empty <ITaskItem>())
{
IDependencyInfo dependencyInfo;
string type = info.GetMetadata("DependencyType");
switch (type)
{
case "Build":
SetVersionsCommitOverride(info, versionsCommit);
dependencyInfo = CreateBuildInfoDependency(info, BuildInfoCacheDir);
break;
case "Submodule":
dependencyInfo = SubmoduleDependencyInfo.Create(
GetRequiredMetadata(info, "Repository"),
GetRequiredMetadata(info, "Ref"),
GetRequiredMetadata(info, "Path"),
remote);
break;
case "Orchestrated build":
SetVersionsCommitOverride(info, versionsCommit);
dependencyInfo = OrchestratedBuildDependencyInfo.CreateAsync(
info.ItemSpec,
new GitHubProject(
GetRequiredMetadata(info, "VersionsRepo"),
GetRequiredMetadata(info, "VersionsRepoOwner")),
GetRequiredMetadata(info, CurrentRefMetadataName),
GetRequiredMetadata(info, "BasePath"),
new BuildManifestClient(GitHubClient)).Result;
break;
case "Orchestrated build file":
dependencyInfo = new OrchestratedBuildDependencyInfo(
info.ItemSpec,
OrchestratedBuildModel.Parse(
XElement.Parse(
File.ReadAllText(
GetRequiredMetadata(info, "Path")))));
break;
default:
throw new NotSupportedException(
$"Unsupported DependencyInfo '{info.ItemSpec}': DependencyType '{type}'.");
}
DependencyInfoConfigItems[dependencyInfo] = info;
yield return(dependencyInfo);
}
}
public async Task <OrchestratedBuildModel> FetchManifestAsync(
GitHubProject project,
string @ref,
string basePath)
{
XElement contents = await FetchModelXmlAsync(project, @ref, basePath);
if (contents == null)
{
return(null);
}
return(OrchestratedBuildModel.Parse(contents));
}
private static async Task <IEnumerable <IDependencyInfo> > GetBuildInfoAsync()
{
Trace.TraceInformation($"Retrieving build info from '{Options.BuildInfoUrl}'");
using (HttpClient client = new HttpClient())
using (Stream stream = await client.GetStreamAsync(Options.BuildInfoUrl))
{
XDocument buildInfoXml = XDocument.Load(stream);
OrchestratedBuildModel buildInfo = OrchestratedBuildModel.Parse(buildInfoXml.Root);
BuildIdentity sdkBuild = buildInfo.Builds
.First(build => string.Equals(build.Name, "cli", StringComparison.OrdinalIgnoreCase));
BuildIdentity coreSetupBuild = buildInfo.Builds
.First(build => string.Equals(build.Name, "core-setup", StringComparison.OrdinalIgnoreCase));
return(new[]
{
CreateDependencyBuildInfo(SdkBuildInfoName, sdkBuild.ProductVersion),
CreateDependencyBuildInfo(RuntimeBuildInfoName, coreSetupBuild.ProductVersion),
});
};
}
private async Task <IDependencyInfo> GetBuildInfoAsync()
{
Trace.TraceInformation($"Retrieving build info from '{BuildInfoUrl}'");
using (var client = new HttpClient())
using (var stream = await client.GetStreamAsync(BuildInfoUrl))
{
var buildInfoXml = XDocument.Load(stream);
var buildInfo = OrchestratedBuildModel.Parse(buildInfoXml.Root);
var aspnetBuild = buildInfo.Builds
.First(build => string.Equals(build.Name, "aspnet", StringComparison.OrdinalIgnoreCase));
return(new BuildDependencyInfo(
new BuildInfo()
{
Name = RuntimeBuildInfo,
LatestReleaseVersion = aspnetBuild.ProductVersion,
LatestPackages = new Dictionary <string, string>()
},
false,
Enumerable.Empty <string>()));
}
}
public async Task PushChangeAsync(BuildManifestChange change)
{
await Retry.RunAsync(async attempt =>
{
BuildManifestLocation location = change.Location;
// Get the current commit. Use this throughout to ensure a clean transaction.
GitReference remoteRef = await _github.GetReferenceAsync(
location.GitHubProject,
location.GitHubRef);
string remoteCommit = remoteRef.Object.Sha;
Trace.TraceInformation($"Creating update on remote commit: {remoteCommit}");
XElement remoteModelXml = await FetchModelXmlAsync(
location.GitHubProject,
remoteCommit,
location.GitHubBasePath);
OrchestratedBuildModel remoteModel = OrchestratedBuildModel.Parse(remoteModelXml);
// This is a subsequent publish step: make sure a new build hasn't happened already.
if (change.OrchestratedBuildId != remoteModel.Identity.BuildId)
{
throw new ManifestChangeOutOfDateException(
change.OrchestratedBuildId,
remoteModel.Identity.BuildId);
}
OrchestratedBuildModel modifiedModel = OrchestratedBuildModel.Parse(remoteModelXml);
change.ApplyModelChanges(modifiedModel);
if (modifiedModel.Identity.BuildId != change.OrchestratedBuildId)
{
throw new ArgumentException(
"Change action shouldn't modify BuildId. Changed from " +
$"'{change.OrchestratedBuildId}' to '{modifiedModel.Identity.BuildId}'.",
nameof(change));
}
XElement modifiedModelXml = modifiedModel.ToXml();
string[] changedSemaphorePaths = change.SemaphorePaths.ToArray();
// Check if any join groups are completed by this change.
var joinCompleteCheckTasks = change.JoinSemaphoreGroups.NullAsEmpty()
.Select(async g => new
{
Group = g,
Joinable = await IsGroupJoinableAsync(
location,
remoteCommit,
change.OrchestratedBuildId,
changedSemaphorePaths,
g)
});
var completeJoinedSemaphores = (await Task.WhenAll(joinCompleteCheckTasks))
.Where(g => g.Joinable)
.Select(g => g.Group.JoinSemaphorePath)
.ToArray();
IEnumerable <SupplementaryUploadRequest> semaphoreUploads = completeJoinedSemaphores
.Concat(changedSemaphorePaths)
.Select(p => new SupplementaryUploadRequest
{
Path = p,
Contents = new SemaphoreModel
{
BuildId = change.OrchestratedBuildId
}.ToFileContent()
});
IEnumerable <SupplementaryUploadRequest> uploads =
semaphoreUploads.Concat(change.SupplementaryUploads.NullAsEmpty());
if (!XNode.DeepEquals(modifiedModelXml, remoteModelXml))
{
uploads = uploads.Concat(new[]
{
new SupplementaryUploadRequest
{
Path = BuildManifestXmlName,
Contents = modifiedModelXml.ToString()
}
});
}
return(await PushUploadsAsync(
location,
change.CommitMessage,
remoteCommit,
uploads));
});
}
public override bool Execute()
{
string contents = System.IO.File.ReadAllText(ManifestFile);
OrchestratedBuildModel model = OrchestratedBuildModel.Parse(XElement.Parse(contents));
EndpointModel blobFeed = model.Endpoints.First(e => e.IsOrchestratedBlobFeed);
string feedAssetsRoot = blobFeed.Url.Replace("/index.json", "/assets");
string sdkProductVersion = model.Builds
.FirstOrDefault(b => b.Name == "cli")
?.ProductVersion;
string runtimeProductVersion = model.Builds
.FirstOrDefault(b => b.Name == "core-setup")
?.ProductVersion;
string aspnetProductVersion = model.Builds
.FirstOrDefault(b => b.Name == "aspnet")
?.ProductVersion;
var builder = new StringBuilder();
builder.Append("## Product build: ");
builder.AppendLine(model.Identity.ToString());
if (!string.IsNullOrEmpty(SdkTableTemplateFile) && sdkProductVersion != null)
{
builder.AppendLine();
builder.AppendLine(FillTemplate(
SdkTableTemplateFile,
feedAssetsRoot,
sdkProductVersion));
}
if (!string.IsNullOrEmpty(DotNetRuntimeTableTemplateFile) && runtimeProductVersion != null)
{
builder.AppendLine();
builder.AppendLine(FillTemplate(
DotNetRuntimeTableTemplateFile,
feedAssetsRoot,
runtimeProductVersion));
}
if (!string.IsNullOrEmpty(AspNetCoreRuntimeTableTemplateFile) && aspnetProductVersion != null)
{
builder.AppendLine();
builder.AppendLine(FillTemplate(
AspNetCoreRuntimeTableTemplateFile,
feedAssetsRoot,
aspnetProductVersion));
}
builder.AppendLine();
builder.AppendLine("### Built Repositories");
foreach (BuildIdentity build in model.Builds
.Where(b => b.Name != "anonymous")
.OrderBy(b => b.Name))
{
builder.Append(" * ");
builder.AppendLine(build.ToString());
}
System.IO.File.WriteAllText(File, builder.ToString());
return(!Log.HasLoggedErrors);
}
public static OrchestratedBuild Load(string xmlText, string branch)
{
var model = OrchestratedBuildModel.Parse(XDocument.Parse(xmlText).Root);
var orchestratedBuild = new OrchestratedBuild
{
BuildNumber = model.Identity.BuildId,
Branch = branch,
OrchestratedBuildId = $"{branch}/{model.Identity.BuildId}",
Name = model.Identity.Name,
IsStable = string.IsNullOrEmpty(model.Identity.IsStable) ? false : bool.Parse(model.Identity.IsStable),
VersionStamp = model.Identity.VersionStamp
};
var buildsIndex = new Dictionary <string, Build>();
foreach (var buildModel in model.Builds.Where(b => b.Name != "anonymous"))
{
var build = new Build
{
OrchestratedBuildId = orchestratedBuild.OrchestratedBuildId,
Name = buildModel.Name,
BuildNumber = buildModel.BuildId,
BuildId = $"{orchestratedBuild.OrchestratedBuildId}/builds/{buildModel.Name}/{buildModel.BuildId}",
Branch = buildModel.Branch,
ProductVersion = buildModel.ProductVersion,
Commit = buildModel.Commit,
};
buildsIndex[build.Name] = build;
orchestratedBuild.Builds.Add(build);
}
var endpointIndex = new Dictionary <string, Endpoint>();
var packageIndex = new Dictionary <string, Package>();
var blobIndex = new Dictionary <string, Blob>();
foreach (var endpoint in model.Endpoints)
{
var baseUrl = endpoint.Url;
if (baseUrl.EndsWith("/index.json"))
{
baseUrl = baseUrl.Substring(0, baseUrl.Length - 11);
}
if (endpoint.IsOrchestratedBlobFeed)
{
baseUrl += "/assets";
}
var ep = new Endpoint
{
EndpointId = $"{orchestratedBuild.OrchestratedBuildId}/endpoints/{endpoint.Id}",
Id = endpoint.Id,
Type = endpoint.Type,
Url = endpoint.Url
};
endpointIndex[ep.EndpointId] = ep;
orchestratedBuild.Endpoints.Add(ep);
foreach (var artifact in endpoint.Artifacts.Packages)
{
if (packageIndex.TryGetValue(artifact.Id, out var existingRef))
{
existingRef.Endpoints.Add(new EndpointRef()
{
EndpointRefId = $"{orchestratedBuild.OrchestratedBuildId}/endpoints/{ep.EndpointId}/packages/{artifact.Id}",
EndpointId = ep.EndpointId,
ArtifactUrl = $"{baseUrl}/flatcontainer/{artifact.Id}/{artifact.Version}/{artifact.Id}.{artifact.Version}.nupkg",
});
}
else
{
var packageRef = new Package
{
PackageId = $"{orchestratedBuild.OrchestratedBuildId}/packages/{artifact.Id}",
Id = artifact.Id,
Version = artifact.Version,
NonShipping = bool.Parse(artifact.Attributes.TryGetValue("NonShipping", bool.TrueString))
};
if (!string.IsNullOrEmpty(artifact.OriginBuildName) && buildsIndex.TryGetValue(artifact.OriginBuildName, out var build))
{
packageRef.OriginBuildId = build.BuildId;
}
packageIndex[artifact.Id] = packageRef;
orchestratedBuild.Packages.Add(packageRef);
}
}
foreach (var artifact in endpoint.Artifacts.Blobs)
{
if (blobIndex.TryGetValue(artifact.Id, out var existingRef))
{
existingRef.Endpoints.Add(new EndpointRef()
{
EndpointRefId = $"{orchestratedBuild.OrchestratedBuildId}/endpoints/{ep.EndpointId}/blobs/{artifact.Id}",
EndpointId = ep.EndpointId,
ArtifactUrl = $"{baseUrl}/assets/{artifact.Id}",
});
}
else
{
var blobRef = new Blob
{
BlobId = $"{orchestratedBuild.OrchestratedBuildId}/blobs/{artifact.Id}",
Id = artifact.Id,
Type = artifact.Attributes.TryGetValue("Type"),
ShipInstaller = artifact.Attributes.TryGetValue("ShipInstaller")
};
blobIndex[artifact.Id] = blobRef;
orchestratedBuild.Blobs.Add(blobRef);
}
}
}
return(orchestratedBuild);
}
public async Task PushChangeAsync(
GitHubProject project,
string @ref,
string basePath,
string orchestratedBuildId,
Action <OrchestratedBuildModel> changeModel,
IEnumerable <string> semaphorePaths,
IEnumerable <SupplementaryUploadRequest> supplementaryUploads,
string message)
{
await Retry.RunAsync(async attempt =>
{
// Get the current commit. Use this throughout to ensure a clean transaction.
string remoteCommit = (await _github.GetReferenceAsync(project, @ref)).Object.Sha;
Trace.TraceInformation($"Creating update on remote commit: {remoteCommit}");
// This is a subsequent publish step: check to make sure the build id matches.
XElement remoteModelXml = await FetchModelXmlAsync(project, remoteCommit, basePath);
OrchestratedBuildModel remoteModel = OrchestratedBuildModel.Parse(remoteModelXml);
if (orchestratedBuildId != remoteModel.Identity.BuildId)
{
throw new ManifestChangeOutOfDateException(
orchestratedBuildId,
remoteModel.Identity.BuildId);
}
OrchestratedBuildModel modifiedModel = OrchestratedBuildModel.Parse(remoteModelXml);
changeModel(modifiedModel);
if (modifiedModel.Identity.BuildId != orchestratedBuildId)
{
throw new ArgumentException(
"Change action shouldn't modify BuildId. Changed from " +
$"'{orchestratedBuildId}' to '{modifiedModel.Identity.BuildId}'.",
nameof(changeModel));
}
XElement modifiedModelXml = modifiedModel.ToXml();
IEnumerable <SupplementaryUploadRequest> uploads = semaphorePaths.NullAsEmpty()
.Select(p => new SupplementaryUploadRequest
{
Path = p,
Contents = new SemaphoreModel
{
BuildId = orchestratedBuildId
}.ToFileContent()
})
.Concat(supplementaryUploads.NullAsEmpty())
.ToArray();
if (!XNode.DeepEquals(modifiedModelXml, remoteModelXml))
{
uploads = uploads.Concat(new[]
{
new SupplementaryUploadRequest
{
Path = BuildManifestXmlName,
Contents = modifiedModelXml.ToString()
}
});
}
return(await PushUploadsAsync(project, @ref, basePath, message, remoteCommit, uploads));
});
}