public async Task <bool> PushMetadataAsync()
{
try
{
Log.LogMessage(MessageImportance.High, "Starting build metadata push to the Build Asset Registry...");
if (!Directory.Exists(ManifestsPath))
{
Log.LogError($"Required folder '{ManifestsPath}' does not exist.");
}
else
{
List <BuildData> buildsManifestMetadata = GetBuildManifestsMetadata(ManifestsPath);
BuildData finalBuild = MergeBuildManifests(buildsManifestMetadata);
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
Client.Models.Build recordedBuild = await client.Builds.CreateAsync(finalBuild, s_cancellationToken);
Log.LogMessage(MessageImportance.High, $"Metadata has been pushed. Build id in the Build Asset Registry is '{recordedBuild.Id}'");
}
}
catch (Exception exc)
{
Log.LogErrorFromException(exc, true);
}
return(!Log.HasLoggedErrors);
}
private async Task PublishBlobsToAzDoNugetFeedAsync(
List <BlobArtifactModel> blobsToPublish,
IMaestroApi client,
Maestro.Client.Models.Build buildInformation,
FeedConfig feedConfig)
{
foreach (var blob in blobsToPublish)
{
var assetRecord = buildInformation.Assets
.Where(a => a.Name.Equals(blob.Id))
.FirstOrDefault();
if (assetRecord == null)
{
Log.LogError($"Asset with Id {blob.Id} isn't registered on the BAR Build with ID {BARBuildId}");
continue;
}
var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id);
if (assetWithLocations?.Locations.Any(al => al.Location.Equals(feedConfig.TargetFeedURL, StringComparison.OrdinalIgnoreCase)) ?? false)
{
Log.LogMessage($"Asset with Id {blob.Id} already has location {feedConfig.TargetFeedURL}");
continue;
}
await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.Container, feedConfig.TargetFeedURL);
}
}
private async Task PublishPackagesToAzDoNugetFeedAsync(
List <PackageArtifactModel> packagesToPublish,
IMaestroApi client,
Maestro.Client.Models.Build buildInformation,
FeedConfig feedConfig)
{
foreach (var package in packagesToPublish)
{
var assetRecord = buildInformation.Assets
.Where(a => a.Name.Equals(package.Id) && a.Version.Equals(package.Version))
.FirstOrDefault();
if (assetRecord == null)
{
Log.LogError($"Asset with Id {package.Id}, Version {package.Version} isn't registered on the BAR Build with ID {BARBuildId}");
continue;
}
var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id);
if (assetWithLocations?.Locations.Any(al => al.Location.Equals(feedConfig.TargetFeedURL, StringComparison.OrdinalIgnoreCase)) ?? false)
{
Log.LogMessage($"Asset with Id {package.Id}, Version {package.Version} already has location {feedConfig.TargetFeedURL}");
continue;
}
await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.NugetFeed, feedConfig.TargetFeedURL);
}
await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients : MaxClients);
}
public Remote(DarcSettings settings, ILogger logger)
{
ValidateSettings(settings);
_logger = logger;
if (settings.GitType == GitRepoType.GitHub)
{
_gitClient = new GitHubClient(settings.PersonalAccessToken, _logger);
}
else if (settings.GitType == GitRepoType.AzureDevOps)
{
_gitClient = new AzureDevOpsClient(settings.PersonalAccessToken, _logger);
}
// Only initialize the file manager if we have a git client, which excludes "None"
if (_gitClient != null)
{
_fileManager = new GitFileManager(_gitClient, _logger);
}
// Initialize the bar client if there is a password
if (!string.IsNullOrEmpty(settings.BuildAssetRegistryPassword))
{
if (!string.IsNullOrEmpty(settings.BuildAssetRegistryBaseUri))
{
_barClient = ApiFactory.GetAuthenticated(settings.BuildAssetRegistryBaseUri, settings.BuildAssetRegistryPassword);
}
else
{
_barClient = ApiFactory.GetAuthenticated(settings.BuildAssetRegistryPassword);
}
}
}
private async Task HandleBlobPublishingAsync(IMaestroApi client, Maestro.Client.Models.Build buildInformation)
{
foreach (var blobsPerCategory in BlobsByCategory)
{
var category = blobsPerCategory.Key;
var blobs = blobsPerCategory.Value;
if (FeedConfigs.TryGetValue(category, out FeedConfig feedConfig))
{
var feedType = feedConfig.Type.ToUpper();
if (feedType.Equals("AZDONUGETFEED"))
{
await PublishBlobsToAzDoNugetFeedAsync(blobs, client, buildInformation, feedConfig);
}
else if (feedType.Equals("AZURESTORAGEFEED"))
{
await PublishBlobsToAzureStorageNugetFeedAsync(blobs, client, buildInformation, feedConfig);
}
else
{
Log.LogError($"Unknown target feed type for category '{category}': '{feedType}'.");
}
}
else
{
Log.LogError($"No target feed configuration found for artifact category: '{category}'.");
}
}
}
public async Task <bool> PushMetadataAsync(CancellationToken cancellationToken)
{
try
{
cancellationToken.ThrowIfCancellationRequested();
Log.LogMessage(MessageImportance.High, "Starting build metadata push to the Build Asset Registry...");
if (!Directory.Exists(ManifestsPath))
{
Log.LogError($"Required folder '{ManifestsPath}' does not exist.");
}
else
{
List <BuildData> buildsManifestMetadata = GetBuildManifestsMetadata(ManifestsPath, cancellationToken);
BuildData finalBuild = MergeBuildManifests(buildsManifestMetadata);
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
var deps = await GetBuildDependenciesAsync(client, cancellationToken);
Log.LogMessage(MessageImportance.High, "Calculated Dependencies:");
foreach (var dep in deps)
{
Log.LogMessage(MessageImportance.High, $" {dep.BuildId}, IsProduct: {dep.IsProduct}");
}
finalBuild.Dependencies = deps;
Client.Models.Build recordedBuild = await client.Builds.CreateAsync(finalBuild, cancellationToken);
Log.LogMessage(MessageImportance.High, $"Metadata has been pushed. Build id in the Build Asset Registry is '{recordedBuild.Id}'");
// Only 'create' the AzDO (VSO) variables if running in an AzDO build
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_BUILDID")))
{
var defaultChannels = await client.DefaultChannels.ListAsync(
recordedBuild.GitHubBranch ?? recordedBuild.AzureDevOpsBranch,
channelId : null,
enabled : true,
recordedBuild.GitHubRepository ?? recordedBuild.AzureDevOpsRepository);
var defaultChannelsStr = "[" + string.Join("][", defaultChannels.Select(x => x.Channel.Id)) + "]";
Console.WriteLine($"##vso[task.setvariable variable=BARBuildId]{recordedBuild.Id}");
Console.WriteLine($"##vso[task.setvariable variable=DefaultChannels]{defaultChannelsStr}");
Console.WriteLine($"##vso[task.setvariable variable=IsStableBuild]{IsStableBuild}");
}
}
}
catch (Exception exc)
{
Log.LogErrorFromException(exc, true, true, null);
}
return(!Log.HasLoggedErrors);
}
public async Task <bool> ExecuteAsync()
{
try
{
var tasks = AssetManifestPaths
.Select(manifestParam => WhichPublishingTask(manifestParam.ItemSpec))
.ToArray();
// Check that was possible to construct a publishing task for all manifests
if (tasks.Any(t => t == null))
{
return(false);
}
// Process all manifests in parallel
var results = await Task.WhenAll(
tasks.Select(t => t.ExecuteAsync())
);
// Check that all tasks returned true
if (results.All(t => t))
{
// Currently a build can produce several build manifests and publish them independently.
// It's also possible that somehow we have manifests using different versions of the publishing infra.
//
// The V3 infra, once all assets have been published, promotes the build to the target channels informed.
// Since we can have multiple manifests (perhaps using different versions), things
// get a bit more complicated. For now, we are going to just promote the build to the target
// channels if it creates at least one V3 manifest.
//
// There is an issue to merge all build manifests into a single one before publishing:
// https://github.com/dotnet/arcade/issues/5489
if (PublishedV3Manifest)
{
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId);
var targetChannelsIds = TargetChannels.Split('-').Select(ci => int.Parse(ci));
foreach (var targetChannelId in targetChannelsIds)
{
await client.Channels.AddBuildToChannelAsync(BARBuildId, targetChannelId);
}
}
return(true);
}
return(false);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
return(false);
}
}
public async Task <bool> ExecuteAsync()
{
try
{
Log.LogMessage(MessageImportance.High, "Publishing artifacts to feed.");
if (string.IsNullOrWhiteSpace(AssetManifestPath) || !File.Exists(AssetManifestPath))
{
Log.LogError($"Problem reading asset manifest path from '{AssetManifestPath}'");
}
if (!Directory.Exists(BlobAssetsBasePath))
{
Log.LogError($"Problem reading blob assets from {BlobAssetsBasePath}");
}
if (!Directory.Exists(PackageAssetsBasePath))
{
Log.LogError($"Problem reading package assets from {PackageAssetsBasePath}");
}
var buildModel = BuildManifestUtil.ManifestFileToModel(AssetManifestPath, Log);
// Parsing the manifest may fail for several reasons
if (Log.HasLoggedErrors)
{
return(false);
}
// Fetch Maestro record of the build. We're going to use it to get the BAR ID
// of the assets being published so we can add a new location for them.
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId);
ParseTargetFeedConfig();
// Return errors from parsing FeedConfig
if (Log.HasLoggedErrors)
{
return(false);
}
SplitArtifactsInCategories(buildModel);
await HandlePackagePublishingAsync(client, buildInformation);
await HandleBlobPublishingAsync(client, buildInformation);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
public override async Task <bool> ExecuteAsync()
{
if (AnyMissingRequiredProperty())
{
Log.LogError("Missing required properties. Aborting execution.");
return(false);
}
try
{
// Fetch Maestro record of the build. We're going to use it to get the BAR ID
// of the assets being published so we can add a new location for them.
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId);
Dictionary <string, HashSet <Asset> > buildAssets = CreateBuildAssetDictionary(buildInformation);
await ParseTargetFeedConfigAsync();
// Return errors from parsing FeedConfig
if (Log.HasLoggedErrors)
{
return(false);
}
SplitArtifactsInCategories(BuildModel);
// Return errors from the safety checks
if (Log.HasLoggedErrors)
{
return(false);
}
CheckForStableAssetsInNonIsolatedFeeds();
if (Log.HasLoggedErrors)
{
return(false);
}
await Task.WhenAll(new Task[] {
HandlePackagePublishingAsync(buildAssets),
HandleBlobPublishingAsync(buildAssets)
});
await PersistPendingAssetLocationAsync(client);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
private async Task PublishBlobsToAzureStorageNugetFeedAsync(
List <BlobArtifactModel> blobsToPublish,
IMaestroApi client,
Maestro.Client.Models.Build buildInformation,
FeedConfig feedConfig)
{
BlobAssetsBasePath = BlobAssetsBasePath.TrimEnd(
Path.DirectorySeparatorChar,
Path.AltDirectorySeparatorChar)
+ Path.DirectorySeparatorChar;
var blobs = blobsToPublish
.Select(blob =>
{
var fileName = Path.GetFileName(blob.Id);
return(new MSBuild.TaskItem($"{BlobAssetsBasePath}{fileName}", new Dictionary <string, string>
{
{ "RelativeBlobPath", blob.Id }
}));
})
.ToArray();
var blobFeedAction = CreateBlobFeedAction(feedConfig);
var pushOptions = new PushOptions
{
AllowOverwrite = false,
PassIfExistingItemIdentical = true
};
foreach (var blob in blobsToPublish)
{
var assetRecord = buildInformation.Assets
.Where(a => a.Name.Equals(blob.Id))
.SingleOrDefault();
if (assetRecord == null)
{
Log.LogError($"Asset with Id {blob.Id} isn't registered on the BAR Build with ID {BARBuildId}");
continue;
}
var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id);
if (assetWithLocations?.Locations.Any(al => al.Location.Equals(feedConfig.TargetFeedURL, StringComparison.OrdinalIgnoreCase)) ?? false)
{
Log.LogMessage($"Asset with Id {blob.Id} already has location {feedConfig.TargetFeedURL}");
continue;
}
await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.Container, feedConfig.TargetFeedURL);
}
await blobFeedAction.PublishToFlatContainerAsync(blobs, maxClients : 8, pushOptions);
}
private TestParameters(string darcExePath, string gitExePath, string maestroBaseUri, string maestroToken, string gitHubToken, IMaestroApi maestroApi, GitHubClient gitHubApi, TemporaryDirectory dir)
{
_dir = dir;
DarcExePath = darcExePath;
GitExePath = gitExePath;
MaestroBaseUri = maestroBaseUri;
MaestroToken = maestroToken;
GitHubToken = gitHubToken;
MaestroApi = maestroApi;
GitHubApi = gitHubApi;
}
public MaestroApiBarClient(string buildAssetRegistryPat, string buildAssetRegistryBaseUri = null)
{
if (!string.IsNullOrEmpty(buildAssetRegistryBaseUri))
{
_barClient = ApiFactory.GetAuthenticated(
buildAssetRegistryBaseUri,
buildAssetRegistryPat);
}
else
{
_barClient = ApiFactory.GetAuthenticated(buildAssetRegistryPat);
}
}
private TestParameters(string darcExePath, string gitExePath, string maestroBaseUri, string maestroToken, string gitHubToken,
IMaestroApi maestroApi, GitHubClient gitHubApi, Microsoft.DotNet.DarcLib.AzureDevOpsClient azdoClient, TemporaryDirectory dir, string azdoToken)
{
_dir = dir;
DarcExePath = darcExePath;
GitExePath = gitExePath;
MaestroBaseUri = maestroBaseUri;
MaestroToken = maestroToken;
GitHubToken = gitHubToken;
MaestroApi = maestroApi;
GitHubApi = gitHubApi;
AzDoClient = azdoClient;
AzDoToken = azdoToken;
}
public static async Task <TestParameters> GetAsync()
{
IConfiguration userSecrets = new ConfigurationBuilder()
.AddUserSecrets <TestParameters>()
.Build();
string maestroBaseUri = Environment.GetEnvironmentVariable("MAESTRO_BASEURI") ?? userSecrets["MAESTRO_BASEURI"] ?? "https://maestro-int.westus2.cloudapp.azure.com";
string maestroToken = Environment.GetEnvironmentVariable("MAESTRO_TOKEN") ?? userSecrets["MAESTRO_TOKEN"];
string githubToken = Environment.GetEnvironmentVariable("GITHUB_TOKEN") ?? userSecrets["GITHUB_TOKEN"];
string darcPackageSource = Environment.GetEnvironmentVariable("DARC_PACKAGE_SOURCE");
string azdoToken = Environment.GetEnvironmentVariable("AZDO_TOKEN") ?? userSecrets["AZDO_TOKEN"];
var testDir = TemporaryDirectory.Get();
var testDirSharedWrapper = Shareable.Create(testDir);
IMaestroApi maestroApi = maestroToken == null
? ApiFactory.GetAnonymous(maestroBaseUri)
: ApiFactory.GetAuthenticated(maestroBaseUri, maestroToken);
string darcVersion = await maestroApi.Assets.GetDarcVersionAsync();
string dotnetExe = await TestHelpers.Which("dotnet");
var toolInstallArgs = new List <string>
{
"tool", "install",
"--tool-path", testDirSharedWrapper.Peek() !.Directory,
"--version", darcVersion,
"Microsoft.DotNet.Darc",
};
if (!string.IsNullOrEmpty(darcPackageSource))
{
toolInstallArgs.Add("--add-source");
toolInstallArgs.Add(darcPackageSource);
}
await TestHelpers.RunExecutableAsync(dotnetExe, toolInstallArgs.ToArray());
string darcExe = Path.Join(testDirSharedWrapper.Peek() !.Directory, RuntimeInformation.IsOSPlatform(OSPlatform.Windows) ? "darc.exe" : "darc");
Assembly assembly = typeof(TestParameters).Assembly;
var githubApi =
new GitHubClient(
new ProductHeaderValue(assembly.GetName().Name, assembly.GetCustomAttribute <AssemblyInformationalVersionAttribute>()?.InformationalVersion),
new InMemoryCredentialStore(new Credentials(githubToken)));
var azDoClient =
new Microsoft.DotNet.DarcLib.AzureDevOpsClient(await TestHelpers.Which("git"), azdoToken, new NUnitLogger(), testDirSharedWrapper.TryTake() !.Directory);
return(new TestParameters(darcExe, await TestHelpers.Which("git"), maestroBaseUri, maestroToken !, githubToken, maestroApi, githubApi, azDoClient, testDir, azdoToken));
}
private async Task PublishPackagesToAzureStorageNugetFeedAsync(
List <PackageArtifactModel> packagesToPublish,
IMaestroApi client,
Maestro.Client.Models.Build buildInformation,
FeedConfig feedConfig)
{
PackageAssetsBasePath = PackageAssetsBasePath.TrimEnd(
Path.DirectorySeparatorChar,
Path.AltDirectorySeparatorChar)
+ Path.DirectorySeparatorChar;
var packages = packagesToPublish.Select(p => $"{PackageAssetsBasePath}{p.Id}.{p.Version}.nupkg");
var blobFeedAction = CreateBlobFeedAction(feedConfig);
var pushOptions = new PushOptions
{
AllowOverwrite = false,
PassIfExistingItemIdentical = true
};
foreach (var package in packagesToPublish)
{
var assetRecord = buildInformation.Assets
.Where(a => a.Name.Equals(package.Id) && a.Version.Equals(package.Version))
.FirstOrDefault();
if (assetRecord == null)
{
Log.LogError($"Asset with Id {package.Id}, Version {package.Version} isn't registered on the BAR Build with ID {BARBuildId}");
continue;
}
var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id);
if (assetWithLocations?.Locations.Any(al => al.Location.Equals(feedConfig.TargetFeedURL, StringComparison.OrdinalIgnoreCase)) ?? false)
{
Log.LogMessage($"Asset with Id {package.Id}, Version {package.Version} already has location {feedConfig.TargetFeedURL}");
continue;
}
await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.NugetFeed, feedConfig.TargetFeedURL);
}
await blobFeedAction.PushToFeedAsync(packages, pushOptions);
}
private async Task <IImmutableList <BuildRef> > GetBuildDependenciesAsync(
IMaestroApi client,
CancellationToken cancellationToken)
{
var logger = new MSBuildLogger(Log);
var local = new Local(logger, RepoRoot);
IEnumerable <DependencyDetail> dependencies = await local.GetDependenciesAsync();
var builds = new Dictionary <int, bool>();
var assetCache = new Dictionary <(string name, string version, string commit), int>();
var buildCache = new Dictionary <int, Client.Models.Build>();
foreach (var dep in dependencies)
{
var buildId = await GetBuildId(dep, client, buildCache, assetCache, cancellationToken);
if (buildId == null)
{
Log.LogMessage(
MessageImportance.High,
$"Asset '{dep.Name}@{dep.Version}' not found in BAR, most likely this is an external dependency, ignoring...");
continue;
}
Log.LogMessage(
MessageImportance.Normal,
$"Dependency '{dep.Name}@{dep.Version}' found in build {buildId.Value}");
var isProduct = dep.Type == DependencyType.Product;
if (!builds.ContainsKey(buildId.Value))
{
builds[buildId.Value] = isProduct;
}
else
{
builds[buildId.Value] = isProduct || builds[buildId.Value];
}
}
return(builds.Select(t => new BuildRef(t.Key, t.Value, 0)).ToImmutableList());
}
public async Task <bool> PushMetadataAsync(CancellationToken cancellationToken)
{
try
{
cancellationToken.ThrowIfCancellationRequested();
Log.LogMessage(MessageImportance.High, "Starting build metadata push to the Build Asset Registry...");
if (!Directory.Exists(ManifestsPath))
{
Log.LogError($"Required folder '{ManifestsPath}' does not exist.");
}
else
{
List <BuildData> buildsManifestMetadata = GetBuildManifestsMetadata(ManifestsPath, cancellationToken);
BuildData finalBuild = MergeBuildManifests(buildsManifestMetadata);
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
var deps = await GetBuildDependenciesAsync(client, cancellationToken);
Log.LogMessage(MessageImportance.High, "Calculated Dependencies:");
foreach (var dep in deps)
{
Log.LogMessage(MessageImportance.High, $" {dep.BuildId}, IsProduct: {dep.IsProduct}");
}
finalBuild.Dependencies = deps;
Client.Models.Build recordedBuild = await client.Builds.CreateAsync(finalBuild, cancellationToken);
Log.LogMessage(MessageImportance.High, $"Metadata has been pushed. Build id in the Build Asset Registry is '{recordedBuild.Id}'");
}
}
catch (Exception exc)
{
Log.LogErrorFromException(exc, true, true, null);
}
return(!Log.HasLoggedErrors);
}
private async Task HandleBlobPublishingAsync(IMaestroApi client, Maestro.Client.Models.Build buildInformation)
{
foreach (var blobsPerCategory in BlobsByCategory)
{
var category = blobsPerCategory.Key;
var blobs = blobsPerCategory.Value;
if (FeedConfigs.TryGetValue(category, out List <FeedConfig> feedConfigsForCategory))
{
foreach (var feedConfig in feedConfigsForCategory)
{
List <BlobArtifactModel> filteredBlobs = FilterBlobs(blobs, feedConfig);
switch (feedConfig.Type)
{
case FeedType.AzDoNugetFeed:
await PublishBlobsToAzDoNugetFeedAsync(filteredBlobs, client, buildInformation, feedConfig);
break;
case FeedType.AzureStorageFeed:
await PublishBlobsToAzureStorageNugetFeedAsync(filteredBlobs, client, buildInformation, feedConfig);
break;
default:
Log.LogError($"Unknown target feed type for category '{category}': '{feedConfig.Type}'.");
break;
}
}
}
else
{
Log.LogError($"No target feed configuration found for artifact category: '{category}'.");
}
}
}
public async Task <bool> ExecuteAsync()
{
try
{
Log.LogMessage(MessageImportance.High, "Performing push feeds.");
if (string.IsNullOrWhiteSpace(ExpectedFeedUrl) || string.IsNullOrWhiteSpace(AccountKey))
{
Log.LogError($"{nameof(ExpectedFeedUrl)} / {nameof(AccountKey)} is not set properly.");
}
else if (string.IsNullOrWhiteSpace(AssetManifestPath) || !File.Exists(AssetManifestPath))
{
Log.LogError($"Problem reading asset manifest path from {AssetManifestPath}");
}
else if (MaxClients <= 0)
{
Log.LogError($"{nameof(MaxClients)} should be greater than zero.");
}
else if (UploadTimeoutInMinutes <= 0)
{
Log.LogError($"{nameof(UploadTimeoutInMinutes)} should be greater than zero.");
}
var buildModel = BuildManifestUtil.ManifestFileToModel(AssetManifestPath, Log);
// Parsing the manifest may fail for several reasons
if (Log.HasLoggedErrors)
{
return(false);
}
// Fetch Maestro record of the build. We're going to use it to get the BAR ID
// of the assets being published so we can add a new location for them.
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId);
var blobFeedAction = new BlobFeedAction(ExpectedFeedUrl, AccountKey, Log);
var pushOptions = new PushOptions
{
AllowOverwrite = Overwrite,
PassIfExistingItemIdentical = PassIfExistingItemIdentical
};
if (buildModel.Artifacts.Packages.Any())
{
if (!Directory.Exists(PackageAssetsBasePath))
{
Log.LogError($"Invalid {nameof(PackageAssetsBasePath)} was supplied: {PackageAssetsBasePath}");
return(false);
}
PackageAssetsBasePath = PackageAssetsBasePath.TrimEnd(Path.DirectorySeparatorChar,
Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar;
var packages = buildModel.Artifacts.Packages.Select(p => $"{PackageAssetsBasePath}{p.Id}.{p.Version}.nupkg");
await blobFeedAction.PushToFeedAsync(packages, pushOptions);
foreach (var package in buildModel.Artifacts.Packages)
{
var assetRecord = buildInformation.Assets
.Where(a => a.Name.Equals(package.Id) && a.Version.Equals(package.Version))
.Single();
if (assetRecord == null)
{
Log.LogError($"Asset with Id {package.Id}, Version {package.Version} isn't registered on the BAR Build with ID {BARBuildId}");
continue;
}
await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id.Value, ExpectedFeedUrl, "NugetFeed");
}
}
if (buildModel.Artifacts.Blobs.Any())
{
if (!Directory.Exists(BlobAssetsBasePath))
{
Log.LogError($"Invalid {nameof(BlobAssetsBasePath)} was supplied: {BlobAssetsBasePath}");
return(false);
}
BlobAssetsBasePath = BlobAssetsBasePath.TrimEnd(Path.DirectorySeparatorChar,
Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar;
var blobs = buildModel.Artifacts.Blobs
.Select(blob =>
{
var fileName = Path.GetFileName(blob.Id);
return(new MSBuild.TaskItem($"{BlobAssetsBasePath}{fileName}", new Dictionary <string, string>
{
{ "RelativeBlobPath", $"{BuildManifestUtil.AssetsVirtualDir}{blob.Id}" }
}));
})
.ToArray();
await blobFeedAction.PublishToFlatContainerAsync(blobs, MaxClients, UploadTimeoutInMinutes, pushOptions);
foreach (var package in buildModel.Artifacts.Blobs)
{
var assetRecord = buildInformation.Assets
.Where(a => a.Name.Equals(package.Id))
.SingleOrDefault();
if (assetRecord == null)
{
Log.LogError($"Asset with Id {package.Id} isn't registered on the BAR Build with ID {BARBuildId}");
continue;
}
await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id.Value, ExpectedFeedUrl, "NugetFeed");
}
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
public async Task <bool> PushMetadataAsync(CancellationToken cancellationToken)
{
try
{
cancellationToken.ThrowIfCancellationRequested();
Log.LogMessage(MessageImportance.High, "Starting build metadata push to the Build Asset Registry...");
if (!Directory.Exists(ManifestsPath))
{
Log.LogError($"Required folder '{ManifestsPath}' does not exist.");
}
else
{
List <BuildData> buildsManifestMetadata = GetBuildManifestsMetadata(ManifestsPath, cancellationToken);
if (buildsManifestMetadata.Count == 0)
{
Log.LogError($"No build manifests found matching the search pattern {SearchPattern} in {ManifestsPath}");
return(!Log.HasLoggedErrors);
}
BuildData finalBuild = MergeBuildManifests(buildsManifestMetadata);
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
var deps = await GetBuildDependenciesAsync(client, cancellationToken);
Log.LogMessage(MessageImportance.High, "Calculated Dependencies:");
foreach (var dep in deps)
{
Log.LogMessage(MessageImportance.High, $" {dep.BuildId}, IsProduct: {dep.IsProduct}");
}
finalBuild.Dependencies = deps;
Client.Models.Build recordedBuild = await client.Builds.CreateAsync(finalBuild, cancellationToken);
BuildId = recordedBuild.Id;
Log.LogMessage(MessageImportance.High, $"Metadata has been pushed. Build id in the Build Asset Registry is '{recordedBuild.Id}'");
Console.WriteLine($"##vso[build.addbuildtag]BAR ID - {recordedBuild.Id}");
// Only 'create' the AzDO (VSO) variables if running in an AzDO build
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_BUILDID")))
{
IEnumerable <DefaultChannel> defaultChannels = await GetBuildDefaultChannelsAsync(client, recordedBuild);
HashSet <int> targetChannelIds = new HashSet <int>(defaultChannels.Select(dc => dc.Channel.Id));
var defaultChannelsStr = "[" + string.Join("][", targetChannelIds) + "]";
Log.LogMessage(MessageImportance.High, $"Determined build will be added to the following channels: { defaultChannelsStr}");
Console.WriteLine($"##vso[task.setvariable variable=BARBuildId]{recordedBuild.Id}");
Console.WriteLine($"##vso[task.setvariable variable=DefaultChannels]{defaultChannelsStr}");
Console.WriteLine($"##vso[task.setvariable variable=IsStableBuild]{IsStableBuild}");
}
}
}
catch (Exception exc)
{
Log.LogErrorFromException(exc, true, true, null);
}
return(!Log.HasLoggedErrors);
}
private static async Task <int?> GetBuildId(DependencyDetail dep, IMaestroApi client, Dictionary <int, Client.Models.Build> buildCache,
Dictionary <(string name, string version, string commit), int> assetCache, CancellationToken cancellationToken)
public override async Task<bool> ExecuteAsync()
{
if (AnyMissingRequiredProperty())
{
Log.LogError("Missing required properties. Aborting execution.");
return false;
}
try
{
List<int> targetChannelsIds = new List<int>();
foreach (var channelIdStr in TargetChannels.Split(','))
{
if (!int.TryParse(channelIdStr, out var channelId))
{
Log.LogError($"Value '{channelIdStr}' isn't recognized as a valid Maestro++ channel ID.");
continue;
}
targetChannelsIds.Add(channelId);
}
if (Log.HasLoggedErrors)
{
Log.LogError($"Could not parse the target channels list '{TargetChannels}'. It should be a comma separated list of integers.");
return false;
}
SplitArtifactsInCategories(BuildModel);
if (Log.HasLoggedErrors)
{
return false;
}
// Fetch Maestro record of the build. We're going to use it to get the BAR ID
// of the assets being published so we can add a new location for them.
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId);
Dictionary<string, HashSet<Asset>> buildAssets = CreateBuildAssetDictionary(buildInformation);
foreach (var targetChannelId in targetChannelsIds)
{
TargetChannelConfig targetChannelConfig = PublishingConstants.ChannelInfos
.Where(ci =>
ci.Id == targetChannelId &&
(ci.PublishingInfraVersion == PublishingInfraVersion.All || ci.PublishingInfraVersion == PublishingInfraVersion.Next))
.FirstOrDefault();
// Invalid channel ID was supplied
if (targetChannelConfig.Equals(default(TargetChannelConfig)))
{
Log.LogError($"Channel with ID '{targetChannelId}' is not configured to be published to.");
return false;
}
if (await client.Channels.GetChannelAsync(targetChannelId) == null)
{
Log.LogError($"Channel with ID '{targetChannelId}' does not exist in BAR.");
return false;
}
Log.LogMessage(MessageImportance.High, $"Publishing to this target channel: {targetChannelConfig}");
var targetFeedsSetup = new SetupTargetFeedConfigV3(
InternalBuild,
BuildModel.Identity.IsStable.Equals("true", System.StringComparison.OrdinalIgnoreCase),
BuildModel.Identity.Name,
BuildModel.Identity.Commit,
AzureStorageTargetFeedKey,
PublishInstallersAndChecksums,
targetChannelConfig.InstallersFeed,
InstallersFeedKey,
targetChannelConfig.ChecksumsFeed,
CheckSumsFeedKey,
targetChannelConfig.ShippingFeed,
targetChannelConfig.TransportFeed,
targetChannelConfig.SymbolsFeed,
$"dotnet/{targetChannelConfig.AkaMSChannelName}",
AzureDevOpsFeedsKey,
BuildEngine = this.BuildEngine);
var targetFeedConfigs = targetFeedsSetup.Setup();
// No target feeds to publish to, very likely this is an error
if (targetFeedConfigs.Count() == 0)
{
Log.LogError($"No target feeds were found to publish the assets to.");
return false;
}
foreach (var feedConfig in targetFeedConfigs)
{
Log.LogMessage(MessageImportance.High, $"Target feed config: {feedConfig}");
TargetFeedContentType categoryKey = feedConfig.ContentType;
if (!FeedConfigs.TryGetValue(categoryKey, out _))
{
FeedConfigs[categoryKey] = new HashSet<TargetFeedConfig>();
}
FeedConfigs[categoryKey].Add(feedConfig);
}
}
CheckForStableAssetsInNonIsolatedFeeds();
if (Log.HasLoggedErrors)
{
return false;
}
await Task.WhenAll(new Task[] {
HandlePackagePublishingAsync(buildAssets),
HandleBlobPublishingAsync(buildAssets)
});
await PersistPendingAssetLocationAsync(client);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
if (!Log.HasLoggedErrors)
{
Log.LogMessage(MessageImportance.High, "Publishing finished with success.");
}
return !Log.HasLoggedErrors;
}
public override async Task <bool> ExecuteAsync()
{
if (AnyMissingRequiredProperty())
{
Log.LogError("Missing required properties. Aborting execution.");
return(false);
}
try
{
List <int> targetChannelsIds = new List <int>();
foreach (var channelIdStr in TargetChannels.Split('-'))
{
if (!int.TryParse(channelIdStr, out var channelId))
{
Log.LogError(
$"Value '{channelIdStr}' isn't recognized as a valid Maestro++ channel ID. To add a channel refer to https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Publishing.md#how-to-add-a-new-channel-to-use-v3-publishing.");
continue;
}
targetChannelsIds.Add(channelId);
}
if (Log.HasLoggedErrors)
{
Log.LogError(
$"Could not parse the target channels list '{TargetChannels}'. It should be a comma separated list of integers.");
return(false);
}
SplitArtifactsInCategories(BuildModel);
if (Log.HasLoggedErrors)
{
return(false);
}
// Fetch Maestro record of the build. We're going to use it to get the BAR ID
// of the assets being published so we can add a new location for them.
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId);
Dictionary <string, HashSet <Asset> > buildAssets = CreateBuildAssetDictionary(buildInformation);
foreach (var targetChannelId in targetChannelsIds)
{
TargetChannelConfig targetChannelConfig = PublishingConstants.ChannelInfos
.Where(ci =>
ci.Id == targetChannelId &&
(ci.PublishingInfraVersion == PublishingInfraVersion.All ||
ci.PublishingInfraVersion == PublishingInfraVersion.Next))
.FirstOrDefault();
// Invalid channel ID was supplied
if (targetChannelConfig.Equals(default(TargetChannelConfig)))
{
Log.LogError($"Channel with ID '{targetChannelId}' is not configured to be published to.");
return(false);
}
if (await client.Channels.GetChannelAsync(targetChannelId) == null)
{
Log.LogError($"Channel with ID '{targetChannelId}' does not exist in BAR.");
return(false);
}
Log.LogMessage(MessageImportance.High, $"Publishing to this target channel: {targetChannelConfig}");
string shortLinkUrl = string.IsNullOrEmpty(targetChannelConfig.AkaMSChannelName)
? $"dotnet/"
: $"dotnet/{targetChannelConfig.AkaMSChannelName}/{BuildQuality}";
var targetFeedsSetup = new SetupTargetFeedConfigV3(
targetChannelConfig.IsInternal,
BuildModel.Identity.IsStable,
BuildModel.Identity.Name,
BuildModel.Identity.Commit,
AzureStorageTargetFeedKey,
PublishInstallersAndChecksums,
GetFeed(targetChannelConfig.InstallersFeed, InstallersFeedOverride),
targetChannelConfig.IsInternal ? InternalInstallersFeedKey : InstallersFeedKey,
GetFeed(targetChannelConfig.ChecksumsFeed, ChecksumsFeedOverride),
targetChannelConfig.IsInternal ? InternalCheckSumsFeedKey : CheckSumsFeedKey,
GetFeed(targetChannelConfig.ShippingFeed, ShippingFeedOverride),
GetFeed(targetChannelConfig.TransportFeed, TransportFeedOverride),
GetFeed(targetChannelConfig.SymbolsFeed, SymbolsFeedOverride),
shortLinkUrl,
AzureDevOpsFeedsKey,
BuildEngine = this.BuildEngine,
targetChannelConfig.SymbolTargetType,
azureDevOpsPublicStaticSymbolsFeed: GetFeed(null, PublicSymbolsFeedOverride),
filesToExclude: targetChannelConfig.FilenamesToExclude,
flatten: targetChannelConfig.Flatten);
var targetFeedConfigs = targetFeedsSetup.Setup();
// No target feeds to publish to, very likely this is an error
if (targetFeedConfigs.Count() == 0)
{
Log.LogError($"No target feeds were found to publish the assets to.");
return(false);
}
foreach (var feedConfig in targetFeedConfigs)
{
Log.LogMessage(MessageImportance.High, $"Target feed config: {feedConfig}");
TargetFeedContentType categoryKey = feedConfig.ContentType;
if (!FeedConfigs.TryGetValue(categoryKey, out _))
{
FeedConfigs[categoryKey] = new HashSet <TargetFeedConfig>();
}
FeedConfigs[categoryKey].Add(feedConfig);
}
}
CheckForStableAssetsInNonIsolatedFeeds();
if (Log.HasLoggedErrors)
{
return(false);
}
string temporarySymbolsLocation = "";
if (!UseStreamingPublishing)
{
temporarySymbolsLocation =
Path.GetFullPath(Path.Combine(BlobAssetsBasePath, @"..\", "tempSymbols"));
EnsureTemporaryDirectoryExists(temporarySymbolsLocation);
DeleteTemporaryFiles(temporarySymbolsLocation);
// Copying symbol files to temporary location is required because the symUploader API needs read/write access to the files,
// since we publish blobs and symbols in parallel this will cause IO exceptions.
CopySymbolFilesToTemporaryLocation(BuildModel, temporarySymbolsLocation);
}
using var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients);
await Task.WhenAll(new Task[]
{
HandlePackagePublishingAsync(buildAssets, clientThrottle),
HandleBlobPublishingAsync(buildAssets, clientThrottle),
HandleSymbolPublishingAsync(
PdbArtifactsBasePath,
MsdlToken,
SymWebToken,
SymbolPublishingExclusionsFile,
PublishSpecialClrFiles,
buildAssets,
clientThrottle,
temporarySymbolsLocation)
});
DeleteTemporaryFiles(temporarySymbolsLocation);
DeleteTemporaryDirectory(temporarySymbolsLocation);
await PersistPendingAssetLocationAsync(client);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
if (!Log.HasLoggedErrors)
{
Log.LogMessage(MessageImportance.High, "Publishing finished with success.");
}
return(!Log.HasLoggedErrors);
}
public async Task <bool> ExecuteAsync()
{
try
{
Log.LogMessage(MessageImportance.High, "Publishing artifacts to feed.");
if (string.IsNullOrWhiteSpace(AssetManifestPath) || !File.Exists(AssetManifestPath))
{
Log.LogError($"Problem reading asset manifest path from '{AssetManifestPath}'");
}
if (!Directory.Exists(BlobAssetsBasePath))
{
Log.LogError($"Problem reading blob assets from {BlobAssetsBasePath}");
}
if (!Directory.Exists(PackageAssetsBasePath))
{
Log.LogError($"Problem reading package assets from {PackageAssetsBasePath}");
}
var buildModel = BuildManifestUtil.ManifestFileToModel(AssetManifestPath, Log);
// Parsing the manifest may fail for several reasons
if (Log.HasLoggedErrors)
{
return(false);
}
// Fetch Maestro record of the build. We're going to use it to get the BAR ID
// of the assets being published so we can add a new location for them.
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId);
foreach (var fc in TargetFeedConfig)
{
var feedConfig = new FeedConfig()
{
TargetFeedURL = fc.GetMetadata("TargetURL"),
Type = fc.GetMetadata("Type"),
FeedKey = fc.GetMetadata("Token")
};
if (string.IsNullOrEmpty(feedConfig.TargetFeedURL) ||
string.IsNullOrEmpty(feedConfig.Type) ||
string.IsNullOrEmpty(feedConfig.FeedKey))
{
Log.LogError($"Invalid FeedConfig entry. TargetURL='{feedConfig.TargetFeedURL}' Type='{feedConfig.Type}' Token='{feedConfig.FeedKey}'");
}
FeedConfigs.Add(fc.ItemSpec.Trim().ToUpper(), feedConfig);
}
// Return errors from parsing FeedConfig
if (Log.HasLoggedErrors)
{
return(false);
}
SplitArtifactsInCategories(buildModel);
await HandlePackagePublishingAsync(client, buildInformation);
await HandleBlobPublishingAsync(client, buildInformation);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
public async Task <bool> PushMetadataAsync(CancellationToken cancellationToken)
{
try
{
cancellationToken.ThrowIfCancellationRequested();
Log.LogMessage(MessageImportance.High, "Starting build metadata push to the Build Asset Registry...");
if (!Directory.Exists(ManifestsPath))
{
Log.LogError($"Required folder '{ManifestsPath}' does not exist.");
}
else
{
(List <BuildData> buildsManifestMetadata,
List <SigningInformation> signingInformation,
ManifestBuildData manifestBuildData) = GetBuildManifestsMetadata(ManifestsPath, cancellationToken);
if (buildsManifestMetadata.Count == 0)
{
Log.LogError($"No build manifests found matching the search pattern {SearchPattern} in {ManifestsPath}");
return(!Log.HasLoggedErrors);
}
BuildData finalBuild = MergeBuildManifests(buildsManifestMetadata);
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
var deps = await GetBuildDependenciesAsync(client, cancellationToken);
Log.LogMessage(MessageImportance.High, "Calculated Dependencies:");
foreach (var dep in deps)
{
Log.LogMessage(MessageImportance.High, $" {dep.BuildId}, IsProduct: {dep.IsProduct}");
}
finalBuild.Dependencies = deps;
// Based on the in-memory merged manifest, create a physical XML file and
// upload it to the BlobArtifacts folder only when publishingVersion >= 3
if (manifestBuildData.PublishingVersion >= 3)
{
SigningInformation finalSigningInfo = MergeSigningInfo(signingInformation);
// Inject an entry of MergedManifest.xml to the in-memory merged manifest
string location = null;
AssetData assetData = finalBuild.Assets.FirstOrDefault();
if (assetData != null)
{
AssetLocationData assetLocationData = assetData.Locations.FirstOrDefault();
if (assetLocationData != null)
{
location = assetLocationData.Location;
}
}
finalBuild.Assets = finalBuild.Assets.Add(GetManifestAsAsset(finalBuild.Assets, location, MergedManifestFileName));
BuildModel modelForManifest = CreateMergedManifestBuildModel(finalBuild.Assets, manifestBuildData);
PushMergedManifest(modelForManifest, finalSigningInfo);
}
Client.Models.Build recordedBuild = await client.Builds.CreateAsync(finalBuild, cancellationToken);
BuildId = recordedBuild.Id;
Log.LogMessage(MessageImportance.High, $"Metadata has been pushed. Build id in the Build Asset Registry is '{recordedBuild.Id}'");
Console.WriteLine($"##vso[build.addbuildtag]BAR ID - {recordedBuild.Id}");
// Only 'create' the AzDO (VSO) variables if running in an AzDO build
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_BUILDID")))
{
IEnumerable <DefaultChannel> defaultChannels = await GetBuildDefaultChannelsAsync(client, recordedBuild);
HashSet <int> targetChannelIds = new HashSet <int>(defaultChannels.Select(dc => dc.Channel.Id));
var defaultChannelsStr = "[" + string.Join("][", targetChannelIds) + "]";
Log.LogMessage(MessageImportance.High, $"Determined build will be added to the following channels: { defaultChannelsStr}");
Console.WriteLine($"##vso[task.setvariable variable=BARBuildId]{recordedBuild.Id}");
Console.WriteLine($"##vso[task.setvariable variable=DefaultChannels]{defaultChannelsStr}");
Console.WriteLine($"##vso[task.setvariable variable=IsStableBuild]{IsStableBuild}");
}
}
}
catch (Exception exc)
{
Log.LogErrorFromException(exc, true, true, null);
}
return(!Log.HasLoggedErrors);
}
private async Task <IEnumerable <DefaultChannel> > GetBuildDefaultChannelsAsync(IMaestroApi client, Client.Models.Build recordedBuild)
{
var defaultChannels = new List <DefaultChannel>();
if (recordedBuild.GitHubBranch != null && recordedBuild.GitHubRepository != null)
{
defaultChannels.AddRange(
await client.DefaultChannels.ListAsync(
branch: recordedBuild.GitHubBranch,
channelId: null,
enabled: true,
repository: recordedBuild.GitHubRepository
));
}
if (recordedBuild.AzureDevOpsBranch != null && recordedBuild.AzureDevOpsRepository != null)
{
defaultChannels.AddRange(
await client.DefaultChannels.ListAsync(
branch: recordedBuild.AzureDevOpsBranch,
channelId: null,
enabled: true,
repository: recordedBuild.AzureDevOpsRepository
));
}
Log.LogMessage(MessageImportance.High, "Found the following default channels:");
foreach (var defaultChannel in defaultChannels)
{
Log.LogMessage(
MessageImportance.High,
$" {defaultChannel.Repository}@{defaultChannel.Branch} " +
$"=> ({defaultChannel.Channel.Id}) {defaultChannel.Channel.Name}");
}
return(defaultChannels);
}
public async Task <bool> PushMetadataAsync(CancellationToken cancellationToken)
{
try
{
cancellationToken.ThrowIfCancellationRequested();
Log.LogMessage(MessageImportance.High, "Starting build metadata push to the Build Asset Registry...");
if (!Directory.Exists(ManifestsPath))
{
Log.LogError($"Required folder '{ManifestsPath}' does not exist.");
}
else
{
//get the list of manifests
List <Manifest> parsedManifests = GetParsedManifests(ManifestsPath, cancellationToken);
if (parsedManifests.Count == 0)
{
Log.LogError(
$"No manifests found matching the search pattern {SearchPattern} in {ManifestsPath}");
return(!Log.HasLoggedErrors);
}
Manifest manifest;
//check if the manifest have any duplicate packages and blobs
if (parsedManifests.Count > 1)
{
manifest = MergeManifests(parsedManifests);
}
else
{
manifest = parsedManifests[0];
}
List <SigningInformation> signingInformation = new List <SigningInformation>();
foreach (var m in parsedManifests)
{
if (m.SigningInformation != null)
{
signingInformation.Add(m.SigningInformation);
}
}
//get packages blobs and signing info
(List <PackageArtifactModel> packages,
List <BlobArtifactModel> blobs) = GetPackagesAndBlobsInfo(manifest);
//create merged buildModel to create the merged manifest
BuildModel modelForManifest =
CreateMergedManifestBuildModel(packages, blobs, manifest);
//add manifest as an asset to the buildModel
var mergedManifestAsset = GetManifestAsAsset(blobs, MergedManifestFileName);
modelForManifest.Artifacts.Blobs.Add(mergedManifestAsset);
SigningInformation finalSigningInfo = MergeSigningInfo(signingInformation);
// push the merged manifest, this is required for only publishingVersion 3 and above
if (manifest.PublishingVersion >= 3)
{
PushMergedManifest(modelForManifest, finalSigningInfo);
}
// populate buildData and assetData using merged manifest data
BuildData buildData = GetMaestroBuildDataFromMergedManifest(modelForManifest, manifest, cancellationToken);
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
var deps = await GetBuildDependenciesAsync(client, cancellationToken);
Log.LogMessage(MessageImportance.High, "Calculated Dependencies:");
foreach (var dep in deps)
{
Log.LogMessage(MessageImportance.High, $" {dep.BuildId}, IsProduct: {dep.IsProduct}");
}
buildData.Dependencies = deps;
LookupForMatchingGitHubRepository(manifest);
buildData.GitHubBranch = GitHubBranch;
buildData.GitHubRepository = GitHubRepository;
Client.Models.Build recordedBuild = await client.Builds.CreateAsync(buildData, cancellationToken);
BuildId = recordedBuild.Id;
Log.LogMessage(MessageImportance.High,
$"Metadata has been pushed. Build id in the Build Asset Registry is '{recordedBuild.Id}'");
Console.WriteLine($"##vso[build.addbuildtag]BAR ID - {recordedBuild.Id}");
// Only 'create' the AzDO (VSO) variables if running in an AzDO build
if (!string.IsNullOrEmpty(Environment.GetEnvironmentVariable("BUILD_BUILDID")))
{
IEnumerable <DefaultChannel> defaultChannels =
await GetBuildDefaultChannelsAsync(client, recordedBuild);
HashSet <int> targetChannelIds = new HashSet <int>(defaultChannels.Select(dc => dc.Channel.Id));
var defaultChannelsStr = "[" + string.Join("][", targetChannelIds) + "]";
Log.LogMessage(MessageImportance.High,
$"Determined build will be added to the following channels: {defaultChannelsStr}");
Console.WriteLine($"##vso[task.setvariable variable=BARBuildId]{recordedBuild.Id}");
Console.WriteLine($"##vso[task.setvariable variable=DefaultChannels]{defaultChannelsStr}");
Console.WriteLine($"##vso[task.setvariable variable=IsStableBuild]{IsStableBuild}");
}
}
}
catch (Exception exc)
{
Log.LogErrorFromException(exc, true, true, null);
}
return(!Log.HasLoggedErrors);
}
