public override async Task <bool> ExecuteAsync()
{
if (AnyMissingRequiredProperty())
{
Log.LogError("Missing required properties. Aborting execution.");
return(false);
}
try
{
List <int> targetChannelsIds = new List <int>();
foreach (var channelIdStr in TargetChannels.Split('-'))
{
if (!int.TryParse(channelIdStr, out var channelId))
{
Log.LogError(
$"Value '{channelIdStr}' isn't recognized as a valid Maestro++ channel ID. To add a channel refer to https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Publishing.md#how-to-add-a-new-channel-to-use-v3-publishing.");
continue;
}
targetChannelsIds.Add(channelId);
}
if (Log.HasLoggedErrors)
{
Log.LogError(
$"Could not parse the target channels list '{TargetChannels}'. It should be a comma separated list of integers.");
return(false);
}
SplitArtifactsInCategories(BuildModel);
if (Log.HasLoggedErrors)
{
return(false);
}
// Fetch Maestro record of the build. We're going to use it to get the BAR ID
// of the assets being published so we can add a new location for them.
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId);
Dictionary <string, HashSet <Asset> > buildAssets = CreateBuildAssetDictionary(buildInformation);
foreach (var targetChannelId in targetChannelsIds)
{
TargetChannelConfig targetChannelConfig = PublishingConstants.ChannelInfos
.Where(ci =>
ci.Id == targetChannelId &&
(ci.PublishingInfraVersion == PublishingInfraVersion.All ||
ci.PublishingInfraVersion == PublishingInfraVersion.Next))
.FirstOrDefault();
// Invalid channel ID was supplied
if (targetChannelConfig.Equals(default(TargetChannelConfig)))
{
Log.LogError($"Channel with ID '{targetChannelId}' is not configured to be published to.");
return(false);
}
if (await client.Channels.GetChannelAsync(targetChannelId) == null)
{
Log.LogError($"Channel with ID '{targetChannelId}' does not exist in BAR.");
return(false);
}
Log.LogMessage(MessageImportance.High, $"Publishing to this target channel: {targetChannelConfig}");
string shortLinkUrl = string.IsNullOrEmpty(targetChannelConfig.AkaMSChannelName)
? $"dotnet/"
: $"dotnet/{targetChannelConfig.AkaMSChannelName}/{BuildQuality}";
var targetFeedsSetup = new SetupTargetFeedConfigV3(
targetChannelConfig.IsInternal,
BuildModel.Identity.IsStable,
BuildModel.Identity.Name,
BuildModel.Identity.Commit,
AzureStorageTargetFeedKey,
PublishInstallersAndChecksums,
GetFeed(targetChannelConfig.InstallersFeed, InstallersFeedOverride),
targetChannelConfig.IsInternal ? InternalInstallersFeedKey : InstallersFeedKey,
GetFeed(targetChannelConfig.ChecksumsFeed, ChecksumsFeedOverride),
targetChannelConfig.IsInternal ? InternalCheckSumsFeedKey : CheckSumsFeedKey,
GetFeed(targetChannelConfig.ShippingFeed, ShippingFeedOverride),
GetFeed(targetChannelConfig.TransportFeed, TransportFeedOverride),
GetFeed(targetChannelConfig.SymbolsFeed, SymbolsFeedOverride),
shortLinkUrl,
AzureDevOpsFeedsKey,
BuildEngine = this.BuildEngine,
targetChannelConfig.SymbolTargetType,
filesToExclude: targetChannelConfig.FilenamesToExclude,
flatten: targetChannelConfig.Flatten);
var targetFeedConfigs = targetFeedsSetup.Setup();
// No target feeds to publish to, very likely this is an error
if (targetFeedConfigs.Count() == 0)
{
Log.LogError($"No target feeds were found to publish the assets to.");
return(false);
}
foreach (var feedConfig in targetFeedConfigs)
{
Log.LogMessage(MessageImportance.High, $"Target feed config: {feedConfig}");
TargetFeedContentType categoryKey = feedConfig.ContentType;
if (!FeedConfigs.TryGetValue(categoryKey, out _))
{
FeedConfigs[categoryKey] = new HashSet <TargetFeedConfig>();
}
FeedConfigs[categoryKey].Add(feedConfig);
}
}
CheckForStableAssetsInNonIsolatedFeeds();
if (Log.HasLoggedErrors)
{
return(false);
}
string temporarySymbolsLocation = "";
if (!UseStreamingPublishing)
{
temporarySymbolsLocation =
Path.GetFullPath(Path.Combine(BlobAssetsBasePath, @"..\", "tempSymbols"));
EnsureTemporaryDirectoryExists(temporarySymbolsLocation);
DeleteTemporaryFiles(temporarySymbolsLocation);
// Copying symbol files to temporary location is required because the symUploader API needs read/write access to the files,
// since we publish blobs and symbols in parallel this will cause IO exceptions.
CopySymbolFilesToTemporaryLocation(BuildModel, temporarySymbolsLocation);
}
using var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients);
await Task.WhenAll(new Task[]
{
HandlePackagePublishingAsync(buildAssets, clientThrottle),
HandleBlobPublishingAsync(buildAssets, clientThrottle),
HandleSymbolPublishingAsync(
PdbArtifactsBasePath,
MsdlToken,
SymWebToken,
SymbolPublishingExclusionsFile,
PublishSpecialClrFiles,
buildAssets,
clientThrottle,
temporarySymbolsLocation)
});
DeleteTemporaryFiles(temporarySymbolsLocation);
DeleteTemporaryDirectory(temporarySymbolsLocation);
await PersistPendingAssetLocationAsync(client);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
if (!Log.HasLoggedErrors)
{
Log.LogMessage(MessageImportance.High, "Publishing finished with success.");
}
return(!Log.HasLoggedErrors);
}
public async Task <bool> ExecuteAsync()
{
try
{
Log.LogMessage(MessageImportance.High, "Performing push feeds.");
if (string.IsNullOrWhiteSpace(ExpectedFeedUrl) || string.IsNullOrWhiteSpace(AccountKey))
{
Log.LogError($"{nameof(ExpectedFeedUrl)} / {nameof(AccountKey)} is not set properly.");
}
else if (string.IsNullOrWhiteSpace(AssetManifestPath) || !File.Exists(AssetManifestPath))
{
Log.LogError($"Problem reading asset manifest path from {AssetManifestPath}");
}
else if (MaxClients <= 0)
{
Log.LogError($"{nameof(MaxClients)} should be greater than zero.");
}
else if (UploadTimeoutInMinutes <= 0)
{
Log.LogError($"{nameof(UploadTimeoutInMinutes)} should be greater than zero.");
}
var buildModel = BuildManifestUtil.ManifestFileToModel(AssetManifestPath, Log);
// Parsing the manifest may fail for several reasons
if (Log.HasLoggedErrors)
{
return(false);
}
// Fetch Maestro record of the build. We're going to use it to get the BAR ID
// of the assets being published so we can add a new location for them.
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId);
var blobFeedAction = new BlobFeedAction(ExpectedFeedUrl, AccountKey, Log);
var pushOptions = new PushOptions
{
AllowOverwrite = Overwrite,
PassIfExistingItemIdentical = PassIfExistingItemIdentical
};
if (buildModel.Artifacts.Packages.Any())
{
if (!Directory.Exists(PackageAssetsBasePath))
{
Log.LogError($"Invalid {nameof(PackageAssetsBasePath)} was supplied: {PackageAssetsBasePath}");
return(false);
}
PackageAssetsBasePath = PackageAssetsBasePath.TrimEnd(Path.DirectorySeparatorChar,
Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar;
var packages = buildModel.Artifacts.Packages.Select(p => $"{PackageAssetsBasePath}{p.Id}.{p.Version}.nupkg");
await blobFeedAction.PushToFeedAsync(packages, pushOptions);
foreach (var package in buildModel.Artifacts.Packages)
{
var assetRecord = buildInformation.Assets
.Where(a => a.Name.Equals(package.Id) && a.Version.Equals(package.Version))
.FirstOrDefault();
if (assetRecord == null)
{
Log.LogError($"Asset with Id {package.Id}, Version {package.Version} isn't registered on the BAR Build with ID {BARBuildId}");
continue;
}
var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id);
if (assetWithLocations?.Locations.Any(al => al.Location.Equals(ExpectedFeedUrl, StringComparison.OrdinalIgnoreCase)) ?? false)
{
Log.LogMessage($"Asset with Id {package.Id}, Version {package.Version} already has location {ExpectedFeedUrl}");
continue;
}
await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.NugetFeed, ExpectedFeedUrl);
}
}
if (buildModel.Artifacts.Blobs.Any())
{
if (!Directory.Exists(BlobAssetsBasePath))
{
Log.LogError($"Invalid {nameof(BlobAssetsBasePath)} was supplied: {BlobAssetsBasePath}");
return(false);
}
BlobAssetsBasePath = BlobAssetsBasePath.TrimEnd(Path.DirectorySeparatorChar,
Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar;
var blobs = buildModel.Artifacts.Blobs
.Select(blob =>
{
var fileName = Path.GetFileName(blob.Id);
return(new MSBuild.TaskItem($"{BlobAssetsBasePath}{fileName}", new Dictionary <string, string>
{
{ "RelativeBlobPath", $"{BuildManifestUtil.AssetsVirtualDir}{blob.Id}" }
}));
})
.ToArray();
await blobFeedAction.PublishToFlatContainerAsync(blobs, MaxClients, pushOptions);
foreach (var package in buildModel.Artifacts.Blobs)
{
var assetRecord = buildInformation.Assets
.Where(a => a.Name.Equals(package.Id))
.SingleOrDefault();
if (assetRecord == null)
{
Log.LogError($"Asset with Id {package.Id} isn't registered on the BAR Build with ID {BARBuildId}");
continue;
}
var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id);
if (assetWithLocations?.Locations.Any(al => al.Location.Equals(ExpectedFeedUrl, StringComparison.OrdinalIgnoreCase)) ?? false)
{
Log.LogMessage($"Asset with Id {package.Id} already has location {ExpectedFeedUrl}");
continue;
}
await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.Container, ExpectedFeedUrl);
}
}
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
