/// <summary>
/// Parse out the input TargetFeedConfig into a dictionary of FeedConfig types
/// </summary>
public async Task ParseTargetFeedConfigAsync()
{
using (HttpClient httpClient = new HttpClient(new HttpClientHandler {
CheckCertificateRevocationList = true
}))
{
foreach (var fc in TargetFeedConfig)
{
string targetFeedUrl = fc.GetMetadata(nameof(Model.TargetFeedConfig.TargetURL));
string feedKey = fc.GetMetadata(nameof(Model.TargetFeedConfig.Token));
string type = fc.GetMetadata(nameof(Model.TargetFeedConfig.Type));
AssetSelection assetSelection = AssetSelection.All;
bool isInternalFeed;
bool isIsolatedFeed = false;
bool isOverridableFeed = false;
if (string.IsNullOrEmpty(targetFeedUrl) ||
string.IsNullOrEmpty(feedKey) ||
string.IsNullOrEmpty(type))
{
Log.LogError($"Invalid FeedConfig entry. {nameof(Model.TargetFeedConfig.TargetURL)}='{targetFeedUrl}' {nameof(Model.TargetFeedConfig.Type)}='{type}' {nameof(Model.TargetFeedConfig.Token)}='{feedKey}'");
continue;
}
if (!targetFeedUrl.EndsWith(PublishingConstants.ExpectedFeedUrlSuffix))
{
Log.LogError($"Exepcted that feed '{targetFeedUrl}' would end in {PublishingConstants.ExpectedFeedUrlSuffix}");
continue;
}
if (!Enum.TryParse <FeedType>(type, true, out FeedType feedType))
{
Log.LogError($"Invalid feed config type '{type}'. Possible values are: {string.Join(", ", Enum.GetNames(typeof(FeedType)))}");
continue;
}
string assetSelectionStr = fc.GetMetadata(nameof(Model.TargetFeedConfig.AssetSelection));
if (!string.IsNullOrEmpty(assetSelectionStr))
{
if (!Enum.TryParse <AssetSelection>(assetSelectionStr, true, out assetSelection))
{
Log.LogError($"Invalid feed config asset selection '{type}'. Possible values are: {string.Join(", ", Enum.GetNames(typeof(AssetSelection)))}");
continue;
}
}
// To determine whether a feed is internal, we allow the user to
// specify the value explicitly.
string isInternalFeedStr = fc.GetMetadata(nameof(Model.TargetFeedConfig.Internal));
if (!string.IsNullOrEmpty(isInternalFeedStr))
{
if (!bool.TryParse(isInternalFeedStr, out isInternalFeed))
{
Log.LogError($"Invalid feed config '{nameof(Model.TargetFeedConfig.Internal)}' setting. Must be 'true' or 'false'.");
continue;
}
}
else
{
bool?isPublicFeed = await GeneralUtils.IsFeedPublicAsync(targetFeedUrl, httpClient, Log);
if (!isPublicFeed.HasValue)
{
continue;
}
else
{
isInternalFeed = !isPublicFeed.Value;
}
}
string isIsolatedFeedStr = fc.GetMetadata(nameof(Model.TargetFeedConfig.Isolated));
if (!string.IsNullOrEmpty(isIsolatedFeedStr))
{
if (!bool.TryParse(isIsolatedFeedStr, out isIsolatedFeed))
{
Log.LogError($"Invalid feed config '{nameof(Model.TargetFeedConfig.Isolated)}' setting. Must be 'true' or 'false'.");
continue;
}
}
string allowOverwriteOnFeed = fc.GetMetadata(nameof(Model.TargetFeedConfig.AllowOverwrite));
if (!string.IsNullOrEmpty(allowOverwriteOnFeed))
{
if (!bool.TryParse(allowOverwriteOnFeed, out isOverridableFeed))
{
Log.LogError($"Invalid feed config '{nameof(Model.TargetFeedConfig.AllowOverwrite)}' setting. Must be 'true' or 'false'.");
continue;
}
}
string latestLinkShortUrlPrefix = fc.GetMetadata(nameof(Model.TargetFeedConfig.LatestLinkShortUrlPrefixes));
if (!string.IsNullOrEmpty(latestLinkShortUrlPrefix))
{
// Verify other inputs are provided
if (string.IsNullOrEmpty(AkaMSClientId) ||
string.IsNullOrEmpty(AkaMSClientSecret) ||
string.IsNullOrEmpty(AkaMSTenant) ||
string.IsNullOrEmpty(AkaMsOwners) ||
string.IsNullOrEmpty(AkaMSCreatedBy))
{
Log.LogError($"If a short url path is provided, please provide {nameof(AkaMSClientId)}, {nameof(AkaMSClientSecret)}, " +
$"{nameof(AkaMSTenant)}, {nameof(AkaMsOwners)}, {nameof(AkaMSCreatedBy)}");
continue;
}
// Set up the link manager if it hasn't already been done
if (LinkManager == null)
{
LinkManager = new LatestLinksManager(AkaMSClientId, AkaMSClientSecret, AkaMSTenant, AkaMSGroupOwner, AkaMSCreatedBy, AkaMsOwners, Log);
}
}
if (!Enum.TryParse(fc.ItemSpec, ignoreCase: true, out TargetFeedContentType categoryKey))
{
Log.LogError($"Invalid target feed config category '{fc.ItemSpec}'.");
}
if (!FeedConfigs.TryGetValue(categoryKey, out _))
{
FeedConfigs[categoryKey] = new HashSet <TargetFeedConfig>();
}
TargetFeedConfig feedConfig = new TargetFeedConfig(
contentType: categoryKey,
targetURL: targetFeedUrl,
type: feedType,
token: feedKey,
latestLinkShortUrlPrefixes: new List <string>()
{
latestLinkShortUrlPrefix
},
assetSelection: assetSelection,
isolated: isIsolatedFeed,
@internal: isInternalFeed,
allowOverwrite: isOverridableFeed);
CheckForInternalBuildsOnPublicFeeds(feedConfig);
FeedConfigs[categoryKey].Add(feedConfig);
}
}
}
public override async Task <bool> ExecuteAsync()
{
if (AnyMissingRequiredProperty())
{
Log.LogError("Missing required properties. Aborting execution.");
return(false);
}
try
{
List <int> targetChannelsIds = new List <int>();
foreach (var channelIdStr in TargetChannels.Split('-'))
{
if (!int.TryParse(channelIdStr, out var channelId))
{
Log.LogError(
$"Value '{channelIdStr}' isn't recognized as a valid Maestro++ channel ID. To add a channel refer to https://github.com/dotnet/arcade/blob/master/Documentation/CorePackages/Publishing.md#how-to-add-a-new-channel-to-use-v3-publishing.");
continue;
}
targetChannelsIds.Add(channelId);
}
if (Log.HasLoggedErrors)
{
Log.LogError(
$"Could not parse the target channels list '{TargetChannels}'. It should be a comma separated list of integers.");
return(false);
}
SplitArtifactsInCategories(BuildModel);
if (Log.HasLoggedErrors)
{
return(false);
}
// Fetch Maestro record of the build. We're going to use it to get the BAR ID
// of the assets being published so we can add a new location for them.
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId);
Dictionary <string, HashSet <Asset> > buildAssets = CreateBuildAssetDictionary(buildInformation);
foreach (var targetChannelId in targetChannelsIds)
{
TargetChannelConfig targetChannelConfig = PublishingConstants.ChannelInfos
.Where(ci =>
ci.Id == targetChannelId &&
(ci.PublishingInfraVersion == PublishingInfraVersion.All ||
ci.PublishingInfraVersion == PublishingInfraVersion.Next))
.FirstOrDefault();
// Invalid channel ID was supplied
if (targetChannelConfig.Equals(default(TargetChannelConfig)))
{
Log.LogError($"Channel with ID '{targetChannelId}' is not configured to be published to.");
return(false);
}
if (await client.Channels.GetChannelAsync(targetChannelId) == null)
{
Log.LogError($"Channel with ID '{targetChannelId}' does not exist in BAR.");
return(false);
}
Log.LogMessage(MessageImportance.High, $"Publishing to this target channel: {targetChannelConfig}");
string shortLinkUrl = string.IsNullOrEmpty(targetChannelConfig.AkaMSChannelName)
? $"dotnet/"
: $"dotnet/{targetChannelConfig.AkaMSChannelName}/{BuildQuality}";
var targetFeedsSetup = new SetupTargetFeedConfigV3(
targetChannelConfig.IsInternal,
BuildModel.Identity.IsStable,
BuildModel.Identity.Name,
BuildModel.Identity.Commit,
AzureStorageTargetFeedKey,
PublishInstallersAndChecksums,
GetFeed(targetChannelConfig.InstallersFeed, InstallersFeedOverride),
targetChannelConfig.IsInternal ? InternalInstallersFeedKey : InstallersFeedKey,
GetFeed(targetChannelConfig.ChecksumsFeed, ChecksumsFeedOverride),
targetChannelConfig.IsInternal ? InternalCheckSumsFeedKey : CheckSumsFeedKey,
GetFeed(targetChannelConfig.ShippingFeed, ShippingFeedOverride),
GetFeed(targetChannelConfig.TransportFeed, TransportFeedOverride),
GetFeed(targetChannelConfig.SymbolsFeed, SymbolsFeedOverride),
shortLinkUrl,
AzureDevOpsFeedsKey,
BuildEngine = this.BuildEngine,
targetChannelConfig.SymbolTargetType,
azureDevOpsPublicStaticSymbolsFeed: GetFeed(null, PublicSymbolsFeedOverride),
filesToExclude: targetChannelConfig.FilenamesToExclude,
flatten: targetChannelConfig.Flatten);
var targetFeedConfigs = targetFeedsSetup.Setup();
// No target feeds to publish to, very likely this is an error
if (targetFeedConfigs.Count() == 0)
{
Log.LogError($"No target feeds were found to publish the assets to.");
return(false);
}
foreach (var feedConfig in targetFeedConfigs)
{
Log.LogMessage(MessageImportance.High, $"Target feed config: {feedConfig}");
TargetFeedContentType categoryKey = feedConfig.ContentType;
if (!FeedConfigs.TryGetValue(categoryKey, out _))
{
FeedConfigs[categoryKey] = new HashSet <TargetFeedConfig>();
}
FeedConfigs[categoryKey].Add(feedConfig);
}
}
CheckForStableAssetsInNonIsolatedFeeds();
if (Log.HasLoggedErrors)
{
return(false);
}
string temporarySymbolsLocation = "";
if (!UseStreamingPublishing)
{
temporarySymbolsLocation =
Path.GetFullPath(Path.Combine(BlobAssetsBasePath, @"..\", "tempSymbols"));
EnsureTemporaryDirectoryExists(temporarySymbolsLocation);
DeleteTemporaryFiles(temporarySymbolsLocation);
// Copying symbol files to temporary location is required because the symUploader API needs read/write access to the files,
// since we publish blobs and symbols in parallel this will cause IO exceptions.
CopySymbolFilesToTemporaryLocation(BuildModel, temporarySymbolsLocation);
}
using var clientThrottle = new SemaphoreSlim(MaxClients, MaxClients);
await Task.WhenAll(new Task[]
{
HandlePackagePublishingAsync(buildAssets, clientThrottle),
HandleBlobPublishingAsync(buildAssets, clientThrottle),
HandleSymbolPublishingAsync(
PdbArtifactsBasePath,
MsdlToken,
SymWebToken,
SymbolPublishingExclusionsFile,
PublishSpecialClrFiles,
buildAssets,
clientThrottle,
temporarySymbolsLocation)
});
DeleteTemporaryFiles(temporarySymbolsLocation);
DeleteTemporaryDirectory(temporarySymbolsLocation);
await PersistPendingAssetLocationAsync(client);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
if (!Log.HasLoggedErrors)
{
Log.LogMessage(MessageImportance.High, "Publishing finished with success.");
}
return(!Log.HasLoggedErrors);
}
public override async Task<bool> ExecuteAsync()
{
if (AnyMissingRequiredProperty())
{
Log.LogError("Missing required properties. Aborting execution.");
return false;
}
try
{
List<int> targetChannelsIds = new List<int>();
foreach (var channelIdStr in TargetChannels.Split(','))
{
if (!int.TryParse(channelIdStr, out var channelId))
{
Log.LogError($"Value '{channelIdStr}' isn't recognized as a valid Maestro++ channel ID.");
continue;
}
targetChannelsIds.Add(channelId);
}
if (Log.HasLoggedErrors)
{
Log.LogError($"Could not parse the target channels list '{TargetChannels}'. It should be a comma separated list of integers.");
return false;
}
SplitArtifactsInCategories(BuildModel);
if (Log.HasLoggedErrors)
{
return false;
}
// Fetch Maestro record of the build. We're going to use it to get the BAR ID
// of the assets being published so we can add a new location for them.
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId);
Dictionary<string, HashSet<Asset>> buildAssets = CreateBuildAssetDictionary(buildInformation);
foreach (var targetChannelId in targetChannelsIds)
{
TargetChannelConfig targetChannelConfig = PublishingConstants.ChannelInfos
.Where(ci =>
ci.Id == targetChannelId &&
(ci.PublishingInfraVersion == PublishingInfraVersion.All || ci.PublishingInfraVersion == PublishingInfraVersion.Next))
.FirstOrDefault();
// Invalid channel ID was supplied
if (targetChannelConfig.Equals(default(TargetChannelConfig)))
{
Log.LogError($"Channel with ID '{targetChannelId}' is not configured to be published to.");
return false;
}
if (await client.Channels.GetChannelAsync(targetChannelId) == null)
{
Log.LogError($"Channel with ID '{targetChannelId}' does not exist in BAR.");
return false;
}
Log.LogMessage(MessageImportance.High, $"Publishing to this target channel: {targetChannelConfig}");
var targetFeedsSetup = new SetupTargetFeedConfigV3(
InternalBuild,
BuildModel.Identity.IsStable.Equals("true", System.StringComparison.OrdinalIgnoreCase),
BuildModel.Identity.Name,
BuildModel.Identity.Commit,
AzureStorageTargetFeedKey,
PublishInstallersAndChecksums,
targetChannelConfig.InstallersFeed,
InstallersFeedKey,
targetChannelConfig.ChecksumsFeed,
CheckSumsFeedKey,
targetChannelConfig.ShippingFeed,
targetChannelConfig.TransportFeed,
targetChannelConfig.SymbolsFeed,
$"dotnet/{targetChannelConfig.AkaMSChannelName}",
AzureDevOpsFeedsKey,
BuildEngine = this.BuildEngine);
var targetFeedConfigs = targetFeedsSetup.Setup();
// No target feeds to publish to, very likely this is an error
if (targetFeedConfigs.Count() == 0)
{
Log.LogError($"No target feeds were found to publish the assets to.");
return false;
}
foreach (var feedConfig in targetFeedConfigs)
{
Log.LogMessage(MessageImportance.High, $"Target feed config: {feedConfig}");
TargetFeedContentType categoryKey = feedConfig.ContentType;
if (!FeedConfigs.TryGetValue(categoryKey, out _))
{
FeedConfigs[categoryKey] = new HashSet<TargetFeedConfig>();
}
FeedConfigs[categoryKey].Add(feedConfig);
}
}
CheckForStableAssetsInNonIsolatedFeeds();
if (Log.HasLoggedErrors)
{
return false;
}
await Task.WhenAll(new Task[] {
HandlePackagePublishingAsync(buildAssets),
HandleBlobPublishingAsync(buildAssets)
});
await PersistPendingAssetLocationAsync(client);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
if (!Log.HasLoggedErrors)
{
Log.LogMessage(MessageImportance.High, "Publishing finished with success.");
}
return !Log.HasLoggedErrors;
}