private async Task PublishPackagesToAzDoNugetFeedAsync(
List <PackageArtifactModel> packagesToPublish,
IMaestroApi client,
Maestro.Client.Models.Build buildInformation,
FeedConfig feedConfig)
{
foreach (var package in packagesToPublish)
{
var assetRecord = buildInformation.Assets
.Where(a => a.Name.Equals(package.Id) && a.Version.Equals(package.Version))
.FirstOrDefault();
if (assetRecord == null)
{
Log.LogError($"Asset with Id {package.Id}, Version {package.Version} isn't registered on the BAR Build with ID {BARBuildId}");
continue;
}
var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id);
if (assetWithLocations?.Locations.Any(al => al.Location.Equals(feedConfig.TargetFeedURL, StringComparison.OrdinalIgnoreCase)) ?? false)
{
Log.LogMessage($"Asset with Id {package.Id}, Version {package.Version} already has location {feedConfig.TargetFeedURL}");
continue;
}
await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.NugetFeed, feedConfig.TargetFeedURL);
}
await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients : MaxClients);
}
private async Task PublishBlobsToAzDoNugetFeedAsync(
List <BlobArtifactModel> blobsToPublish,
IMaestroApi client,
Maestro.Client.Models.Build buildInformation,
FeedConfig feedConfig)
{
foreach (var blob in blobsToPublish)
{
var assetRecord = buildInformation.Assets
.Where(a => a.Name.Equals(blob.Id))
.FirstOrDefault();
if (assetRecord == null)
{
Log.LogError($"Asset with Id {blob.Id} isn't registered on the BAR Build with ID {BARBuildId}");
continue;
}
var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id);
if (assetWithLocations?.Locations.Any(al => al.Location.Equals(feedConfig.TargetFeedURL, StringComparison.OrdinalIgnoreCase)) ?? false)
{
Log.LogMessage($"Asset with Id {blob.Id} already has location {feedConfig.TargetFeedURL}");
continue;
}
await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.Container, feedConfig.TargetFeedURL);
}
}
/// <summary>
/// Filter the blobs by the feed config information
/// </summary>
/// <param name="blobs"></param>
/// <param name="feedConfig"></param>
/// <returns></returns>
private List <BlobArtifactModel> FilterBlobs(List <BlobArtifactModel> blobs, FeedConfig feedConfig)
{
// If the feed config wants further filtering, do that now.
List <BlobArtifactModel> filteredBlobs = null;
switch (feedConfig.AssetSelection)
{
case AssetSelection.All:
// No filtering needed
filteredBlobs = blobs;
break;
case AssetSelection.NonShippingOnly:
filteredBlobs = blobs.Where(p => p.NonShipping).ToList();
break;
case AssetSelection.ShippingOnly:
filteredBlobs = blobs.Where(p => !p.NonShipping).ToList();
break;
default:
// Throw NYI here instead of logging an error because error would have already been logged in the
// parser for the user.
throw new NotImplementedException("Unknown asset selection type '{feedConfig.AssetSelection}'");
}
return(filteredBlobs);
}
/// <summary>
/// Push a single package to the azure devops nuget feed.
/// </summary>
/// <param name="feedConfig">Feed</param>
/// <param name="packageToPublish">Package to push</param>
/// <returns>Task</returns>
/// <remarks>
/// This method attempts to take the most efficient path to push the package.
/// There are two cases:
/// - The package does not exist, and is pushed normally
/// - The package exists, and its contents may or may not be equivalent.
/// The second case is is by far the most common. So, we first attempt to push the package normally using nuget.exe.
/// If this fails, this could mean any number of things (like failed auth). But in normal circumstances, this might
/// mean the package already exists. This either means that we are attempting to push the same package, or attemtping to push
/// a different package with the same id and version. The second case is an error, as azure devops feeds are immutable, the former
/// is simply a case where we should continue onward.
/// </remarks>
private async Task PushNugetPackageAsync(FeedConfig feedConfig, HttpClient client, PackageArtifactModel packageToPublish,
string feedAccount, string feedVisibility, string feedName)
{
Log.LogMessage(MessageImportance.High, $"Pushing package '{packageToPublish.Id}' to feed {feedConfig.TargetFeedURL}");
PackageAssetsBasePath = PackageAssetsBasePath.TrimEnd(
Path.DirectorySeparatorChar,
Path.AltDirectorySeparatorChar)
+ Path.DirectorySeparatorChar;
string localPackageLocation = $"{PackageAssetsBasePath}{packageToPublish.Id}.{packageToPublish.Version}.nupkg";
if (!File.Exists(localPackageLocation))
{
Log.LogError($"Could not locate '{packageToPublish.Id}.{packageToPublish.Version}' at '{localPackageLocation}'");
return;
}
try
{
// The feed key when pushing to AzDo feeds is "AzureDevOps" (works with the credential helper).
int result = await StartProcessAsync(NugetPath, $"push \"{localPackageLocation}\" -Source \"{feedConfig.TargetFeedURL}\" -NonInteractive -ApiKey AzureDevOps");
if (result != 0)
{
Log.LogMessage(MessageImportance.Low, $"Failed to push {localPackageLocation}, attempting to determine whether the package already exists on the feed with the same content.");
try
{
string packageContentUrl = $"https://pkgs.dev.azure.com/{feedAccount}/{feedVisibility}_apis/packaging/feeds/{feedName}/nuget/packages/{packageToPublish.Id}/versions/{packageToPublish.Version}/content";
if (await IsLocalPackageIdenticalToFeedPackage(localPackageLocation, packageContentUrl, client))
{
Log.LogMessage(MessageImportance.Normal, $"Package '{packageToPublish.Id}@{packageToPublish.Version}' already exists on '{feedConfig.TargetFeedURL}' but has the same content. Skipping.");
}
else
{
Log.LogError($"Package '{packageToPublish.Id}@{packageToPublish.Version}' already exists on '{feedConfig.TargetFeedURL}' with different content.");
}
return;
}
catch (Exception e)
{
// This is an error. It means we were unable to push using nuget, and then could not access to the package otherwise.
Log.LogWarning($"Failed to determine whether an existing package on the feed has the same content as '{localPackageLocation}': {e.Message}");
}
Log.LogError($"Failed to push '{packageToPublish.Id}@{packageToPublish.Version}'. Result code '{result}'.");
}
}
catch (Exception e)
{
Log.LogError($"Unexpected exception pushing package '{packageToPublish.Id}@{packageToPublish.Version}': {e.Message}");
}
}
private async Task PublishBlobsToAzureStorageNugetFeedAsync(
List <BlobArtifactModel> blobsToPublish,
IMaestroApi client,
Maestro.Client.Models.Build buildInformation,
FeedConfig feedConfig)
{
BlobAssetsBasePath = BlobAssetsBasePath.TrimEnd(
Path.DirectorySeparatorChar,
Path.AltDirectorySeparatorChar)
+ Path.DirectorySeparatorChar;
var blobs = blobsToPublish
.Select(blob =>
{
var fileName = Path.GetFileName(blob.Id);
return(new MSBuild.TaskItem($"{BlobAssetsBasePath}{fileName}", new Dictionary <string, string>
{
{ "RelativeBlobPath", blob.Id }
}));
})
.ToArray();
var blobFeedAction = CreateBlobFeedAction(feedConfig);
var pushOptions = new PushOptions
{
AllowOverwrite = false,
PassIfExistingItemIdentical = true
};
foreach (var blob in blobsToPublish)
{
var assetRecord = buildInformation.Assets
.Where(a => a.Name.Equals(blob.Id))
.SingleOrDefault();
if (assetRecord == null)
{
Log.LogError($"Asset with Id {blob.Id} isn't registered on the BAR Build with ID {BARBuildId}");
continue;
}
var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id);
if (assetWithLocations?.Locations.Any(al => al.Location.Equals(feedConfig.TargetFeedURL, StringComparison.OrdinalIgnoreCase)) ?? false)
{
Log.LogMessage($"Asset with Id {blob.Id} already has location {feedConfig.TargetFeedURL}");
continue;
}
await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.Container, feedConfig.TargetFeedURL);
}
await blobFeedAction.PublishToFlatContainerAsync(blobs, maxClients : 8, pushOptions);
}
private BlobFeedAction CreateBlobFeedAction(FeedConfig feedConfig)
{
// Matches package feeds like
// https://dotnet-feed-internal.azurewebsites.net/container/dotnet-core-internal/sig/dsdfasdfasdf234234s/se/2020-02-02/darc-int-dotnet-arcade-services-babababababe-08/index.json
const string azureStorageProxyFeedPattern =
@"(?<feedURL>https://([a-z-]+).azurewebsites.net/container/(?<container>[^/]+)/sig/\w+/se/([0-9]{4}-[0-9]{2}-[0-9]{2})/(?<baseFeedName>darc-(?<type>int|pub)-(?<repository>.+?)-(?<sha>[A-Fa-f0-9]{7,40})-?(?<subversion>\d*)/))index.json";
// Matches package feeds like the one below. Special case for static internal proxy-backed feed
// https://dotnet-feed-internal.azurewebsites.net/container/dotnet-core-internal/sig/dsdfasdfasdf234234s/se/2020-02-02/darc-int-dotnet-arcade-services-babababababe-08/index.json
const string azureStorageProxyFeedStaticPattern =
@"(?<feedURL>https://([a-z-]+).azurewebsites.net/container/(?<container>[^/]+)/sig/\w+/se/([0-9]{4}-[0-9]{2}-[0-9]{2})/(?<baseFeedName>[^/]+/))index.json";
// Matches package feeds like
// https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json
const string azureStorageStaticBlobFeedPattern =
@"https://([a-z-]+).blob.core.windows.net/[^/]+/index.json";
var proxyBackedFeedMatch = Regex.Match(feedConfig.TargetFeedURL, azureStorageProxyFeedPattern);
var proxyBackedStaticFeedMatch = Regex.Match(feedConfig.TargetFeedURL, azureStorageProxyFeedStaticPattern);
var azureStorageStaticBlobFeedMatch = Regex.Match(feedConfig.TargetFeedURL, azureStorageStaticBlobFeedPattern);
if (proxyBackedFeedMatch.Success || proxyBackedStaticFeedMatch.Success)
{
var regexMatch = (proxyBackedFeedMatch.Success) ? proxyBackedFeedMatch : proxyBackedStaticFeedMatch;
var containerName = regexMatch.Groups["container"].Value;
var baseFeedName = regexMatch.Groups["baseFeedName"].Value;
var feedURL = regexMatch.Groups["feedURL"].Value;
var storageAccountName = "dotnetfeed";
// Initialize the feed using sleet
SleetSource sleetSource = new SleetSource()
{
Name = baseFeedName,
Type = "azure",
BaseUri = feedURL,
AccountName = storageAccountName,
Container = containerName,
FeedSubPath = baseFeedName,
ConnectionString = $"DefaultEndpointsProtocol=https;AccountName={storageAccountName};AccountKey={feedConfig.FeedKey};EndpointSuffix=core.windows.net"
};
return(new BlobFeedAction(sleetSource, feedConfig.FeedKey, Log));
}
else if (azureStorageStaticBlobFeedMatch.Success)
{
return(new BlobFeedAction(feedConfig.TargetFeedURL, feedConfig.FeedKey, Log));
}
else
{
Log.LogError($"Could not parse Azure feed URL: '{feedConfig.TargetFeedURL}'");
return(null);
}
}
/// <summary>
/// Parse out the input TargetFeedConfig into a dictionary of FeedConfig types
/// </summary>
public void ParseTargetFeedConfig()
{
foreach (var fc in TargetFeedConfig)
{
string targetFeedUrl = fc.GetMetadata("TargetURL");
string feedKey = fc.GetMetadata("Token");
string type = fc.GetMetadata("Type");
if (string.IsNullOrEmpty(targetFeedUrl) ||
string.IsNullOrEmpty(feedKey) ||
string.IsNullOrEmpty(type))
{
Log.LogError($"Invalid FeedConfig entry. TargetURL='{targetFeedUrl}' Type='{type}' Token='{feedKey}'");
continue;
}
if (!Enum.TryParse <FeedType>(type, true, out FeedType feedType))
{
Log.LogError($"Invalid feed config type '{type}'. Possible values are: {string.Join(", ", Enum.GetNames(typeof(FeedType)))}");
continue;
}
var feedConfig = new FeedConfig()
{
TargetFeedURL = targetFeedUrl,
Type = feedType,
FeedKey = feedKey
};
string assetSelection = fc.GetMetadata("AssetSelection");
if (!string.IsNullOrEmpty(assetSelection))
{
if (!Enum.TryParse <AssetSelection>(assetSelection, true, out AssetSelection selection))
{
Log.LogError($"Invalid feed config asset selection '{type}'. Possible values are: {string.Join(", ", Enum.GetNames(typeof(AssetSelection)))}");
continue;
}
feedConfig.AssetSelection = selection;
}
string categoryKey = fc.ItemSpec.Trim().ToUpper();
if (!FeedConfigs.TryGetValue(categoryKey, out var feedsList))
{
FeedConfigs[categoryKey] = new List <FeedConfig>();
}
FeedConfigs[categoryKey].Add(feedConfig);
}
}
private async Task PublishPackagesToAzureStorageNugetFeedAsync(
List <PackageArtifactModel> packagesToPublish,
IMaestroApi client,
Maestro.Client.Models.Build buildInformation,
FeedConfig feedConfig)
{
PackageAssetsBasePath = PackageAssetsBasePath.TrimEnd(
Path.DirectorySeparatorChar,
Path.AltDirectorySeparatorChar)
+ Path.DirectorySeparatorChar;
var packages = packagesToPublish.Select(p => $"{PackageAssetsBasePath}{p.Id}.{p.Version}.nupkg");
var blobFeedAction = CreateBlobFeedAction(feedConfig);
var pushOptions = new PushOptions
{
AllowOverwrite = false,
PassIfExistingItemIdentical = true
};
foreach (var package in packagesToPublish)
{
var assetRecord = buildInformation.Assets
.Where(a => a.Name.Equals(package.Id) && a.Version.Equals(package.Version))
.FirstOrDefault();
if (assetRecord == null)
{
Log.LogError($"Asset with Id {package.Id}, Version {package.Version} isn't registered on the BAR Build with ID {BARBuildId}");
continue;
}
var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id);
if (assetWithLocations?.Locations.Any(al => al.Location.Equals(feedConfig.TargetFeedURL, StringComparison.OrdinalIgnoreCase)) ?? false)
{
Log.LogMessage($"Asset with Id {package.Id}, Version {package.Version} already has location {feedConfig.TargetFeedURL}");
continue;
}
await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.NugetFeed, feedConfig.TargetFeedURL);
}
await blobFeedAction.PushToFeedAsync(packages, pushOptions);
}
private Task <int> PublishWithNugetAsync(FeedConfig feedConfig, PackageArtifactModel package)
{
var packageFullPath = $"{PackageAssetsBasePath}{Path.DirectorySeparatorChar}{package.Id}.{package.Version}.nupkg";
var tcs = new TaskCompletionSource <int>();
Log.LogMessage($"Publishing package {packageFullPath} to target feed {feedConfig.TargetFeedURL} with nuget.exe push");
var process = new Process
{
StartInfo = new ProcessStartInfo()
{
FileName = NugetPath,
Arguments = $"push -Source {feedConfig.TargetFeedURL} -apikey {feedConfig.FeedKey} {packageFullPath}",
UseShellExecute = false,
RedirectStandardOutput = true,
RedirectStandardError = true,
CreateNoWindow = true
},
EnableRaisingEvents = true
};
process.Exited += (sender, args) =>
{
tcs.SetResult(process.ExitCode);
if (process.ExitCode != 0)
{
Log.LogError($"Nuget push failed with exit code {process.ExitCode}. Standard error output: {process.StandardError.ReadToEnd()}");
}
else
{
Log.LogMessage($"Successfully published package {packageFullPath} to {feedConfig.TargetFeedURL}");
}
process.Dispose();
};
process.Start();
return(tcs.Task);
}
private BlobFeedAction CreateBlobFeedAction(FeedConfig feedConfig)
{
var proxyBackedFeedMatch = Regex.Match(feedConfig.TargetFeedURL, AzureStorageProxyFeedPattern);
var proxyBackedStaticFeedMatch = Regex.Match(feedConfig.TargetFeedURL, AzureStorageProxyFeedStaticPattern);
var azureStorageStaticBlobFeedMatch = Regex.Match(feedConfig.TargetFeedURL, AzureStorageStaticBlobFeedPattern);
if (proxyBackedFeedMatch.Success || proxyBackedStaticFeedMatch.Success)
{
var regexMatch = (proxyBackedFeedMatch.Success) ? proxyBackedFeedMatch : proxyBackedStaticFeedMatch;
var containerName = regexMatch.Groups["container"].Value;
var baseFeedName = regexMatch.Groups["baseFeedName"].Value;
var feedURL = regexMatch.Groups["feedURL"].Value;
var storageAccountName = "dotnetfeed";
// Initialize the feed using sleet
SleetSource sleetSource = new SleetSource()
{
Name = baseFeedName,
Type = "azure",
BaseUri = feedURL,
AccountName = storageAccountName,
Container = containerName,
FeedSubPath = baseFeedName,
ConnectionString = $"DefaultEndpointsProtocol=https;AccountName={storageAccountName};AccountKey={feedConfig.FeedKey};EndpointSuffix=core.windows.net"
};
return(new BlobFeedAction(sleetSource, feedConfig.FeedKey, Log));
}
else if (azureStorageStaticBlobFeedMatch.Success)
{
return(new BlobFeedAction(feedConfig.TargetFeedURL, feedConfig.FeedKey, Log));
}
else
{
Log.LogError($"Could not parse Azure feed URL: '{feedConfig.TargetFeedURL}'");
return(null);
}
}
public async Task <bool> ExecuteAsync()
{
try
{
Log.LogMessage(MessageImportance.High, "Publishing artifacts to feed.");
if (string.IsNullOrWhiteSpace(AssetManifestPath) || !File.Exists(AssetManifestPath))
{
Log.LogError($"Problem reading asset manifest path from '{AssetManifestPath}'");
}
if (!Directory.Exists(BlobAssetsBasePath))
{
Log.LogError($"Problem reading blob assets from {BlobAssetsBasePath}");
}
if (!Directory.Exists(PackageAssetsBasePath))
{
Log.LogError($"Problem reading package assets from {PackageAssetsBasePath}");
}
var buildModel = BuildManifestUtil.ManifestFileToModel(AssetManifestPath, Log);
// Parsing the manifest may fail for several reasons
if (Log.HasLoggedErrors)
{
return(false);
}
// Fetch Maestro record of the build. We're going to use it to get the BAR ID
// of the assets being published so we can add a new location for them.
IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken);
Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId);
foreach (var fc in TargetFeedConfig)
{
var feedConfig = new FeedConfig()
{
TargetFeedURL = fc.GetMetadata("TargetURL"),
Type = fc.GetMetadata("Type"),
FeedKey = fc.GetMetadata("Token")
};
if (string.IsNullOrEmpty(feedConfig.TargetFeedURL) ||
string.IsNullOrEmpty(feedConfig.Type) ||
string.IsNullOrEmpty(feedConfig.FeedKey))
{
Log.LogError($"Invalid FeedConfig entry. TargetURL='{feedConfig.TargetFeedURL}' Type='{feedConfig.Type}' Token='{feedConfig.FeedKey}'");
}
FeedConfigs.Add(fc.ItemSpec.Trim().ToUpper(), feedConfig);
}
// Return errors from parsing FeedConfig
if (Log.HasLoggedErrors)
{
return(false);
}
SplitArtifactsInCategories(buildModel);
await HandlePackagePublishingAsync(client, buildInformation);
await HandleBlobPublishingAsync(client, buildInformation);
}
catch (Exception e)
{
Log.LogErrorFromException(e, true);
}
return(!Log.HasLoggedErrors);
}
/// <summary>
/// Push nuget packages to the azure devops feed.
/// </summary>
/// <param name="packagesToPublish">List of packages to publish</param>
/// <param name="feedConfig">Information about feed to publish ot</param>
/// <returns>Async task.</returns>
public async Task PushNugetPackagesAsync(List <PackageArtifactModel> packagesToPublish, FeedConfig feedConfig, int maxClients)
{
var parsedUri = Regex.Match(feedConfig.TargetFeedURL, PublishArtifactsInManifest.AzDoNuGetFeedPattern);
if (!parsedUri.Success)
{
Log.LogError($"Azure DevOps NuGetFeed was not in the expected format '{PublishArtifactsInManifest.AzDoNuGetFeedPattern}'");
return;
}
string feedAccount = parsedUri.Groups["account"].Value;
string feedVisibility = parsedUri.Groups["visibility"].Value;
string feedName = parsedUri.Groups["feed"].Value;
using (var clientThrottle = new SemaphoreSlim(maxClients, maxClients))
{
using (HttpClient httpClient = new HttpClient(new HttpClientHandler {
CheckCertificateRevocationList = true
}))
{
httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue(
"Basic",
Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.FeedKey))));
Log.LogMessage(MessageImportance.High, $"Pushing {packagesToPublish.Count()} packages.");
await System.Threading.Tasks.Task.WhenAll(packagesToPublish.Select(async packageToPublish =>
{
try
{
// Wait to avoid starting too many processes.
await clientThrottle.WaitAsync();
await PushNugetPackageAsync(feedConfig, httpClient, packageToPublish, feedAccount, feedVisibility, feedName);
}
finally
{
clientThrottle.Release();
}
}));
}
}
}
private List <PackageArtifactModel> FilterPackages(List <PackageArtifactModel> packages, FeedConfig feedConfig)
{
switch (feedConfig.AssetSelection)
{
case AssetSelection.All:
// No filtering needed
return(packages);
case AssetSelection.NonShippingOnly:
return(packages.Where(p => p.NonShipping).ToList());
case AssetSelection.ShippingOnly:
return(packages.Where(p => !p.NonShipping).ToList());
default:
// Throw NYI here instead of logging an error because error would have already been logged in the
// parser for the user.
throw new NotImplementedException("Unknown asset selection type '{feedConfig.AssetSelection}'");
}
}
