private async Task PublishPackagesToAzDoNugetFeedAsync( List <PackageArtifactModel> packagesToPublish, IMaestroApi client, Maestro.Client.Models.Build buildInformation, FeedConfig feedConfig) { foreach (var package in packagesToPublish) { var assetRecord = buildInformation.Assets .Where(a => a.Name.Equals(package.Id) && a.Version.Equals(package.Version)) .FirstOrDefault(); if (assetRecord == null) { Log.LogError($"Asset with Id {package.Id}, Version {package.Version} isn't registered on the BAR Build with ID {BARBuildId}"); continue; } var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id); if (assetWithLocations?.Locations.Any(al => al.Location.Equals(feedConfig.TargetFeedURL, StringComparison.OrdinalIgnoreCase)) ?? false) { Log.LogMessage($"Asset with Id {package.Id}, Version {package.Version} already has location {feedConfig.TargetFeedURL}"); continue; } await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.NugetFeed, feedConfig.TargetFeedURL); } await PushNugetPackagesAsync(packagesToPublish, feedConfig, maxClients : MaxClients); }
private async Task PublishBlobsToAzDoNugetFeedAsync( List <BlobArtifactModel> blobsToPublish, IMaestroApi client, Maestro.Client.Models.Build buildInformation, FeedConfig feedConfig) { foreach (var blob in blobsToPublish) { var assetRecord = buildInformation.Assets .Where(a => a.Name.Equals(blob.Id)) .FirstOrDefault(); if (assetRecord == null) { Log.LogError($"Asset with Id {blob.Id} isn't registered on the BAR Build with ID {BARBuildId}"); continue; } var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id); if (assetWithLocations?.Locations.Any(al => al.Location.Equals(feedConfig.TargetFeedURL, StringComparison.OrdinalIgnoreCase)) ?? false) { Log.LogMessage($"Asset with Id {blob.Id} already has location {feedConfig.TargetFeedURL}"); continue; } await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.Container, feedConfig.TargetFeedURL); } }
/// <summary> /// Filter the blobs by the feed config information /// </summary> /// <param name="blobs"></param> /// <param name="feedConfig"></param> /// <returns></returns> private List <BlobArtifactModel> FilterBlobs(List <BlobArtifactModel> blobs, FeedConfig feedConfig) { // If the feed config wants further filtering, do that now. List <BlobArtifactModel> filteredBlobs = null; switch (feedConfig.AssetSelection) { case AssetSelection.All: // No filtering needed filteredBlobs = blobs; break; case AssetSelection.NonShippingOnly: filteredBlobs = blobs.Where(p => p.NonShipping).ToList(); break; case AssetSelection.ShippingOnly: filteredBlobs = blobs.Where(p => !p.NonShipping).ToList(); break; default: // Throw NYI here instead of logging an error because error would have already been logged in the // parser for the user. throw new NotImplementedException("Unknown asset selection type '{feedConfig.AssetSelection}'"); } return(filteredBlobs); }
/// <summary> /// Push a single package to the azure devops nuget feed. /// </summary> /// <param name="feedConfig">Feed</param> /// <param name="packageToPublish">Package to push</param> /// <returns>Task</returns> /// <remarks> /// This method attempts to take the most efficient path to push the package. /// There are two cases: /// - The package does not exist, and is pushed normally /// - The package exists, and its contents may or may not be equivalent. /// The second case is is by far the most common. So, we first attempt to push the package normally using nuget.exe. /// If this fails, this could mean any number of things (like failed auth). But in normal circumstances, this might /// mean the package already exists. This either means that we are attempting to push the same package, or attemtping to push /// a different package with the same id and version. The second case is an error, as azure devops feeds are immutable, the former /// is simply a case where we should continue onward. /// </remarks> private async Task PushNugetPackageAsync(FeedConfig feedConfig, HttpClient client, PackageArtifactModel packageToPublish, string feedAccount, string feedVisibility, string feedName) { Log.LogMessage(MessageImportance.High, $"Pushing package '{packageToPublish.Id}' to feed {feedConfig.TargetFeedURL}"); PackageAssetsBasePath = PackageAssetsBasePath.TrimEnd( Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; string localPackageLocation = $"{PackageAssetsBasePath}{packageToPublish.Id}.{packageToPublish.Version}.nupkg"; if (!File.Exists(localPackageLocation)) { Log.LogError($"Could not locate '{packageToPublish.Id}.{packageToPublish.Version}' at '{localPackageLocation}'"); return; } try { // The feed key when pushing to AzDo feeds is "AzureDevOps" (works with the credential helper). int result = await StartProcessAsync(NugetPath, $"push \"{localPackageLocation}\" -Source \"{feedConfig.TargetFeedURL}\" -NonInteractive -ApiKey AzureDevOps"); if (result != 0) { Log.LogMessage(MessageImportance.Low, $"Failed to push {localPackageLocation}, attempting to determine whether the package already exists on the feed with the same content."); try { string packageContentUrl = $"https://pkgs.dev.azure.com/{feedAccount}/{feedVisibility}_apis/packaging/feeds/{feedName}/nuget/packages/{packageToPublish.Id}/versions/{packageToPublish.Version}/content"; if (await IsLocalPackageIdenticalToFeedPackage(localPackageLocation, packageContentUrl, client)) { Log.LogMessage(MessageImportance.Normal, $"Package '{packageToPublish.Id}@{packageToPublish.Version}' already exists on '{feedConfig.TargetFeedURL}' but has the same content. Skipping."); } else { Log.LogError($"Package '{packageToPublish.Id}@{packageToPublish.Version}' already exists on '{feedConfig.TargetFeedURL}' with different content."); } return; } catch (Exception e) { // This is an error. It means we were unable to push using nuget, and then could not access to the package otherwise. Log.LogWarning($"Failed to determine whether an existing package on the feed has the same content as '{localPackageLocation}': {e.Message}"); } Log.LogError($"Failed to push '{packageToPublish.Id}@{packageToPublish.Version}'. Result code '{result}'."); } } catch (Exception e) { Log.LogError($"Unexpected exception pushing package '{packageToPublish.Id}@{packageToPublish.Version}': {e.Message}"); } }
private async Task PublishBlobsToAzureStorageNugetFeedAsync( List <BlobArtifactModel> blobsToPublish, IMaestroApi client, Maestro.Client.Models.Build buildInformation, FeedConfig feedConfig) { BlobAssetsBasePath = BlobAssetsBasePath.TrimEnd( Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; var blobs = blobsToPublish .Select(blob => { var fileName = Path.GetFileName(blob.Id); return(new MSBuild.TaskItem($"{BlobAssetsBasePath}{fileName}", new Dictionary <string, string> { { "RelativeBlobPath", blob.Id } })); }) .ToArray(); var blobFeedAction = CreateBlobFeedAction(feedConfig); var pushOptions = new PushOptions { AllowOverwrite = false, PassIfExistingItemIdentical = true }; foreach (var blob in blobsToPublish) { var assetRecord = buildInformation.Assets .Where(a => a.Name.Equals(blob.Id)) .SingleOrDefault(); if (assetRecord == null) { Log.LogError($"Asset with Id {blob.Id} isn't registered on the BAR Build with ID {BARBuildId}"); continue; } var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id); if (assetWithLocations?.Locations.Any(al => al.Location.Equals(feedConfig.TargetFeedURL, StringComparison.OrdinalIgnoreCase)) ?? false) { Log.LogMessage($"Asset with Id {blob.Id} already has location {feedConfig.TargetFeedURL}"); continue; } await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.Container, feedConfig.TargetFeedURL); } await blobFeedAction.PublishToFlatContainerAsync(blobs, maxClients : 8, pushOptions); }
private BlobFeedAction CreateBlobFeedAction(FeedConfig feedConfig) { // Matches package feeds like // https://dotnet-feed-internal.azurewebsites.net/container/dotnet-core-internal/sig/dsdfasdfasdf234234s/se/2020-02-02/darc-int-dotnet-arcade-services-babababababe-08/index.json const string azureStorageProxyFeedPattern = @"(?<feedURL>https://([a-z-]+).azurewebsites.net/container/(?<container>[^/]+)/sig/\w+/se/([0-9]{4}-[0-9]{2}-[0-9]{2})/(?<baseFeedName>darc-(?<type>int|pub)-(?<repository>.+?)-(?<sha>[A-Fa-f0-9]{7,40})-?(?<subversion>\d*)/))index.json"; // Matches package feeds like the one below. Special case for static internal proxy-backed feed // https://dotnet-feed-internal.azurewebsites.net/container/dotnet-core-internal/sig/dsdfasdfasdf234234s/se/2020-02-02/darc-int-dotnet-arcade-services-babababababe-08/index.json const string azureStorageProxyFeedStaticPattern = @"(?<feedURL>https://([a-z-]+).azurewebsites.net/container/(?<container>[^/]+)/sig/\w+/se/([0-9]{4}-[0-9]{2}-[0-9]{2})/(?<baseFeedName>[^/]+/))index.json"; // Matches package feeds like // https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json const string azureStorageStaticBlobFeedPattern = @"https://([a-z-]+).blob.core.windows.net/[^/]+/index.json"; var proxyBackedFeedMatch = Regex.Match(feedConfig.TargetFeedURL, azureStorageProxyFeedPattern); var proxyBackedStaticFeedMatch = Regex.Match(feedConfig.TargetFeedURL, azureStorageProxyFeedStaticPattern); var azureStorageStaticBlobFeedMatch = Regex.Match(feedConfig.TargetFeedURL, azureStorageStaticBlobFeedPattern); if (proxyBackedFeedMatch.Success || proxyBackedStaticFeedMatch.Success) { var regexMatch = (proxyBackedFeedMatch.Success) ? proxyBackedFeedMatch : proxyBackedStaticFeedMatch; var containerName = regexMatch.Groups["container"].Value; var baseFeedName = regexMatch.Groups["baseFeedName"].Value; var feedURL = regexMatch.Groups["feedURL"].Value; var storageAccountName = "dotnetfeed"; // Initialize the feed using sleet SleetSource sleetSource = new SleetSource() { Name = baseFeedName, Type = "azure", BaseUri = feedURL, AccountName = storageAccountName, Container = containerName, FeedSubPath = baseFeedName, ConnectionString = $"DefaultEndpointsProtocol=https;AccountName={storageAccountName};AccountKey={feedConfig.FeedKey};EndpointSuffix=core.windows.net" }; return(new BlobFeedAction(sleetSource, feedConfig.FeedKey, Log)); } else if (azureStorageStaticBlobFeedMatch.Success) { return(new BlobFeedAction(feedConfig.TargetFeedURL, feedConfig.FeedKey, Log)); } else { Log.LogError($"Could not parse Azure feed URL: '{feedConfig.TargetFeedURL}'"); return(null); } }
/// <summary> /// Parse out the input TargetFeedConfig into a dictionary of FeedConfig types /// </summary> public void ParseTargetFeedConfig() { foreach (var fc in TargetFeedConfig) { string targetFeedUrl = fc.GetMetadata("TargetURL"); string feedKey = fc.GetMetadata("Token"); string type = fc.GetMetadata("Type"); if (string.IsNullOrEmpty(targetFeedUrl) || string.IsNullOrEmpty(feedKey) || string.IsNullOrEmpty(type)) { Log.LogError($"Invalid FeedConfig entry. TargetURL='{targetFeedUrl}' Type='{type}' Token='{feedKey}'"); continue; } if (!Enum.TryParse <FeedType>(type, true, out FeedType feedType)) { Log.LogError($"Invalid feed config type '{type}'. Possible values are: {string.Join(", ", Enum.GetNames(typeof(FeedType)))}"); continue; } var feedConfig = new FeedConfig() { TargetFeedURL = targetFeedUrl, Type = feedType, FeedKey = feedKey }; string assetSelection = fc.GetMetadata("AssetSelection"); if (!string.IsNullOrEmpty(assetSelection)) { if (!Enum.TryParse <AssetSelection>(assetSelection, true, out AssetSelection selection)) { Log.LogError($"Invalid feed config asset selection '{type}'. Possible values are: {string.Join(", ", Enum.GetNames(typeof(AssetSelection)))}"); continue; } feedConfig.AssetSelection = selection; } string categoryKey = fc.ItemSpec.Trim().ToUpper(); if (!FeedConfigs.TryGetValue(categoryKey, out var feedsList)) { FeedConfigs[categoryKey] = new List <FeedConfig>(); } FeedConfigs[categoryKey].Add(feedConfig); } }
private async Task PublishPackagesToAzureStorageNugetFeedAsync( List <PackageArtifactModel> packagesToPublish, IMaestroApi client, Maestro.Client.Models.Build buildInformation, FeedConfig feedConfig) { PackageAssetsBasePath = PackageAssetsBasePath.TrimEnd( Path.DirectorySeparatorChar, Path.AltDirectorySeparatorChar) + Path.DirectorySeparatorChar; var packages = packagesToPublish.Select(p => $"{PackageAssetsBasePath}{p.Id}.{p.Version}.nupkg"); var blobFeedAction = CreateBlobFeedAction(feedConfig); var pushOptions = new PushOptions { AllowOverwrite = false, PassIfExistingItemIdentical = true }; foreach (var package in packagesToPublish) { var assetRecord = buildInformation.Assets .Where(a => a.Name.Equals(package.Id) && a.Version.Equals(package.Version)) .FirstOrDefault(); if (assetRecord == null) { Log.LogError($"Asset with Id {package.Id}, Version {package.Version} isn't registered on the BAR Build with ID {BARBuildId}"); continue; } var assetWithLocations = await client.Assets.GetAssetAsync(assetRecord.Id); if (assetWithLocations?.Locations.Any(al => al.Location.Equals(feedConfig.TargetFeedURL, StringComparison.OrdinalIgnoreCase)) ?? false) { Log.LogMessage($"Asset with Id {package.Id}, Version {package.Version} already has location {feedConfig.TargetFeedURL}"); continue; } await client.Assets.AddAssetLocationToAssetAsync(assetRecord.Id, AddAssetLocationToAssetAssetLocationType.NugetFeed, feedConfig.TargetFeedURL); } await blobFeedAction.PushToFeedAsync(packages, pushOptions); }
private Task <int> PublishWithNugetAsync(FeedConfig feedConfig, PackageArtifactModel package) { var packageFullPath = $"{PackageAssetsBasePath}{Path.DirectorySeparatorChar}{package.Id}.{package.Version}.nupkg"; var tcs = new TaskCompletionSource <int>(); Log.LogMessage($"Publishing package {packageFullPath} to target feed {feedConfig.TargetFeedURL} with nuget.exe push"); var process = new Process { StartInfo = new ProcessStartInfo() { FileName = NugetPath, Arguments = $"push -Source {feedConfig.TargetFeedURL} -apikey {feedConfig.FeedKey} {packageFullPath}", UseShellExecute = false, RedirectStandardOutput = true, RedirectStandardError = true, CreateNoWindow = true }, EnableRaisingEvents = true }; process.Exited += (sender, args) => { tcs.SetResult(process.ExitCode); if (process.ExitCode != 0) { Log.LogError($"Nuget push failed with exit code {process.ExitCode}. Standard error output: {process.StandardError.ReadToEnd()}"); } else { Log.LogMessage($"Successfully published package {packageFullPath} to {feedConfig.TargetFeedURL}"); } process.Dispose(); }; process.Start(); return(tcs.Task); }
private BlobFeedAction CreateBlobFeedAction(FeedConfig feedConfig) { var proxyBackedFeedMatch = Regex.Match(feedConfig.TargetFeedURL, AzureStorageProxyFeedPattern); var proxyBackedStaticFeedMatch = Regex.Match(feedConfig.TargetFeedURL, AzureStorageProxyFeedStaticPattern); var azureStorageStaticBlobFeedMatch = Regex.Match(feedConfig.TargetFeedURL, AzureStorageStaticBlobFeedPattern); if (proxyBackedFeedMatch.Success || proxyBackedStaticFeedMatch.Success) { var regexMatch = (proxyBackedFeedMatch.Success) ? proxyBackedFeedMatch : proxyBackedStaticFeedMatch; var containerName = regexMatch.Groups["container"].Value; var baseFeedName = regexMatch.Groups["baseFeedName"].Value; var feedURL = regexMatch.Groups["feedURL"].Value; var storageAccountName = "dotnetfeed"; // Initialize the feed using sleet SleetSource sleetSource = new SleetSource() { Name = baseFeedName, Type = "azure", BaseUri = feedURL, AccountName = storageAccountName, Container = containerName, FeedSubPath = baseFeedName, ConnectionString = $"DefaultEndpointsProtocol=https;AccountName={storageAccountName};AccountKey={feedConfig.FeedKey};EndpointSuffix=core.windows.net" }; return(new BlobFeedAction(sleetSource, feedConfig.FeedKey, Log)); } else if (azureStorageStaticBlobFeedMatch.Success) { return(new BlobFeedAction(feedConfig.TargetFeedURL, feedConfig.FeedKey, Log)); } else { Log.LogError($"Could not parse Azure feed URL: '{feedConfig.TargetFeedURL}'"); return(null); } }
public async Task <bool> ExecuteAsync() { try { Log.LogMessage(MessageImportance.High, "Publishing artifacts to feed."); if (string.IsNullOrWhiteSpace(AssetManifestPath) || !File.Exists(AssetManifestPath)) { Log.LogError($"Problem reading asset manifest path from '{AssetManifestPath}'"); } if (!Directory.Exists(BlobAssetsBasePath)) { Log.LogError($"Problem reading blob assets from {BlobAssetsBasePath}"); } if (!Directory.Exists(PackageAssetsBasePath)) { Log.LogError($"Problem reading package assets from {PackageAssetsBasePath}"); } var buildModel = BuildManifestUtil.ManifestFileToModel(AssetManifestPath, Log); // Parsing the manifest may fail for several reasons if (Log.HasLoggedErrors) { return(false); } // Fetch Maestro record of the build. We're going to use it to get the BAR ID // of the assets being published so we can add a new location for them. IMaestroApi client = ApiFactory.GetAuthenticated(MaestroApiEndpoint, BuildAssetRegistryToken); Maestro.Client.Models.Build buildInformation = await client.Builds.GetBuildAsync(BARBuildId); foreach (var fc in TargetFeedConfig) { var feedConfig = new FeedConfig() { TargetFeedURL = fc.GetMetadata("TargetURL"), Type = fc.GetMetadata("Type"), FeedKey = fc.GetMetadata("Token") }; if (string.IsNullOrEmpty(feedConfig.TargetFeedURL) || string.IsNullOrEmpty(feedConfig.Type) || string.IsNullOrEmpty(feedConfig.FeedKey)) { Log.LogError($"Invalid FeedConfig entry. TargetURL='{feedConfig.TargetFeedURL}' Type='{feedConfig.Type}' Token='{feedConfig.FeedKey}'"); } FeedConfigs.Add(fc.ItemSpec.Trim().ToUpper(), feedConfig); } // Return errors from parsing FeedConfig if (Log.HasLoggedErrors) { return(false); } SplitArtifactsInCategories(buildModel); await HandlePackagePublishingAsync(client, buildInformation); await HandleBlobPublishingAsync(client, buildInformation); } catch (Exception e) { Log.LogErrorFromException(e, true); } return(!Log.HasLoggedErrors); }
/// <summary> /// Push nuget packages to the azure devops feed. /// </summary> /// <param name="packagesToPublish">List of packages to publish</param> /// <param name="feedConfig">Information about feed to publish ot</param> /// <returns>Async task.</returns> public async Task PushNugetPackagesAsync(List <PackageArtifactModel> packagesToPublish, FeedConfig feedConfig, int maxClients) { var parsedUri = Regex.Match(feedConfig.TargetFeedURL, PublishArtifactsInManifest.AzDoNuGetFeedPattern); if (!parsedUri.Success) { Log.LogError($"Azure DevOps NuGetFeed was not in the expected format '{PublishArtifactsInManifest.AzDoNuGetFeedPattern}'"); return; } string feedAccount = parsedUri.Groups["account"].Value; string feedVisibility = parsedUri.Groups["visibility"].Value; string feedName = parsedUri.Groups["feed"].Value; using (var clientThrottle = new SemaphoreSlim(maxClients, maxClients)) { using (HttpClient httpClient = new HttpClient(new HttpClientHandler { CheckCertificateRevocationList = true })) { httpClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue( "Basic", Convert.ToBase64String(Encoding.ASCII.GetBytes(string.Format("{0}:{1}", "", feedConfig.FeedKey)))); Log.LogMessage(MessageImportance.High, $"Pushing {packagesToPublish.Count()} packages."); await System.Threading.Tasks.Task.WhenAll(packagesToPublish.Select(async packageToPublish => { try { // Wait to avoid starting too many processes. await clientThrottle.WaitAsync(); await PushNugetPackageAsync(feedConfig, httpClient, packageToPublish, feedAccount, feedVisibility, feedName); } finally { clientThrottle.Release(); } })); } } }
private List <PackageArtifactModel> FilterPackages(List <PackageArtifactModel> packages, FeedConfig feedConfig) { switch (feedConfig.AssetSelection) { case AssetSelection.All: // No filtering needed return(packages); case AssetSelection.NonShippingOnly: return(packages.Where(p => p.NonShipping).ToList()); case AssetSelection.ShippingOnly: return(packages.Where(p => !p.NonShipping).ToList()); default: // Throw NYI here instead of logging an error because error would have already been logged in the // parser for the user. throw new NotImplementedException("Unknown asset selection type '{feedConfig.AssetSelection}'"); } }