private async Task <DownloadedAsset> DownloadBlobAsync(HttpClient client,
Build build,
Asset asset,
AssetLocation assetLocation,
string subPath,
List <string> errors)
{
// Normalize the asset name. Sometimes the upload to the BAR will have
// "assets/" prepended to it and sometimes not (depending on the Maestro tasks version).
// Remove assets/ if it exists so we get consistent target paths.
string normalizedAssetName = asset.Name;
if (asset.Name.StartsWith("assets/"))
{
normalizedAssetName = asset.Name.Substring("assets/".Length);
}
string fullTargetPath = Path.Combine(subPath, assetsSubPath, normalizedAssetName);
DownloadedAsset downloadedAsset = new DownloadedAsset()
{
Successful = false,
Asset = asset,
TargetLocation = fullTargetPath
};
// If the location is a blob storage account ending in index.json, as would be expected
// if PushToBlobFeed was used, strip off the index.json and append the asset name. If that doesn't work,
// prepend "assets/" to the asset name and try that.
// When uploading assets via the PushToBlobFeed task, assets/ may be prepended (e.g. assets/symbols/)
// upon upload, but may not be reported to the BAR, or may be reported to BAR.
// Either way, normalize so that we end up with only one assets/ prepended.
if (IsBlobFeedUrl(assetLocation.Location))
{
string finalBaseUri = assetLocation.Location.Substring(0, assetLocation.Location.Length - "index.json".Length);
string finalUri1 = $"{finalBaseUri}{asset.Name}";
string finalUri2 = $"{finalBaseUri}assets/{asset.Name}";
if (await DownloadFileAsync(client, finalUri1, fullTargetPath, errors))
{
downloadedAsset.Successful = true;
downloadedAsset.SourceLocation = finalUri1;
return(downloadedAsset);
}
if (await DownloadFileAsync(client, finalUri2, fullTargetPath, errors))
{
downloadedAsset.Successful = true;
downloadedAsset.SourceLocation = finalUri2;
return(downloadedAsset);
}
// Could be under assets/assets/ in some recent builds due to a bug in the release
// pipeline.
if (!_options.NoWorkarounds)
{
string finalUri3 = $"{finalBaseUri}assets/assets/{asset.Name}";
if (await DownloadFileAsync(client, finalUri3, fullTargetPath, errors))
{
downloadedAsset.Successful = true;
downloadedAsset.SourceLocation = finalUri3;
return(downloadedAsset);
}
}
return(downloadedAsset);
}
else if (IsAzureDevOpsArtifactsUrl(assetLocation.Location))
{
// Do not attempt to download from AzDO.
return(downloadedAsset);
}
if (string.IsNullOrEmpty(assetLocation.Location))
{
errors.Add($"Asset location for {asset.Name} is not available.");
}
else
{
errors.Add($"Blob uri '{assetLocation.Location} for {asset.Name} is of an unknown type");
}
return(downloadedAsset);
}
/// <summary>
///
/// </summary>
/// <param name="client"></param>
/// <param name="asset"></param>
/// <param name="rootOutputDirectory"></param>
/// <returns></returns>
/// <remarks>
/// Layout:
/// {root dir}\shipping\assets - blobs
/// {root dir}\shipping\packages - packages
/// {root dir}\nonshipping\assets - blobs
/// {root dir}\nonshipping\packages - packages
/// </remarks>
private async Task <DownloadedAsset> DownloadAssetAsync(HttpClient client,
Build build,
Asset asset,
string rootOutputDirectory)
{
string assetNameAndVersion = GetAssetNameForLogging(asset);
if (_options.IncludeNonShipping || !asset.NonShipping)
{
Console.WriteLine($" Downloading asset {assetNameAndVersion}");
}
else
{
Console.WriteLine($" Skipping non-shipping asset {assetNameAndVersion}");
return(null);
}
DownloadedAsset downloadedAsset = new DownloadedAsset()
{
Successful = false,
Asset = asset
};
List <string> errors = new List <string>();
List <AssetLocation> assetLocations = new List <AssetLocation>(asset.Locations);
if (assetLocations.Count == 0)
{
// If there is no location information and the user wants workarounds, add a bunch
// of feeds.
if (!_options.NoWorkarounds)
{
if (asset.Name.Contains("/"))
{
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetcli.blob.core.windows.net/dotnet/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore-tooling/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/aspnet-entityframeworkcore/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/dotnet-sdk/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json"));
}
else
{
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore-tooling/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/aspnet-entityframeworkcore/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/dotnet-sdk/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json"));
}
}
else
{
errors.Add($"Asset '{assetNameAndVersion}' has no known location information.");
}
}
if (assetLocations.Count != 0)
{
string subPath = Path.Combine(rootOutputDirectory, asset.NonShipping ? nonShippingSubPath : shippingSubPath);
// Walk the locations and attempt to gather the asset at each one, setting the output
// path based on the type.
foreach (AssetLocation location in assetLocations)
{
var locationType = location.Type;
// Make sure the location type ends up correct. Currently in some cases,
// we end up with 'none' or a symbol package ends up with nuget feed.
// Generally we can make an assumption that if the path doesn't have a
// '/' then it's a package. Nuget packages also don't have '.nupkg' suffix
// (they are just the package name).
if (!_options.NoWorkarounds)
{
if (!asset.Name.Contains("/") && !asset.Name.Contains(".nupkg"))
{
locationType = LocationType.NugetFeed;
}
else
{
locationType = LocationType.Container;
}
}
switch (locationType)
{
case LocationType.NugetFeed:
downloadedAsset = await DownloadNugetPackageAsync(client, build, asset, location, subPath, errors);
break;
case LocationType.Container:
downloadedAsset = await DownloadBlobAsync(client, build, asset, location, subPath, errors);
break;
case LocationType.None:
default:
errors.Add($"Unexpected location type {locationType}");
break;
}
if (downloadedAsset.Successful)
{
return(downloadedAsset);
}
}
}
// If none of the download attempts succeeded, then we should print out all the error
// information.
Console.WriteLine($" Failed to download asset, errors shown below:");
foreach (string error in errors)
{
Console.WriteLine($" {error}");
}
return(downloadedAsset);
}
/// <summary>
/// Gather the drop for a specific build.
/// </summary>
/// <param name="build">Build to gather drop for</param>
/// <param name="rootOutputDirectory">Output directory. Must exist.</param>
private async Task <DownloadedBuild> GatherDropForBuildAsync(Build build, string rootOutputDirectory)
{
IRemote remote = RemoteFactory.GetBarOnlyRemote(_options, Logger);
bool success = true;
// If the drop is separated, calculate the directory name based on the last element of the build
// repo uri plus the build number (to disambiguate overlapping builds)
string outputDirectory = rootOutputDirectory;
string repoUri = build.GitHubRepository ?? build.AzureDevOpsRepository;
if (_options.ReleaseLayout)
{
int lastSlash = repoUri.LastIndexOf("/");
if (lastSlash != -1 && lastSlash != repoUri.Length - 1)
{
outputDirectory = Path.Combine(rootOutputDirectory, repoUri.Substring(lastSlash + 1), build.AzureDevOpsBuildNumber);
}
else
{
// Might contain invalid path chars, this is currently unhandled.
outputDirectory = Path.Combine(rootOutputDirectory, repoUri, build.AzureDevOpsBuildNumber);
}
Directory.CreateDirectory(outputDirectory);
}
List <DownloadedAsset> downloadedAssets = new List <DownloadedAsset>();
bool anyShipping = false;
Console.WriteLine($"Gathering drop for build {build.AzureDevOpsBuildNumber} of {repoUri}");
using (HttpClient client = new HttpClient(new HttpClientHandler {
CheckCertificateRevocationList = true
}))
{
var assets = await remote.GetAssetsAsync(buildId : build.Id, nonShipping : (!_options.IncludeNonShipping ? (bool?)false : null));
foreach (var asset in assets)
{
DownloadedAsset downloadedAsset = await DownloadAssetAsync(client, build, asset, outputDirectory);
if (downloadedAsset == null)
{
continue;
}
else if (!downloadedAsset.Successful)
{
success = false;
if (!_options.ContinueOnError)
{
Console.WriteLine($"Aborting download.");
break;
}
}
else
{
anyShipping |= !asset.NonShipping;
downloadedAssets.Add(downloadedAsset);
}
}
}
DownloadedBuild newBuild = new DownloadedBuild
{
Successful = success,
Build = build,
DownloadedAssets = downloadedAssets,
OutputDirectory = outputDirectory,
AnyShippingAssets = anyShipping
};
// If separated drop, generate a manifest per build
if (_options.ReleaseLayout)
{
await WriteDropManifest(new List <DownloadedBuild>() { newBuild }, outputDirectory);
}
return(newBuild);
}
private async Task <DownloadedAsset> DownloadBlobAsync(HttpClient client,
Build build,
Asset asset,
AssetLocation assetLocation,
string subPath,
List <string> errors)
{
// Normalize the asset name. Sometimes the upload to the BAR will have
// "assets/" prepended to it and sometimes not (depending on the Maestro tasks version).
// Remove assets/ if it exists so we get consistent target paths.
string normalizedAssetName = asset.Name;
if (asset.Name.StartsWith("assets/"))
{
normalizedAssetName = asset.Name.Substring("assets/".Length);
}
string fullTargetPath = Path.Combine(subPath, assetsSubPath, normalizedAssetName);
DownloadedAsset downloadedAsset = new DownloadedAsset()
{
Successful = false,
Asset = asset,
TargetLocation = fullTargetPath
};
// If the location is a blob storage account ending in index.json, as would be expected
// if PushToBlobFeed was used, strip off the index.json and append the asset name. If that doesn't work,
// prepend "assets/" to the asset name and try that.
// When uploading assets via the PushToBlobFeed task, assets/ may be prepended (e.g. assets/symbols/)
// upon upload, but may not be reported to the BAR, or may be reported to BAR.
// Either way, normalize so that we end up with only one assets/ prepended.
if (IsBlobFeedUrl(assetLocation.Location))
{
string finalBaseUri = assetLocation.Location.Substring(0, assetLocation.Location.Length - "index.json".Length);
string finalUri1 = $"{finalBaseUri}{asset.Name}";
string finalUri2 = $"{finalBaseUri}assets/{asset.Name}";
if (await DownloadFileAsync(client, finalUri1, fullTargetPath, errors))
{
downloadedAsset.Successful = true;
downloadedAsset.SourceLocation = finalUri1;
return(downloadedAsset);
}
if (await DownloadFileAsync(client, finalUri2, fullTargetPath, errors))
{
downloadedAsset.Successful = true;
downloadedAsset.SourceLocation = finalUri2;
return(downloadedAsset);
}
return(downloadedAsset);
}
// WORKAROUND: Right now we don't have the ability to have multiple root build locations
// So the BAR location gets reported as the overall manifest location. This isn't correct,
// but we're stuck with it for now until we redesign how the manifest merging is done.
// So if we see a myget url here, just look up the asset in the dotnetcli storage account.
if (IsMyGetUrl(assetLocation.Location) && assetLocation.Location.Contains("aspnetcore-dev"))
{
// First try to grab the asset from the dotnetcli storage account
string dotnetcliStorageUri = $"https://dotnetcli.blob.core.windows.net/dotnet/{asset.Name}";
if (await DownloadFileAsync(client, $"{dotnetcliStorageUri}", fullTargetPath, errors))
{
downloadedAsset.Successful = true;
downloadedAsset.SourceLocation = dotnetcliStorageUri;
return(downloadedAsset);
}
// AspNet symbol packages have incorrect names right now. They are found on the drop share.
if (asset.Name.EndsWith(".symbols.nupkg"))
{
string symbolPackageName = asset.Name;
int lastSlash = asset.Name.LastIndexOf("/");
if (lastSlash != -1)
{
symbolPackageName = asset.Name.Substring(lastSlash);
}
string shippingNonShippingFolder = asset.NonShipping ? "NonShipping" : "Shipping";
string aspnetciSymbolSharePath = $@"\\aspnetci\drops\AspNetCore\master\{build.AzureDevOpsBuildNumber}\packages\Release\{shippingNonShippingFolder}\{symbolPackageName}";
if (await DownloadFromShareAsync(aspnetciSymbolSharePath, fullTargetPath, errors))
{
downloadedAsset.Successful = true;
downloadedAsset.SourceLocation = aspnetciSymbolSharePath;
return(downloadedAsset);
}
}
return(downloadedAsset);
}
else
{
if (string.IsNullOrEmpty(assetLocation.Location))
{
errors.Add($"Asset location for {asset.Name} is not available.");
}
else
{
errors.Add($"Blob uri '{assetLocation.Location} for {asset.Name} is of an unknown type");
}
return(downloadedAsset);
}
}
/// <summary>
///
/// </summary>
/// <param name="client"></param>
/// <param name="asset"></param>
/// <param name="rootOutputDirectory"></param>
/// <returns></returns>
/// <remarks>
/// Layout:
/// {root dir}\shipping\assets - blobs
/// {root dir}\shipping\packages - packages
/// {root dir}\nonshipping\assets - blobs
/// {root dir}\nonshipping\packages - packages
/// </remarks>
private async Task <DownloadedAsset> DownloadAssetAsync(HttpClient client,
Build build,
Asset asset,
string rootOutputDirectory)
{
string assetNameAndVersion = GetAssetNameForLogging(asset);
if (_options.IncludeNonShipping || !asset.NonShipping)
{
Console.WriteLine($" Downloading asset {assetNameAndVersion}");
}
else
{
Console.WriteLine($" Skipping non-shipping asset {assetNameAndVersion}");
return(null);
}
DownloadedAsset downloadedAsset = new DownloadedAsset()
{
Successful = false,
Asset = asset
};
List <string> errors = new List <string>();
if (asset.Locations.Count == 0)
{
errors.Add($"Asset '{assetNameAndVersion}' has no known location information.");
}
else
{
string subPath = Path.Combine(rootOutputDirectory, asset.NonShipping ? nonShippingSubPath : shippingSubPath);
// Walk the locations and attempt to gather the asset at each one, setting the output
// path based on the type. Note that if there are multiple locations and their types don't
// match, consider this an error.
LocationType locationType = asset.Locations[0].Type;
foreach (AssetLocation location in asset.Locations)
{
if (locationType != location.Type)
{
errors.Add($"Asset '{assetNameAndVersion}' has inconsistent location types ({locationType} vs. {location.Type})");
break;
}
switch (locationType)
{
case LocationType.NugetFeed:
downloadedAsset = await DownloadNugetPackageAsync(client, build, asset, location, subPath, errors);
break;
case LocationType.Container:
downloadedAsset = await DownloadBlobAsync(client, build, asset, location, subPath, errors);
break;
default:
errors.Add($"Unexpected location type {locationType}");
break;
}
if (downloadedAsset.Successful)
{
return(downloadedAsset);
}
}
}
// If none of the download attempts succeeded, then we should print out all the error
// information.
Console.WriteLine($" Failed to download asset, errors shown below:");
foreach (string error in errors)
{
Console.WriteLine($" {error}");
}
return(downloadedAsset);
}
/// <summary>
///
/// </summary>
/// <param name="client"></param>
/// <param name="asset"></param>
/// <param name="rootOutputDirectory"></param>
/// <returns></returns>
/// <remarks>
/// Layout:
/// {root dir}\shipping\assets - blobs
/// {root dir}\shipping\packages - packages
/// {root dir}\nonshipping\assets - blobs
/// {root dir}\nonshipping\packages - packages
/// </remarks>
private async Task <DownloadedAsset> DownloadAssetAsync(HttpClient client,
Build build,
Asset asset,
string rootOutputDirectory)
{
string assetNameAndVersion = GetAssetNameForLogging(asset);
if (_options.IncludeNonShipping || !asset.NonShipping)
{
Console.WriteLine($" Downloading asset {assetNameAndVersion}");
}
else
{
Console.WriteLine($" Skipping non-shipping asset {assetNameAndVersion}");
return(null);
}
DownloadedAsset downloadedAsset = new DownloadedAsset()
{
Successful = false,
Asset = asset
};
List <string> errors = new List <string>();
List <AssetLocation> assetLocations = new List <AssetLocation>(asset.Locations);
if (assetLocations.Count == 0)
{
// If there is no location information and the user wants workarounds, add a bunch
// of feeds.
if (!_options.NoWorkarounds)
{
if (asset.Name.Contains("/"))
{
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetcli.blob.core.windows.net/dotnet/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetclichecksums.blob.core.windows.net/dotnet/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore-tooling/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/aspnet-entityframeworkcore/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/dotnet-sdk/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.Container, "https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json"));
}
else
{
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/aspnet-aspnetcore-tooling/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/aspnet-entityframeworkcore/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/aspnet-extensions/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/dotnet-core/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/dotnet-coreclr/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/dotnet-sdk/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/dotnet-toolset/index.json"));
assetLocations.Add(new AssetLocation(0, LocationType.NugetFeed, "https://dotnetfeed.blob.core.windows.net/dotnet-windowsdesktop/index.json"));
}
}
else
{
errors.Add($"Asset '{assetNameAndVersion}' has no known location information.");
}
}
else
{
if (_options.LatestLocation)
{
downloadedAsset = await DownloadAssetFromLatestLocation(client, build, asset, assetLocations, rootOutputDirectory, errors);
}
else
{
downloadedAsset = await DownloadAssetFromAnyLocationAsync(client, build, asset, assetLocations, rootOutputDirectory, errors);
}
if (downloadedAsset.Successful)
{
return(downloadedAsset);
}
}
// If none of the download attempts succeeded, then we should print out all the error
// information.
Console.WriteLine($" Failed to download asset, errors shown below:");
foreach (string error in errors)
{
Console.WriteLine($" {error}");
}
return(downloadedAsset);
}
