public async Task <bool> DoesPackageExistAsync(string id, NuGetVersion version)
{
var indexUrl = RegistrationUrlBuilder.GetIndexUrl(_endpoint, id);
var index = await _registrationClient.GetIndexOrNullAsync(indexUrl);
if (index == null)
{
return(false);
}
var matchingPageItem = index
.Items
.FirstOrDefault(x => NuGetVersion.Parse(x.Lower) <= version && version <= NuGetVersion.Parse(x.Upper));
if (matchingPageItem == null)
{
return(false);
}
var items = matchingPageItem.Items;
if (items == null)
{
try
{
var page = await _registrationClient.GetPageAsync(matchingPageItem.Url);
items = page.Items;
}
catch (SimpleHttpClientException ex) when(ex.StatusCode == HttpStatusCode.NotFound)
{
return(false);
}
}
var matchingItem = items
.FirstOrDefault(x => NuGetVersion.Parse(x.CatalogEntry.Version) == version);
if (matchingItem == null)
{
return(false);
}
return(true);
}
private V3SearchPackage ToV3SearchPackage(SearchDocument.Full result, string registrationsBaseUrl)
{
var registrationIndexUrl = RegistrationUrlBuilder.GetIndexUrl(registrationsBaseUrl, result.PackageId);
return(new V3SearchPackage
{
AtId = registrationIndexUrl,
Type = "Package",
Registration = registrationIndexUrl,
Id = result.PackageId,
Version = result.FullVersion,
Description = result.Description ?? string.Empty,
Summary = result.Summary ?? string.Empty,
Title = TitleThenId(result),
IconUrl = GetIconUrl(result),
LicenseUrl = result.LicenseUrl,
ProjectUrl = result.ProjectUrl,
Tags = result.Tags ?? Array.Empty <string>(),
Authors = new[] { result.Authors ?? string.Empty },
TotalDownloads = AuxiliaryData.GetTotalDownloadCount(result.PackageId),
Verified = AuxiliaryData.IsVerified(result.PackageId),
PackageTypes = GetV3SearchPackageTypes(result),
Versions = result
.Versions
.Select(x =>
{
// Each of these versions is the full version.
var lowerVersion = NuGetVersion.Parse(x).ToNormalizedString().ToLowerInvariant();
return new V3SearchVersion
{
Version = x,
Downloads = AuxiliaryData.GetDownloadCount(result.PackageId, lowerVersion),
AtId = RegistrationUrlBuilder.GetLeafUrl(registrationsBaseUrl, result.PackageId, x),
};
})
.ToList(),
});
}
public async Task CopyAsync(ICloudBlobContainer container, string oldBaseUrl, string newBaseUrl, string id, bool gzip)
{
var normalizedOldBaseUrl = oldBaseUrl.TrimEnd('/');
var normalizedNewBaseUrl = newBaseUrl.TrimEnd('/');
var indexUrl = RegistrationUrlBuilder.GetIndexUrl(normalizedOldBaseUrl, id);
_logger.LogInformation("Downloading index {IndexUrl}...", indexUrl);
await Throttle.WaitAsync();
RegistrationIndex index;
try
{
index = await _registrationClient.GetIndexOrNullAsync(indexUrl);
}
finally
{
Throttle.Release();
}
if (index == null)
{
return;
}
var pageUrls = new List <string>();
var itemUrls = new List <string>();
var pageTasks = index
.Items
.Select(async pageItem =>
{
await Task.Yield();
await Throttle.WaitAsync();
try
{
List <RegistrationLeafItem> leafItems;
if (pageItem.Items != null)
{
leafItems = pageItem.Items;
}
else
{
pageUrls.Add(pageItem.Url);
_logger.LogInformation("Downloading page {PageUrl}...", pageItem.Url);
var page = await _registrationClient.GetPageAsync(pageItem.Url);
leafItems = page.Items;
}
foreach (var leafItem in leafItems)
{
itemUrls.Add(leafItem.Url);
}
}
finally
{
Throttle.Release();
}
})
.ToList();
await Task.WhenAll(pageTasks);
var copyTasks = itemUrls
.Concat(pageUrls)
.Concat(new[] { indexUrl })
.Select(async url =>
{
await Task.Yield();
await CopyUrlAsync(container, normalizedOldBaseUrl, normalizedNewBaseUrl, url, gzip);
})
.ToList();
await Task.WhenAll(copyTasks);
}
public async Task <LatestCatalogLeaves> GetLatestLeavesAsync(
string packageId,
IReadOnlyList <IReadOnlyList <NuGetVersion> > versions)
{
if (packageId == null)
{
throw new ArgumentNullException(nameof(packageId));
}
if (versions == null)
{
throw new ArgumentNullException(nameof(versions));
}
if (!versions.Any())
{
throw new ArgumentException("At least one version list must be provided.", nameof(versions));
}
using (_telemetryService.TrackGetLatestLeaves(packageId, versions.SelectMany(v => v).Distinct().Count()))
{
var unavailable = new HashSet <NuGetVersion>();
var fetched = new Dictionary <NuGetVersion, PackageDetailsCatalogLeaf>();
var unlisted = new Dictionary <NuGetVersion, string>();
var registrationIndexUrl = RegistrationUrlBuilder.GetIndexUrl(_options.Value.RegistrationsBaseUrl, packageId);
var registrationIndex = await _registrationClient.GetIndexOrNullAsync(registrationIndexUrl);
if (registrationIndex == null)
{
_logger.LogWarning(
"No registation index was found. ID: {PackageId}, registration index URL: {RegistrationIndexUrl}",
packageId,
registrationIndexUrl);
foreach (var version in versions.SelectMany(x => x))
{
unavailable.Add(version);
}
return(new LatestCatalogLeaves(unavailable, fetched));
}
var pageUrlToInfo = registrationIndex
.Items
.ToDictionary(x => x.Url, x => new RegistrationPageInfo(x));
// Make a list of ranges for logging purposes.
var ranges = pageUrlToInfo
.OrderBy(x => x.Value.Range.MinVersion)
.Select(x => x.Value.RangeString)
.ToList();
foreach (var versionList in versions)
{
await AddLatestLeafAsync(
packageId,
versionList,
pageUrlToInfo,
ranges,
unavailable,
fetched,
unlisted);
}
// Fetch the unlisted version's metadata in parallel. This can be many versions in the case of a bulk
// unlist.
var allWork = new ConcurrentBag <KeyValuePair <NuGetVersion, string> >(unlisted);
var allResults = new ConcurrentBag <KeyValuePair <NuGetVersion, PackageDetailsCatalogLeaf> >();
var tasks = Enumerable
.Range(0, _options.Value.MaxConcurrentBatches)
.Select(async x =>
{
await Task.Yield();
while (allWork.TryTake(out var work))
{
var leaf = await _catalogClient.GetPackageDetailsLeafAsync(work.Value);
allResults.Add(KeyValuePair.Create(work.Key, leaf));
}
})
.ToList();
await Task.WhenAll(tasks);
foreach (var pair in allResults)
{
fetched.Add(pair.Key, pair.Value);
}
return(new LatestCatalogLeaves(unavailable, fetched));
}
}
