private CatalogCommitItem GetLatestPerSingleIdentity(IEnumerable <CatalogCommitItem> entries)
{
CatalogCommitItem max = null;
foreach (var entry in entries)
{
if (max == null)
{
max = entry;
continue;
}
if (!StringComparer.OrdinalIgnoreCase.Equals(max.PackageIdentity, entry.PackageIdentity))
{
throw new InvalidOperationException("The entries compared should have the same package identity.");
}
if (entry.CommitTimeStamp > max.CommitTimeStamp)
{
max = entry;
}
else if (entry.CommitTimeStamp == max.CommitTimeStamp)
{
const string message = "There are multiple catalog leaves for a single package at one time.";
_logger.LogError(
message + " ID: {PackageId}, version: {PackageVersion}, commit timestamp: {CommitTimestamp:O}",
entry.PackageIdentity.Id,
entry.PackageIdentity.Version.ToFullString(),
entry.CommitTimeStamp);
throw new InvalidOperationException(message);
}
}
return(max);
}
public StartProcessingBatchesIfNoFailures()
{
_commitItem0 = TestUtility.CreateCatalogCommitItem(_now, _packageIdentitya);
_commitItem1 = TestUtility.CreateCatalogCommitItem(_now, _packageIdentityb);
_commitItem2 = TestUtility.CreateCatalogCommitItem(_now.AddMinutes(1), _packageIdentityc);
_commitItem3 = TestUtility.CreateCatalogCommitItem(_now.AddMinutes(2), _packageIdentityd);
}
protected void SetVersion(string version)
{
var parsedVersion = NuGetVersion.Parse(version);
_packageVersion = version;
_commitItem = new CatalogCommitItem(
new Uri("https://example/uri"),
"29e5c582-c1ef-4a5c-a053-d86c7381466b",
new DateTime(2018, 11, 1),
new List <string> {
Schema.DataTypes.PackageDetails.AbsoluteUri
},
new List <Uri> {
Schema.DataTypes.PackageDetails
},
new PackageIdentity(_packageId, parsedVersion));
_leaf = new PackageDetailsCatalogLeaf
{
PackageId = _packageId,
PackageVersion = _commitItem.PackageIdentity.Version.ToFullString(),
VerbatimVersion = _commitItem.PackageIdentity.Version.OriginalVersion,
IsPrerelease = parsedVersion.IsPrerelease,
Listed = true,
};
_latestEntries = new List <CatalogCommitItem> {
_commitItem
};
_entryToLeaf = new Dictionary <CatalogCommitItem, PackageDetailsCatalogLeaf>(
ReferenceEqualityComparer <CatalogCommitItem> .Default)
{
{ _commitItem, _leaf },
};
}
private async Task InsertAsync(
Context context,
PageInfo pageInfo,
int index,
CatalogCommitItem entry)
{
if (entry.IsPackageDelete)
{
// No matching version was found for this delete. No more work is necessary.
_logger.LogInformation(
"Version {Version} does not exist. The delete from commit {CommitId} will have no affect.",
entry.PackageIdentity.Version.ToNormalizedString(),
entry.CommitId);
}
else
{
// Insert the new registration leaf item.
_logger.LogInformation(
"Version {Version} will be added by commit {CommitId}.",
entry.PackageIdentity.Version.ToNormalizedString(),
entry.CommitId);
var leafInfo = LeafInfo.New(entry.PackageIdentity.Version);
await pageInfo.InsertAsync(index, leafInfo);
context.ModifiedLeaves.Add(leafInfo);
context.ModifiedPages.Add(pageInfo);
}
}
private VersionListChange GetVersionListChange(
Context context,
CatalogCommitItem entry)
{
if (entry.IsPackageDetails && !entry.IsPackageDelete)
{
var leaf = context.EntryToLeaf[entry];
return(VersionListChange.Upsert(
leaf.VerbatimVersion ?? leaf.PackageVersion,
new VersionPropertiesData(
listed: leaf.IsListed(),
semVer2: leaf.IsSemVer2())));
}
else if (entry.IsPackageDelete && !entry.IsPackageDetails)
{
return(VersionListChange.Delete(entry.PackageIdentity.Version));
}
else
{
const string message = "An unsupported leaf type was encountered.";
_logger.LogError(
message + " ID: {PackageId}, version: {PackageVersion}, commit timestamp: {CommitTimestamp:O}, " +
"types: {EntryTypeUris}, leaf URL: {Url}",
entry.PackageIdentity.Id,
entry.PackageIdentity.Version.ToFullString(),
entry.CommitTimeStamp,
entry.TypeUris,
entry.Uri.AbsoluteUri);
throw new ArgumentException("An unsupported leaf type was encountered.");
}
}
public async Task ProcessPackageDeleteLeafAsync(Storage storage, CatalogCommitItem item, CancellationToken cancellationToken)
{
var targetStoragePath = GetTargetStorageIconPath(item);
await _iconProcessor.DeleteIconAsync(storage, targetStoragePath, cancellationToken, item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString());
// it would be nice to remove the icon copy result from cache for this item, but we don't have an icon URL here,
// so can't remove anything. Will rely on the copy code to catch the copy failure and cleanup the cache appropriately.
}
public void CompareTo_WhenObjIsNotCatalogCommit_Throws()
{
var commitItem = CatalogCommitItem.Create(_context, _commitItem);
var exception = Assert.Throws <ArgumentException>(() => commitItem.CompareTo(new object()));
Assert.Equal("obj", exception.ParamName);
}
private static CatalogCommitItem CreatePackage(string id, string version)
{
var context = TestUtility.CreateCatalogContextJObject();
var packageIdentity = new PackageIdentity(id, new NuGetVersion(version));
var commitItem = TestUtility.CreateCatalogCommitItemJObject(DateTime.UtcNow, packageIdentity);
return(CatalogCommitItem.Create(context, commitItem));
}
public async Task ReturnsEmptyItemLIsResultsInEmptyBatchList()
{
var items = new CatalogCommitItem[0];
var batches = await _target.CreateBatchesAsync(items);
Assert.Empty(batches);
}
public void Create_WhenContextIsNull_Throws()
{
const JObject context = null;
var exception = Assert.Throws <ArgumentNullException>(() => CatalogCommitItem.Create(context, _commitItem));
Assert.Equal("context", exception.ParamName);
}
public void Create_WhenTypeIsEmpty_Throws()
{
_commitItem[CatalogConstants.TypeKeyword] = new JArray();
var exception = Assert.Throws <ArgumentException>(() => CatalogCommitItem.Create(_context, _commitItem));
Assert.Equal("commitItem", exception.ParamName);
Assert.StartsWith($"The value of property '{CatalogConstants.TypeKeyword}' must be non-null and non-empty.", exception.Message);
}
private async Task <int> MergeEntryIntoPageAsync(
Context context,
CatalogCommitItem entry,
int pageIndex,
int itemIndex)
{
var pageInfo = context.IndexInfo.Items[pageIndex];
var items = await pageInfo.GetLeafInfosAsync();
while (itemIndex < items.Count)
{
if (entry.PackageIdentity.Version > items[itemIndex].Version)
{
// The current position in the item list is too low for the catalog version. Keep looking.
itemIndex++;
}
else if (entry.PackageIdentity.Version == items[itemIndex].Version)
{
if (entry.IsPackageDelete)
{
// Remove the registration leaf item. The current catalog commit item represents a
// delete for this version.
_logger.LogInformation(
"Version {Version} will be deleted by commit {CommitId}.",
entry.PackageIdentity.Version.ToNormalizedString(),
entry.CommitId);
context.DeletedLeaves.Add(await pageInfo.RemoveAtAsync(itemIndex));
context.ModifiedPages.Add(pageInfo);
RemovePageAtIfEmpty(context, pageIndex);
}
else
{
// Update the metadata of the existing registration leaf item.
_logger.LogInformation(
"Version {Version} will be updated by commit {CommitId}.",
entry.PackageIdentity.Version.ToNormalizedString(),
entry.CommitId);
context.ModifiedLeaves.Add(items[itemIndex]);
context.ModifiedPages.Add(pageInfo);
}
// The version has been matched with an existing version. Leave the item index as-is. The next item
// is now at the current position. No more work is necessary.
return(itemIndex);
}
else
{
break;
}
}
await InsertAsync(context, pageInfo, itemIndex, entry);
return(itemIndex);
}
public async Task FinishAsync()
{
if (!_urls.Any())
{
return;
}
// Download all of the leaves.
var urlsToDownload = new ConcurrentBag <string>(_urls);
var leaves = new ConcurrentBag <CatalogLeaf>();
await ParallelAsync.Repeat(
async() =>
{
await Task.Yield();
while (urlsToDownload.TryTake(out var url))
{
var leaf = await _catalogClient.GetLeafAsync(url);
leaves.Add(leaf);
}
});
// Build the input to hive updaters.
var entries = new List <CatalogCommitItem>();
var entryToLeaf = new Dictionary <CatalogCommitItem, PackageDetailsCatalogLeaf>(
ReferenceEqualityComparer <CatalogCommitItem> .Default);
foreach (var leaf in leaves)
{
if (leaf.IsPackageDelete() == leaf.IsPackageDetails())
{
throw new InvalidOperationException("A catalog leaf must be either a package delete or a package details leaf.");
}
var typeUri = leaf.IsPackageDetails() ? Schema.DataTypes.PackageDetails : Schema.DataTypes.PackageDelete;
var catalogCommitItem = new CatalogCommitItem(
new Uri(leaf.Url),
leaf.CommitId,
leaf.CommitTimestamp.UtcDateTime,
types: null,
typeUris: new[] { typeUri },
packageIdentity: new PackageIdentity(_packageId, leaf.ParsePackageVersion()));
entries.Add(catalogCommitItem);
if (leaf.IsPackageDetails())
{
entryToLeaf.Add(catalogCommitItem, (PackageDetailsCatalogLeaf)leaf);
}
}
// Update the hives.
await _updater.UpdateAsync(_packageId, entries, entryToLeaf);
_urls.Clear();
}
public static CatalogCommitItem CreateCatalogCommitItem(
DateTime commitTimeStamp,
PackageIdentity packageIdentity,
string commitId = null)
{
var context = CreateCatalogContextJObject();
var commitItem = CreateCatalogCommitItemJObject(commitTimeStamp, packageIdentity, commitId);
return(CatalogCommitItem.Create(context, commitItem));
}
public void Create_WhenArgumentsAreValid_ReturnsInstance()
{
var commitItem = CatalogCommitItem.Create(_context, _commitItem);
Assert.Equal($"https://nuget.test/{_packageIdentity.Id}", commitItem.Uri.AbsoluteUri);
Assert.Equal(_now, commitItem.CommitTimeStamp.ToUniversalTime());
Assert.True(Guid.TryParse(commitItem.CommitId, out var commitId));
Assert.Equal(_packageIdentity, commitItem.PackageIdentity);
Assert.Equal(CatalogConstants.NuGetPackageDetails, commitItem.Types.Single());
Assert.Equal(Schema.DataTypes.PackageDetails.AbsoluteUri, commitItem.TypeUris.Single().AbsoluteUri);
}
internal static CatalogEntry Create(CatalogCommitItem item)
{
var typeUri = item.TypeUris.Single(uri =>
uri.AbsoluteUri == Schema.DataTypes.PackageDetails.AbsoluteUri ||
uri.AbsoluteUri == Schema.DataTypes.PackageDelete.AbsoluteUri);
return(new CatalogEntry(
item.CommitTimeStamp,
item.PackageIdentity.Id.ToLowerInvariant(),
item.PackageIdentity.Version.ToNormalizedString().ToLowerInvariant(),
typeUri));
}
public void Create_WhenArgumentIsValid_ReturnsInstance()
{
var contextJObject = TestUtility.CreateCatalogContextJObject();
var commitItemJObject = TestUtility.CreateCatalogCommitItemJObject(_commitTimeStamp, _packageIdentity);
var commitItem = CatalogCommitItem.Create(contextJObject, commitItemJObject);
var entry = CatalogIndexEntry.Create(commitItem);
Assert.Equal(_uri.AbsoluteUri, entry.Uri.AbsoluteUri);
Assert.Equal(CatalogConstants.NuGetPackageDetails, entry.Types.Single());
Assert.Equal(commitItemJObject[CatalogConstants.CommitId].ToString(), entry.CommitId);
Assert.Equal(_commitTimeStamp, entry.CommitTimeStamp.ToUniversalTime());
Assert.Equal(_packageId, entry.Id);
Assert.Equal(_packageVersion, entry.Version);
}
private static List <CatalogCommitItem> MakeSortedCatalog(ICollection <VersionAction> sortedVersionActions)
{
var output = new List <CatalogCommitItem>();
foreach (var versionAction in sortedVersionActions)
{
var item = new CatalogCommitItem(
uri: null,
commitId: null,
commitTimeStamp: DateTime.MinValue,
types: null,
typeUris: new[] { versionAction.IsDelete?Schema.DataTypes.PackageDelete: Schema.DataTypes.PackageDetails },
packageIdentity: new PackageIdentity("NuGet.Versioning", versionAction.Version));
output.Add(item);
}
return(output);
}
private async Task ProcessEmbeddedIconAsync(IStorage destinationStorage, CatalogCommitItem item, string iconFile, CancellationToken cancellationToken)
{
var packageFilename = PackageUtility.GetPackageFileName(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()).ToLowerInvariant();
var packageUri = _packageStorage.ResolveUri(packageFilename);
var packageBlobReference = await _packageStorage.GetCloudBlockBlobReferenceAsync(packageUri);
using (_telemetryService.TrackEmbeddedIconProcessingDuration(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()))
{
Stream packageStream;
try
{
packageStream = await packageBlobReference.GetStreamAsync(cancellationToken);
}
catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound)
{
_logger.LogWarning("Package blob not found at {PackageUrl}: {Exception}. Will assume package was deleted and skip",
packageUri.AbsoluteUri,
ex);
return;
}
catch (Exception ex)
{
// logging other exceptions here to have proper scope in log message
_logger.LogError("Exception while trying to access package blob {PackageUrl}: {Exception}",
packageUri.AbsoluteUri,
ex);
throw;
}
using (packageStream)
{
var targetStoragePath = GetTargetStorageIconPath(item);
var resultUrl = await _iconProcessor.CopyEmbeddedIconFromPackageAsync(
packageStream,
iconFile,
destinationStorage,
targetStoragePath,
cancellationToken,
item.PackageIdentity.Id,
item.PackageIdentity.Version.ToNormalizedString());
}
}
}
private void ExcludeSemVer2(
HiveType hive,
List <CatalogCommitItem> sortedCatalog,
Dictionary <NuGetVersion, PackageDetailsCatalogLeaf> versionToCatalogLeaf)
{
Guard.Assert(
hive == HiveType.Legacy || hive == HiveType.Gzipped,
"Only the legacy and gzipped hives should exclude SemVer 2.0.0 versions.");
for (int i = 0; i < sortedCatalog.Count; i++)
{
var catalogCommitItem = sortedCatalog[i];
if (catalogCommitItem.IsPackageDelete)
{
continue;
}
var version = catalogCommitItem.PackageIdentity.Version;
var catalogLeaf = versionToCatalogLeaf[version];
if (catalogLeaf.IsSemVer2())
{
// Turn the PackageDetails into a PackageDelete to ensure that a known SemVer 2.0.0 version is not
// in a hive that should not have SemVer 2.0.0. This may cause a little bit more work (like a
// non-inlined page getting downloaded) but it's worth it to allow reflows of SemVer 2.0.0 packages
// to fix up problems. In general, reflow should be a powerful fix-up tool. If this causes
// performance issues later, we can revisit this extra work at that time.
sortedCatalog[i] = new CatalogCommitItem(
catalogCommitItem.Uri,
catalogCommitItem.CommitId,
catalogCommitItem.CommitTimeStamp,
types: null,
typeUris: DeleteUris,
packageIdentity: catalogCommitItem.PackageIdentity);
versionToCatalogLeaf.Remove(version);
_logger.LogInformation(
"Version {Version} is SemVer 2.0.0 so it will be treated as a package delete.",
catalogLeaf.ParsePackageVersion().ToFullString(),
hive);
}
}
}
private async Task ProcessEmbeddedIconAsync(IStorage destinationStorage, CatalogCommitItem item, string iconFile, CancellationToken cancellationToken)
{
var packageFilename = PackageUtility.GetPackageFileName(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()).ToLowerInvariant();
var packageUri = _packageStorage.ResolveUri(packageFilename);
var packageBlobReference = await _packageStorage.GetCloudBlockBlobReferenceAsync(packageUri);
using (_telemetryService.TrackEmbeddedIconProcessingDuration(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()))
using (var packageStream = await packageBlobReference.GetStreamAsync(cancellationToken))
{
var targetStoragePath = GetTargetStorageIconPath(item);
var resultUrl = await _iconProcessor.CopyEmbeddedIconFromPackageAsync(
packageStream,
iconFile,
destinationStorage,
targetStoragePath,
cancellationToken,
item.PackageIdentity.Id,
item.PackageIdentity.Version.ToNormalizedString());
}
}
public void CreateCommitItemBatches_WhenMultipleCommitItemsShareCommitTimeStampButNotCommitId_Throws()
{
var commitTimeStamp = DateTime.UtcNow;
var context = TestUtility.CreateCatalogContextJObject();
var commitItem0 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp, _packageIdentitya));
var commitItem1 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp, _packageIdentityb));
var commitItems = new[] { commitItem0, commitItem1 };
var exception = Assert.Throws <ArgumentException>(
() => CatalogCommitUtilities.CreateCommitItemBatches(
commitItems,
CatalogCommitUtilities.GetPackageIdKey));
Assert.Equal("catalogItems", exception.ParamName);
Assert.StartsWith("Multiple commits exist with the same commit timestamp but different commit ID's: " +
$"{{ CommitId = {commitItem0.CommitId}, CommitTimeStamp = {commitItem0.CommitTimeStamp.ToString("O")} }}, " +
$"{{ CommitId = {commitItem1.CommitId}, CommitTimeStamp = {commitItem1.CommitTimeStamp.ToString("O")} }}.",
exception.Message);
}
public void CreateCommitItemBatches_WhenMultipleCommitItemsShareCommitTimeStampButNotCommitIdAndLaterCommitExists_DoesNotThrow()
{
var commitTimeStamp0 = DateTime.UtcNow;
var commitTimeStamp1 = commitTimeStamp0.AddMinutes(1);
var context = TestUtility.CreateCatalogContextJObject();
var commitItem0 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp0, _packageIdentitya));
var commitItem1 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp0, _packageIdentityb));
var commitItem2 = CatalogCommitItem.Create(
context,
TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp1, _packageIdentitya));
var commitItems = new[] { commitItem0, commitItem1, commitItem2 };
var batches = CatalogCommitUtilities.CreateCommitItemBatches(
commitItems,
CatalogCommitUtilities.GetPackageIdKey);
Assert.Collection(
batches,
batch =>
{
Assert.Equal(commitTimeStamp1, batch.CommitTimeStamp.ToUniversalTime());
Assert.Collection(
batch.Items,
commit => Assert.True(ReferenceEquals(commit, commitItem2)));
},
batch =>
{
Assert.Equal(commitTimeStamp0, batch.CommitTimeStamp.ToUniversalTime());
Assert.Collection(
batch.Items,
commit => Assert.True(ReferenceEquals(commit, commitItem1)));
});
}
private async Task CopyIcon(Uri iconUrl, IStorage destinationStorage, IStorage iconCacheStorage, CatalogCommitItem item, CancellationToken cancellationToken)
{
var ingestionResult = await Retry.IncrementalAsync(
async() => await TryIngestExternalIconAsync(item, iconUrl, destinationStorage, cancellationToken),
e => false,
r => r.Result == AttemptResult.FailCanRetry,
MaxExternalIconIngestAttempts,
initialWaitInterval : TimeSpan.FromSeconds(5),
waitIncrement : TimeSpan.FromSeconds(1));
if (ingestionResult.Result == AttemptResult.Success)
{
try
{
await _iconCopyResultCache.SaveExternalIcon(iconUrl, ingestionResult.ResultUrl, destinationStorage, iconCacheStorage, cancellationToken);
}
catch (Exception e)
{
// we will report and ignore such exceptions. Failure to store icon will cause the original icon
// to be re-retrieved next time it is encountered.
_logger.LogWarning(0, e, "Failed to store icon in the cache");
}
}
else
{
var destinationStoragePath = GetTargetStorageIconPath(item);
await _iconProcessor.DeleteIconAsync(destinationStorage, destinationStoragePath, cancellationToken, item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString());
_telemetryService.TrackExternalIconIngestionFailure(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString());
_iconCopyResultCache.SaveExternalCopyFailure(iconUrl);
}
}
protected override string GetKey(CatalogCommitItem item)
{
return(CatalogCommitUtilities.GetPackageIdKey(item));
}
private static string GetTargetStorageIconPath(CatalogCommitItem item)
{
return($"{item.PackageIdentity.Id.ToLowerInvariant()}/{item.PackageIdentity.Version.ToNormalizedString().ToLowerInvariant()}/icon");
}
public async Task ProcessPackageDetailsLeafAsync(IStorage destinationStorage, IStorage iconCacheStorage, CatalogCommitItem item, string iconUrlString, string iconFile, CancellationToken cancellationToken)
{
var hasExternalIconUrl = !string.IsNullOrWhiteSpace(iconUrlString);
var hasEmbeddedIcon = !string.IsNullOrWhiteSpace(iconFile);
if (hasExternalIconUrl && !hasEmbeddedIcon && Uri.TryCreate(iconUrlString, UriKind.Absolute, out var iconUrl))
{
using (_logger.BeginScope("Processing icon url {IconUrl}", iconUrl))
{
await ProcessExternalIconUrlAsync(destinationStorage, iconCacheStorage, item, iconUrl, cancellationToken);
}
}
else if (hasEmbeddedIcon)
{
await ProcessEmbeddedIconAsync(destinationStorage, item, iconFile, cancellationToken);
}
}
public void Create_WhenCommitItemIsNull_Throws()
{
var exception = Assert.Throws <ArgumentNullException>(() => CatalogCommitItem.Create(_context, commitItem: null));
Assert.Equal("commitItem", exception.ParamName);
}
public static bool IsOnlyPackageDetails(CatalogCommitItem e)
{
return(e.IsPackageDetails && !e.IsPackageDelete);
}
private async Task ProcessExternalIconUrlAsync(IStorage destinationStorage, IStorage iconCacheStorage, CatalogCommitItem item, Uri iconUrl, CancellationToken cancellationToken)
{
_logger.LogInformation("Found external icon url {IconUrl} for {PackageId} {PackageVersion}",
iconUrl,
item.PackageIdentity.Id,
item.PackageIdentity.Version);
if (!IsValidIconUrl(iconUrl))
{
_logger.LogInformation("Invalid icon URL {IconUrl}", iconUrl);
return;
}
var cachedResult = _iconCopyResultCache.Get(iconUrl);
if (cachedResult != null && await TryTakeFromCache(iconUrl, cachedResult, iconCacheStorage, destinationStorage, item, cancellationToken))
{
return;
}
using (_telemetryService.TrackExternalIconProcessingDuration(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()))
{
await CopyIcon(iconUrl, destinationStorage, iconCacheStorage, item, cancellationToken);
}
}
