private CatalogCommitItem GetLatestPerSingleIdentity(IEnumerable <CatalogCommitItem> entries) { CatalogCommitItem max = null; foreach (var entry in entries) { if (max == null) { max = entry; continue; } if (!StringComparer.OrdinalIgnoreCase.Equals(max.PackageIdentity, entry.PackageIdentity)) { throw new InvalidOperationException("The entries compared should have the same package identity."); } if (entry.CommitTimeStamp > max.CommitTimeStamp) { max = entry; } else if (entry.CommitTimeStamp == max.CommitTimeStamp) { const string message = "There are multiple catalog leaves for a single package at one time."; _logger.LogError( message + " ID: {PackageId}, version: {PackageVersion}, commit timestamp: {CommitTimestamp:O}", entry.PackageIdentity.Id, entry.PackageIdentity.Version.ToFullString(), entry.CommitTimeStamp); throw new InvalidOperationException(message); } } return(max); }
public StartProcessingBatchesIfNoFailures() { _commitItem0 = TestUtility.CreateCatalogCommitItem(_now, _packageIdentitya); _commitItem1 = TestUtility.CreateCatalogCommitItem(_now, _packageIdentityb); _commitItem2 = TestUtility.CreateCatalogCommitItem(_now.AddMinutes(1), _packageIdentityc); _commitItem3 = TestUtility.CreateCatalogCommitItem(_now.AddMinutes(2), _packageIdentityd); }
protected void SetVersion(string version) { var parsedVersion = NuGetVersion.Parse(version); _packageVersion = version; _commitItem = new CatalogCommitItem( new Uri("https://example/uri"), "29e5c582-c1ef-4a5c-a053-d86c7381466b", new DateTime(2018, 11, 1), new List <string> { Schema.DataTypes.PackageDetails.AbsoluteUri }, new List <Uri> { Schema.DataTypes.PackageDetails }, new PackageIdentity(_packageId, parsedVersion)); _leaf = new PackageDetailsCatalogLeaf { PackageId = _packageId, PackageVersion = _commitItem.PackageIdentity.Version.ToFullString(), VerbatimVersion = _commitItem.PackageIdentity.Version.OriginalVersion, IsPrerelease = parsedVersion.IsPrerelease, Listed = true, }; _latestEntries = new List <CatalogCommitItem> { _commitItem }; _entryToLeaf = new Dictionary <CatalogCommitItem, PackageDetailsCatalogLeaf>( ReferenceEqualityComparer <CatalogCommitItem> .Default) { { _commitItem, _leaf }, }; }
private async Task InsertAsync( Context context, PageInfo pageInfo, int index, CatalogCommitItem entry) { if (entry.IsPackageDelete) { // No matching version was found for this delete. No more work is necessary. _logger.LogInformation( "Version {Version} does not exist. The delete from commit {CommitId} will have no affect.", entry.PackageIdentity.Version.ToNormalizedString(), entry.CommitId); } else { // Insert the new registration leaf item. _logger.LogInformation( "Version {Version} will be added by commit {CommitId}.", entry.PackageIdentity.Version.ToNormalizedString(), entry.CommitId); var leafInfo = LeafInfo.New(entry.PackageIdentity.Version); await pageInfo.InsertAsync(index, leafInfo); context.ModifiedLeaves.Add(leafInfo); context.ModifiedPages.Add(pageInfo); } }
private VersionListChange GetVersionListChange( Context context, CatalogCommitItem entry) { if (entry.IsPackageDetails && !entry.IsPackageDelete) { var leaf = context.EntryToLeaf[entry]; return(VersionListChange.Upsert( leaf.VerbatimVersion ?? leaf.PackageVersion, new VersionPropertiesData( listed: leaf.IsListed(), semVer2: leaf.IsSemVer2()))); } else if (entry.IsPackageDelete && !entry.IsPackageDetails) { return(VersionListChange.Delete(entry.PackageIdentity.Version)); } else { const string message = "An unsupported leaf type was encountered."; _logger.LogError( message + " ID: {PackageId}, version: {PackageVersion}, commit timestamp: {CommitTimestamp:O}, " + "types: {EntryTypeUris}, leaf URL: {Url}", entry.PackageIdentity.Id, entry.PackageIdentity.Version.ToFullString(), entry.CommitTimeStamp, entry.TypeUris, entry.Uri.AbsoluteUri); throw new ArgumentException("An unsupported leaf type was encountered."); } }
public async Task ProcessPackageDeleteLeafAsync(Storage storage, CatalogCommitItem item, CancellationToken cancellationToken) { var targetStoragePath = GetTargetStorageIconPath(item); await _iconProcessor.DeleteIconAsync(storage, targetStoragePath, cancellationToken, item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()); // it would be nice to remove the icon copy result from cache for this item, but we don't have an icon URL here, // so can't remove anything. Will rely on the copy code to catch the copy failure and cleanup the cache appropriately. }
public void CompareTo_WhenObjIsNotCatalogCommit_Throws() { var commitItem = CatalogCommitItem.Create(_context, _commitItem); var exception = Assert.Throws <ArgumentException>(() => commitItem.CompareTo(new object())); Assert.Equal("obj", exception.ParamName); }
private static CatalogCommitItem CreatePackage(string id, string version) { var context = TestUtility.CreateCatalogContextJObject(); var packageIdentity = new PackageIdentity(id, new NuGetVersion(version)); var commitItem = TestUtility.CreateCatalogCommitItemJObject(DateTime.UtcNow, packageIdentity); return(CatalogCommitItem.Create(context, commitItem)); }
public async Task ReturnsEmptyItemLIsResultsInEmptyBatchList() { var items = new CatalogCommitItem[0]; var batches = await _target.CreateBatchesAsync(items); Assert.Empty(batches); }
public void Create_WhenContextIsNull_Throws() { const JObject context = null; var exception = Assert.Throws <ArgumentNullException>(() => CatalogCommitItem.Create(context, _commitItem)); Assert.Equal("context", exception.ParamName); }
public void Create_WhenTypeIsEmpty_Throws() { _commitItem[CatalogConstants.TypeKeyword] = new JArray(); var exception = Assert.Throws <ArgumentException>(() => CatalogCommitItem.Create(_context, _commitItem)); Assert.Equal("commitItem", exception.ParamName); Assert.StartsWith($"The value of property '{CatalogConstants.TypeKeyword}' must be non-null and non-empty.", exception.Message); }
private async Task <int> MergeEntryIntoPageAsync( Context context, CatalogCommitItem entry, int pageIndex, int itemIndex) { var pageInfo = context.IndexInfo.Items[pageIndex]; var items = await pageInfo.GetLeafInfosAsync(); while (itemIndex < items.Count) { if (entry.PackageIdentity.Version > items[itemIndex].Version) { // The current position in the item list is too low for the catalog version. Keep looking. itemIndex++; } else if (entry.PackageIdentity.Version == items[itemIndex].Version) { if (entry.IsPackageDelete) { // Remove the registration leaf item. The current catalog commit item represents a // delete for this version. _logger.LogInformation( "Version {Version} will be deleted by commit {CommitId}.", entry.PackageIdentity.Version.ToNormalizedString(), entry.CommitId); context.DeletedLeaves.Add(await pageInfo.RemoveAtAsync(itemIndex)); context.ModifiedPages.Add(pageInfo); RemovePageAtIfEmpty(context, pageIndex); } else { // Update the metadata of the existing registration leaf item. _logger.LogInformation( "Version {Version} will be updated by commit {CommitId}.", entry.PackageIdentity.Version.ToNormalizedString(), entry.CommitId); context.ModifiedLeaves.Add(items[itemIndex]); context.ModifiedPages.Add(pageInfo); } // The version has been matched with an existing version. Leave the item index as-is. The next item // is now at the current position. No more work is necessary. return(itemIndex); } else { break; } } await InsertAsync(context, pageInfo, itemIndex, entry); return(itemIndex); }
public async Task FinishAsync() { if (!_urls.Any()) { return; } // Download all of the leaves. var urlsToDownload = new ConcurrentBag <string>(_urls); var leaves = new ConcurrentBag <CatalogLeaf>(); await ParallelAsync.Repeat( async() => { await Task.Yield(); while (urlsToDownload.TryTake(out var url)) { var leaf = await _catalogClient.GetLeafAsync(url); leaves.Add(leaf); } }); // Build the input to hive updaters. var entries = new List <CatalogCommitItem>(); var entryToLeaf = new Dictionary <CatalogCommitItem, PackageDetailsCatalogLeaf>( ReferenceEqualityComparer <CatalogCommitItem> .Default); foreach (var leaf in leaves) { if (leaf.IsPackageDelete() == leaf.IsPackageDetails()) { throw new InvalidOperationException("A catalog leaf must be either a package delete or a package details leaf."); } var typeUri = leaf.IsPackageDetails() ? Schema.DataTypes.PackageDetails : Schema.DataTypes.PackageDelete; var catalogCommitItem = new CatalogCommitItem( new Uri(leaf.Url), leaf.CommitId, leaf.CommitTimestamp.UtcDateTime, types: null, typeUris: new[] { typeUri }, packageIdentity: new PackageIdentity(_packageId, leaf.ParsePackageVersion())); entries.Add(catalogCommitItem); if (leaf.IsPackageDetails()) { entryToLeaf.Add(catalogCommitItem, (PackageDetailsCatalogLeaf)leaf); } } // Update the hives. await _updater.UpdateAsync(_packageId, entries, entryToLeaf); _urls.Clear(); }
public static CatalogCommitItem CreateCatalogCommitItem( DateTime commitTimeStamp, PackageIdentity packageIdentity, string commitId = null) { var context = CreateCatalogContextJObject(); var commitItem = CreateCatalogCommitItemJObject(commitTimeStamp, packageIdentity, commitId); return(CatalogCommitItem.Create(context, commitItem)); }
public void Create_WhenArgumentsAreValid_ReturnsInstance() { var commitItem = CatalogCommitItem.Create(_context, _commitItem); Assert.Equal($"https://nuget.test/{_packageIdentity.Id}", commitItem.Uri.AbsoluteUri); Assert.Equal(_now, commitItem.CommitTimeStamp.ToUniversalTime()); Assert.True(Guid.TryParse(commitItem.CommitId, out var commitId)); Assert.Equal(_packageIdentity, commitItem.PackageIdentity); Assert.Equal(CatalogConstants.NuGetPackageDetails, commitItem.Types.Single()); Assert.Equal(Schema.DataTypes.PackageDetails.AbsoluteUri, commitItem.TypeUris.Single().AbsoluteUri); }
internal static CatalogEntry Create(CatalogCommitItem item) { var typeUri = item.TypeUris.Single(uri => uri.AbsoluteUri == Schema.DataTypes.PackageDetails.AbsoluteUri || uri.AbsoluteUri == Schema.DataTypes.PackageDelete.AbsoluteUri); return(new CatalogEntry( item.CommitTimeStamp, item.PackageIdentity.Id.ToLowerInvariant(), item.PackageIdentity.Version.ToNormalizedString().ToLowerInvariant(), typeUri)); }
public void Create_WhenArgumentIsValid_ReturnsInstance() { var contextJObject = TestUtility.CreateCatalogContextJObject(); var commitItemJObject = TestUtility.CreateCatalogCommitItemJObject(_commitTimeStamp, _packageIdentity); var commitItem = CatalogCommitItem.Create(contextJObject, commitItemJObject); var entry = CatalogIndexEntry.Create(commitItem); Assert.Equal(_uri.AbsoluteUri, entry.Uri.AbsoluteUri); Assert.Equal(CatalogConstants.NuGetPackageDetails, entry.Types.Single()); Assert.Equal(commitItemJObject[CatalogConstants.CommitId].ToString(), entry.CommitId); Assert.Equal(_commitTimeStamp, entry.CommitTimeStamp.ToUniversalTime()); Assert.Equal(_packageId, entry.Id); Assert.Equal(_packageVersion, entry.Version); }
private static List <CatalogCommitItem> MakeSortedCatalog(ICollection <VersionAction> sortedVersionActions) { var output = new List <CatalogCommitItem>(); foreach (var versionAction in sortedVersionActions) { var item = new CatalogCommitItem( uri: null, commitId: null, commitTimeStamp: DateTime.MinValue, types: null, typeUris: new[] { versionAction.IsDelete?Schema.DataTypes.PackageDelete: Schema.DataTypes.PackageDetails }, packageIdentity: new PackageIdentity("NuGet.Versioning", versionAction.Version)); output.Add(item); } return(output); }
private async Task ProcessEmbeddedIconAsync(IStorage destinationStorage, CatalogCommitItem item, string iconFile, CancellationToken cancellationToken) { var packageFilename = PackageUtility.GetPackageFileName(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()).ToLowerInvariant(); var packageUri = _packageStorage.ResolveUri(packageFilename); var packageBlobReference = await _packageStorage.GetCloudBlockBlobReferenceAsync(packageUri); using (_telemetryService.TrackEmbeddedIconProcessingDuration(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString())) { Stream packageStream; try { packageStream = await packageBlobReference.GetStreamAsync(cancellationToken); } catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound) { _logger.LogWarning("Package blob not found at {PackageUrl}: {Exception}. Will assume package was deleted and skip", packageUri.AbsoluteUri, ex); return; } catch (Exception ex) { // logging other exceptions here to have proper scope in log message _logger.LogError("Exception while trying to access package blob {PackageUrl}: {Exception}", packageUri.AbsoluteUri, ex); throw; } using (packageStream) { var targetStoragePath = GetTargetStorageIconPath(item); var resultUrl = await _iconProcessor.CopyEmbeddedIconFromPackageAsync( packageStream, iconFile, destinationStorage, targetStoragePath, cancellationToken, item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()); } } }
private void ExcludeSemVer2( HiveType hive, List <CatalogCommitItem> sortedCatalog, Dictionary <NuGetVersion, PackageDetailsCatalogLeaf> versionToCatalogLeaf) { Guard.Assert( hive == HiveType.Legacy || hive == HiveType.Gzipped, "Only the legacy and gzipped hives should exclude SemVer 2.0.0 versions."); for (int i = 0; i < sortedCatalog.Count; i++) { var catalogCommitItem = sortedCatalog[i]; if (catalogCommitItem.IsPackageDelete) { continue; } var version = catalogCommitItem.PackageIdentity.Version; var catalogLeaf = versionToCatalogLeaf[version]; if (catalogLeaf.IsSemVer2()) { // Turn the PackageDetails into a PackageDelete to ensure that a known SemVer 2.0.0 version is not // in a hive that should not have SemVer 2.0.0. This may cause a little bit more work (like a // non-inlined page getting downloaded) but it's worth it to allow reflows of SemVer 2.0.0 packages // to fix up problems. In general, reflow should be a powerful fix-up tool. If this causes // performance issues later, we can revisit this extra work at that time. sortedCatalog[i] = new CatalogCommitItem( catalogCommitItem.Uri, catalogCommitItem.CommitId, catalogCommitItem.CommitTimeStamp, types: null, typeUris: DeleteUris, packageIdentity: catalogCommitItem.PackageIdentity); versionToCatalogLeaf.Remove(version); _logger.LogInformation( "Version {Version} is SemVer 2.0.0 so it will be treated as a package delete.", catalogLeaf.ParsePackageVersion().ToFullString(), hive); } } }
private async Task ProcessEmbeddedIconAsync(IStorage destinationStorage, CatalogCommitItem item, string iconFile, CancellationToken cancellationToken) { var packageFilename = PackageUtility.GetPackageFileName(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()).ToLowerInvariant(); var packageUri = _packageStorage.ResolveUri(packageFilename); var packageBlobReference = await _packageStorage.GetCloudBlockBlobReferenceAsync(packageUri); using (_telemetryService.TrackEmbeddedIconProcessingDuration(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString())) using (var packageStream = await packageBlobReference.GetStreamAsync(cancellationToken)) { var targetStoragePath = GetTargetStorageIconPath(item); var resultUrl = await _iconProcessor.CopyEmbeddedIconFromPackageAsync( packageStream, iconFile, destinationStorage, targetStoragePath, cancellationToken, item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()); } }
public void CreateCommitItemBatches_WhenMultipleCommitItemsShareCommitTimeStampButNotCommitId_Throws() { var commitTimeStamp = DateTime.UtcNow; var context = TestUtility.CreateCatalogContextJObject(); var commitItem0 = CatalogCommitItem.Create( context, TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp, _packageIdentitya)); var commitItem1 = CatalogCommitItem.Create( context, TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp, _packageIdentityb)); var commitItems = new[] { commitItem0, commitItem1 }; var exception = Assert.Throws <ArgumentException>( () => CatalogCommitUtilities.CreateCommitItemBatches( commitItems, CatalogCommitUtilities.GetPackageIdKey)); Assert.Equal("catalogItems", exception.ParamName); Assert.StartsWith("Multiple commits exist with the same commit timestamp but different commit ID's: " + $"{{ CommitId = {commitItem0.CommitId}, CommitTimeStamp = {commitItem0.CommitTimeStamp.ToString("O")} }}, " + $"{{ CommitId = {commitItem1.CommitId}, CommitTimeStamp = {commitItem1.CommitTimeStamp.ToString("O")} }}.", exception.Message); }
public void CreateCommitItemBatches_WhenMultipleCommitItemsShareCommitTimeStampButNotCommitIdAndLaterCommitExists_DoesNotThrow() { var commitTimeStamp0 = DateTime.UtcNow; var commitTimeStamp1 = commitTimeStamp0.AddMinutes(1); var context = TestUtility.CreateCatalogContextJObject(); var commitItem0 = CatalogCommitItem.Create( context, TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp0, _packageIdentitya)); var commitItem1 = CatalogCommitItem.Create( context, TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp0, _packageIdentityb)); var commitItem2 = CatalogCommitItem.Create( context, TestUtility.CreateCatalogCommitItemJObject(commitTimeStamp1, _packageIdentitya)); var commitItems = new[] { commitItem0, commitItem1, commitItem2 }; var batches = CatalogCommitUtilities.CreateCommitItemBatches( commitItems, CatalogCommitUtilities.GetPackageIdKey); Assert.Collection( batches, batch => { Assert.Equal(commitTimeStamp1, batch.CommitTimeStamp.ToUniversalTime()); Assert.Collection( batch.Items, commit => Assert.True(ReferenceEquals(commit, commitItem2))); }, batch => { Assert.Equal(commitTimeStamp0, batch.CommitTimeStamp.ToUniversalTime()); Assert.Collection( batch.Items, commit => Assert.True(ReferenceEquals(commit, commitItem1))); }); }
private async Task CopyIcon(Uri iconUrl, IStorage destinationStorage, IStorage iconCacheStorage, CatalogCommitItem item, CancellationToken cancellationToken) { var ingestionResult = await Retry.IncrementalAsync( async() => await TryIngestExternalIconAsync(item, iconUrl, destinationStorage, cancellationToken), e => false, r => r.Result == AttemptResult.FailCanRetry, MaxExternalIconIngestAttempts, initialWaitInterval : TimeSpan.FromSeconds(5), waitIncrement : TimeSpan.FromSeconds(1)); if (ingestionResult.Result == AttemptResult.Success) { try { await _iconCopyResultCache.SaveExternalIcon(iconUrl, ingestionResult.ResultUrl, destinationStorage, iconCacheStorage, cancellationToken); } catch (Exception e) { // we will report and ignore such exceptions. Failure to store icon will cause the original icon // to be re-retrieved next time it is encountered. _logger.LogWarning(0, e, "Failed to store icon in the cache"); } } else { var destinationStoragePath = GetTargetStorageIconPath(item); await _iconProcessor.DeleteIconAsync(destinationStorage, destinationStoragePath, cancellationToken, item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()); _telemetryService.TrackExternalIconIngestionFailure(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString()); _iconCopyResultCache.SaveExternalCopyFailure(iconUrl); } }
protected override string GetKey(CatalogCommitItem item) { return(CatalogCommitUtilities.GetPackageIdKey(item)); }
private static string GetTargetStorageIconPath(CatalogCommitItem item) { return($"{item.PackageIdentity.Id.ToLowerInvariant()}/{item.PackageIdentity.Version.ToNormalizedString().ToLowerInvariant()}/icon"); }
public async Task ProcessPackageDetailsLeafAsync(IStorage destinationStorage, IStorage iconCacheStorage, CatalogCommitItem item, string iconUrlString, string iconFile, CancellationToken cancellationToken) { var hasExternalIconUrl = !string.IsNullOrWhiteSpace(iconUrlString); var hasEmbeddedIcon = !string.IsNullOrWhiteSpace(iconFile); if (hasExternalIconUrl && !hasEmbeddedIcon && Uri.TryCreate(iconUrlString, UriKind.Absolute, out var iconUrl)) { using (_logger.BeginScope("Processing icon url {IconUrl}", iconUrl)) { await ProcessExternalIconUrlAsync(destinationStorage, iconCacheStorage, item, iconUrl, cancellationToken); } } else if (hasEmbeddedIcon) { await ProcessEmbeddedIconAsync(destinationStorage, item, iconFile, cancellationToken); } }
public void Create_WhenCommitItemIsNull_Throws() { var exception = Assert.Throws <ArgumentNullException>(() => CatalogCommitItem.Create(_context, commitItem: null)); Assert.Equal("commitItem", exception.ParamName); }
public static bool IsOnlyPackageDetails(CatalogCommitItem e) { return(e.IsPackageDetails && !e.IsPackageDelete); }
private async Task ProcessExternalIconUrlAsync(IStorage destinationStorage, IStorage iconCacheStorage, CatalogCommitItem item, Uri iconUrl, CancellationToken cancellationToken) { _logger.LogInformation("Found external icon url {IconUrl} for {PackageId} {PackageVersion}", iconUrl, item.PackageIdentity.Id, item.PackageIdentity.Version); if (!IsValidIconUrl(iconUrl)) { _logger.LogInformation("Invalid icon URL {IconUrl}", iconUrl); return; } var cachedResult = _iconCopyResultCache.Get(iconUrl); if (cachedResult != null && await TryTakeFromCache(iconUrl, cachedResult, iconCacheStorage, destinationStorage, item, cancellationToken)) { return; } using (_telemetryService.TrackExternalIconProcessingDuration(item.PackageIdentity.Id, item.PackageIdentity.Version.ToNormalizedString())) { await CopyIcon(iconUrl, destinationStorage, iconCacheStorage, item, cancellationToken); } }