public async Task DowngradeToUnlist() { var existingVersion = "0.0.1"; var existingLeaf = new PackageDetailsCatalogLeaf { CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero), Url = "http://example/leaf/0.0.1", PackageId = _packageId, VerbatimVersion = existingVersion, PackageVersion = existingVersion, Listed = false, }; _leaf.Listed = false; _versionListDataResult = new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData> { { existingVersion, new VersionPropertiesData(listed: true, semVer2: false) }, { _packageVersion, new VersionPropertiesData(listed: true, semVer2: false) }, }), _versionListDataResult.AccessCondition); _latestCatalogLeaves = new LatestCatalogLeaves( new HashSet <NuGetVersion>(), new Dictionary <NuGetVersion, PackageDetailsCatalogLeaf> { { NuGetVersion.Parse(existingVersion), existingLeaf }, }); var indexActions = await _target.AddCatalogEntriesAsync( _packageId, _latestEntries, _entryToLeaf); Assert.Equal(4, indexActions.Search.Count); Assert.All(indexActions.Search, x => Assert.IsType <KeyedDocument>(x.Document)); Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.Delete, x.ActionType)); Assert.Equal(2, indexActions.Hijack.Count); var existing = indexActions.Hijack.Single(x => x.Document.Key == existingVersion); Assert.IsType <HijackDocument.Full>(existing.Document); Assert.Equal(IndexActionType.MergeOrUpload, existing.ActionType); var added = indexActions.Hijack.Single(x => x.Document.Key == _packageVersion); Assert.IsType <HijackDocument.Full>(added.Document); Assert.Equal(IndexActionType.MergeOrUpload, added.ActionType); Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); var properties = indexActions.VersionListDataResult.Result.VersionProperties; Assert.Equal( new[] { existingVersion, _packageVersion }, properties.Keys.ToArray()); Assert.False(properties[existingVersion].Listed); Assert.False(properties[existingVersion].SemVer2); Assert.False(properties[_packageVersion].Listed); Assert.False(properties[_packageVersion].SemVer2); _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny <string>()), Times.Never); }
public async Task <ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > > > ReadLatestIndexedAsync() { var stopwatch = Stopwatch.StartNew(); var blobName = GetLatestIndexedBlobName(); var blobReference = Container.GetBlobReference(blobName); _logger.LogInformation("Reading the latest indexed owners from {BlobName}.", blobName); var builder = new PackageIdToOwnersBuilder(_logger); IAccessCondition accessCondition; try { using (var stream = await blobReference.OpenReadAsync(AccessCondition.GenerateEmptyCondition())) { accessCondition = AccessConditionWrapper.GenerateIfMatchCondition(blobReference.ETag); ReadStream(stream, builder.Add); } } catch (StorageException ex) when(ex.RequestInformation.HttpStatusCode == (int)HttpStatusCode.NotFound) { accessCondition = AccessConditionWrapper.GenerateIfNotExistsCondition(); _logger.LogInformation("The blob {BlobName} does not exist.", blobName); } var output = new ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > >( builder.GetResult(), accessCondition); stopwatch.Stop(); _telemetryService.TrackReadLatestIndexedOwners(output.Result.Count, stopwatch.Elapsed); return(output); }
public async Task AddNewLatestVersionForOnlySomeSearchFilters() { var existingVersion = "0.0.1"; _versionListDataResult = new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData> { { existingVersion, new VersionPropertiesData(listed: true, semVer2: false) }, }), _versionListDataResult.AccessCondition); SetVersion("1.0.0-beta"); var indexActions = await _target.AddCatalogEntriesAsync( _packageId, _latestEntries, _entryToLeaf); Assert.Equal(4, indexActions.Search.Count); var isPrerelease = indexActions.Search.ToLookup(x => x.Document.Key.Contains("Prerelease")); Assert.All(isPrerelease[false], x => Assert.IsType <SearchDocument.UpdateVersionListAndOwners>(x.Document)); Assert.All(isPrerelease[false], x => Assert.Equal(IndexActionType.Merge, x.ActionType)); Assert.All(isPrerelease[true], x => Assert.IsType <SearchDocument.UpdateLatest>(x.Document)); Assert.All(isPrerelease[true], x => Assert.Equal(IndexActionType.MergeOrUpload, x.ActionType)); Assert.Equal(2, indexActions.Hijack.Count); var existing = indexActions.Hijack.Single(x => x.Document.Key == existingVersion); Assert.IsType <HijackDocument.Latest>(existing.Document); Assert.Equal(IndexActionType.Merge, existing.ActionType); var added = indexActions.Hijack.Single(x => x.Document.Key == _packageVersion); Assert.IsType <HijackDocument.Full>(added.Document); Assert.Equal(IndexActionType.MergeOrUpload, added.ActionType); Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); var properties = indexActions.VersionListDataResult.Result.VersionProperties; Assert.Equal( new[] { existingVersion, _packageVersion }, properties.Keys.ToArray()); Assert.True(properties[existingVersion].Listed); Assert.False(properties[existingVersion].SemVer2); Assert.True(properties[_packageVersion].Listed); Assert.False(properties[_packageVersion].SemVer2); _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny <string>()), Times.Once); _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(_packageId), Times.Once); }
public Context( string packageId, ResultAndAccessCondition <VersionListData> versionListDataResult, IEnumerable <CatalogCommitItem> latestEntries, IReadOnlyDictionary <CatalogCommitItem, PackageDetailsCatalogLeaf> entryToLeaf) { PackageId = packageId; VersionListDataResult = versionListDataResult; VersionToEntry = latestEntries.ToDictionary(x => x.PackageIdentity.Version); EntryToLeaf = entryToLeaf.ToDictionary( x => x.Key, x => x.Value, ReferenceEqualityComparer <CatalogCommitItem> .Default); var lastCommit = latestEntries .GroupBy(x => new { x.CommitTimeStamp, x.CommitId }) .Select(x => x.Key) .OrderByDescending(x => x.CommitTimeStamp) .First(); // Assume UTC on the commit timestamp. LastCommitTimestamp = new DateTimeOffset(lastCommit.CommitTimeStamp.Ticks, TimeSpan.Zero); LastCommitId = lastCommit.CommitId; }
public async Task AssumesDateTimeIsUtc() { var existingVersion = "1.0.1"; _versionListDataResult = new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData> { { existingVersion, new VersionPropertiesData(listed: true, semVer2: false) }, }), _versionListDataResult.AccessCondition); await _target.AddCatalogEntriesAsync( _packageId, _latestEntries, _entryToLeaf); _search.Verify( x => x.UpdateVersionListFromCatalog( It.IsAny <string>(), It.IsAny <SearchFilters>(), new DateTimeOffset(_commitItem.CommitTimeStamp.Ticks, TimeSpan.Zero), It.IsAny <string>(), It.IsAny <string[]>(), It.IsAny <bool>(), It.IsAny <bool>()), Times.AtLeastOnce); _hijack.Verify( x => x.LatestFromCatalog( It.IsAny <string>(), It.IsAny <string>(), new DateTimeOffset(_commitItem.CommitTimeStamp.Ticks, TimeSpan.Zero), It.IsAny <string>(), It.IsAny <HijackDocumentChanges>()), Times.AtLeastOnce); }
public Facts(ITestOutputHelper output) { DatabaseOwnerFetcher = new Mock <IDatabaseAuxiliaryDataFetcher>(); OwnerDataClient = new Mock <IOwnerDataClient>(); OwnerSetComparer = new Mock <IDataSetComparer>(); SearchDocumentBuilder = new Mock <ISearchDocumentBuilder>(); SearchIndexActionBuilder = new Mock <ISearchIndexActionBuilder>(); Pusher = new Mock <IBatchPusher>(); Options = new Mock <IOptionsSnapshot <AzureSearchJobConfiguration> >(); TelemetryService = new Mock <IAzureSearchTelemetryService>(); Logger = output.GetLogger <UpdateOwnersCommand>(); Configuration = new AzureSearchJobConfiguration { MaxConcurrentBatches = 1, }; DatabaseResult = new SortedDictionary <string, SortedSet <string> >(); StorageResult = new ResultAndAccessCondition <SortedDictionary <string, SortedSet <string> > >( new SortedDictionary <string, SortedSet <string> >(), new Mock <IAccessCondition>().Object); Changes = new SortedDictionary <string, string[]>(); IndexActions = new IndexActions( new List <IndexAction <KeyedDocument> > { IndexAction.Merge(new KeyedDocument()) }, new List <IndexAction <KeyedDocument> > { IndexAction.Merge(new KeyedDocument()) }, new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), new Mock <IAccessCondition>().Object)); Pusher.SetReturnsDefault(Task.FromResult(new BatchPusherResult())); Options .Setup(x => x.Value) .Returns(() => Configuration); DatabaseOwnerFetcher .Setup(x => x.GetPackageIdToOwnersAsync()) .ReturnsAsync(() => DatabaseResult); OwnerDataClient .Setup(x => x.ReadLatestIndexedAsync()) .ReturnsAsync(() => StorageResult); OwnerSetComparer .Setup(x => x.CompareOwners( It.IsAny <SortedDictionary <string, SortedSet <string> > >(), It.IsAny <SortedDictionary <string, SortedSet <string> > >())) .Returns(() => Changes); SearchIndexActionBuilder .Setup(x => x.UpdateAsync(It.IsAny <string>(), It.IsAny <Func <SearchFilters, KeyedDocument> >())) .ReturnsAsync(() => IndexActions); Target = new UpdateOwnersCommand( DatabaseOwnerFetcher.Object, OwnerDataClient.Object, OwnerSetComparer.Object, SearchDocumentBuilder.Object, SearchIndexActionBuilder.Object, () => Pusher.Object, Options.Object, TelemetryService.Object, Logger); }
public BaseFacts(ITestOutputHelper output) { _versionListDataClient = new Mock <IVersionListDataClient>(); _leafFetcher = new Mock <ICatalogLeafFetcher>(); _ownerFetcher = new Mock <IDatabaseAuxiliaryDataFetcher>(); _search = new Mock <ISearchDocumentBuilder>(); _hijack = new Mock <IHijackDocumentBuilder>(); _logger = output.GetLogger <CatalogIndexActionBuilder>(); _packageId = Data.PackageId; SetVersion("1.0.0"); _versionListDataResult = new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData>()), AccessConditionWrapper.GenerateIfNotExistsCondition()); _owners = Data.Owners; _latestCatalogLeaves = new LatestCatalogLeaves( new HashSet <NuGetVersion>(), new Dictionary <NuGetVersion, PackageDetailsCatalogLeaf>()); _versionListDataClient .Setup(x => x.ReadAsync(It.IsAny <string>())) .ReturnsAsync(() => _versionListDataResult); _search .Setup(x => x.LatestFlagsOrNull(It.IsAny <VersionLists>(), It.IsAny <SearchFilters>())) .Returns <VersionLists, SearchFilters>((vl, sf) => new SearchDocument.LatestFlags( vl.GetLatestVersionInfoOrNull(sf), isLatestStable: true, isLatest: true)); _search .Setup(x => x.Keyed(It.IsAny <string>(), It.IsAny <SearchFilters>())) .Returns <string, SearchFilters>( (i, sf) => new KeyedDocument { Key = sf.ToString() }); _search .Setup(x => x.UpdateVersionListFromCatalog( It.IsAny <string>(), It.IsAny <SearchFilters>(), It.IsAny <DateTimeOffset>(), It.IsAny <string>(), It.IsAny <string[]>(), It.IsAny <bool>(), It.IsAny <bool>())) .Returns <string, SearchFilters, DateTimeOffset, string, string[], bool, bool>( (i, ct, ci, sf, v, ls, l) => new SearchDocument.UpdateVersionList { Key = sf.ToString() }); _search .Setup(x => x.UpdateVersionListAndOwnersFromCatalog( It.IsAny <string>(), It.IsAny <SearchFilters>(), It.IsAny <DateTimeOffset>(), It.IsAny <string>(), It.IsAny <string[]>(), It.IsAny <bool>(), It.IsAny <bool>(), It.IsAny <string[]>())) .Returns <string, SearchFilters, DateTimeOffset, string, string[], bool, bool, string[]>( (i, ct, ci, sf, v, ls, l, o) => new SearchDocument.UpdateVersionListAndOwners { Key = sf.ToString() }); _search .Setup(x => x.UpdateLatestFromCatalog( It.IsAny <SearchFilters>(), It.IsAny <string[]>(), It.IsAny <bool>(), It.IsAny <bool>(), It.IsAny <string>(), It.IsAny <string>(), It.IsAny <PackageDetailsCatalogLeaf>(), It.IsAny <string[]>())) .Returns <SearchFilters, string[], bool, bool, string, string, PackageDetailsCatalogLeaf, string[]>( (sf, v, ls, l, nv, fv, lf, o) => new SearchDocument.UpdateLatest { Key = sf.ToString() }); _hijack .Setup(x => x.Keyed(It.IsAny <string>(), It.IsAny <string>())) .Returns <string, string>( (i, v) => new KeyedDocument { Key = v }); _hijack .Setup(x => x.LatestFromCatalog( It.IsAny <string>(), It.IsAny <string>(), It.IsAny <DateTimeOffset>(), It.IsAny <string>(), It.IsAny <HijackDocumentChanges>())) .Returns <string, string, DateTimeOffset, string, HijackDocumentChanges>( (i, v, ct, ci, c) => new HijackDocument.Latest { Key = v }); _hijack .Setup(x => x.FullFromCatalog(It.IsAny <string>(), It.IsAny <HijackDocumentChanges>(), It.IsAny <PackageDetailsCatalogLeaf>())) .Returns <string, HijackDocumentChanges, PackageDetailsCatalogLeaf>( (v, c, l) => new HijackDocument.Full { Key = v }); _leafFetcher .Setup(x => x.GetLatestLeavesAsync(It.IsAny <string>(), It.IsAny <IReadOnlyList <IReadOnlyList <NuGetVersion> > >())) .ReturnsAsync(() => _latestCatalogLeaves); _ownerFetcher .Setup(x => x.GetOwnersOrEmptyAsync(It.IsAny <string>())) .ReturnsAsync(() => _owners); _target = new CatalogIndexActionBuilder( _versionListDataClient.Object, _leafFetcher.Object, _ownerFetcher.Object, _search.Object, _hijack.Object, _logger); }
public async Task DowngradeUnlistsOtherSearchFilterLatest() { var existingVersion1 = "2.5.11"; var existingVersion2 = "3.0.107-pre"; var existingVersion3 = "3.1.0+sha.8e3b68e"; var existingLeaf1 = new PackageDetailsCatalogLeaf // This version is still listed. { CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero), Url = "http://example/leaf/1", PackageId = _packageId, VerbatimVersion = existingVersion1, PackageVersion = existingVersion1, Listed = true, }; var existingLeaf2 = new PackageDetailsCatalogLeaf // This version is still listed. { CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero), Url = "http://example/leaf/2", PackageId = _packageId, VerbatimVersion = existingVersion2, PackageVersion = existingVersion2, Listed = true, }; var newLeaf3 = new PackageDetailsCatalogLeaf // This version is no longer listed. { CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero), Url = "http://example/leaf/3", PackageId = _packageId, VerbatimVersion = existingVersion3, PackageVersion = existingVersion3, Listed = false, }; SetVersion("3.2.0-dev.1+sha.ad6878e"); _leaf.Listed = false; _versionListDataResult = new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData> { { existingVersion1, new VersionPropertiesData(listed: true, semVer2: false) }, { existingVersion2, new VersionPropertiesData(listed: true, semVer2: false) }, { existingVersion3, new VersionPropertiesData(listed: true, semVer2: true) }, { _packageVersion, new VersionPropertiesData(listed: true, semVer2: true) }, }), _versionListDataResult.AccessCondition); _leafFetcher .SetupSequence(x => x.GetLatestLeavesAsync(It.IsAny <string>(), It.IsAny <IReadOnlyList <IReadOnlyList <NuGetVersion> > >())) .ReturnsAsync(new LatestCatalogLeaves( new HashSet <NuGetVersion>(), new Dictionary <NuGetVersion, PackageDetailsCatalogLeaf> { { NuGetVersion.Parse(existingVersion2), existingLeaf2 }, { NuGetVersion.Parse(existingVersion3), newLeaf3 }, })) .ReturnsAsync(new LatestCatalogLeaves( new HashSet <NuGetVersion>(), new Dictionary <NuGetVersion, PackageDetailsCatalogLeaf> { { NuGetVersion.Parse(existingVersion1), existingLeaf1 }, })) .Throws <NotImplementedException>(); var indexActions = await _target.AddCatalogEntriesAsync( _packageId, _latestEntries, _entryToLeaf); Assert.Equal(4, indexActions.Search.Count); Assert.All(indexActions.Search, x => Assert.IsType <SearchDocument.UpdateLatest>(x.Document)); Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.MergeOrUpload, x.ActionType)); Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); var properties = indexActions.VersionListDataResult.Result.VersionProperties; Assert.Equal( new[] { existingVersion1, existingVersion2, existingVersion3, _packageVersion }, properties.Keys.ToArray()); Assert.True(properties[existingVersion1].Listed); Assert.False(properties[existingVersion1].SemVer2); Assert.True(properties[existingVersion2].Listed); Assert.False(properties[existingVersion2].SemVer2); Assert.False(properties[existingVersion3].Listed); Assert.True(properties[existingVersion3].SemVer2); Assert.False(properties[_packageVersion].Listed); Assert.True(properties[_packageVersion].SemVer2); _leafFetcher.Verify( x => x.GetLatestLeavesAsync(_packageId, It.Is <IReadOnlyList <IReadOnlyList <NuGetVersion> > >(y => y.Count == 1 && y[0].Count == 3 && y[0][0] == NuGetVersion.Parse(existingVersion1) && y[0][1] == NuGetVersion.Parse(existingVersion2) && y[0][2] == NuGetVersion.Parse(existingVersion3))), Times.Once); _leafFetcher.Verify( x => x.GetLatestLeavesAsync(_packageId, It.Is <IReadOnlyList <IReadOnlyList <NuGetVersion> > >(y => y.Count == 1 && y[0].Count == 1 && y[0][0] == NuGetVersion.Parse(existingVersion1))), Times.Once); _leafFetcher.Verify( x => x.GetLatestLeavesAsync(It.IsAny <string>(), It.IsAny <IReadOnlyList <IReadOnlyList <NuGetVersion> > >()), Times.Exactly(2)); _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny <string>()), Times.Once); _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(_packageId), Times.Once); }
public async Task DowngradeToDifferent() { var existingVersion1 = "0.0.1"; var existingVersion2 = "0.0.2-alpha"; var existingLeaf1 = new PackageDetailsCatalogLeaf { CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero), Url = "http://example/leaf/0.0.1", PackageId = _packageId, VerbatimVersion = existingVersion1, PackageVersion = existingVersion1, Listed = true, }; var existingLeaf2 = new PackageDetailsCatalogLeaf { CommitTimestamp = new DateTimeOffset(2018, 12, 1, 0, 0, 0, TimeSpan.Zero), Url = "http://example/leaf/0.0.2-alpha", PackageId = _packageId, VerbatimVersion = existingVersion2, PackageVersion = existingVersion2, Listed = true, IsPrerelease = true, }; _leaf.Listed = false; _versionListDataResult = new ResultAndAccessCondition <VersionListData>( new VersionListData(new Dictionary <string, VersionPropertiesData> { { existingVersion1, new VersionPropertiesData(listed: true, semVer2: false) }, { existingVersion2, new VersionPropertiesData(listed: true, semVer2: false) }, { _packageVersion, new VersionPropertiesData(listed: true, semVer2: false) }, }), _versionListDataResult.AccessCondition); _latestCatalogLeaves = new LatestCatalogLeaves( new HashSet <NuGetVersion>(), new Dictionary <NuGetVersion, PackageDetailsCatalogLeaf> { { NuGetVersion.Parse(existingVersion1), existingLeaf1 }, { NuGetVersion.Parse(existingVersion2), existingLeaf2 }, }); var indexActions = await _target.AddCatalogEntriesAsync( _packageId, _latestEntries, _entryToLeaf); Assert.Equal(4, indexActions.Search.Count); Assert.All(indexActions.Search, x => Assert.IsType <SearchDocument.UpdateLatest>(x.Document)); Assert.All(indexActions.Search, x => Assert.Equal(IndexActionType.MergeOrUpload, x.ActionType)); Assert.Equal(3, indexActions.Hijack.Count); var existing1 = indexActions.Hijack.Single(x => x.Document.Key == existingVersion1); Assert.IsType <HijackDocument.Full>(existing1.Document); Assert.Equal(IndexActionType.MergeOrUpload, existing1.ActionType); var existing2 = indexActions.Hijack.Single(x => x.Document.Key == existingVersion2); Assert.IsType <HijackDocument.Full>(existing2.Document); Assert.Equal(IndexActionType.MergeOrUpload, existing2.ActionType); var added = indexActions.Hijack.Single(x => x.Document.Key == _packageVersion); Assert.IsType <HijackDocument.Full>(added.Document); Assert.Equal(IndexActionType.MergeOrUpload, added.ActionType); Assert.Same(_versionListDataResult.AccessCondition, indexActions.VersionListDataResult.AccessCondition); var properties = indexActions.VersionListDataResult.Result.VersionProperties; Assert.Equal( new[] { existingVersion1, existingVersion2, _packageVersion }, properties.Keys.ToArray()); Assert.True(properties[existingVersion1].Listed); Assert.False(properties[existingVersion1].SemVer2); Assert.True(properties[existingVersion2].Listed); Assert.False(properties[existingVersion2].SemVer2); Assert.False(properties[_packageVersion].Listed); Assert.False(properties[_packageVersion].SemVer2); _leafFetcher.Verify( x => x.GetLatestLeavesAsync( It.IsAny <string>(), It.Is <IReadOnlyList <IReadOnlyList <NuGetVersion> > >(y => y.Count == 2)), Times.Once); _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(It.IsAny <string>()), Times.Once); _ownerFetcher.Verify(x => x.GetOwnersOrEmptyAsync(_packageId), Times.Once); }