public async Task RunAsync_WhenMultipleEntriesWithSamePackageIdentityInSameBatch_Throws() { var zipWithWrongNameNuspec = CreateZipStreamWithEntry("Newtonsoft.Json.nuspec", _nuspecData); var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg"); var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec"); var nupkgStream = File.OpenRead(@"Packages\ListedPackage.1.0.1.zip"); var expectedNupkg = GetStreamBytes(nupkgStream); var catalogStorage = Catalogs.CreateTestCatalogWithMultipleEntriesWithSamePackageIdentityInSameBatch(); await _mockServer.AddStorageAsync(catalogStorage); _mockServer.SetAction( "/packages/listedpackage.1.0.0.nupkg", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(nupkgStream) })); var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); var exception = await Assert.ThrowsAsync <InvalidOperationException>( () => _target.RunAsync(front, back, CancellationToken.None)); Assert.Equal("The catalog batch 10/13/2015 6:40:07 AM contains multiple entries for the same package identity. Package(s): listedpackage 1.0.0", exception.Message); }
public async Task RunAsync_WhenDownloadingPackage_RejectsUnexpectedHttpStatusCode(HttpStatusCode statusCode) { var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); await _mockServer.AddStorageAsync(catalogStorage); _mockServer.Return404OnUnknownAction = true; _mockServer.SetAction( "/packages/listedpackage.1.0.0.nupkg", request => Task.FromResult(new HttpResponseMessage(statusCode) { Content = _noContent })); var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); var exception = await Assert.ThrowsAsync <InvalidOperationException>( () => _target.RunAsync(front, back, CancellationToken.None)); Assert.Equal( $"Expected status code OK for package download, actual: {statusCode}", exception.Message); Assert.Equal(0, _catalogToDnxStorage.Content.Count); }
public async Task RunAsync_WithPackageCreatedThenDeleted_LeavesNoArtifacts() { var indexJsonUri = _catalogToDnxStorage.ResolveUri("/otherpackage/index.json"); var nupkgUri = _catalogToDnxStorage.ResolveUri("/otherpackage/1.0.0/otherpackage.1.0.0.nupkg"); var nuspecUri = _catalogToDnxStorage.ResolveUri("/otherpackage/1.0.0/otherpackage.nuspec"); var catalogStorage = Catalogs.CreateTestCatalogWithPackageCreatedThenDeleted(); await _mockServer.AddStorageAsync(catalogStorage); _mockServer.SetAction( "/packages/otherpackage.1.0.0.nupkg", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\OtherPackage.1.0.0.zip")) })); var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(1, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); Assert.False(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(indexJsonUri)); Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); }
public async Task PutsSemVer2PackagesInLegacyStorageWhenSemVer2IsDisabled() { // Arrange SharedInit(useLegacy: true, useSemVer2: false); var catalogStorage = Catalogs.CreateTestCatalogWithSemVer2Package(); await _mockServer.AddStorageAsync(catalogStorage); var front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); var back = MemoryCursor.CreateMax(); // Act await _target.RunAsync(front, back, CancellationToken.None); // Assert var legacyCursor = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); Assert.NotNull(legacyCursor.Key); var legacyIndex = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/index.json")); Assert.NotNull(legacyIndex.Key); var legacyLeaf = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/1.0.0-alpha.1.json")); Assert.NotNull(legacyLeaf.Key); Assert.Equal(3, _legacyStorage.Content.Count); }
public async void CanReadWhatItSaves() { StorageContent savedContent = null; var storageMock = CreateStorageMock(); storageMock .Protected().Setup <Task>("OnSave", ItExpr.IsAny <Uri>(), ItExpr.IsAny <StorageContent>(), ItExpr.IsAny <CancellationToken>()) .Callback <Uri, StorageContent, CancellationToken>((uri, content, token) => { savedContent = content; }) .Returns(Task.FromResult(0)); storageMock .Protected().Setup <Task <StorageContent> >("OnLoad", ItExpr.IsAny <Uri>(), ItExpr.IsAny <CancellationToken>()) .Returns <Uri, CancellationToken>((uri, token) => Task.FromResult(savedContent)); DateTimeOffset defaultValue = new DateTimeOffset(2017, 5, 5, 17, 8, 42, TimeSpan.Zero); DateTimeOffset actualValue = new DateTimeOffset(2017, 5, 5, 17, 49, 42, TimeSpan.Zero); var durableCursorSaver = new DurableCursor(new Uri("http://localhost/cursor.json"), storageMock.Object, defaultValue); durableCursorSaver.Value = actualValue; await durableCursorSaver.Save(CancellationToken.None); var durableCursorLoader = new DurableCursor(new Uri("http://localhost/cursor.json"), storageMock.Object, defaultValue); await durableCursorLoader.Load(CancellationToken.None); Assert.Equal(actualValue, durableCursorLoader.Value); }
public async Task RunAsync_WhenSourceNupkgIsNotFound_SkipsPackage() { var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); var nupkgUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/1.0.0/unlistedpackage.1.0.0.nupkg"); var nuspecUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/1.0.0/unlistedpackage.nuspec"); var catalogStorage = Catalogs.CreateTestCatalogWithThreePackages(); await _mockServer.AddStorageAsync(catalogStorage); _mockServer.SetAction( "/packages/listedpackage.1.0.0.nupkg", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.NotFound) { Content = _noContent })); var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(1, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); Assert.False(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(indexJsonUri)); Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); }
static async Task Test1Async() { await MakeTestCatalog(); Func <HttpMessageHandler> handlerFunc = () => { return(new FileSystemEmulatorHandler { BaseAddress = new Uri("http://*****:*****@"c:\data\site", InnerHandler = new HttpClientHandler() }); }; TestCollector collector = new TestCollector("Test1", new Uri("http://*****:*****@"c:\data\site\cursor"; Storage storage = new FileStorage(baseAddress, path); DurableCursor front = new DurableCursor(new Uri("http://localhost:8000/cursor/front.json"), storage, MemoryCursor.Min.Value); //DurableCursor back = new DurableCursor(new Uri("http://localhost:8000/cursor/back.json"), storage); MemoryCursor back = MemoryCursor.Max; bool didWork = await collector.Run(front, back); if (!didWork) { Console.WriteLine("executed but no work was done"); } }
public async Task RunAsync_WhenPackageDoesNotHaveNuspec_SkipsPackage() { var zipWithNoNuspec = CreateZipStreamWithEntry("readme.txt", "content"); var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); var catalogStorage = Catalogs.CreateTestCatalogWithThreePackages(); await _mockServer.AddStorageAsync(catalogStorage); _mockServer.SetAction( "/packages/listedpackage.1.0.0.nupkg", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(zipWithNoNuspec) })); var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(1, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); Assert.False(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(indexJsonUri)); }
public async Task RunAsync_WhenDownloadingPackage_OnlyDownloadsNupkgOncePerCatalogLeaf() { // Arrange var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); await _mockServer.AddStorageAsync(catalogStorage); _mockServer.SetAction( "/packages/listedpackage.1.0.0.nupkg", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.0.zip")) })); _mockServer.SetAction( "/packages/listedpackage.1.0.1.nupkg", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.1.zip")) })); _mockServer.SetAction( "/packages/unlistedpackage.1.0.0.nupkg", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\UnlistedPackage.1.0.0.zip")) })); _mockServer.SetAction( "/packages/otherpackage.1.0.0.nupkg", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\OtherPackage.1.0.0.zip")) })); ReadWriteCursor front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); // Act await _target.RunAsync(front, back, CancellationToken.None); // Assert Assert.Equal(9, _catalogToDnxStorage.Content.Count); Assert.Equal(5, _mockServer.Requests.Count); Assert.EndsWith("/index.json", _mockServer.Requests[0].RequestUri.AbsoluteUri); Assert.EndsWith("/page0.json", _mockServer.Requests[1].RequestUri.AbsoluteUri); // The packages were processed in random order. var remainingRequests = _mockServer.Requests .Skip(2) .Take(3) .Select(request => request.RequestUri.AbsoluteUri) .OrderBy(uri => uri) .ToArray(); Assert.Contains("/listedpackage.1.0.0.nupkg", remainingRequests[0]); Assert.Contains("/listedpackage.1.0.1.nupkg", remainingRequests[1]); Assert.Contains("/unlistedpackage.1.0.0.nupkg", remainingRequests[2]); }
public async Task RunAsync_WithFakeIAzureStorage_WhenPackageIsAlreadySynchronizedButDoesNotHaveRequiredProperties_ProcessesPackage() { _catalogToDnxStorage = new AzureSynchronizedMemoryStorageStub(new[] { new Uri("http://tempuri.org/packages/listedpackage.1.0.0.nupkg") }, areRequiredPropertiesPresentAsync: false); _catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name)); var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg"); var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec"); var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.0.zip"); var expectedNupkg = GetStreamBytes(nupkgStream); await _catalogToDnxStorage.SaveAsync( new Uri("http://tempuri.org/listedpackage/index.json"), new StringStorageContent(GetExpectedIndexJsonContent("1.0.0")), CancellationToken.None); _target = new DnxCatalogCollector( new Uri("http://tempuri.org/index.json"), _catalogToDnxStorageFactory, _nullPreferredPackageSourceStorage, _contentBaseAddress, Mock.Of <ITelemetryService>(), _logger, _maxDegreeOfParallelism, () => _mockServer); var catalogStorage = Catalogs.CreateTestCatalogWithOnePackage(); await _mockServer.AddStorageAsync(catalogStorage); _mockServer.SetAction( "/packages/listedpackage.1.0.0.nupkg", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(nupkgStream) })); var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(4, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson)); Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson)); }
public static void CreateNewCursor(string[] args) { IDictionary <string, string> arguments = GetArguments(args, 0); StorageFactory storageFactory = CreateStorageFactory(arguments, verbose: true); Storage storage = storageFactory.Create(); DurableCursor cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, GetDefaultValue(arguments)); cursor.Load().Wait(); cursor.Save().Wait(); }
public async Task ThrowsIfCommitTimesOut() { // Arrange var storage = new MemoryStorage(); var storageFactory = new TestStorageFactory(name => storage.WithName(name)); MockServerHttpClientHandler mockServer; mockServer = new MockServerHttpClientHandler(); mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK))); var catalogStorage = Catalogs.CreateTestCatalogWithOnePackage(); await mockServer.AddStorageAsync(catalogStorage); ReadWriteCursor front = new DurableCursor( storage.ResolveUri("cursor.json"), storage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); var commitTimeout = TimeSpan.FromSeconds(1); var stuckDuration = TimeSpan.FromMinutes(1); var telemetryService = new Mock <ITelemetryService>(); var indexCommitDurationMetric = new Mock <IDisposable>(); telemetryService.Setup(t => t.TrackIndexCommitDuration()).Returns(indexCommitDurationMetric.Object); using (var testDirectory = TestDirectory.Create()) { var luceneDirectory = new SimpleFSDirectory(new DirectoryInfo(testDirectory)); using (var indexWriter = Catalog2LuceneJob.CreateIndexWriter(luceneDirectory)) using (var stuckIndexWriter = StuckIndexWriter.FromIndexWriter(indexWriter, stuckDuration)) { var target = new SearchIndexFromCatalogCollector( new Uri("http://tempuri.org/index.json"), stuckIndexWriter, commitEachBatch: true, commitTimeout: commitTimeout, baseAddress: null, telemetryService: telemetryService.Object, logger: new TestLogger(), handlerFunc: () => mockServer, httpRetryStrategy: new NoRetryStrategy()); // Act & Assert await Assert.ThrowsAsync <OperationCanceledException>(() => target.RunAsync(front, back, CancellationToken.None)); telemetryService.Verify(t => t.TrackIndexCommitDuration(), Times.Once); telemetryService.Verify(t => t.TrackIndexCommitTimeout(), Times.Once); indexCommitDurationMetric.Verify(m => m.Dispose(), Times.Never); } } }
private async Task ExecuteAsync(CancellationToken token) { using (var cancelledCts = new CancellationTokenSource()) using (var produceWorkCts = new CancellationTokenSource()) { // Initialize the indexes, container and excluded packages data. await InitializeAsync(); // Here, we fetch the current catalog timestamp to use as the initial cursor value for // catalog2azuresearch. The idea here is that database is always more up-to-date than the catalog. // We're about to read the database so if we capture a catalog timestamp now, we are guaranteed that // any data we get from a database query will be more recent than the data represented by this catalog // timestamp. When catalog2azuresearch starts up for the first time to update the index produced by this // job, it will probably encounter some duplicate packages, but this is okay. // // Note that we could capture any dependency cursors here instead of catalog cursor, but this is // pointless because there is no reliable way to filter out data fetched from the database based on a // catalog-based cursor value. Suppose the dependency cursor is catalog2registration. If // catalog2registration is very behind, then the index produced by this job will include packages that // are not yet restorable (since they are not in the registration hives). This could lead to a case // where a user is able to search for a package that he cannot restore. We mitigate this risk by // trusting that our end-to-end tests will fail when catalog2registration (or any other V3 component) is // broken, this blocking the deployment of new Azure Search indexes. var catalogIndex = await _catalogClient.GetIndexAsync(_options.Value.CatalogIndexUrl); var initialCursorValue = catalogIndex.CommitTimestamp; _logger.LogInformation("The initial cursor value will be {CursorValue:O}.", initialCursorValue); var initialAuxiliaryData = await PushAllPackageRegistrationsAsync(cancelledCts, produceWorkCts); // Write the owner data file. await WriteOwnerDataAsync(initialAuxiliaryData.Owners); // Write the download data file. await WriteDownloadDataAsync(initialAuxiliaryData.Downloads); // Write the verified packages data file. await WriteVerifiedPackagesDataAsync(initialAuxiliaryData.VerifiedPackages); // Write popularity transfers data file. await WritePopularityTransfersDataAsync(initialAuxiliaryData.PopularityTransfers); // Write the cursor. _logger.LogInformation("Writing the initial cursor value to be {CursorValue:O}.", initialCursorValue); var frontCursorStorage = _storageFactory.Create(); var frontCursor = new DurableCursor( frontCursorStorage.ResolveUri(Catalog2AzureSearchCommand.CursorRelativeUri), frontCursorStorage, DateTime.MinValue); frontCursor.Value = initialCursorValue.UtcDateTime; await frontCursor.SaveAsync(token); } }
public async Task RunAsync_WhenPackageIsAlreadySynchronizedButNotInIndex_ProcessesPackage() { _catalogToDnxStorage = new SynchronizedMemoryStorage(new[] { new Uri("http://tempuri.org/packages/listedpackage.1.0.1.nupkg"), }); _catalogToDnxStorageFactory = new TestStorageFactory(name => _catalogToDnxStorage.WithName(name)); _mockServer = new MockServerHttpClientHandler(); _mockServer.SetAction("/", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK))); var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg"); var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.1/listedpackage.nuspec"); _target = new DnxCatalogCollector( new Uri("http://tempuri.org/index.json"), _catalogToDnxStorageFactory, _nullPreferredPackageSourceStorage, _contentBaseAddress, Mock.Of <ITelemetryService>(), new Mock <ILogger>().Object, _maxDegreeOfParallelism, () => _mockServer); var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); await _mockServer.AddStorageAsync(catalogStorage); _mockServer.SetAction( "/packages/listedpackage.1.0.1.nupkg", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.1.zip")) })); var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(2, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); Assert.False(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); Assert.False(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson)); Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nupkgUri)); Assert.False(_catalogToDnxStorage.ContentBytes.ContainsKey(nuspecUri)); Assert.Equal(GetExpectedIndexJsonContent("1.0.1"), Encoding.UTF8.GetString(indexJson)); }
static async Task Test2Async() { await MakeTestCatalog(); Func <HttpMessageHandler> handlerFunc = () => { return(new FileSystemEmulatorHandler { BaseAddress = new Uri("http://*****:*****@"c:\data\site", InnerHandler = new HttpClientHandler() }); }; TestCollector collectorA = new TestCollector("A", new Uri("http://*****:*****@"c:\data\site\cursor"; Storage storage = new FileStorage(baseAddress, path); DurableCursor cursorA = new DurableCursor(new Uri("http://localhost:8000/cursor/cursorA.json"), storage, MemoryCursor.Min.Value); DurableCursor cursorB = new DurableCursor(new Uri("http://localhost:8000/cursor/cursorB.json"), storage, MemoryCursor.Min.Value); Console.WriteLine("check catalog..."); bool run = false; do { run = false; run |= await collectorA.Run(cursorA, MemoryCursor.Max); run |= await collectorB.Run(cursorB, cursorA); }while (run); Console.WriteLine("ADDING MORE CATALOG"); await MoreTestCatalog(); do { run = false; run |= await collectorA.Run(cursorA, MemoryCursor.Max); run |= await collectorB.Run(cursorB, cursorA); }while (run); Console.WriteLine("ALL DONE"); }
public async Task DoesNotSkipPackagesWhenExceptionOccurs(string catalogUri, string expectedCursorBeforeRetry) { // Arrange SharedInit(useLegacy: true, useSemVer2: false); var catalogStorage = Catalogs.CreateTestCatalogWithCommitThenTwoPackageCommit(); await _mockServer.AddStorageAsync(catalogStorage); // Make the first request for a catalog leaf node fail. This will cause the registration collector // to fail the first time but pass the second time. FailFirstRequest(catalogUri); expectedCursorBeforeRetry = expectedCursorBeforeRetry ?? MemoryCursor.MinValue.ToString("O"); ReadWriteCursor front = new DurableCursor( _legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); // Act await Assert.ThrowsAsync <BatchProcessingException>( () => _target.RunAsync(front, back, CancellationToken.None)); var cursorBeforeRetry = front.Value; await _target.RunAsync(front, back, CancellationToken.None); var cursorAfterRetry = front.Value; // Assert var unlistedPackage100 = _legacyStorage .Content .FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0.json")); Assert.NotNull(unlistedPackage100.Key); var listedPackage101 = _legacyStorage .Content .FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1.json")); Assert.NotNull(listedPackage101.Key); var anotherPackage100 = _legacyStorage .Content .FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/anotherpackage/1.0.0.json")); Assert.NotNull(anotherPackage100.Key); Assert.Equal(MemoryCursor.MinValue, cursorBeforeRetry); Assert.Equal(DateTime.Parse("2015-10-12T10:08:55.3335317Z").ToUniversalTime(), cursorAfterRetry); }
private async Task ExecuteAsync(CancellationToken token) { // Initialize the cursors. ReadCursor backCursor; if (_options.Value.DependencyCursorUrls != null && _options.Value.DependencyCursorUrls.Any()) { _logger.LogInformation("Depending on cursors: {DependencyCursorUrls}", _options.Value.DependencyCursorUrls); backCursor = new AggregateCursor(_options .Value .DependencyCursorUrls.Select(r => new HttpReadCursor(new Uri(r), _handlerFunc))); } else { _logger.LogInformation("Depending on no cursors, meaning the job will process up to the latest catalog information."); backCursor = MemoryCursor.CreateMax(); } var frontCursorStorage = _storageFactory.Create(); var frontCursorUri = frontCursorStorage.ResolveUri(CursorRelativeUri); var frontCursor = new DurableCursor(frontCursorUri, frontCursorStorage, DateTime.MinValue); _logger.LogInformation("Using cursor: {CursurUrl}", frontCursorUri.AbsoluteUri); LogContainerUrl(HiveType.Legacy, c => c.LegacyStorageContainer); LogContainerUrl(HiveType.Gzipped, c => c.GzippedStorageContainer); LogContainerUrl(HiveType.SemVer2, c => c.SemVer2StorageContainer); // Optionally create the containers. if (_options.Value.CreateContainers) { await CreateContainerIfNotExistsAsync(c => c.LegacyStorageContainer); await CreateContainerIfNotExistsAsync(c => c.GzippedStorageContainer); await CreateContainerIfNotExistsAsync(c => c.SemVer2StorageContainer); } await frontCursor.LoadAsync(token); await backCursor.LoadAsync(token); _logger.LogInformation( "The cursors have been loaded. Front: {FrontCursor}. Back: {BackCursor}.", frontCursor.Value, backCursor.Value); // Run the collector. await _collector.RunAsync( frontCursor, backCursor, token); }
public async void UsesDefaultValue() { var storageMock = CreateStorageMock(); storageMock .Protected().Setup <Task>("OnLoad", ItExpr.IsAny <Uri>(), ItExpr.IsAny <CancellationToken>()) .Returns(Task.FromResult <StorageContent>(null)); DateTimeOffset defaultValue = new DateTimeOffset(2017, 5, 5, 17, 8, 42, TimeSpan.Zero); var durableCursor = new DurableCursor(new Uri("http://localhost/cursor.json"), storageMock.Object, defaultValue); await durableCursor.Load(CancellationToken.None); Assert.Equal(defaultValue, durableCursor.Value); }
public async void SavesToStorage() { var storageMock = CreateStorageMock(); storageMock .Protected().Setup <Task>("OnSave", ItExpr.IsAny <Uri>(), ItExpr.IsAny <StorageContent>(), ItExpr.IsAny <CancellationToken>()) .Returns(Task.FromResult(0)) .Verifiable(); var durableCursor = new DurableCursor(new Uri("http://localhost/cursor.json"), storageMock.Object, new DateTimeOffset(2017, 5, 5, 17, 8, 42, TimeSpan.Zero)); await durableCursor.Save(CancellationToken.None); storageMock.Verify(); }
protected override void Init(IDictionary <string, string> arguments, CancellationToken cancellationToken) { ServicePointManager.DefaultConnectionLimit = DegreeOfParallelism; var verbose = arguments.GetOrDefault(Arguments.Verbose, false); var packageStorageBase = arguments.GetOrThrow <string>(Arguments.ContentBaseAddress); var failCacheTime = arguments.GetOrDefault(FailCacheTime, TimeSpan.FromHours(1)); var auxStorageFactory = CreateAuxStorageFactory(arguments, verbose); var targetStorageFactory = CreateTargetStorageFactory(arguments, verbose); var packageStorage = new AzureStorage( storageBaseUri: new Uri(packageStorageBase), maxExecutionTime: TimeSpan.FromMinutes(15), serverTimeout: TimeSpan.FromMinutes(10), useServerSideCopy: true, compressContent: false, verbose: true, throttle: null); var source = arguments.GetOrThrow <string>(Arguments.Source); var iconProcessor = new IconProcessor(TelemetryService, LoggerFactory.CreateLogger <IconProcessor>()); var httpHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose); var httpMessageHandler = httpHandlerFactory(); var httpClient = new HttpClient(httpMessageHandler); var simpleHttpClient = new SimpleHttpClient(httpClient, LoggerFactory.CreateLogger <SimpleHttpClient>()); var catalogClient = new CatalogClient(simpleHttpClient, LoggerFactory.CreateLogger <CatalogClient>()); var httpResponseProvider = new HttpClientWrapper(httpClient); var externalIconProvider = new ExternalIconContentProvider(httpResponseProvider, LoggerFactory.CreateLogger <ExternalIconContentProvider>()); var iconCopyResultCache = new IconCopyResultCache(auxStorageFactory.Create(), failCacheTime, LoggerFactory.CreateLogger <IconCopyResultCache>()); var leafProcessor = new CatalogLeafDataProcessor( packageStorage, iconProcessor, externalIconProvider, iconCopyResultCache, TelemetryService, LoggerFactory.CreateLogger <CatalogLeafDataProcessor>()); _collector = new IconsCollector( new Uri(source), TelemetryService, targetStorageFactory, catalogClient, leafProcessor, iconCopyResultCache, auxStorageFactory, CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, verbose), LoggerFactory.CreateLogger <IconsCollector>()); var cursorStorage = auxStorageFactory.Create(); _front = new DurableCursor(cursorStorage.ResolveUri("c2icursor.json"), cursorStorage, DateTime.MinValue.ToUniversalTime()); }
public async Task CreatesRegistrationsWithSemVer2() { // Arrange SharedInit(useLegacy: true, useSemVer2: true); var catalogStorage = Catalogs.CreateTestCatalogWithSemVer2Package(); await _mockServer.AddStorageAsync(catalogStorage); var front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); var back = MemoryCursor.CreateMax(); // Act await _target.RunAsync(front, back, CancellationToken.None); // Assert // Verify the contents of the legacy (non-SemVer 2.0.0) storage Assert.Equal(1, _legacyStorage.Content.Count); var legacyCursorJson = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); Assert.NotNull(legacyCursorJson.Key); // Verify the contents of the SemVer 2.0.0 storage Assert.Equal(2, _semVer2Storage.Content.Count); var semVer2CursorJson = _semVer2Storage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); Assert.Null(semVer2CursorJson.Key); var index = _semVer2Storage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/index.json")); Assert.NotNull(index.Key); Assert.Contains("\"catalog:CatalogRoot\"", index.Value.GetContentString()); Assert.Contains("\"PackageRegistration\"", index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/testpackage.semver2.1.0.0-alpha.1.json\"", index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/packages/testpackage.semver2.1.0.0-alpha.1.nupkg\"", index.Value.GetContentString()); Assert.Contains("\"version\":\"1.0.0-alpha.1+githash\"", index.Value.GetContentString()); Assert.Contains("1.0.0-alpha.1/1.0.0-alpha.1", index.Value.GetContentString()); Assert.Contains("\"lower\":\"1.0.0-alpha.1\",", index.Value.GetContentString()); Assert.Contains("\"upper\":\"1.0.0-alpha.1\"", index.Value.GetContentString()); var package = _semVer2Storage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/testpackage.semver2/1.0.0-alpha.1.json")); Assert.NotNull(package.Key); Assert.Contains("\"Package\"", package.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/testpackage.semver2.1.0.0-alpha.1.json\"", package.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/packages/testpackage.semver2.1.0.0-alpha.1.nupkg\"", package.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/testpackage.semver2/index.json\"", package.Value.GetContentString()); }
public PackageHashProcessor( IEntityRepository <Package> packageRepository, IBatchProcessor batchProcessor, IResultRecorder resultRecorder, IOptionsSnapshot <PackageHashConfiguration> configuration, DurableCursor cursor, ILogger <PackageHashProcessor> logger) { _packageRepository = packageRepository ?? throw new ArgumentNullException(nameof(packageRepository)); _batchProcessor = batchProcessor ?? throw new ArgumentNullException(nameof(batchProcessor)); _resultRecorder = resultRecorder ?? throw new ArgumentNullException(nameof(resultRecorder)); _configuration = configuration ?? throw new ArgumentNullException(nameof(configuration)); _cursor = cursor ?? throw new ArgumentNullException(nameof(cursor)); _logger = logger ?? throw new ArgumentNullException(nameof(logger)); }
public async Task HandlesDeleteCatalogItemWithNonNormalizedVersion() { // Arrange SharedInit(useLegacy: true, useSemVer2: false); var catalogStorage = Catalogs.CreateTestCatalogWithNonNormalizedDelete(); await _mockServer.AddStorageAsync(catalogStorage); ReadWriteCursor front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); // Act await _target.RunAsync( front, new MemoryCursor(DateTime.Parse("2015-10-12T10:08:54.1506742")), CancellationToken.None); var intermediateUris = _legacyStorage .Content .Select(pair => pair.Key.ToString()) .OrderBy(uri => uri) .ToList(); await _target.RunAsync( front, MemoryCursor.CreateMax(), CancellationToken.None); // Assert Assert.Equal(3, intermediateUris.Count); Assert.Equal("http://tempuri.org/cursor.json", intermediateUris[0]); Assert.Equal("http://tempuri.org/otherpackage/1.0.0.json", intermediateUris[1]); Assert.Equal("http://tempuri.org/otherpackage/index.json", intermediateUris[2]); // This should really be 1, but see: // https://github.com/NuGet/Engineering/issues/404 var finalUris = _legacyStorage .Content .Select(pair => pair.Key.ToString()) .OrderBy(uri => uri) .ToList(); Assert.Equal(2, finalUris.Count); Assert.Equal("http://tempuri.org/cursor.json", finalUris[0]); Assert.Equal("http://tempuri.org/otherpackage/1.0.0.json", finalUris[1]); }
public async Task WhenPackageHasMultipleCommitsRespectsOrder(string pageContent) { // Arrange SharedInit(useLegacy: true, useSemVer2: false); var catalogStorage = Catalogs.CreateTestCatalogWithThreeItemsForSamePackage(pageContent); await _mockServer.AddStorageAsync(catalogStorage); ReadWriteCursor front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); // Act await _target.RunAsync(front, back, CancellationToken.None); // Assert Assert.Equal(3, _legacyStorage.Content.Count); // Ensure storage has cursor.json var cursorJson = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); Assert.NotNull(cursorJson.Key); // Check package entries - ListedPackage var myPackageIndexFile = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/mypackage/index.json")); Assert.NotNull(myPackageIndexFile.Key); Assert.Contains("\"catalog:CatalogRoot\"", myPackageIndexFile.Value.GetContentString()); Assert.Contains("\"PackageRegistration\"", myPackageIndexFile.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/data/2017.02.08.17.16.18/mypackage.3.0.0.json\"", myPackageIndexFile.Value.GetContentString()); Assert.Contains("\"packageContent\":\"http://tempuri.org/packages/mypackage.3.0.0.nupkg\"", myPackageIndexFile.Value.GetContentString()); Assert.Contains("\"lower\":\"3.0.0\",", myPackageIndexFile.Value.GetContentString()); Assert.Contains("\"upper\":\"3.0.0\"", myPackageIndexFile.Value.GetContentString()); var myPackageVersionFile = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/mypackage/3.0.0.json")); Assert.NotNull(myPackageVersionFile.Key); Assert.Contains("\"catalogEntry\":\"http://tempuri.org/data/2017.02.08.17.16.18/mypackage.3.0.0.json\"", myPackageVersionFile.Value.GetContentString()); Assert.Contains("\"listed\":true", myPackageVersionFile.Value.GetContentString()); Assert.Contains("\"packageContent\":\"http://tempuri.org/packages/mypackage.3.0.0.nupkg\"", myPackageIndexFile.Value.GetContentString()); }
static async Task Loop(string source, StorageFactory storageFactory, string contentBaseAddress, bool verbose, int interval) { CommitCollector collector = new DnxCatalogCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose)) { ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress) }; Storage storage = storageFactory.Create(); ReadWriteCursor front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.Min.Value); ReadCursor back = MemoryCursor.Max; while (true) { bool run = false; do { run = await collector.Run(front, back); }while (run); Thread.Sleep(interval * 1000); } }
public async Task RunAsync_WithValidPackage_CreatesFlatContainer() { var indexJsonUri = _catalogToDnxStorage.ResolveUri("/listedpackage/index.json"); var nupkgUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg"); var nuspecUri = _catalogToDnxStorage.ResolveUri("/listedpackage/1.0.0/listedpackage.nuspec"); var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); var nupkgStream = File.OpenRead("Packages\\ListedPackage.1.0.0.zip"); var expectedNupkg = GetStreamBytes(nupkgStream); await _mockServer.AddStorageAsync(catalogStorage); _mockServer.SetAction( "/packages/listedpackage.1.0.0.nupkg", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(nupkgStream) })); var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(4, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson)); Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nupkgUri, out var nupkg)); Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nuspecUri, out var nuspec)); Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson)); Assert.Equal(expectedNupkg, nupkg); Assert.Equal( "<?xml version=\"1.0\" encoding=\"utf-8\"?>\r\n<package xmlns=\"http://schemas.microsoft.com/packaging/2010/07/nuspec.xsd\">\r\n <metadata>\r\n <id>ListedPackage</id>\r\n <version>1.0.0</version>\r\n <authors>NuGet</authors>\r\n <requireLicenseAcceptance>false</requireLicenseAcceptance>\r\n <description>Package description.</description>\r\n </metadata>\r\n</package>", Encoding.UTF8.GetString(nuspec)); }
public async Task ExecuteAsync(bool restart) { var fileSystemStorage = new FileStorageFactory( new Uri("http://localhost/"), Directory.GetCurrentDirectory(), verbose: false); var front = new DurableCursor( new Uri("http://localhost/cursor.json"), fileSystemStorage.Create(), DateTime.MinValue); if (restart) { await front.LoadAsync(CancellationToken.None); front.Value = DateTime.MinValue; await front.SaveAsync(CancellationToken.None); } var back = MemoryCursor.CreateMax(); await _collector.RunAsync(front, back, CancellationToken.None); }
public async Task RunAsync_WhenPackageHasNuspecWithWrongName_ProcessesPackage() { var zipWithWrongNameNuspec = CreateZipStreamWithEntry("Newtonsoft.Json.nuspec", _nuspecData); var indexJsonUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/index.json"); var nupkgUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/1.0.0/unlistedpackage.1.0.0.nupkg"); var nuspecUri = _catalogToDnxStorage.ResolveUri("/unlistedpackage/1.0.0/unlistedpackage.nuspec"); var catalogStorage = Catalogs.CreateTestCatalogWithThreePackages(); await _mockServer.AddStorageAsync(catalogStorage); _mockServer.SetAction( "/packages/unlistedpackage.1.0.0.nupkg", request => Task.FromResult(new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(zipWithWrongNameNuspec) })); var front = new DurableCursor(_cursorJsonUri, _catalogToDnxStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); await _target.RunAsync(front, back, CancellationToken.None); Assert.Equal(4, _catalogToDnxStorage.Content.Count); Assert.True(_catalogToDnxStorage.Content.ContainsKey(_cursorJsonUri)); Assert.True(_catalogToDnxStorage.Content.ContainsKey(indexJsonUri)); Assert.True(_catalogToDnxStorage.Content.ContainsKey(nupkgUri)); Assert.True(_catalogToDnxStorage.Content.ContainsKey(nuspecUri)); Assert.True(_catalogToDnxStorage.ContentBytes.ContainsKey(_cursorJsonUri)); Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(indexJsonUri, out var indexJson)); Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nupkgUri, out var nupkg)); Assert.True(_catalogToDnxStorage.ContentBytes.TryGetValue(nuspecUri, out var nuspec)); Assert.Equal(GetExpectedIndexJsonContent("1.0.0"), Encoding.UTF8.GetString(indexJson)); Assert.Equal(zipWithWrongNameNuspec.ToArray(), nupkg); Assert.Equal(_nuspecData, Encoding.UTF8.GetString(nuspec)); }
private async Task PrepareAsync() { _log.WriteLine("Making sure folder {0} exists.", _outputFolder); if (!Directory.Exists(_outputFolder)) { Directory.CreateDirectory(_outputFolder); } // Create reindex file _log.WriteLine("Start preparing lightning reindex file..."); var latestCommit = DateTime.MinValue; int numberOfEntries = 0; string indexFile = Path.Combine(_outputFolder, "index.txt"); string optionalArgumentsTemplate = "optionalArguments"; using (var streamWriter = new StreamWriter(indexFile, false)) { var httpMessageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, _verbose); var collectorHttpClient = new CollectorHttpClient(httpMessageHandlerFactory()); var catalogIndexReader = new CatalogIndexReader(new Uri(_catalogIndex), collectorHttpClient, TelemetryService); var catalogIndexEntries = await catalogIndexReader.GetEntries(); foreach (var packageRegistrationGroup in catalogIndexEntries .OrderBy(x => x.CommitTimeStamp) .ThenBy(x => x.Id, StringComparer.OrdinalIgnoreCase) .ThenBy(x => x.Version) .GroupBy(x => x.Id, StringComparer.OrdinalIgnoreCase)) { streamWriter.WriteLine("Element@{0}. {1}", numberOfEntries++, packageRegistrationGroup.Key); var latestCatalogPages = new Dictionary <string, Uri>(); foreach (CatalogIndexEntry catalogIndexEntry in packageRegistrationGroup) { string key = catalogIndexEntry.Version.ToNormalizedString(); if (latestCatalogPages.ContainsKey(key)) { latestCatalogPages[key] = catalogIndexEntry.Uri; } else { latestCatalogPages.Add(key, catalogIndexEntry.Uri); } if (latestCommit < catalogIndexEntry.CommitTimeStamp) { latestCommit = catalogIndexEntry.CommitTimeStamp; } } foreach (var latestCatalogPage in latestCatalogPages) { streamWriter.WriteLine("{0}", latestCatalogPage.Value); } } } _log.WriteLine("Finished preparing lightning reindex file. Output file: {0}", indexFile); // Write cursor to storage _log.WriteLine("Start writing new cursor..."); var storage = _storageFactories.LegacyStorageFactory.Create(); var cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, latestCommit) { Value = latestCommit }; await cursor.SaveAsync(CancellationToken.None); _log.WriteLine("Finished writing new cursor."); // Ensure the SemVer 2.0.0 storage containers is created, if applicable. The gzipped storage account is // created above when we write the cursor. _storageFactories.SemVer2StorageFactory?.Create(); // Write command files _log.WriteLine("Start preparing lightning reindex command files..."); string templateFileContents; using (var templateStreamReader = new StreamReader(_templateFile)) { templateFileContents = await templateStreamReader.ReadToEndAsync(); } int batchNumber = 0; int batchSizeValue = int.Parse(_batchSize); for (int batchStart = 0; batchStart < numberOfEntries; batchStart += batchSizeValue) { var batchEnd = (batchStart + batchSizeValue - 1); if (batchEnd >= numberOfEntries) { batchEnd = numberOfEntries - 1; } var cursorCommandFileName = "cursor" + batchNumber + ".cmd"; var cursorTextFileName = "cursor" + batchNumber + ".txt"; using (var cursorCommandStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorCommandFileName))) using (var cursorTextStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorTextFileName))) { var commandStreamContents = templateFileContents; var replacements = _arguments .Concat(new[] { new KeyValuePair <string, string>("indexFile", indexFile), new KeyValuePair <string, string>("cursorFile", cursorTextFileName) }); foreach (var replacement in replacements) { commandStreamContents = commandStreamContents .Replace($"[{replacement.Key}]", replacement.Value); } //the not required arguments need to be added only if they were passed in //they cannot be hardcoded in the template var optionalArguments = new StringBuilder(); AppendOptionalArgument(optionalArguments, Arguments.ContentIsFlatContainer); AppendOptionalArgument(optionalArguments, Arguments.FlatContainerName); AppendOptionalArgument(optionalArguments, Arguments.StorageSuffix); AppendOptionalArgument(optionalArguments, Arguments.AllIconsInFlatContainer); AppendOptionalArgument(optionalArguments, Arguments.Driver); AppendOptionalArgument(optionalArguments, Arguments.Verbose); commandStreamContents = commandStreamContents .Replace($"[{optionalArgumentsTemplate}]", optionalArguments.ToString()); await cursorCommandStreamWriter.WriteLineAsync(commandStreamContents); await cursorTextStreamWriter.WriteLineAsync(batchStart + "," + batchEnd); } batchNumber++; } _log.WriteLine("Finished preparing lightning reindex command files."); _log.WriteLine("You can now copy the {0} file and all cursor*.cmd, cursor*.txt", indexFile); _log.WriteLine("to multiple machines and run the cursor*.cmd files in parallel."); }
public async Task CreatesRegistrationsAndRespectsDeletes() { // Arrange SharedInit(useLegacy: true, useSemVer2: false); var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); await _mockServer.AddStorageAsync(catalogStorage); ReadWriteCursor front = new DurableCursor(_legacyStorage.ResolveUri("cursor.json"), _legacyStorage, MemoryCursor.MinValue); ReadCursor back = MemoryCursor.CreateMax(); // Act await _target.RunAsync(front, back, CancellationToken.None); // Assert Assert.Equal(6, _legacyStorage.Content.Count); // Ensure storage has cursor.json var cursorJson = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); Assert.NotNull(cursorJson.Key); // Check package entries - ListedPackage var package1Index = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/index.json")); Assert.NotNull(package1Index.Key); Assert.Contains("\"listed\":true,", package1Index.Value.GetContentString()); Assert.Contains("\"catalog:CatalogRoot\"", package1Index.Value.GetContentString()); Assert.Contains("\"PackageRegistration\"", package1Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json\"", package1Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.0.nupkg\"", package1Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json\"", package1Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.1.nupkg\"", package1Index.Value.GetContentString()); Assert.Contains("\"lower\":\"1.0.0\",", package1Index.Value.GetContentString()); Assert.Contains("\"upper\":\"1.0.1\"", package1Index.Value.GetContentString()); var package1 = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.0.json")); Assert.Contains("\"listed\":true,", package1.Value.GetContentString()); Assert.NotNull(package1.Key); var package2 = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1.json")); Assert.Contains("\"listed\":true,", package2.Value.GetContentString()); Assert.NotNull(package2.Key); // Check package entries - UnlistedPackage var package2Index = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/index.json")); Assert.NotNull(package2Index.Key); Assert.Contains("\"listed\":false,", package2Index.Value.GetContentString()); Assert.Contains("\"catalog:CatalogRoot\"", package2Index.Value.GetContentString()); Assert.Contains("\"PackageRegistration\"", package2Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json\"", package2Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/packages/unlistedpackage.1.0.0.nupkg\"", package2Index.Value.GetContentString()); Assert.Contains("\"lower\":\"1.0.0\",", package2Index.Value.GetContentString()); Assert.Contains("\"upper\":\"1.0.0\"", package2Index.Value.GetContentString()); var package3 = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0.json")); Assert.Contains("\"listed\":false,", package3.Value.GetContentString()); Assert.NotNull(package3.Key); // Ensure storage does not have the deleted "OtherPackage" var otherPackageIndex = _legacyStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage/index.json")); Assert.Null(otherPackageIndex.Key); }