public async Task Loop(string source, StorageFactory storageFactory, string contentBaseAddress, bool unlistShouldDelete, bool verbose, int interval, CancellationToken cancellationToken) { CommitCollector collector = new RegistrationCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose)) { ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress), UnlistShouldDelete = unlistShouldDelete }; Storage storage = storageFactory.Create(); ReadWriteCursor front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.Min.Value); ReadCursor back = MemoryCursor.Max; while (true) { bool run = false; do { run = await collector.Run(front, back, cancellationToken); } while (run); Thread.Sleep(interval * 1000); } }
public static async Task Test4Async() { //Uri catalogUri = new Uri("https://nugetdevstorage.blob.core.windows.net/catalog/index.json"); Uri catalogUri = new Uri("https://nugetjohtaylo.blob.core.windows.net/catalog/index.json"); //Uri catalogUri = new Uri("https://api.nuget.org/v3/catalog0/index.json"); string path = "c:\\data\\registration20150421"; DirectoryInfo directoryInfo = new DirectoryInfo(path); if (directoryInfo.Exists) { directoryInfo.Delete(true); } directoryInfo.Create(); FileStorageFactory factory = new FileStorageFactory(new Uri("http://tempuri.org"), path); CollectorBase collector = new RegistrationCollector(catalogUri, factory) { UnlistShouldDelete = true, Concurrent = false }; await collector.Run(CancellationToken.None); }
public async Task CreatesRegistrationsAndRespectsDeletes() { // Arrange var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); var catalogToRegistrationStorage = new MemoryStorage(); var catalogToRegistrationStorageFactory = new TestStorageFactory(name => catalogToRegistrationStorage.WithName(name)); var mockServer = new MockServerHttpClientHandler(); mockServer.SetAction("/", async request => new HttpResponseMessage(HttpStatusCode.OK)); await mockServer.AddStorage(catalogStorage); // Setup collector var target = new RegistrationCollector(new Uri("http://tempuri.org/index.json"), catalogToRegistrationStorageFactory, () => mockServer) { ContentBaseAddress = new Uri("http://tempuri.org/packages"), UnlistShouldDelete = false }; ReadWriteCursor front = new DurableCursor(catalogToRegistrationStorage.ResolveUri("cursor.json"), catalogToRegistrationStorage, MemoryCursor.Min.Value); ReadCursor back = MemoryCursor.Max; // Act await target.Run(front, back, CancellationToken.None); // Assert Assert.Equal(6, catalogToRegistrationStorage.Content.Count); // Ensure storage has cursor.json var cursorJson = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); Assert.NotNull(cursorJson.Key); // Check package entries - ListedPackage var package1Index = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/index.json")); Assert.NotNull(package1Index.Key); Assert.Contains("\"catalog:CatalogRoot\"", package1Index.Value.GetContentString()); Assert.Contains("\"PackageRegistration\"", package1Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json\"", package1Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.0.nupkg\"", package1Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json\"", package1Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.1.nupkg\"", package1Index.Value.GetContentString()); Assert.Contains("\"lower\": \"1.0.0\",", package1Index.Value.GetContentString()); Assert.Contains("\"upper\": \"1.0.1\"", package1Index.Value.GetContentString()); var package1 = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.0.json")); Assert.NotNull(package1.Key); var package2 = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1.json")); Assert.NotNull(package2.Key); // Check package entries - UnlistedPackage var package2Index = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/index.json")); Assert.NotNull(package1Index.Key); Assert.Contains("\"catalog:CatalogRoot\"", package2Index.Value.GetContentString()); Assert.Contains("\"PackageRegistration\"", package2Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json\"", package2Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/packages/unlistedpackage.1.0.0.nupkg\"", package2Index.Value.GetContentString()); Assert.Contains("\"lower\": \"1.0.0\",", package2Index.Value.GetContentString()); Assert.Contains("\"upper\": \"1.0.0\"", package2Index.Value.GetContentString()); var package3 = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0.json")); Assert.NotNull(package3.Key); // Ensure storage does not have the deleted "OtherPackage" var otherPackageIndex = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage/index.json")); Assert.Null(otherPackageIndex.Key); }
async Task CreateRegistrationBlobs(Uri catalogIndex, Func<StorageHttpMessageHandler> handlerFunc, CancellationToken cancellationToken) { var factory = new MemoryStorageFactory(new Uri(_baseAddress, Registration), _store); CommitCollector collector = new RegistrationCollector(catalogIndex, factory, handlerFunc) { ContentBaseAddress = new Uri("http://tempuri.org/content/"), UnlistShouldDelete = false }; await collector.Run( new MemoryCursor(DateTime.MinValue.ToUniversalTime()), new MemoryCursor(DateTime.MaxValue.ToUniversalTime()), cancellationToken); }