public async Task Loop(string source, StorageFactory storageFactory, string contentBaseAddress, bool unlistShouldDelete, bool verbose, int interval, CancellationToken cancellationToken) { CommitCollector collector = new RegistrationCollector(new Uri(source), storageFactory, CommandHelpers.GetHttpMessageHandlerFactory(verbose)) { ContentBaseAddress = contentBaseAddress == null ? null : new Uri(contentBaseAddress), UnlistShouldDelete = unlistShouldDelete }; Storage storage = storageFactory.Create(); ReadWriteCursor front = new DurableCursor(storage.ResolveUri("cursor.json"), storage, MemoryCursor.Min.Value); ReadCursor back = MemoryCursor.Max; while (true) { bool run = false; do { run = await collector.Run(front, back, cancellationToken); } while (run); Thread.Sleep(interval * 1000); } }
public async Task CreatesFlatContainerAndRespectsDeletes() { // Arrange var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); var catalogToDnxStorage = new MemoryStorage(); var catalogToDnxStorageFactory = new TestStorageFactory(name => catalogToDnxStorage.WithName(name)); var mockServer = new MockServerHttpClientHandler(); mockServer.SetAction("/", async request => new HttpResponseMessage(HttpStatusCode.OK)); await mockServer.AddStorage(catalogStorage); mockServer.SetAction("/listedpackage.1.0.0.nupkg", async request => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.0.zip")) }); mockServer.SetAction("/listedpackage.1.0.1.nupkg", async request => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\ListedPackage.1.0.1.zip")) }); mockServer.SetAction("/unlistedpackage.1.0.0.nupkg", async request => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\UnlistedPackage.1.0.0.zip")) }); mockServer.SetAction("/otherpackage.1.0.0.nupkg", async request => new HttpResponseMessage(HttpStatusCode.OK) { Content = new StreamContent(File.OpenRead("Packages\\OtherPackage.1.0.0.zip")) }); // Setup collector var target = new DnxCatalogCollector(new Uri("http://tempuri.org/index.json"), catalogToDnxStorageFactory, () => mockServer) { ContentBaseAddress = new Uri("http://tempuri.org/packages") }; ReadWriteCursor front = new DurableCursor(catalogToDnxStorage.ResolveUri("cursor.json"), catalogToDnxStorage, MemoryCursor.Min.Value); ReadCursor back = MemoryCursor.Max; // Act await target.Run(front, back, CancellationToken.None); // Assert Assert.Equal(9, catalogToDnxStorage.Content.Count); // Ensure storage has cursor.json var cursorJson = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); Assert.NotNull(cursorJson.Key); // Check package entries - ListedPackage var package1Index = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/index.json")); Assert.NotNull(package1Index.Key); Assert.Contains("\"1.0.0\"", package1Index.Value.GetContentString()); Assert.Contains("\"1.0.1\"", package1Index.Value.GetContentString()); var package1Nuspec = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.0/listedpackage.nuspec")); var package1Nupkg = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.0/listedpackage.1.0.0.nupkg")); Assert.NotNull(package1Nuspec.Key); Assert.NotNull(package1Nupkg.Key); var package2Nuspec = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1/listedpackage.nuspec")); var package2Nupkg = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1/listedpackage.1.0.1.nupkg")); Assert.NotNull(package2Nuspec.Key); Assert.NotNull(package2Nupkg.Key); // Check package entries - UnlistedPackage var package3Index = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/index.json")); Assert.NotNull(package3Index.Key); Assert.Contains("\"1.0.0\"", package3Index.Value.GetContentString()); var package3Nuspec = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0/unlistedpackage.nuspec")); var package3Nupkg = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0/unlistedpackage.1.0.0.nupkg")); Assert.NotNull(package3Nuspec.Key); Assert.NotNull(package3Nupkg.Key); // Ensure storage does not have the deleted "OtherPackage" var otherPackageIndex = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage/index.json")); var otherPackageNuspec = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage/1.0.0/otherpackage.nuspec")); var otherPackageNupkg = catalogToDnxStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage/1.0.0/otherpackage.1.0.0.nupkg")); Assert.Null(otherPackageIndex.Key); Assert.Null(otherPackageNuspec.Key); Assert.Null(otherPackageNupkg.Key); }
public static void CreateNewCursor(string[] args) { IDictionary<string, string> arguments = GetArguments(args, 0); StorageFactory storageFactory = CreateStorageFactory(arguments, verbose: true); Storage storage = storageFactory.Create(); DurableCursor cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, GetDefaultValue(arguments)); cursor.Load(CancellationToken.None).Wait(); cursor.Save(CancellationToken.None).Wait(); }
static async Task Test2Async() { await MakeTestCatalog(); Func<HttpMessageHandler> handlerFunc = () => { return new FileSystemEmulatorHandler { BaseAddress = new Uri("http://*****:*****@"c:\data\site", InnerHandler = new HttpClientHandler() }; }; TestCollector collectorA = new TestCollector("A", new Uri("http://*****:*****@"c:\data\site\cursor"; Storage storage = new FileStorage(baseAddress, path); DurableCursor cursorA = new DurableCursor(new Uri("http://localhost:8000/cursor/cursorA.json"), storage, MemoryCursor.MinValue); DurableCursor cursorB = new DurableCursor(new Uri("http://localhost:8000/cursor/cursorB.json"), storage, MemoryCursor.MinValue); Console.WriteLine("check catalog..."); bool run = false; do { run = false; run |= await collectorA.Run(cursorA, MemoryCursor.CreateMax(), CancellationToken.None); run |= await collectorB.Run(cursorB, cursorA, CancellationToken.None); } while (run); Console.WriteLine("ADDING MORE CATALOG"); await MoreTestCatalog(); do { run = false; run |= await collectorA.Run(cursorA, MemoryCursor.CreateMax(), CancellationToken.None); run |= await collectorB.Run(cursorB, cursorA, CancellationToken.None); } while (run); Console.WriteLine("ALL DONE"); }
static async Task Test1Async() { await MakeTestCatalog(); Func<HttpMessageHandler> handlerFunc = () => { return new FileSystemEmulatorHandler { BaseAddress = new Uri("http://*****:*****@"c:\data\site", InnerHandler = new HttpClientHandler() }; }; TestCollector collector = new TestCollector("Test1", new Uri("http://*****:*****@"c:\data\site\cursor"; Storage storage = new FileStorage(baseAddress, path); DurableCursor front = new DurableCursor(new Uri("http://localhost:8000/cursor/front.json"), storage, MemoryCursor.MinValue); //DurableCursor back = new DurableCursor(new Uri("http://localhost:8000/cursor/back.json"), storage); MemoryCursor back = MemoryCursor.CreateMax(); bool didWork = await collector.Run(front, back, CancellationToken.None); if (!didWork) { Console.WriteLine("executed but no work was done"); } }
private async Task PrepareAsync() { _log.WriteLine("Making sure folder {0} exists.", _outputFolder); if (!Directory.Exists(_outputFolder)) { Directory.CreateDirectory(_outputFolder); } // Create reindex file _log.WriteLine("Start preparing lightning reindex file..."); var latestCommit = DateTime.MinValue; int numberOfEntries = 0; string indexFile = Path.Combine(_outputFolder, "index.txt"); using (var streamWriter = new StreamWriter(indexFile, false)) { var collectorHttpClient = new CollectorHttpClient(); var catalogIndexReader = new CatalogIndexReader(new Uri(_catalogIndex), collectorHttpClient); var catalogIndexEntries = await catalogIndexReader.GetEntries(); foreach (var packageRegistrationGroup in catalogIndexEntries .OrderBy(x => x.CommitTimeStamp) .ThenBy(x => x.Id) .ThenBy(x => x.Version) .GroupBy(x => x.Id)) { streamWriter.WriteLine("Element@{0}. {1}", numberOfEntries++, packageRegistrationGroup.Key); var latestCatalogPages = new Dictionary<string, Uri>(); foreach (CatalogIndexEntry catalogIndexEntry in packageRegistrationGroup) { string key = catalogIndexEntry.Version.ToNormalizedString(); if (latestCatalogPages.ContainsKey(key)) { latestCatalogPages[key] = catalogIndexEntry.Uri; } else { latestCatalogPages.Add(key, catalogIndexEntry.Uri); } if (latestCommit < catalogIndexEntry.CommitTimeStamp) { latestCommit = catalogIndexEntry.CommitTimeStamp; } } foreach (var latestCatalogPage in latestCatalogPages) { streamWriter.WriteLine("{0}", latestCatalogPage.Value); } } } _log.WriteLine("Finished preparing lightning reindex file. Output file: {0}", indexFile); // Write cursor to storage _log.WriteLine("Start writing new cursor..."); var account = CloudStorageAccount.Parse(_storageAccount); var storageFactory = (StorageFactory)new AzureStorageFactory(account, _storageContainer); var storage = storageFactory.Create(); var cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, latestCommit); cursor.Value = latestCommit; await cursor.Save(CancellationToken.None); _log.WriteLine("Finished writing new cursor."); // Write command files _log.WriteLine("Start preparing lightning reindex command files..."); string templateFileContents; using (var templateStreamReader = new StreamReader(_templateFile)) { templateFileContents = await templateStreamReader.ReadToEndAsync(); } int batchNumber = 0; int batchSizeValue = int.Parse(_batchSize); for (int batchStart = 0; batchStart < numberOfEntries; batchStart += batchSizeValue) { var batchEnd = (batchStart + batchSizeValue - 1); if (batchEnd >= numberOfEntries) { batchEnd = numberOfEntries - 1; } var cursorCommandFileName = "cursor" + batchNumber + ".cmd"; var cursorTextFileName = "cursor" + batchNumber + ".txt"; using (var cursorCommandStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorCommandFileName))) { using (var cursorTextStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorTextFileName))) { var commandStreamContents = templateFileContents .Replace("[index]", indexFile) .Replace("[cursor]", cursorTextFileName) .Replace("[contentbaseaddress]", _contentBaseAddress) .Replace("[storageaccount]", _storageAccount) .Replace("[storagecontainer]", _storageContainer) .Replace("[storagebaseaddress]", _storageBaseAddress) .Replace("[compress]", _compress.ToString().ToLowerInvariant()); await cursorCommandStreamWriter.WriteLineAsync(commandStreamContents); await cursorTextStreamWriter.WriteLineAsync(batchStart + "," + batchEnd); } } batchNumber++; } _log.WriteLine("Finished preparing lightning reindex command files."); _log.WriteLine("You can now copy the {0} file and all cursor*.cmd, cursor*.txt", indexFile); _log.WriteLine("to multiple machines and run the cursor*.cmd files in parallel."); }
public async Task CreatesRegistrationsAndRespectsDeletes() { // Arrange var catalogStorage = Catalogs.CreateTestCatalogWithThreePackagesAndDelete(); var catalogToRegistrationStorage = new MemoryStorage(); var catalogToRegistrationStorageFactory = new TestStorageFactory(name => catalogToRegistrationStorage.WithName(name)); var mockServer = new MockServerHttpClientHandler(); mockServer.SetAction("/", async request => new HttpResponseMessage(HttpStatusCode.OK)); await mockServer.AddStorage(catalogStorage); // Setup collector var target = new RegistrationCollector(new Uri("http://tempuri.org/index.json"), catalogToRegistrationStorageFactory, () => mockServer) { ContentBaseAddress = new Uri("http://tempuri.org/packages"), UnlistShouldDelete = false }; ReadWriteCursor front = new DurableCursor(catalogToRegistrationStorage.ResolveUri("cursor.json"), catalogToRegistrationStorage, MemoryCursor.Min.Value); ReadCursor back = MemoryCursor.Max; // Act await target.Run(front, back, CancellationToken.None); // Assert Assert.Equal(6, catalogToRegistrationStorage.Content.Count); // Ensure storage has cursor.json var cursorJson = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("cursor.json")); Assert.NotNull(cursorJson.Key); // Check package entries - ListedPackage var package1Index = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/index.json")); Assert.NotNull(package1Index.Key); Assert.Contains("\"catalog:CatalogRoot\"", package1Index.Value.GetContentString()); Assert.Contains("\"PackageRegistration\"", package1Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json\"", package1Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.0.nupkg\"", package1Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json\"", package1Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/packages/listedpackage.1.0.1.nupkg\"", package1Index.Value.GetContentString()); Assert.Contains("\"lower\": \"1.0.0\",", package1Index.Value.GetContentString()); Assert.Contains("\"upper\": \"1.0.1\"", package1Index.Value.GetContentString()); var package1 = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.0.json")); Assert.NotNull(package1.Key); var package2 = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/listedpackage/1.0.1.json")); Assert.NotNull(package2.Key); // Check package entries - UnlistedPackage var package2Index = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/index.json")); Assert.NotNull(package1Index.Key); Assert.Contains("\"catalog:CatalogRoot\"", package2Index.Value.GetContentString()); Assert.Contains("\"PackageRegistration\"", package2Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json\"", package2Index.Value.GetContentString()); Assert.Contains("\"http://tempuri.org/packages/unlistedpackage.1.0.0.nupkg\"", package2Index.Value.GetContentString()); Assert.Contains("\"lower\": \"1.0.0\",", package2Index.Value.GetContentString()); Assert.Contains("\"upper\": \"1.0.0\"", package2Index.Value.GetContentString()); var package3 = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/unlistedpackage/1.0.0.json")); Assert.NotNull(package3.Key); // Ensure storage does not have the deleted "OtherPackage" var otherPackageIndex = catalogToRegistrationStorage.Content.FirstOrDefault(pair => pair.Key.PathAndQuery.EndsWith("/otherpackage/index.json")); Assert.Null(otherPackageIndex.Key); }