public async static Task Run([TimerTrigger("0 */1 * * * *")] TimerInfo myTimer, ILogger log) { log.LogInformation($"IndexNuGetPackages started, next run: {myTimer.ScheduleStatus.Next}"); var reader = new CatalogIndexReader(httpClient); var catalogIndex = await reader.ReadUrl("https://api.nuget.org/v3/catalog0/index.json"); var lastPage = catalogIndex.Items.OrderBy(i => i.CommitTimeStamp).Last(); log.LogInformation($"IndexNuGetPackages complete, last page {lastPage.Id}, commited at: {lastPage.CommitTimeStamp}"); }
public async Task ParseCatalogIndex() { var cursor = DateTime.UtcNow - TimeSpan.FromDays(1); var reader = new CatalogIndexReader(new HttpClient()); var catalogIndex = await reader.ReadUrl("https://api.nuget.org/v3/catalog0/index.json"); Console.Out.WriteLine($"All pages since: {cursor}"); foreach (var page in catalogIndex.Items.Where(i => i.CommitTimeStamp > cursor).OrderByDescending(i => i.CommitTimeStamp)) { Console.Out.WriteLine($"{page.CommitTimeStamp} - {page.Id} ({page.Count})"); } }
public async Task GetEntries() { // Arrange var indexUri = "http://tempuri.org/index.json"; var responses = new Dictionary <string, string>() { { "http://tempuri.org/index.json", TestCatalogEntries.TestCatalogStorageWithThreePackagesIndex }, { "http://tempuri.org/page0.json", TestCatalogEntries.TestCatalogStorageWithThreePackagesPage }, }; var reader = new CatalogIndexReader( new Uri(indexUri), new CollectorHttpClient(new InMemoryHttpHandler(responses)), new Mock <ITelemetryService>().Object); // Act var entries = await reader.GetEntries(); // Assert var entryList = entries.ToList(); Assert.Equal(3, entryList.Count); Assert.Equal("http://tempuri.org/data/2015.10.12.10.08.55/listedpackage.1.0.1.json", entryList[0].Uri.ToString()); Assert.Equal("http://tempuri.org/data/2015.10.12.10.08.54/listedpackage.1.0.0.json", entryList[1].Uri.ToString()); Assert.Equal("http://tempuri.org/data/2015.10.12.10.08.54/unlistedpackage.1.0.0.json", entryList[2].Uri.ToString()); Assert.Equal("2015-10-12T10:08:55.3335317", entryList[0].CommitTimeStamp.ToString("O")); Assert.Equal("2015-10-12T10:08:54.1506742", entryList[1].CommitTimeStamp.ToString("O")); Assert.Equal("2015-10-12T10:08:54.1506742", entryList[2].CommitTimeStamp.ToString("O")); Assert.Equal("8a9e7694-73d4-4775-9b7a-20aa59b9773e", entryList[0].CommitId); Assert.Equal("9a37734f-1960-4c07-8934-c8bc797e35c1", entryList[1].CommitId); Assert.Equal("9a37734f-1960-4c07-8934-c8bc797e35c1", entryList[2].CommitId); Assert.Equal("ListedPackage", entryList[0].Id); Assert.Equal("ListedPackage", entryList[1].Id); Assert.Equal("UnlistedPackage", entryList[2].Id); Assert.Equal(new NuGetVersion("1.0.1"), entryList[0].Version); Assert.Equal(new NuGetVersion("1.0.0"), entryList[1].Version); Assert.Equal(new NuGetVersion("1.0.0"), entryList[2].Version); Assert.Equal(new[] { "nuget:PackageDetails" }, entryList[0].Types); Assert.Equal(new[] { "nuget:PackageDetails" }, entryList[1].Types); Assert.Equal(new[] { "nuget:PackageDetails" }, entryList[2].Types); Assert.Same(entryList[0].Id, entryList[1].Id); }
public static HashSet <PackageEntry> GetCatalogPackages(string catalogRootUrl, string storageConnectionString, DateTime startCommitTimeStamp, DateTime endCommitTimeStamp) { CollectorHttpClient client = new CollectorHttpClient(); CloudStorageAccount csa = CloudStorageAccount.Parse(storageConnectionString); var blobClient = csa.CreateCloudBlobClient(); Uri catalogIndex = new Uri(catalogRootUrl); CatalogIndexReader reader = new CatalogIndexReader(catalogIndex, client); //TBD Update CatalogIndexReader to return packages based on commit time stamp.Right now it returns all packages. var task = reader.GetEntries(); task.Wait(); List <CatalogIndexEntry> entries = task.Result.ToList(); entries = entries.Where(e => e.CommitTimeStamp >= startCommitTimeStamp && e.CommitTimeStamp <= endCommitTimeStamp).ToList(); var catalogPackages = new HashSet <PackageEntry>(entries.Select(e => new PackageEntry(e.Id, e.Version.ToNormalizedString())), PackageEntry.Comparer); return(catalogPackages); }
public static async Task Test0Async(string[] args) { const string V2FeedCountQuery = "/Packages/$count"; Console.WriteLine("Simple count test for distinct package ids and version between v2 feed and catalog"); if (args.Length != 2) { Console.WriteLine("Please enter only 2 arguments. First v2gallery feed url, and second catalog index.json url"); return; } else { string v2FeedUrl = args[0].TrimEnd('/'); string v2FeedCountUrl = v2FeedUrl.TrimEnd('/') + V2FeedCountQuery; int v2FeedCount = 0; using (var client = new HttpClient()) { var response = await client.GetAsync(v2FeedCountUrl); string v2FeedCountString = await response.Content.ReadAsStringAsync(); v2FeedCount = Int32.Parse(v2FeedCountString); } string catalog = args[1]; Uri catalogIndex = new Uri(catalog); CatalogIndexReader reader = new CatalogIndexReader(catalogIndex); var task = reader.GetEntries(); task.Wait(); var entries = task.Result; Console.WriteLine("Total packages count from catalog is " + entries.Count()); var distinctCatalogPackages = entries.Distinct(new CatalogIndexEntryIdVersionComparer()); int v3CatalogPackagesCount = distinctCatalogPackages.Count(); Console.WriteLine("Distinct packages count from catalog is " + v3CatalogPackagesCount); Console.WriteLine("Distinct packages count from " + v2FeedUrl + " is " + v2FeedCount); Console.WriteLine("Current difference between v2Feed and v3 catalog is " + (v2FeedCount - v3CatalogPackagesCount)); } }
private async Task PrepareAsync() { _log.WriteLine("Making sure folder {0} exists.", _outputFolder); if (!Directory.Exists(_outputFolder)) { Directory.CreateDirectory(_outputFolder); } // Create reindex file _log.WriteLine("Start preparing lightning reindex file..."); var latestCommit = DateTime.MinValue; int numberOfEntries = 0; string indexFile = Path.Combine(_outputFolder, "index.txt"); string optionalArgumentsTemplate = "optionalArguments"; using (var streamWriter = new StreamWriter(indexFile, false)) { var httpMessageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, _verbose); var collectorHttpClient = new CollectorHttpClient(httpMessageHandlerFactory()); var catalogIndexReader = new CatalogIndexReader(new Uri(_catalogIndex), collectorHttpClient, TelemetryService); var catalogIndexEntries = await catalogIndexReader.GetEntries(); foreach (var packageRegistrationGroup in catalogIndexEntries .OrderBy(x => x.CommitTimeStamp) .ThenBy(x => x.Id, StringComparer.OrdinalIgnoreCase) .ThenBy(x => x.Version) .GroupBy(x => x.Id, StringComparer.OrdinalIgnoreCase)) { streamWriter.WriteLine("Element@{0}. {1}", numberOfEntries++, packageRegistrationGroup.Key); var latestCatalogPages = new Dictionary <string, Uri>(); foreach (CatalogIndexEntry catalogIndexEntry in packageRegistrationGroup) { string key = catalogIndexEntry.Version.ToNormalizedString(); if (latestCatalogPages.ContainsKey(key)) { latestCatalogPages[key] = catalogIndexEntry.Uri; } else { latestCatalogPages.Add(key, catalogIndexEntry.Uri); } if (latestCommit < catalogIndexEntry.CommitTimeStamp) { latestCommit = catalogIndexEntry.CommitTimeStamp; } } foreach (var latestCatalogPage in latestCatalogPages) { streamWriter.WriteLine("{0}", latestCatalogPage.Value); } } } _log.WriteLine("Finished preparing lightning reindex file. Output file: {0}", indexFile); // Write cursor to storage _log.WriteLine("Start writing new cursor..."); var storage = _storageFactories.LegacyStorageFactory.Create(); var cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, latestCommit) { Value = latestCommit }; await cursor.SaveAsync(CancellationToken.None); _log.WriteLine("Finished writing new cursor."); // Ensure the SemVer 2.0.0 storage containers is created, if applicable. The gzipped storage account is // created above when we write the cursor. _storageFactories.SemVer2StorageFactory?.Create(); // Write command files _log.WriteLine("Start preparing lightning reindex command files..."); string templateFileContents; using (var templateStreamReader = new StreamReader(_templateFile)) { templateFileContents = await templateStreamReader.ReadToEndAsync(); } int batchNumber = 0; int batchSizeValue = int.Parse(_batchSize); for (int batchStart = 0; batchStart < numberOfEntries; batchStart += batchSizeValue) { var batchEnd = (batchStart + batchSizeValue - 1); if (batchEnd >= numberOfEntries) { batchEnd = numberOfEntries - 1; } var cursorCommandFileName = "cursor" + batchNumber + ".cmd"; var cursorTextFileName = "cursor" + batchNumber + ".txt"; using (var cursorCommandStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorCommandFileName))) using (var cursorTextStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorTextFileName))) { var commandStreamContents = templateFileContents; var replacements = _arguments .Concat(new[] { new KeyValuePair <string, string>("indexFile", indexFile), new KeyValuePair <string, string>("cursorFile", cursorTextFileName) }); foreach (var replacement in replacements) { commandStreamContents = commandStreamContents .Replace($"[{replacement.Key}]", replacement.Value); } //the not required arguments need to be added only if they were passed in //they cannot be hardcoded in the template var optionalArguments = new StringBuilder(); AppendOptionalArgument(optionalArguments, Arguments.ContentIsFlatContainer); AppendOptionalArgument(optionalArguments, Arguments.FlatContainerName); AppendOptionalArgument(optionalArguments, Arguments.StorageSuffix); AppendOptionalArgument(optionalArguments, Arguments.AllIconsInFlatContainer); AppendOptionalArgument(optionalArguments, Arguments.Driver); AppendOptionalArgument(optionalArguments, Arguments.Verbose); commandStreamContents = commandStreamContents .Replace($"[{optionalArgumentsTemplate}]", optionalArguments.ToString()); await cursorCommandStreamWriter.WriteLineAsync(commandStreamContents); await cursorTextStreamWriter.WriteLineAsync(batchStart + "," + batchEnd); } batchNumber++; } _log.WriteLine("Finished preparing lightning reindex command files."); _log.WriteLine("You can now copy the {0} file and all cursor*.cmd, cursor*.txt", indexFile); _log.WriteLine("to multiple machines and run the cursor*.cmd files in parallel."); }
public override async Task RunCore() { int nextMasterRegId = 0; DateTime position = Cursor.Position; JToken nextMasterRegIdToken = null; if (Cursor.Metadata.TryGetValue("nextMasterRegId", out nextMasterRegIdToken)) { nextMasterRegId = nextMasterRegIdToken.ToObject <int>(); } // Get the catalog index Uri catalogIndexUri = new Uri(Config.GetProperty("CatalogIndex")); Log("Reading index entries"); var indexReader = new CatalogIndexReader(catalogIndexUri); var indexEntries = await indexReader.GetRolledUpEntries(); var context = indexReader.GetContext(); Log("Finding new or editted entries"); var changedEntries = new HashSet <string>(indexEntries.Where(e => e.CommitTimeStamp.CompareTo(position) > 0) .Select(e => e.Id.ToLowerInvariant()), StringComparer.OrdinalIgnoreCase); DateTime newPosition = indexEntries.Select(e => e.CommitTimeStamp).OrderByDescending(e => e).FirstOrDefault(); ConcurrentDictionary <string, ConcurrentBag <Uri> > batches = new ConcurrentDictionary <string, ConcurrentBag <Uri> >(StringComparer.OrdinalIgnoreCase); ParallelOptions options = new ParallelOptions(); options.MaxDegreeOfParallelism = 8; Parallel.ForEach(indexEntries, options, entry => { if (changedEntries.Contains(entry.Id)) { batches.AddOrUpdate(entry.Id, new ConcurrentBag <Uri>() { entry.Uri }, (id, uris) => { uris.Add(entry.Uri); return(uris); }); } }); Uri contentBaseAddress = new Uri(Config.GetProperty("ContentBaseAddress")); if (batches.Count > 0) { Log("Building registrations from: " + position.ToString("O")); options.MaxDegreeOfParallelism = 4; for (int i = 0; i < 3 && batches.Count > 0; i++) { if (i != 0) { options.MaxDegreeOfParallelism = 1; Console.WriteLine("Single batch run."); } var ids = batches.Keys.OrderBy(s => s).ToArray(); Stopwatch buildTimer = new Stopwatch(); buildTimer.Start(); int startingCount = ids.Length; Parallel.ForEach(ids, options, id => { try { BatchRegistrationCollector regCollector = new BatchRegistrationCollector(null, _factory); regCollector.ContentBaseAddress = contentBaseAddress; Stopwatch timer = new Stopwatch(); timer.Start(); var uriGroup = batches[id].ToArray(); regCollector.ProcessGraphs(_client, id, uriGroup, context).Wait(); int rem = batches.Count; timer.Stop(); string log = String.Format("Completed: {0} Duration: {1} Uris: {2} Remaining Ids: {3} Loop: {4}", id, timer.Elapsed, uriGroup.Length, rem, i); Console.WriteLine(log); // stats double perPackage = buildTimer.Elapsed.TotalSeconds / (double)(startingCount - rem + 1); DateTime finish = DateTime.Now.AddSeconds(Math.Ceiling(perPackage * rem)); Console.WriteLine("Estimated Finish: " + finish.ToString("O")); ConcurrentBag <Uri> vals; if (!batches.TryRemove(id, out vals)) { Console.WriteLine("Unable to remove!"); } } catch (Exception ex) { LogError("Registration failed: " + id + " " + ex.ToString()); } }); } // mark this with the last commit we included Cursor.Position = newPosition; await Cursor.Save(); Log("Finished registrations: " + newPosition.ToString("O")); } }
private async Task PrepareAsync() { _log.WriteLine("Making sure folder {0} exists.", _outputFolder); if (!Directory.Exists(_outputFolder)) { Directory.CreateDirectory(_outputFolder); } // Create reindex file _log.WriteLine("Start preparing lightning reindex file..."); var latestCommit = DateTime.MinValue; int numberOfEntries = 0; string indexFile = Path.Combine(_outputFolder, "index.txt"); string storageCredentialArgumentsTemplate = "storageCredentialArguments"; string optionalArgumentsTemplate = "optionalArguments"; using (var streamWriter = new StreamWriter(indexFile, false)) { var httpMessageHandlerFactory = CommandHelpers.GetHttpMessageHandlerFactory(TelemetryService, _verbose); var collectorHttpClient = new CollectorHttpClient(httpMessageHandlerFactory()); var catalogIndexReader = new CatalogIndexReader(new Uri(_catalogIndex), collectorHttpClient, TelemetryService); var catalogIndexEntries = await catalogIndexReader.GetEntries(); foreach (var packageRegistrationGroup in catalogIndexEntries .OrderBy(x => x.CommitTimeStamp) .ThenBy(x => x.Id, StringComparer.OrdinalIgnoreCase) .ThenBy(x => x.Version) .GroupBy(x => x.Id, StringComparer.OrdinalIgnoreCase)) { streamWriter.WriteLine("Element@{0}. {1}", numberOfEntries++, packageRegistrationGroup.Key); var latestCatalogPages = new Dictionary <string, Uri>(); foreach (CatalogIndexEntry catalogIndexEntry in packageRegistrationGroup) { string key = catalogIndexEntry.Version.ToNormalizedString(); if (latestCatalogPages.ContainsKey(key)) { latestCatalogPages[key] = catalogIndexEntry.Uri; } else { latestCatalogPages.Add(key, catalogIndexEntry.Uri); } if (latestCommit < catalogIndexEntry.CommitTimeStamp) { latestCommit = catalogIndexEntry.CommitTimeStamp; } } foreach (var latestCatalogPage in latestCatalogPages) { streamWriter.WriteLine("{0}", latestCatalogPage.Value); } } } _log.WriteLine("Finished preparing lightning reindex file. Output file: {0}", indexFile); // Create the containers _log.WriteLine("Creating the containers..."); var container = GetAutofacContainer(); var blobClient = container.Resolve <ICloudBlobClient>(); var config = container.Resolve <IOptionsSnapshot <Catalog2RegistrationConfiguration> >().Value; foreach (var name in new[] { config.LegacyStorageContainer, config.GzippedStorageContainer, config.SemVer2StorageContainer }) { var reference = blobClient.GetContainerReference(name); var permissions = new BlobContainerPermissions { PublicAccess = BlobContainerPublicAccessType.Blob }; await reference.CreateIfNotExistAsync(permissions); } // Write cursor to storage _log.WriteLine("Start writing new cursor..."); var storageFactory = container.ResolveKeyed <IStorageFactory>(DependencyInjectionExtensions.CursorBindingKey); var storage = storageFactory.Create(); var cursor = new DurableCursor(storage.ResolveUri("cursor.json"), storage, latestCommit) { Value = latestCommit }; await cursor.SaveAsync(CancellationToken.None); _log.WriteLine("Finished writing new cursor."); // Write command files _log.WriteLine("Start preparing lightning reindex command files..."); string templateFileContents; using (var templateStreamReader = new StreamReader(_templateFile)) { templateFileContents = await templateStreamReader.ReadToEndAsync(); } int batchNumber = 0; int batchSizeValue = int.Parse(_batchSize); for (int batchStart = 0; batchStart < numberOfEntries; batchStart += batchSizeValue) { var batchEnd = (batchStart + batchSizeValue - 1); if (batchEnd >= numberOfEntries) { batchEnd = numberOfEntries - 1; } var cursorCommandFileName = "cursor" + batchNumber + ".cmd"; var cursorTextFileName = "cursor" + batchNumber + ".txt"; using (var cursorCommandStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorCommandFileName))) using (var cursorTextStreamWriter = new StreamWriter(Path.Combine(_outputFolder, cursorTextFileName))) { var commandStreamContents = templateFileContents; var replacements = _arguments .Concat(new[] { new KeyValuePair <string, string>("indexFile", indexFile), new KeyValuePair <string, string>("cursorFile", cursorTextFileName) }); foreach (var replacement in replacements) { commandStreamContents = commandStreamContents .Replace($"[{replacement.Key}]", replacement.Value); } // Since we only need to set the storage key or the storage sas token, only one will be added to the template. var storageCredentialArguments = new StringBuilder(); AddStorageCredentialArgument(storageCredentialArguments, Arguments.StorageSasValue, Arguments.StorageKeyValue); AddStorageCredentialArgument(storageCredentialArguments, Arguments.CompressedStorageSasValue, Arguments.CompressedStorageKeyValue); AddStorageCredentialArgument(storageCredentialArguments, Arguments.SemVer2StorageSasValue, Arguments.SemVer2StorageKeyValue); commandStreamContents = commandStreamContents .Replace($"[{storageCredentialArgumentsTemplate}]", storageCredentialArguments.ToString()); //the not required arguments need to be added only if they were passed in //they cannot be hardcoded in the template var optionalArguments = new StringBuilder(); AppendArgument(optionalArguments, Arguments.FlatContainerName); AppendArgument(optionalArguments, Arguments.StorageSuffix); AppendArgument(optionalArguments, Arguments.Verbose); commandStreamContents = commandStreamContents .Replace($"[{optionalArgumentsTemplate}]", optionalArguments.ToString()); await cursorCommandStreamWriter.WriteLineAsync(commandStreamContents); await cursorTextStreamWriter.WriteLineAsync(batchStart + "," + batchEnd); } batchNumber++; } _log.WriteLine("Finished preparing lightning reindex command files."); _log.WriteLine("You can now copy the {0} file and all cursor*.cmd, cursor*.txt", indexFile); _log.WriteLine("to multiple machines and run the cursor*.cmd files in parallel."); }