public static void FetchRange(string sqlConnectionString, Tuple<int, int> range, GalleryExportBatcher batcher) { IDictionary<int, JObject> packages = FetchPackages(sqlConnectionString, range); IDictionary<int, string> registrations = FetchPackageRegistrations(sqlConnectionString, range); IDictionary<int, List<JObject>> dependencies = FetchPackageDependencies(sqlConnectionString, range); IDictionary<int, List<string>> targetFrameworks = FetchPackageFrameworks(sqlConnectionString, range); foreach (int key in packages.Keys) { string registration = null; if (!registrations.TryGetValue(key, out registration)) { Console.WriteLine("could not find registration for {0}", key); continue; } List<JObject> dependency = null; dependencies.TryGetValue(key, out dependency); List<string> targetFramework = null; targetFrameworks.TryGetValue(key, out targetFramework); batcher.Process(packages[key], registration, dependency, targetFramework); } }
public async Task Update(string sqlConnectionString, Uri catalogIndexUrl) { // Collect Database Checksums CatalogUpdaterEventSource.Log.CollectingDatabaseChecksums(); var databaseChecksums = new Dictionary<int, string>(_checksums.Data.Count); int lastKey = 0; int batchSize = DatabaseChecksumBatchSize; // Capture the value to prevent the caller from tinkering with it :) while (true) { var range = await GalleryExport.FetchChecksums(sqlConnectionString, lastKey, batchSize); foreach (var pair in range) { databaseChecksums[pair.Key] = pair.Value; } if (range.Count < batchSize) { break; } lastKey = range.Max(p => p.Key); CatalogUpdaterEventSource.Log.CollectedDatabaseChecksumBatch(databaseChecksums.Count); } CatalogUpdaterEventSource.Log.CollectedDatabaseChecksums(databaseChecksums.Count); // Diff the checksums CatalogUpdaterEventSource.Log.ComparingChecksums(); var diffs = GalleryExport.CompareChecksums(_checksums.Data, databaseChecksums).ToList(); CatalogUpdaterEventSource.Log.ComparedChecksums(diffs.Count); // Update the catalog CatalogUpdaterEventSource.Log.UpdatingCatalog(); var batcher = new GalleryExportBatcher(CatalogAddBatchSize, _writer); foreach (var diff in diffs) { // Temporarily ignoring updates to avoid the resolver collector going a little wonky // diff.Result == ComparisonResult.DifferentInCatalog || if (diff.Result == ComparisonResult.PresentInDatabaseOnly) { CatalogUpdaterEventSource.Log.WritingItem( "Update", diff.Key, diff.Id, diff.Version); await GalleryExport.WritePackage(sqlConnectionString, diff.Key, batcher); } // Also disabling deletes due to some edge cases. //else if(diff.Result == ComparisonResult.PresentInCatalogOnly) //{ // CatalogUpdaterEventSource.Log.WritingItem( // "Delete", diff.Key, diff.Id, diff.Version); // await batcher.Add(new DeletePackageCatalogItem(diff.Id, diff.Version, diff.Key.ToString())); //} CatalogUpdaterEventSource.Log.WroteItem(); } await batcher.Complete(); await _writer.Commit(); CatalogUpdaterEventSource.Log.UpdatedCatalog(); }
public static void Test0() { const int SqlChunkSize = 8000; string sqlConnectionString = ""; const int CatalogBatchSize = 1000; const int CatalogMaxPageSize = 1000; Storage storage = new FileStorage { Path = @"c:\data\site\export2", Container = "export2", BaseAddress = "http://localhost:8000/" }; CatalogWriter writer = new CatalogWriter(storage, new CatalogContext(), CatalogMaxPageSize); GalleryExportBatcher batcher = new GalleryExportBatcher(CatalogBatchSize, writer); int lastHighestPackageKey = 0; int count = 0; while (true) { Tuple<int, int> range = GalleryExport.GetNextRange(sqlConnectionString, lastHighestPackageKey, SqlChunkSize); if (range.Item1 == 0 && range.Item2 == 0) { break; } if (count++ == 3) { break; } Console.WriteLine("{0} {1}", range.Item1, range.Item2); GalleryExport.FetchRange(sqlConnectionString, range, batcher); lastHighestPackageKey = range.Item2; } batcher.Complete(); Console.WriteLine(batcher.Total); }
public void Rebuild(RebuildArgs args) { const int batchSize = 2000; if (Directory.Exists(args.CatalogFolder)) { Console.WriteLine("Catalog folder exists. Deleting!"); Directory.Delete(args.CatalogFolder, recursive: true); } // Load storage Storage storage = new FileStorage(args.BaseAddress, args.CatalogFolder); using (var writer = new CatalogWriter(storage, new CatalogContext())) { if (!String.IsNullOrEmpty(args.DatabaseConnection)) { var batcher = new GalleryExportBatcher(batchSize, writer); int lastHighest = 0; while (true) { var range = GalleryExport.GetNextRange( args.DatabaseConnection, lastHighest, batchSize).Result; if (range.Item1 == 0 && range.Item2 == 0) { break; } Console.WriteLine("Writing packages with Keys {0}-{1} to catalog...", range.Item1, range.Item2); GalleryExport.WriteRange( args.DatabaseConnection, range, batcher).Wait(); lastHighest = range.Item2; } batcher.Complete().Wait(); } else if (!String.IsNullOrEmpty(args.NuPkgFolder)) { Stopwatch timer = new Stopwatch(); timer.Start(); // files are sorted by GetFiles Queue<string> files = new Queue<string>(Directory.GetFiles(args.NuPkgFolder, "*.nu*", SearchOption.TopDirectoryOnly) .Where(s => s.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase) || s.EndsWith(".nuspec", StringComparison.OrdinalIgnoreCase))); int total = files.Count; ParallelOptions options = new ParallelOptions(); options.MaxDegreeOfParallelism = 8; Task commitTask = null; while (files.Count > 0) { Queue<PackageCatalogItem> currentBatch = new Queue<PackageCatalogItem>(batchSize); // create the batch while (currentBatch.Count < batchSize && files.Count > 0) { string file = files.Dequeue(); if (file.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase)) { currentBatch.Enqueue(new NupkgCatalogItem(file)); } else { currentBatch.Enqueue(new NuspecPackageCatalogItem(file)); } } // process the nupkgs and nuspec files in parallel Parallel.ForEach(currentBatch, options, nupkg => { nupkg.Load(); }); // wait for the previous commit to finish before adding more if (commitTask != null) { commitTask.Wait(); } // add everything from the queue foreach(PackageCatalogItem item in currentBatch) { writer.Add(item); } // commit commitTask = Task.Run(async () => await writer.Commit(DateTime.UtcNow)); Console.WriteLine("committing {0}/{1}", total - files.Count, total); } // wait for the final commit if (commitTask != null) { commitTask.Wait(); } timer.Stop(); Console.ForegroundColor = ConsoleColor.Green; Console.WriteLine("Committed {0} catalog items in {1}", total, timer.Elapsed); Console.ResetColor(); } } }