public static void FetchRange(string sqlConnectionString, Tuple<int, int> range, GalleryExportBatcher batcher)
{
IDictionary<int, JObject> packages = FetchPackages(sqlConnectionString, range);
IDictionary<int, string> registrations = FetchPackageRegistrations(sqlConnectionString, range);
IDictionary<int, List<JObject>> dependencies = FetchPackageDependencies(sqlConnectionString, range);
IDictionary<int, List<string>> targetFrameworks = FetchPackageFrameworks(sqlConnectionString, range);
foreach (int key in packages.Keys)
{
string registration = null;
if (!registrations.TryGetValue(key, out registration))
{
Console.WriteLine("could not find registration for {0}", key);
continue;
}
List<JObject> dependency = null;
dependencies.TryGetValue(key, out dependency);
List<string> targetFramework = null;
targetFrameworks.TryGetValue(key, out targetFramework);
batcher.Process(packages[key], registration, dependency, targetFramework);
}
}
public async Task Update(string sqlConnectionString, Uri catalogIndexUrl)
{
// Collect Database Checksums
CatalogUpdaterEventSource.Log.CollectingDatabaseChecksums();
var databaseChecksums = new Dictionary<int, string>(_checksums.Data.Count);
int lastKey = 0;
int batchSize = DatabaseChecksumBatchSize; // Capture the value to prevent the caller from tinkering with it :)
while (true)
{
var range = await GalleryExport.FetchChecksums(sqlConnectionString, lastKey, batchSize);
foreach (var pair in range)
{
databaseChecksums[pair.Key] = pair.Value;
}
if (range.Count < batchSize)
{
break;
}
lastKey = range.Max(p => p.Key);
CatalogUpdaterEventSource.Log.CollectedDatabaseChecksumBatch(databaseChecksums.Count);
}
CatalogUpdaterEventSource.Log.CollectedDatabaseChecksums(databaseChecksums.Count);
// Diff the checksums
CatalogUpdaterEventSource.Log.ComparingChecksums();
var diffs = GalleryExport.CompareChecksums(_checksums.Data, databaseChecksums).ToList();
CatalogUpdaterEventSource.Log.ComparedChecksums(diffs.Count);
// Update the catalog
CatalogUpdaterEventSource.Log.UpdatingCatalog();
var batcher = new GalleryExportBatcher(CatalogAddBatchSize, _writer);
foreach (var diff in diffs)
{
// Temporarily ignoring updates to avoid the resolver collector going a little wonky
// diff.Result == ComparisonResult.DifferentInCatalog ||
if (diff.Result == ComparisonResult.PresentInDatabaseOnly)
{
CatalogUpdaterEventSource.Log.WritingItem(
"Update", diff.Key, diff.Id, diff.Version);
await GalleryExport.WritePackage(sqlConnectionString, diff.Key, batcher);
}
// Also disabling deletes due to some edge cases.
//else if(diff.Result == ComparisonResult.PresentInCatalogOnly)
//{
// CatalogUpdaterEventSource.Log.WritingItem(
// "Delete", diff.Key, diff.Id, diff.Version);
// await batcher.Add(new DeletePackageCatalogItem(diff.Id, diff.Version, diff.Key.ToString()));
//}
CatalogUpdaterEventSource.Log.WroteItem();
}
await batcher.Complete();
await _writer.Commit();
CatalogUpdaterEventSource.Log.UpdatedCatalog();
}
public static void Test0()
{
const int SqlChunkSize = 8000;
string sqlConnectionString = "";
const int CatalogBatchSize = 1000;
const int CatalogMaxPageSize = 1000;
Storage storage = new FileStorage
{
Path = @"c:\data\site\export2",
Container = "export2",
BaseAddress = "http://localhost:8000/"
};
CatalogWriter writer = new CatalogWriter(storage, new CatalogContext(), CatalogMaxPageSize);
GalleryExportBatcher batcher = new GalleryExportBatcher(CatalogBatchSize, writer);
int lastHighestPackageKey = 0;
int count = 0;
while (true)
{
Tuple<int, int> range = GalleryExport.GetNextRange(sqlConnectionString, lastHighestPackageKey, SqlChunkSize);
if (range.Item1 == 0 && range.Item2 == 0)
{
break;
}
if (count++ == 3)
{
break;
}
Console.WriteLine("{0} {1}", range.Item1, range.Item2);
GalleryExport.FetchRange(sqlConnectionString, range, batcher);
lastHighestPackageKey = range.Item2;
}
batcher.Complete();
Console.WriteLine(batcher.Total);
}
public void Rebuild(RebuildArgs args)
{
const int batchSize = 2000;
if (Directory.Exists(args.CatalogFolder))
{
Console.WriteLine("Catalog folder exists. Deleting!");
Directory.Delete(args.CatalogFolder, recursive: true);
}
// Load storage
Storage storage = new FileStorage(args.BaseAddress, args.CatalogFolder);
using (var writer = new CatalogWriter(storage, new CatalogContext()))
{
if (!String.IsNullOrEmpty(args.DatabaseConnection))
{
var batcher = new GalleryExportBatcher(batchSize, writer);
int lastHighest = 0;
while (true)
{
var range = GalleryExport.GetNextRange(
args.DatabaseConnection,
lastHighest,
batchSize).Result;
if (range.Item1 == 0 && range.Item2 == 0)
{
break;
}
Console.WriteLine("Writing packages with Keys {0}-{1} to catalog...", range.Item1, range.Item2);
GalleryExport.WriteRange(
args.DatabaseConnection,
range,
batcher).Wait();
lastHighest = range.Item2;
}
batcher.Complete().Wait();
}
else if (!String.IsNullOrEmpty(args.NuPkgFolder))
{
Stopwatch timer = new Stopwatch();
timer.Start();
// files are sorted by GetFiles
Queue<string> files = new Queue<string>(Directory.GetFiles(args.NuPkgFolder, "*.nu*", SearchOption.TopDirectoryOnly)
.Where(s => s.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase) || s.EndsWith(".nuspec", StringComparison.OrdinalIgnoreCase)));
int total = files.Count;
ParallelOptions options = new ParallelOptions();
options.MaxDegreeOfParallelism = 8;
Task commitTask = null;
while (files.Count > 0)
{
Queue<PackageCatalogItem> currentBatch = new Queue<PackageCatalogItem>(batchSize);
// create the batch
while (currentBatch.Count < batchSize && files.Count > 0)
{
string file = files.Dequeue();
if (file.EndsWith(".nupkg", StringComparison.OrdinalIgnoreCase))
{
currentBatch.Enqueue(new NupkgCatalogItem(file));
}
else
{
currentBatch.Enqueue(new NuspecPackageCatalogItem(file));
}
}
// process the nupkgs and nuspec files in parallel
Parallel.ForEach(currentBatch, options, nupkg =>
{
nupkg.Load();
});
// wait for the previous commit to finish before adding more
if (commitTask != null)
{
commitTask.Wait();
}
// add everything from the queue
foreach(PackageCatalogItem item in currentBatch)
{
writer.Add(item);
}
// commit
commitTask = Task.Run(async () => await writer.Commit(DateTime.UtcNow));
Console.WriteLine("committing {0}/{1}", total - files.Count, total);
}
// wait for the final commit
if (commitTask != null)
{
commitTask.Wait();
}
timer.Stop();
Console.ForegroundColor = ConsoleColor.Green;
Console.WriteLine("Committed {0} catalog items in {1}", total, timer.Elapsed);
Console.ResetColor();
}
}
}
