public async Task ImportAsync(int skip = 0)
{
_logger.LogInformation("Starting import with skip {Skip}...", skip);
var batchCount = 1;
var left = await _context.PackageIds
.Where(p => !p.Done)
.CountAsync();
_logger.LogInformation("{PackageIdsLeft} package ids left to import", left);
while (true)
{
_logger.LogInformation("Importing batch {BatchCount}...", batchCount);
var batch = await _context.PackageIds
.Where(p => !p.Done)
.OrderBy(p => p.Key)
.Skip(skip)
.Take(ImportBatchSize)
.ToListAsync();
if (batch.Count == 0)
{
break;
}
await _indexer.IndexAsync(batch.Select(p => p.Value).ToArray());
foreach (var package in batch)
{
package.Done = true;
}
await _context.SaveChangesAsync();
_logger.LogInformation("Imported batch {BatchCount}", batchCount);
batchCount++;
}
_logger.LogInformation("Finished importing");
}
private async Task InitializeStateAsync()
{
if (await _indexerContext.PackageIds.AnyAsync())
{
_logger.LogInformation("Indexer state is already initialized");
return;
}
_logger.LogInformation("Unitialized state. Finding packages to track in indexer state...");
var packageIds = await _bagetContext.Packages
.Select(p => p.Id)
.Distinct()
.ToListAsync();
_logger.LogInformation("Found {PackageIdCount} package ids to track in indexer state", packageIds.Count);
var batchCount = 1;
foreach (var batch in packageIds.Batch(InitializationBatchSize))
{
foreach (var packageId in batch)
{
_indexerContext.PackageIds.Add(new PackageId
{
Value = packageId,
Done = false,
});
}
_logger.LogInformation("Saving package id batch {BatchCount} to indexer state...", batchCount);
await _indexerContext.SaveChangesAsync();
batchCount++;
}
_logger.LogInformation(
"Finished adding {PackageIdCount} package ids to indexer state",
packageIds.Count);
}