public async Task RunAsync(CancellationToken cancellationToken = default)
{
// Prepare the processing.
ThreadPool.SetMinThreads(ParallelHelper.MaxDegreeOfParallelism, completionPortThreads: 4);
ServicePointManager.DefaultConnectionLimit = ParallelHelper.MaxDegreeOfParallelism;
ServicePointManager.MaxServicePointIdleTime = 10000;
var cursor = (DateTimeOffset?)DateTimeOffset.MinValue;
//var cursor = await _cursor.GetAsync(cancellationToken);
//if (cursor == null)
//{
// cursor = DateTimeOffset.MinValue;
//}
_logger.LogInformation("Finding catalog leafs comitted after time {Cursor}...", cursor);
var catalogClient = await _clientFactory.CreateCatalogClientAsync(cancellationToken);
var catalogIndex = await catalogClient.GetIndexAsync(cancellationToken);
var catalogLeafItems = await GetCatalogLeafItems(catalogClient, catalogIndex, cursor.Value, cancellationToken);
catalogLeafItems = DeduplicateCatalogLeafItems(catalogLeafItems);
_logger.LogInformation("Processing {CatalogLeafs} catalog leafs...", catalogLeafItems.Count());
await ProcessCatalogLeafsAsync(catalogLeafItems, cancellationToken);
//await _cursor.SetAsync(catalogIndex.CommitTimestamp, cancellationToken);
_logger.LogInformation("Finished processing catalog leafs");
}
public async Task <CatalogIndex> GetIndexAsync(CancellationToken cancellationToken = default)
{
var client = await _clientfactory.CreateCatalogClientAsync(cancellationToken);
return(await client.GetIndexAsync(cancellationToken));
}