public async Task RunAsync(CancellationToken cancellationToken = default)
{
if (!_options.Value.Enqueue)
{
_logger.LogError("V3 rebuild does not support direct processing at this time, please use --enqueue");
return;
}
var minCursor = DateTimeOffset.MinValue;
var maxCursor = await _cursor.GetAsync(cancellationToken);
if (maxCursor == null)
{
maxCursor = DateTimeOffset.MinValue;
}
_logger.LogInformation("Finding catalog leafs committed before time {Cursor}...", maxCursor);
var catalogClient = _clientFactory.CreateCatalogClient();
var(catalogIndex, catalogLeafItems) = await catalogClient.LoadCatalogAsync(
minCursor,
maxCursor.Value,
_logger,
cancellationToken);
var messages = catalogLeafItems
.Select(l => l.PackageId.ToLowerInvariant())
.Distinct()
.Select(ToMessage)
.ToList();
await _queue.SendAsync(messages, cancellationToken);
}
protected override async Task ExecuteAsync(CancellationToken cancellationToken)
{
var cursor = await _cursor.GetAsync(cancellationToken) ?? _options.Value.DefaultMinCursor;
var sleepDuration = TimeSpan.FromSeconds(_options.Value.SleepDurationSeconds);
while (!cancellationToken.IsCancellationRequested)
{
cursor = await ExecuteAsync(cursor, cancellationToken);
_logger.LogInformation("Sleeping for {SleepDurationSeconds} seconds...", sleepDuration.TotalSeconds);
await Task.Delay(sleepDuration, cancellationToken);
}
}
private async Task <DateTimeOffset> GetMinCommitTimestamp(CancellationToken cancellationToken)
{
var minCommitTimestamp = await _cursor.GetAsync(cancellationToken);
minCommitTimestamp = minCommitTimestamp
?? _options.DefaultMinCommitTimestamp
?? _options.MinCommitTimestamp;
if (minCommitTimestamp.Value < _options.MinCommitTimestamp)
{
minCommitTimestamp = _options.MinCommitTimestamp;
}
return(minCommitTimestamp.Value);
}
private async Task <DateTimeOffset> GetMinCommitTimestamp()
{
var minCommitTimestamp = await _cursor.GetAsync();
minCommitTimestamp = minCommitTimestamp
?? _settings.DefaultMinCommitTimestamp
?? _settings.MinCommitTimestamp;
if (minCommitTimestamp.Value < _settings.MinCommitTimestamp)
{
minCommitTimestamp = _settings.MinCommitTimestamp;
}
return(minCommitTimestamp.Value);
}
public async Task RunAsync(CancellationToken cancellationToken = default)
{
if (_options.Value.PackageId != null)
{
_logger.LogError("Adding individual packages is not supported at this time");
return;
}
var maxCursor = DateTimeOffset.MaxValue;
var minCursor = await _cursor.GetAsync(cancellationToken);
if (minCursor == null)
{
minCursor = DateTimeOffset.MinValue;
}
_logger.LogInformation("Finding catalog leafs committed after time {Cursor}...", minCursor);
var catalogClient = _clientFactory.CreateCatalogClient();
var(catalogIndex, catalogLeafItems) = await catalogClient.LoadCatalogAsync(
minCursor.Value,
maxCursor,
_logger,
cancellationToken);
_logger.LogInformation("Removing duplicate catalog leafs...");
catalogLeafItems = catalogLeafItems
.GroupBy(l => new PackageIdentity(l.PackageId, l.ParsePackageVersion()))
.Select(g => g.OrderByDescending(l => l.CommitTimestamp).First())
.Where(l => l.IsPackageDetails())
.ToList();
_logger.LogInformation("Processing {CatalogLeafs} catalog leafs...", catalogLeafItems.Count());
await _leafProcessor.ProcessAsync(catalogLeafItems, cancellationToken);
await _cursor.SetAsync(catalogIndex.CommitTimestamp, cancellationToken);
_logger.LogInformation("Finished processing catalog leafs");
}
public async Task RunAsync(CancellationToken cancellationToken)
{
var minCursor = DateTimeOffset.MinValue;
var maxCursor = await _cursor.GetAsync(cancellationToken);
if (maxCursor == null)
{
maxCursor = DateTimeOffset.MinValue;
}
_logger.LogInformation("Finding catalog leafs committed before time {Cursor}...", maxCursor);
var catalogClient = _clientFactory.CreateCatalogClient();
var(catalogIndex, catalogLeafItems) = await catalogClient.LoadCatalogAsync(
minCursor,
maxCursor.Value,
_logger,
cancellationToken);
_logger.LogInformation("Removing duplicate catalog leafs...");
var packageIds = catalogLeafItems
.GroupBy(l => new PackageIdentity(l.PackageId, l.ParsePackageVersion()))
.Select(g => g.OrderByDescending(l => l.CommitTimestamp).First())
.Where(l => l.IsPackageDetails())
.Select(l => l.PackageId)
.Distinct(StringComparer.OrdinalIgnoreCase)
.ToList();
_logger.LogInformation("Processing {PackageCount} packages", packageIds.Count);
var channel = Channel.CreateBounded <IndexAction <KeyedDocument> >(new BoundedChannelOptions(5000)
{
FullMode = BoundedChannelFullMode.Wait,
SingleWriter = false,
SingleReader = false,
});
var produceTask = ProduceIndexActionsAsync(
channel.Writer,
new ConcurrentBag <string>(packageIds),
cancellationToken);
var consumeTask1 = ConsumeIndexActionsAsync(
channel.Reader,
cancellationToken);
var consumeTask2 = ConsumeIndexActionsAsync(
channel.Reader,
cancellationToken);
var consumeTask3 = ConsumeIndexActionsAsync(
channel.Reader,
cancellationToken);
await Task.WhenAll(
produceTask,
consumeTask1,
consumeTask2,
consumeTask3);
_logger.LogInformation("Finished rebuilding search");
}