private async void OnProductChanged(object sender, ProductChangeEventArgs e)
{
await _subscriberStream.WriteAsync(e.Product);
}
public async Task <DateTimeOffset> ProduceAsync(
ChannelWriter <CatalogLeafItem> channel,
DateTimeOffset minCursor,
DateTimeOffset maxCursor,
CancellationToken cancellationToken)
{
_logger.LogInformation("Fetching catalog index...");
var client = _factory.CreateCatalogClient();
var catalogIndex = await client.GetIndexAsync(cancellationToken);
var pages = catalogIndex.GetPagesInBounds(minCursor, maxCursor);
var maxPages = _options.Value.MaxPages;
if (maxPages.HasValue)
{
pages = pages.Take(maxPages.Value).ToList();
}
if (!pages.Any() || minCursor == maxCursor)
{
_logger.LogInformation("No pending leaf items on the catalog.");
channel.Complete();
return(minCursor);
}
var work = new ConcurrentBag <CatalogPageItem>(pages);
var workers = Math.Min(_options.Value.ProducerWorkers, pages.Count);
_logger.LogInformation(
"Fetching {Pages} catalog pages using {ProducerWorkers} workers...",
pages.Count,
workers);
var tasks = Enumerable
.Repeat(0, workers)
.Select(async _ =>
{
await Task.Yield();
while (work.TryTake(out var pageItem))
{
var done = false;
while (!done)
{
try
{
_logger.LogDebug("Processing catalog page {PageUrl}...", pageItem.CatalogPageUrl);
var page = await client.GetPageAsync(pageItem.CatalogPageUrl, cancellationToken);
foreach (var leaf in page.Items)
{
// Don't process leaves that are not within the cursors.
if (leaf.CommitTimestamp <= minCursor)
{
continue;
}
if (leaf.CommitTimestamp > maxCursor)
{
continue;
}
if (!channel.TryWrite(leaf))
{
await channel.WriteAsync(leaf, cancellationToken);
}
}
_logger.LogDebug("Processed catalog page {PageUrl}.", pageItem.CatalogPageUrl);
done = true;
}
catch (Exception e) when(!cancellationToken.IsCancellationRequested)
{
_logger.LogError(e, "Retrying catalog page {PageUrl} in 5 seconds...", pageItem.CatalogPageUrl);
await Task.Delay(TimeSpan.FromSeconds(5), cancellationToken);
}
}
}
});
await Task.WhenAll(tasks);
var cursor = pages.Last().CommitTimestamp;
_logger.LogInformation("Fetched catalog pages up to cursor {Cursor}", cursor);
channel.Complete();
return(cursor);
}
static async Task PingPongWithoutNewTask(ChannelWriter <int> out_, ChannelReader <int> in_)
{
await out_.WriteAsync(1 + await in_.ReadAsync());
}
public static Task <long> WriteAllConcurrentlyAsync <T>(this ChannelWriter <T> target,
int maxConcurrency, IEnumerable <ValueTask <T> > source, bool complete = false, CancellationToken cancellationToken = default)
{
if (target is null)
{
throw new ArgumentNullException(nameof(target));
}
if (source is null)
{
throw new ArgumentNullException(nameof(source));
}
if (maxConcurrency < 1)
{
throw new ArgumentOutOfRangeException(nameof(maxConcurrency), maxConcurrency, "Must be at least 1.");
}
Contract.EndContractBlock();
if (cancellationToken.IsCancellationRequested)
{
return(Task.FromCanceled <long>(cancellationToken));
}
if (maxConcurrency == 1)
{
return(target.WriteAllAsync(source, complete, true, cancellationToken).AsTask());
}
var shouldWait = target
.WaitToWriteAndThrowIfClosedAsync("The target channel was closed before writing could begin.", cancellationToken)
.AsTask(); // ValueTasks can only have a single await.
var errorTokenSource = new CancellationTokenSource();
var errorToken = errorTokenSource.Token;
var enumerator = source.GetEnumerator();
var writers = new Task <long> [maxConcurrency];
for (var w = 0; w < maxConcurrency; w++)
{
writers[w] = WriteAllAsyncCore();
}
return(Task
.WhenAll(writers)
.ContinueWith(t =>
{
errorTokenSource.Dispose();
if (complete)
{
target.Complete(t.Exception);
}
if (t.IsFaulted)
{
return Task.FromException <long>(t.Exception);
}
if (t.IsCanceled)
{
return Task.FromCanceled <long>(cancellationToken);
}
return Task.FromResult(t.Result.Sum());
},
CancellationToken.None,
TaskContinuationOptions.ExecuteSynchronously,
TaskScheduler.Current)
.Unwrap());
// returns false if there's no more (wasn't cancelled).
async Task <long> WriteAllAsyncCore()
{
await Task.Yield();
try
{
await shouldWait.ConfigureAwait(false);
long count = 0;
var next = new ValueTask();
var potentiallyCancelled = true; // if it completed and actually returned false, no need to bubble the cancellation since it actually completed.
while (!errorToken.IsCancellationRequested &&
!cancellationToken.IsCancellationRequested &&
(potentiallyCancelled = TryMoveNextSynchronized(enumerator, out var e)))
{
var value = await e.ConfigureAwait(false);
await next.ConfigureAwait(false);
count++;
next = target.TryWrite(value) // do this to avoid unneccesary early cancel.
? new ValueTask()
: target.WriteAsync(value, cancellationToken);
}
await next.ConfigureAwait(false);
if (potentiallyCancelled)
{
cancellationToken.ThrowIfCancellationRequested();
}
return(count);
}
catch
{
errorTokenSource.Cancel();
throw;
}
}
}
public override async Task RefreshAsync()
{
await writer.WriteAsync(1, cancellationTokenSource.Token);
}