async Task Consume()
{
var prefetchCount = Math.Max(1000, _consumerContext.ReceiveSettings.CheckpointMessageCount / 10);
var executor = new ChannelExecutor(prefetchCount, _consumerContext.ReceiveSettings.ConcurrencyLimit);
await _consumerContext.Subscribe().ConfigureAwait(false);
SetReady();
try
{
while (!_cancellationTokenSource.IsCancellationRequested)
{
ConsumeResult <TKey, TValue> consumeResult = await _consumerContext.Consume(_cancellationTokenSource.Token).ConfigureAwait(false);
await executor.Push(() => Handle(consumeResult), Stopping).ConfigureAwait(false);
}
}
catch (OperationCanceledException exception) when(exception.CancellationToken == Stopping ||
exception.CancellationToken == _cancellationTokenSource.Token)
{
}
catch (Exception exception)
{
LogContext.Error?.Log(exception, "Consume Loop faulted");
}
finally
{
await executor.DisposeAsync().ConfigureAwait(false);
}
SetCompleted(TaskUtil.Completed);
}
async Task <int> ReceiveMessages(int messageLimit, ChannelExecutor executor)
{
IList <Message> messages = await _client.ReceiveMessages(_receiveSettings.EntityName, messageLimit, _receiveSettings.WaitTimeSeconds, Stopping)
.ConfigureAwait(false);
await Task.WhenAll(messages.Select(message => executor.Push(() => HandleMessage(message), Stopping))).ConfigureAwait(false);
return(messages.Count);
}
public async Task Connect(CancellationToken cancellationToken)
{
var logContext = _context.LogContext;
var inputAddress = _context.InputAddress;
LogContext.SetCurrentIfNull(logContext);
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
async Task ProcessEventAsync(ProcessEventArgs arg)
{
LogContext.SetCurrentIfNull(logContext);
try
{
await _executor.Push(() => _transport.Handle(arg, cancellationToken), _cancellationTokenSource.Token).ConfigureAwait(false);
}
catch (OperationCanceledException e) when(e.CancellationToken == _cancellationTokenSource.Token)
{
}
}
async Task ProcessErrorAsync(ProcessErrorEventArgs arg)
{
var faultedEvent = new ReceiveTransportFaultedEvent(inputAddress, arg.Exception);
await _context.TransportObservers.Faulted(faultedEvent).ConfigureAwait(false);
await _context.EndpointObservers.Faulted(new ReceiveEndpointFaultedEvent(faultedEvent, this)).ConfigureAwait(false);
}
_processor.ProcessEventAsync += ProcessEventAsync;
_processor.ProcessErrorAsync += ProcessErrorAsync;
try
{
await _blobContainerClient.CreateIfNotExistsAsync(cancellationToken : cancellationToken).ConfigureAwait(false);
}
catch (RequestFailedException exception)
{
LogContext.Warning?.Log(exception, "Azure Blob Container does not exist: {Address}", _blobContainerClient.Uri);
}
LogContext.Debug?.Log("EventHub processor starting: {EventHubName}", _processor.EventHubName);
await _processor.StartProcessingAsync(cancellationToken).ConfigureAwait(false);
await _context.TransportObservers.Ready(new ReceiveTransportReadyEvent(inputAddress)).ConfigureAwait(false);
var endpointReadyEvent = new ReceiveEndpointReadyEvent(_context.InputAddress, this, true);
_started.TrySetResult(endpointReadyEvent);
await _context.EndpointObservers.Ready(endpointReadyEvent).ConfigureAwait(false);
}
async Task HandleMessage(ProcessEventArgs eventArgs)
{
LogContext.SetCurrentIfNull(_context.LogContext);
try
{
await _executor.Push(() => Handle(eventArgs), Stopping).ConfigureAwait(false);
}
catch (OperationCanceledException e) when(e.CancellationToken == Stopping)
{
}
}
public Task Start(CancellationToken cancellationToken)
{
var logContext = _context.LogContext;
LogContext.SetCurrentIfNull(logContext);
_cancellationTokenSource = CancellationTokenSource.CreateLinkedTokenSource(cancellationToken);
// TODO this feels like it needs to be more resilient, right now one error and it exits
_consumerTask = Task.Run(async() =>
{
var inputAddress = _context.InputAddress;
await _context.TransportObservers.Ready(new ReceiveTransportReadyEvent(inputAddress)).ConfigureAwait(false);
var endpointReadyEvent = new ReceiveEndpointReadyEvent(_context.InputAddress, this, true);
_started.TrySetResult(endpointReadyEvent);
await _context.EndpointObservers.Ready(endpointReadyEvent).ConfigureAwait(false);
LogContext.Debug?.Log("Kafka consumer started: {InputAddress}", _context.InputAddress);
while (!_cancellationTokenSource.IsCancellationRequested)
{
LogContext.SetCurrentIfNull(logContext);
try
{
ConsumeResult <TKey, TValue> message = _consumer.Consume(_cancellationTokenSource.Token);
await _executor.Push(() => _transport.Handle(message, cancellationToken), _cancellationTokenSource.Token).ConfigureAwait(false);
}
catch (OperationCanceledException e) when(e.CancellationToken == _cancellationTokenSource.Token)
{
}
catch (Exception exception)
{
var faultedEvent = new ReceiveTransportFaultedEvent(inputAddress, exception);
await _context.TransportObservers.Faulted(faultedEvent).ConfigureAwait(false);
await _context.EndpointObservers.Faulted(new ReceiveEndpointFaultedEvent(faultedEvent, this)).ConfigureAwait(false);
throw;
}
}
}, cancellationToken);
_consumer.Subscribe(_topic);
return(_consumerTask.IsCompleted ? _consumerTask : TaskUtil.Completed);
}
async Task <int> ReceiveAndHandleMessages(int messageLimit, ChannelExecutor executor)
{
try
{
IList <Message> messages = await _client.ReceiveMessages(_receiveSettings.EntityName, messageLimit, _receiveSettings.WaitTimeSeconds, Stopping)
.ConfigureAwait(false);
foreach (var message in messages)
{
await executor.Push(() => HandleMessage(message), Stopping).ConfigureAwait(false);
}
return(messages.Count);
}
catch (OperationCanceledException)
{
return(0);
}
}
public override void Post(SendOrPostCallback callback, object state)
{
if (callback == null)
{
throw new ArgumentNullException(nameof(callback));
}
if (_completed)
{
throw new TaskSchedulerException("The synchronization context was already completed");
}
try
{
_executor?.Push(async() => callback(state), _cancellationToken);
}
catch (InvalidOperationException)
{
}
}
public async Task Should_be_performant()
{
long value = 0;
var executor = new ChannelExecutor(10);
const int limit = 10000;
for (var i = 0; i < limit; i++)
{
await executor.Push(() =>
{
Interlocked.Increment(ref value);
return(Task.CompletedTask);
});
}
await executor.DisposeAsync().ConfigureAwait(false);
Assert.That(value, Is.EqualTo(10000));
}
public Task Deliver(DeliveryContext <InMemoryTransportMessage> context)
{
return(context.WasAlreadyDelivered(this)
? Task.FromResult(false)
: _executor.Push(() => DispatchMessage(context.Package)));
}