public async void GivenEventConsumer_WhenEventConsumerThrowsException_ThenRetryEventConsumer_Test()
{
var retries = 0;
var initialBatch = new List <EventMessage>()
{
new EventMessage("0", new ReadOnlyMemory <byte>(), 1, 1, new DateTime(2020, 12, 31, 5, 10, 20), new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>()))
};
var logger = Substitute.For <ITelemetryLogger>();
var eventConsumer = Substitute.For <IEventConsumer>();
// fail to consume events 3 times in a row, then succeed
eventConsumer.When(x => x.ConsumeAsync(initialBatch))
.Do(x => { if (retries < 3)
{
retries++; throw new Exception("failure");
}
});
var eventEventConsumers = new List <IEventConsumer>()
{
eventConsumer
};
var eventConsumerService = new EventConsumerService(eventEventConsumers, logger);
// 3 retries followed by a successful call
await eventConsumerService.ConsumeEvents(initialBatch);
await eventConsumer.Received(4).ConsumeAsync(initialBatch);
}
public static async Task Main()
{
var config = GetEnvironmentConfig();
var eventHubName = GetEventHubName(config);
var eventHubOptions = GetEventHubInfo(config, eventHubName);
EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubConnectionString);
EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubName);
var serviceCollection = GetRequiredServiceCollection(config, eventHubName);
var serviceProvider = serviceCollection.BuildServiceProvider();
var logger = serviceProvider.GetRequiredService <ITelemetryLogger>();
var blobContainerClientFactory = new BlobContainerClientFactory();
var eventConsumers = GetEventConsumers(config, eventHubName, serviceProvider, blobContainerClientFactory, logger);
var storageOptions = new StorageCheckpointOptions();
config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions);
var checkpointContainerOptions = new BlobContainerClientOptions();
config.GetSection("CheckpointStorage").Bind(checkpointContainerOptions);
var storageCheckpointClient = GetStorageCheckpointClient(blobContainerClientFactory, checkpointContainerOptions, storageOptions, logger, eventHubName);
var eventConsumerService = new EventConsumerService(eventConsumers, logger);
var incomingEventReader = GetEventProcessorClient(storageCheckpointClient.GetBlobContainerClient(), eventHubOptions);
var eventHubReader = GetEventProcessor(config, eventConsumerService, storageCheckpointClient, logger);
System.Console.WriteLine($"Reading from event hub: {eventHubName}");
var ct = new CancellationToken();
await eventHubReader.RunAsync(incomingEventReader, ct);
}
public static async Task Main()
{
var config = GetEnvironmentConfig();
// determine which event hub to read from
var eventHub = Environment.GetEnvironmentVariable("WEBJOBS_NAME");
if (eventHub == null)
{
eventHub = config.GetSection("Console:EventHub").Value;
}
System.Console.WriteLine($"Reading from event hub: {eventHub}");
System.Console.WriteLine($"Logs and Metrics will be written to Application Insights");
var eventHubOptions = GetEventHubInfo(config, eventHub);
EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubConnectionString);
EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubName);
var eventBatchingOptions = new EventBatchingOptions();
config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions);
var serviceProvider = GetRequiredServiceProvider(config, eventHub);
var logger = serviceProvider.GetRequiredService <ITelemetryLogger>();
var eventConsumers = GetEventConsumers(config, eventHub, serviceProvider, logger);
var storageOptions = new StorageCheckpointOptions();
config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions);
storageOptions.BlobPrefix = eventHub;
var checkpointClient = new StorageCheckpointClient(storageOptions, logger);
var eventConsumerService = new EventConsumerService(eventConsumers, logger);
var ct = new CancellationToken();
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(storageOptions.BlobStorageConnectionString, storageOptions.BlobContainerName);
var eventProcessorClientOptions = new EventProcessorClientOptions();
eventProcessorClientOptions.MaximumWaitTime = TimeSpan.FromSeconds(60);
EventProcessorClient client = new EventProcessorClient(storageClient, consumerGroup, eventHubOptions.EventHubConnectionString, eventHubOptions.EventHubName, eventProcessorClientOptions);
var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger);
var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger);
await eventHubReader.RunAsync(client, ct);
}
public async void GivenEventConsumer_WhenEventConsumerThrowsException_ThenExceptionIsLogged_()
{
var initialBatch = new List <EventMessage>()
{
new EventMessage("0", new ReadOnlyMemory <byte>(), 1, 1, new DateTime(2020, 12, 31, 5, 10, 20), new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>()))
};
var logger = Substitute.For <ITelemetryLogger>();
var eventConsumer = Substitute.For <IEventConsumer>();
eventConsumer.ConsumeAsync(Arg.Any <IEnumerable <IEventMessage> >()).ReturnsForAnyArgs(Task.FromException(new Exception("failure")));
var eventEventConsumers = new List <IEventConsumer>()
{
eventConsumer
};
var eventConsumerService = new EventConsumerService(eventEventConsumers, logger);
await eventConsumerService.ConsumeEvents(initialBatch);
logger.Received(1).LogError(Arg.Is <Exception>(ex => ex.Message == "failure"));
await eventConsumer.Received(1).ConsumeAsync(initialBatch);
}
// Processes two types of events
// 1) Event hub events
// 2) Maximum wait events. These are generated when we have not received an event hub
// event for a certain time period and this event is used to flush events in the current window.
protected virtual async Task ProcessEventHandler(ProcessEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
// The event arguments contain a cancellation token that the EventProcessorClient uses to signal the handler that processing should cease as soon as possible.
// This is most commonly seen when the EventProcessorClient is stopping or has encountered an unrecoverable problem.
Logger.LogTrace($"ProcessEventArgs contain a cancellation request {eventArgs.Partition.PartitionId}");
return;
}
IEventMessage evt;
if (eventArgs.HasEvent)
{
evt = EventMessageFactory.CreateEvent(eventArgs);
}
else
{
evt = new MaximumWaitEvent(eventArgs.Partition.PartitionId, DateTime.UtcNow);
}
await EventConsumerService.ConsumeEvent(evt);
}
public TodosController(ILogger logger, DatastoreService datastoreService, EventConsumerService eventConsumerService)
{
_logger = logger;
_datastoreService = datastoreService;
_eventConsumerService = eventConsumerService;
}