public async void GivenBatchServiceRunning_WhenEventReceivedOutsideWindow_ThenFlushEventsInQueueAndCreateNewWindow_Test()
{
var eventReader = new EventBatchingService(_eventConsumerService, _options, _checkpointClient, _logger);
var firstEventTime = DateTime.UtcNow.AddSeconds(-301);
var firstEvent = new EventMessage("0", new ReadOnlyMemory <byte>(), 1, 1, firstEventTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>()));
await eventReader.ConsumeEvent(firstEvent);
// first window end is: utc - 1 second
// utc - 301 seconds (firstEventTime) + 300 seconds (FlushTimespan)
var endWindow = firstEventTime.Add(TimeSpan.FromSeconds(_options.FlushTimespan));
Assert.Equal(endWindow, eventReader.GetPartition(firstEvent.PartitionId).GetPartitionWindow());
var nextEventTime = DateTime.UtcNow;
var nextEvent = new EventMessage("0", new ReadOnlyMemory <byte>(), 2, 2, nextEventTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>()));
await eventReader.ConsumeEvent(nextEvent);
// flush the 1 event that exists within the first window, verify event outside of window is in queue
await _eventConsumerService.Received(1).ConsumeEvents(Arg.Is <IEnumerable <IEventMessage> >(x => x.Count() == 1));
var expectedQueueCount = 1;
Assert.Equal(expectedQueueCount, eventReader.GetPartition(firstEvent.PartitionId).GetPartitionBatchCount());
// check that the window is incremented
var newEndWindow = endWindow.Add(TimeSpan.FromSeconds(_options.FlushTimespan));
Assert.Equal(newEndWindow, eventReader.GetPartition(firstEvent.PartitionId).GetPartitionWindow());
}
public async void GivenBatchServiceRunning_WhenNewEventConsumed_ThenCheckpointUpdatedLocally_Test()
{
_options.MaxEvents = 1;
var eventReader = new EventBatchingService(_eventConsumerService, _options, _checkpointClient, _logger);
var firstEventTime = DateTime.UtcNow.AddSeconds(-301);
var firstEvent = new EventMessage("0", new ReadOnlyMemory <byte>(), 1, 1, firstEventTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>()));
await eventReader.ConsumeEvent(firstEvent);
await _checkpointClient.Received(1).SetCheckpointAsync(firstEvent);
}
public async void GivenBatchServiceRunning_WhenFlushWindowHasNotPassedAndMaxWaitEventReceived_ThenDoNotFlush_Test()
{
var eventReader = new EventBatchingService(_eventConsumerService, _options, _checkpointClient, _logger);
var firstEventTime = DateTime.UtcNow.AddSeconds(-30);
var firstEvent = new EventMessage("0", new ReadOnlyMemory <byte>(), 1, 1, firstEventTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>()));
await eventReader.ConsumeEvent(firstEvent);
var maxWaitEvent = new MaximumWaitEvent("0", DateTime.UtcNow.AddSeconds(-10));
await eventReader.ConsumeEvent(maxWaitEvent);
await _eventConsumerService.Received(0).ConsumeEvents(Arg.Any <IEnumerable <EventMessage> >());
}
public virtual EventProcessor ResolveEventProcessor(IServiceProvider serviceProvider)
{
var eventConsumerService = serviceProvider.GetRequiredService <IEventConsumerService>();
var checkpointClient = serviceProvider.GetRequiredService <StorageCheckpointClient>();
var logger = serviceProvider.GetRequiredService <ITelemetryLogger>();
var eventBatchingOptions = new EventBatchingOptions();
Configuration.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions);
var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger);
var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger);
return(eventHubReader);
}
public static EventProcessor GetEventProcessor(
IConfiguration config,
IEventConsumerService eventConsumerService,
ICheckpointClient checkpointClient,
ITelemetryLogger logger)
{
var eventBatchingOptions = new EventBatchingOptions();
config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions);
var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger);
var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger);
return(eventHubReader);
}
public static async Task Main()
{
var config = GetEnvironmentConfig();
// determine which event hub to read from
var eventHub = Environment.GetEnvironmentVariable("WEBJOBS_NAME");
if (eventHub == null)
{
eventHub = config.GetSection("Console:EventHub").Value;
}
System.Console.WriteLine($"Reading from event hub: {eventHub}");
System.Console.WriteLine($"Logs and Metrics will be written to Application Insights");
var eventHubOptions = GetEventHubInfo(config, eventHub);
EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubConnectionString);
EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubName);
var eventBatchingOptions = new EventBatchingOptions();
config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions);
var serviceProvider = GetRequiredServiceProvider(config, eventHub);
var logger = serviceProvider.GetRequiredService <ITelemetryLogger>();
var eventConsumers = GetEventConsumers(config, eventHub, serviceProvider, logger);
var storageOptions = new StorageCheckpointOptions();
config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions);
storageOptions.BlobPrefix = eventHub;
var checkpointClient = new StorageCheckpointClient(storageOptions, logger);
var eventConsumerService = new EventConsumerService(eventConsumers, logger);
var ct = new CancellationToken();
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(storageOptions.BlobStorageConnectionString, storageOptions.BlobContainerName);
var eventProcessorClientOptions = new EventProcessorClientOptions();
eventProcessorClientOptions.MaximumWaitTime = TimeSpan.FromSeconds(60);
EventProcessorClient client = new EventProcessorClient(storageClient, consumerGroup, eventHubOptions.EventHubConnectionString, eventHubOptions.EventHubName, eventProcessorClientOptions);
var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger);
var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger);
await eventHubReader.RunAsync(client, ct);
}
public async void GivenBatchServiceCreated_WhenFirstEventReceived_ThenCreateQueueAndWindow_Test()
{
var eventReader = new EventBatchingService(_eventConsumerService, _options, _checkpointClient, _logger);
var enqueuedTime = DateTime.UtcNow;
var event1 = new EventMessage("0", new ReadOnlyMemory <byte>(), 1, 1, enqueuedTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>()));
await eventReader.ConsumeEvent(event1);
var endWindow = enqueuedTime.Add(TimeSpan.FromSeconds(_options.FlushTimespan));
var partitionWindow = eventReader.GetPartition("0").GetPartitionWindow();
Assert.Equal(endWindow, partitionWindow);
}
public async void GivenBatchServiceCreated_WhenNextEventReadAndNoEventsInCurrentWindow_ThenAdvanceWindowToIncludeCurrentEvent_Test()
{
var eventReader = new EventBatchingService(_eventConsumerService, _options, _checkpointClient, _logger);
var firstEventTime = DateTime.UtcNow.AddSeconds(-901);
var firstEvent = new EventMessage("0", new ReadOnlyMemory <byte>(), 1, 1, firstEventTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>()));
await eventReader.ConsumeEvent(firstEvent);
var endWindow = firstEventTime.Add(TimeSpan.FromSeconds(_options.FlushTimespan));
Assert.Equal(endWindow, eventReader.GetPartition("0").GetPartitionWindow());
var nextEventTime = DateTime.UtcNow;
var nextEvent = new EventMessage("0", new ReadOnlyMemory <byte>(), 2, 2, nextEventTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>()));
await eventReader.ConsumeEvent(nextEvent);
// check that the window is incremented up until next event is included in the current window
var currentWindowStart = eventReader.GetPartition(nextEvent.PartitionId).GetPartitionWindow().Add(-TimeSpan.FromSeconds(_options.FlushTimespan));
var currrentWindowEnd = eventReader.GetPartition(nextEvent.PartitionId).GetPartitionWindow();
Assert.InRange(nextEventTime, currentWindowStart, currrentWindowEnd);
}
public async void GivenBatchServiceRunning_WhenMaxEventsReached_ThenFlushEventsAndKeepWindow_Test()
{
_options.MaxEvents = 3;
var eventReader = new EventBatchingService(_eventConsumerService, _options, _checkpointClient, _logger);
var newEventTime = DateTime.UtcNow;
var newEvent = new EventMessage("0", new ReadOnlyMemory <byte>(), 1, 1, newEventTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>()));
await eventReader.ConsumeEvent(newEvent);
await eventReader.ConsumeEvent(newEvent);
var endWindow = newEventTime.Add(TimeSpan.FromSeconds(_options.FlushTimespan));
Assert.Equal(endWindow, eventReader.GetPartition(newEvent.PartitionId).GetPartitionWindow());
await _eventConsumerService.Received(0).ConsumeEvents(Arg.Any <IEnumerable <IEventMessage> >());
await eventReader.ConsumeEvent(newEvent);
await _eventConsumerService.Received(1).ConsumeEvents(Arg.Any <IEnumerable <IEventMessage> >());
Assert.Equal(endWindow, eventReader.GetPartition(newEvent.PartitionId).GetPartitionWindow());
}