public async void GivenBatchServiceRunning_WhenEventReceivedOutsideWindow_ThenFlushEventsInQueueAndCreateNewWindow_Test() { var eventReader = new EventBatchingService(_eventConsumerService, _options, _checkpointClient, _logger); var firstEventTime = DateTime.UtcNow.AddSeconds(-301); var firstEvent = new EventMessage("0", new ReadOnlyMemory <byte>(), 1, 1, firstEventTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>())); await eventReader.ConsumeEvent(firstEvent); // first window end is: utc - 1 second // utc - 301 seconds (firstEventTime) + 300 seconds (FlushTimespan) var endWindow = firstEventTime.Add(TimeSpan.FromSeconds(_options.FlushTimespan)); Assert.Equal(endWindow, eventReader.GetPartition(firstEvent.PartitionId).GetPartitionWindow()); var nextEventTime = DateTime.UtcNow; var nextEvent = new EventMessage("0", new ReadOnlyMemory <byte>(), 2, 2, nextEventTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>())); await eventReader.ConsumeEvent(nextEvent); // flush the 1 event that exists within the first window, verify event outside of window is in queue await _eventConsumerService.Received(1).ConsumeEvents(Arg.Is <IEnumerable <IEventMessage> >(x => x.Count() == 1)); var expectedQueueCount = 1; Assert.Equal(expectedQueueCount, eventReader.GetPartition(firstEvent.PartitionId).GetPartitionBatchCount()); // check that the window is incremented var newEndWindow = endWindow.Add(TimeSpan.FromSeconds(_options.FlushTimespan)); Assert.Equal(newEndWindow, eventReader.GetPartition(firstEvent.PartitionId).GetPartitionWindow()); }
public async void GivenBatchServiceRunning_WhenNewEventConsumed_ThenCheckpointUpdatedLocally_Test() { _options.MaxEvents = 1; var eventReader = new EventBatchingService(_eventConsumerService, _options, _checkpointClient, _logger); var firstEventTime = DateTime.UtcNow.AddSeconds(-301); var firstEvent = new EventMessage("0", new ReadOnlyMemory <byte>(), 1, 1, firstEventTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>())); await eventReader.ConsumeEvent(firstEvent); await _checkpointClient.Received(1).SetCheckpointAsync(firstEvent); }
public async void GivenBatchServiceRunning_WhenFlushWindowHasNotPassedAndMaxWaitEventReceived_ThenDoNotFlush_Test() { var eventReader = new EventBatchingService(_eventConsumerService, _options, _checkpointClient, _logger); var firstEventTime = DateTime.UtcNow.AddSeconds(-30); var firstEvent = new EventMessage("0", new ReadOnlyMemory <byte>(), 1, 1, firstEventTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>())); await eventReader.ConsumeEvent(firstEvent); var maxWaitEvent = new MaximumWaitEvent("0", DateTime.UtcNow.AddSeconds(-10)); await eventReader.ConsumeEvent(maxWaitEvent); await _eventConsumerService.Received(0).ConsumeEvents(Arg.Any <IEnumerable <EventMessage> >()); }
public virtual EventProcessor ResolveEventProcessor(IServiceProvider serviceProvider) { var eventConsumerService = serviceProvider.GetRequiredService <IEventConsumerService>(); var checkpointClient = serviceProvider.GetRequiredService <StorageCheckpointClient>(); var logger = serviceProvider.GetRequiredService <ITelemetryLogger>(); var eventBatchingOptions = new EventBatchingOptions(); Configuration.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions); var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger); var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger); return(eventHubReader); }
public static EventProcessor GetEventProcessor( IConfiguration config, IEventConsumerService eventConsumerService, ICheckpointClient checkpointClient, ITelemetryLogger logger) { var eventBatchingOptions = new EventBatchingOptions(); config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions); var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger); var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger); return(eventHubReader); }
public static async Task Main() { var config = GetEnvironmentConfig(); // determine which event hub to read from var eventHub = Environment.GetEnvironmentVariable("WEBJOBS_NAME"); if (eventHub == null) { eventHub = config.GetSection("Console:EventHub").Value; } System.Console.WriteLine($"Reading from event hub: {eventHub}"); System.Console.WriteLine($"Logs and Metrics will be written to Application Insights"); var eventHubOptions = GetEventHubInfo(config, eventHub); EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubConnectionString); EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubName); var eventBatchingOptions = new EventBatchingOptions(); config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions); var serviceProvider = GetRequiredServiceProvider(config, eventHub); var logger = serviceProvider.GetRequiredService <ITelemetryLogger>(); var eventConsumers = GetEventConsumers(config, eventHub, serviceProvider, logger); var storageOptions = new StorageCheckpointOptions(); config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions); storageOptions.BlobPrefix = eventHub; var checkpointClient = new StorageCheckpointClient(storageOptions, logger); var eventConsumerService = new EventConsumerService(eventConsumers, logger); var ct = new CancellationToken(); string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName; BlobContainerClient storageClient = new BlobContainerClient(storageOptions.BlobStorageConnectionString, storageOptions.BlobContainerName); var eventProcessorClientOptions = new EventProcessorClientOptions(); eventProcessorClientOptions.MaximumWaitTime = TimeSpan.FromSeconds(60); EventProcessorClient client = new EventProcessorClient(storageClient, consumerGroup, eventHubOptions.EventHubConnectionString, eventHubOptions.EventHubName, eventProcessorClientOptions); var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger); var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger); await eventHubReader.RunAsync(client, ct); }
public async void GivenBatchServiceCreated_WhenFirstEventReceived_ThenCreateQueueAndWindow_Test() { var eventReader = new EventBatchingService(_eventConsumerService, _options, _checkpointClient, _logger); var enqueuedTime = DateTime.UtcNow; var event1 = new EventMessage("0", new ReadOnlyMemory <byte>(), 1, 1, enqueuedTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>())); await eventReader.ConsumeEvent(event1); var endWindow = enqueuedTime.Add(TimeSpan.FromSeconds(_options.FlushTimespan)); var partitionWindow = eventReader.GetPartition("0").GetPartitionWindow(); Assert.Equal(endWindow, partitionWindow); }
public async void GivenBatchServiceCreated_WhenNextEventReadAndNoEventsInCurrentWindow_ThenAdvanceWindowToIncludeCurrentEvent_Test() { var eventReader = new EventBatchingService(_eventConsumerService, _options, _checkpointClient, _logger); var firstEventTime = DateTime.UtcNow.AddSeconds(-901); var firstEvent = new EventMessage("0", new ReadOnlyMemory <byte>(), 1, 1, firstEventTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>())); await eventReader.ConsumeEvent(firstEvent); var endWindow = firstEventTime.Add(TimeSpan.FromSeconds(_options.FlushTimespan)); Assert.Equal(endWindow, eventReader.GetPartition("0").GetPartitionWindow()); var nextEventTime = DateTime.UtcNow; var nextEvent = new EventMessage("0", new ReadOnlyMemory <byte>(), 2, 2, nextEventTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>())); await eventReader.ConsumeEvent(nextEvent); // check that the window is incremented up until next event is included in the current window var currentWindowStart = eventReader.GetPartition(nextEvent.PartitionId).GetPartitionWindow().Add(-TimeSpan.FromSeconds(_options.FlushTimespan)); var currrentWindowEnd = eventReader.GetPartition(nextEvent.PartitionId).GetPartitionWindow(); Assert.InRange(nextEventTime, currentWindowStart, currrentWindowEnd); }
public async void GivenBatchServiceRunning_WhenMaxEventsReached_ThenFlushEventsAndKeepWindow_Test() { _options.MaxEvents = 3; var eventReader = new EventBatchingService(_eventConsumerService, _options, _checkpointClient, _logger); var newEventTime = DateTime.UtcNow; var newEvent = new EventMessage("0", new ReadOnlyMemory <byte>(), 1, 1, newEventTime, new Dictionary <string, object>(), new ReadOnlyDictionary <string, object>(new Dictionary <string, object>())); await eventReader.ConsumeEvent(newEvent); await eventReader.ConsumeEvent(newEvent); var endWindow = newEventTime.Add(TimeSpan.FromSeconds(_options.FlushTimespan)); Assert.Equal(endWindow, eventReader.GetPartition(newEvent.PartitionId).GetPartitionWindow()); await _eventConsumerService.Received(0).ConsumeEvents(Arg.Any <IEnumerable <IEventMessage> >()); await eventReader.ConsumeEvent(newEvent); await _eventConsumerService.Received(1).ConsumeEvents(Arg.Any <IEnumerable <IEventMessage> >()); Assert.Equal(endWindow, eventReader.GetPartition(newEvent.PartitionId).GetPartitionWindow()); }