public virtual EventProcessor ResolveEventProcessor(IServiceProvider serviceProvider) { var eventConsumerService = serviceProvider.GetRequiredService <IEventConsumerService>(); var checkpointClient = serviceProvider.GetRequiredService <StorageCheckpointClient>(); var logger = serviceProvider.GetRequiredService <ITelemetryLogger>(); var eventBatchingOptions = new EventBatchingOptions(); Configuration.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions); var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger); var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger); return(eventHubReader); }
public void SendForASpecificPartitionDoesNotAllowAPartitionHashKey() { var batchingOptions = new EventBatchingOptions { PartitionKey = "testKey" }; var events = new[] { new EventData(new byte[] { 0x44, 0x66, 0x88 }) }; var transportSender = new ObservableTransportSenderMock(); var sender = new EventSender(transportSender, "dummy", new EventSenderOptions { PartitionId = "1" }); Assert.That(async() => await sender.SendAsync(events, batchingOptions), Throws.InvalidOperationException); }
public async Task SendInvokesTheTransportSender() { var events = Mock.Of <IEnumerable <EventData> >(); var options = new EventBatchingOptions(); var transportSender = new ObservableTransportSenderMock(); var sender = new EventSender(transportSender, "dummy", new EventSenderOptions()); await sender.SendAsync(events, options); (var calledWithEvents, var calledWithOptions) = transportSender.SendCalledWithParameters; Assert.That(calledWithEvents, Is.SameAs(events), "The events should be the same instance."); Assert.That(calledWithOptions, Is.SameAs(options), "The options should be the same instance"); }
public static EventProcessor GetEventProcessor( IConfiguration config, IEventConsumerService eventConsumerService, ICheckpointClient checkpointClient, ITelemetryLogger logger) { var eventBatchingOptions = new EventBatchingOptions(); config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions); var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger); var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger); return(eventHubReader); }
public static async Task Main() { var config = GetEnvironmentConfig(); // determine which event hub to read from var eventHub = Environment.GetEnvironmentVariable("WEBJOBS_NAME"); if (eventHub == null) { eventHub = config.GetSection("Console:EventHub").Value; } System.Console.WriteLine($"Reading from event hub: {eventHub}"); System.Console.WriteLine($"Logs and Metrics will be written to Application Insights"); var eventHubOptions = GetEventHubInfo(config, eventHub); EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubConnectionString); EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubName); var eventBatchingOptions = new EventBatchingOptions(); config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions); var serviceProvider = GetRequiredServiceProvider(config, eventHub); var logger = serviceProvider.GetRequiredService <ITelemetryLogger>(); var eventConsumers = GetEventConsumers(config, eventHub, serviceProvider, logger); var storageOptions = new StorageCheckpointOptions(); config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions); storageOptions.BlobPrefix = eventHub; var checkpointClient = new StorageCheckpointClient(storageOptions, logger); var eventConsumerService = new EventConsumerService(eventConsumers, logger); var ct = new CancellationToken(); string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName; BlobContainerClient storageClient = new BlobContainerClient(storageOptions.BlobStorageConnectionString, storageOptions.BlobContainerName); var eventProcessorClientOptions = new EventProcessorClientOptions(); eventProcessorClientOptions.MaximumWaitTime = TimeSpan.FromSeconds(60); EventProcessorClient client = new EventProcessorClient(storageClient, consumerGroup, eventHubOptions.EventHubConnectionString, eventHubOptions.EventHubName, eventProcessorClientOptions); var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger); var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger); await eventHubReader.RunAsync(client, ct); }
public async Task SendAsyncForwardsThePartitionHashKey(string expectedHashKey) { var options = new EventBatchingOptions { PartitionKey = expectedHashKey }; var mock = new ObservableSenderMock(new ClientMock(), null); var sender = new TrackOneEventSender(() => mock); await sender.SendAsync(new[] { new EventData(new byte[] { 0x43 }) }, options, CancellationToken.None); Assert.That(mock.SendCalledWithParameters, Is.Not.Null, "The Send request should have been delegated."); (_, var actualHashKey) = mock.SendCalledWithParameters; Assert.That(actualHashKey, Is.EqualTo(expectedHashKey), "The partition hash key should have been forwarded."); }
public async Task SenderCanSendEventsUsingAPartitionHashHey() { await using (var scope = await EventHubScope.CreateAsync(4)) { var events = Enumerable .Range(0, 25) .Select(index => new EventData(Encoding.UTF8.GetBytes(new String('X', index + 5)))); var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var client = new EventHubClient(connectionString)) await using (var sender = client.CreateSender()) { var batchOptions = new EventBatchingOptions { PartitionKey = "some123key-!d" }; Assert.That(async() => await sender.SendAsync(events, batchOptions), Throws.Nothing); } } }
public async Task SenderCanSendMultipleBatchesOfEventsUsingAPartitionHashHey() { await using (var scope = await EventHubScope.CreateAsync(4)) { var batchOptions = new EventBatchingOptions { PartitionKey = "some123key-!d" }; for (var index = 0; index < 5; ++index) { var events = Enumerable .Range(0, 25) .Select(index => new EventData(Encoding.UTF8.GetBytes(new String((char)(65 + index), index + 5)))); var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var client = new EventHubClient(connectionString)) await using (var sender = client.CreateSender()) { Assert.That(async() => await sender.SendAsync(events, batchOptions), Throws.Nothing, $"Batch { index } should not have thrown an exception."); } } } }
/// <summary> /// Sends a set of events to the associated Event Hub using a batched approach. If the size of events exceed the /// maximum size of a single batch, an exception will be triggered and the send will fail. /// </summary> /// /// <param name="events">The set of event data to send.</param> /// <param name="batchOptions">The set of options to consider when sending this batch.</param> /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param> /// public override Task SendAsync(IEnumerable <EventData> events, EventBatchingOptions batchOptions, CancellationToken cancellationToken) {
/// <summary> /// Sends a set of events to the associated Event Hub using a batched approach. If the size of events exceed the /// maximum size of a single batch, an exception will be triggered and the send will fail. /// </summary> /// /// <param name="events">The set of event data to send.</param> /// <param name="batchOptions">The set of options to consider when sending this batch.</param> /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param> /// public abstract Task SendAsync(IEnumerable <EventData> events, EventBatchingOptions batchOptions, CancellationToken cancellationToken);