public virtual EventProcessor ResolveEventProcessor(IServiceProvider serviceProvider)
{
var eventConsumerService = serviceProvider.GetRequiredService <IEventConsumerService>();
var checkpointClient = serviceProvider.GetRequiredService <StorageCheckpointClient>();
var logger = serviceProvider.GetRequiredService <ITelemetryLogger>();
var eventBatchingOptions = new EventBatchingOptions();
Configuration.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions);
var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger);
var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger);
return(eventHubReader);
}
public void SendForASpecificPartitionDoesNotAllowAPartitionHashKey()
{
var batchingOptions = new EventBatchingOptions {
PartitionKey = "testKey"
};
var events = new[] { new EventData(new byte[] { 0x44, 0x66, 0x88 }) };
var transportSender = new ObservableTransportSenderMock();
var sender = new EventSender(transportSender, "dummy", new EventSenderOptions {
PartitionId = "1"
});
Assert.That(async() => await sender.SendAsync(events, batchingOptions), Throws.InvalidOperationException);
}
public async Task SendInvokesTheTransportSender()
{
var events = Mock.Of <IEnumerable <EventData> >();
var options = new EventBatchingOptions();
var transportSender = new ObservableTransportSenderMock();
var sender = new EventSender(transportSender, "dummy", new EventSenderOptions());
await sender.SendAsync(events, options);
(var calledWithEvents, var calledWithOptions) = transportSender.SendCalledWithParameters;
Assert.That(calledWithEvents, Is.SameAs(events), "The events should be the same instance.");
Assert.That(calledWithOptions, Is.SameAs(options), "The options should be the same instance");
}
public static EventProcessor GetEventProcessor(
IConfiguration config,
IEventConsumerService eventConsumerService,
ICheckpointClient checkpointClient,
ITelemetryLogger logger)
{
var eventBatchingOptions = new EventBatchingOptions();
config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions);
var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger);
var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger);
return(eventHubReader);
}
public static async Task Main()
{
var config = GetEnvironmentConfig();
// determine which event hub to read from
var eventHub = Environment.GetEnvironmentVariable("WEBJOBS_NAME");
if (eventHub == null)
{
eventHub = config.GetSection("Console:EventHub").Value;
}
System.Console.WriteLine($"Reading from event hub: {eventHub}");
System.Console.WriteLine($"Logs and Metrics will be written to Application Insights");
var eventHubOptions = GetEventHubInfo(config, eventHub);
EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubConnectionString);
EnsureArg.IsNotNullOrWhiteSpace(eventHubOptions.EventHubName);
var eventBatchingOptions = new EventBatchingOptions();
config.GetSection(EventBatchingOptions.Settings).Bind(eventBatchingOptions);
var serviceProvider = GetRequiredServiceProvider(config, eventHub);
var logger = serviceProvider.GetRequiredService <ITelemetryLogger>();
var eventConsumers = GetEventConsumers(config, eventHub, serviceProvider, logger);
var storageOptions = new StorageCheckpointOptions();
config.GetSection(StorageCheckpointOptions.Settings).Bind(storageOptions);
storageOptions.BlobPrefix = eventHub;
var checkpointClient = new StorageCheckpointClient(storageOptions, logger);
var eventConsumerService = new EventConsumerService(eventConsumers, logger);
var ct = new CancellationToken();
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(storageOptions.BlobStorageConnectionString, storageOptions.BlobContainerName);
var eventProcessorClientOptions = new EventProcessorClientOptions();
eventProcessorClientOptions.MaximumWaitTime = TimeSpan.FromSeconds(60);
EventProcessorClient client = new EventProcessorClient(storageClient, consumerGroup, eventHubOptions.EventHubConnectionString, eventHubOptions.EventHubName, eventProcessorClientOptions);
var eventBatchingService = new EventBatchingService(eventConsumerService, eventBatchingOptions, checkpointClient, logger);
var eventHubReader = new EventProcessor(eventBatchingService, checkpointClient, logger);
await eventHubReader.RunAsync(client, ct);
}
public async Task SendAsyncForwardsThePartitionHashKey(string expectedHashKey)
{
var options = new EventBatchingOptions {
PartitionKey = expectedHashKey
};
var mock = new ObservableSenderMock(new ClientMock(), null);
var sender = new TrackOneEventSender(() => mock);
await sender.SendAsync(new[] { new EventData(new byte[] { 0x43 }) }, options, CancellationToken.None);
Assert.That(mock.SendCalledWithParameters, Is.Not.Null, "The Send request should have been delegated.");
(_, var actualHashKey) = mock.SendCalledWithParameters;
Assert.That(actualHashKey, Is.EqualTo(expectedHashKey), "The partition hash key should have been forwarded.");
}
public async Task SenderCanSendEventsUsingAPartitionHashHey()
{
await using (var scope = await EventHubScope.CreateAsync(4))
{
var events = Enumerable
.Range(0, 25)
.Select(index => new EventData(Encoding.UTF8.GetBytes(new String('X', index + 5))));
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (var sender = client.CreateSender())
{
var batchOptions = new EventBatchingOptions {
PartitionKey = "some123key-!d"
};
Assert.That(async() => await sender.SendAsync(events, batchOptions), Throws.Nothing);
}
}
}
public async Task SenderCanSendMultipleBatchesOfEventsUsingAPartitionHashHey()
{
await using (var scope = await EventHubScope.CreateAsync(4))
{
var batchOptions = new EventBatchingOptions {
PartitionKey = "some123key-!d"
};
for (var index = 0; index < 5; ++index)
{
var events = Enumerable
.Range(0, 25)
.Select(index => new EventData(Encoding.UTF8.GetBytes(new String((char)(65 + index), index + 5))));
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (var sender = client.CreateSender())
{
Assert.That(async() => await sender.SendAsync(events, batchOptions), Throws.Nothing, $"Batch { index } should not have thrown an exception.");
}
}
}
}
/// <summary>
/// Sends a set of events to the associated Event Hub using a batched approach. If the size of events exceed the
/// maximum size of a single batch, an exception will be triggered and the send will fail.
/// </summary>
///
/// <param name="events">The set of event data to send.</param>
/// <param name="batchOptions">The set of options to consider when sending this batch.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
public override Task SendAsync(IEnumerable <EventData> events,
EventBatchingOptions batchOptions,
CancellationToken cancellationToken)
{
/// <summary>
/// Sends a set of events to the associated Event Hub using a batched approach. If the size of events exceed the
/// maximum size of a single batch, an exception will be triggered and the send will fail.
/// </summary>
///
/// <param name="events">The set of event data to send.</param>
/// <param name="batchOptions">The set of options to consider when sending this batch.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
public abstract Task SendAsync(IEnumerable <EventData> events,
EventBatchingOptions batchOptions,
CancellationToken cancellationToken);