public async Task ProducerCanSendAnEventBatchUsingAPartitionHashKey() { await using (EventHubScope scope = await EventHubScope.CreateAsync(2)) { IEnumerable <EventData> events = Enumerable .Range(0, 25) .Select(index => new EventData(Encoding.UTF8.GetBytes(new string('X', index + 5)))); var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); var batchOptions = new BatchOptions { PartitionKey = "some123key-!d" }; await using (var client = new EventHubClient(connectionString)) await using (EventHubProducer producer = client.CreateProducer()) { using EventDataBatch batch = await producer.CreateBatchAsync(batchOptions); foreach (EventData eventData in events) { Assert.That(() => batch.TryAdd(eventData), Is.True, "An event was rejected by the batch; all events should be accepted."); } Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing); } } }
public async Task ProducerCanSendLargeEventBatch() { await using (EventHubScope scope = await EventHubScope.CreateAsync(1)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var client = new EventHubClient(connectionString, new EventHubClientOptions { RetryOptions = new RetryOptions { TryTimeout = TimeSpan.FromMinutes(5) } })) await using (EventHubProducer producer = client.CreateProducer()) { using EventDataBatch batch = await producer.CreateBatchAsync(); // Actual limit is 1046520 for a single event. batch.TryAdd(new EventData(new byte[100000 / 3])); batch.TryAdd(new EventData(new byte[100000 / 3])); batch.TryAdd(new EventData(new byte[100000 / 3])); Assert.That(batch.Count, Is.EqualTo(3), "The batch should contain all 3 events."); Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing); } } }
/// <summary> /// Creates a size-constraint batch to which <see cref="EventData" /> may be added using a try-based pattern. If an event would /// exceed the maximum allowable size of the batch, the batch will not allow adding the event and signal that scenario using its /// return value. /// /// Because events that would violate the size constraint cannot be added, publishing a batch will not trigger an exception when /// attempting to send the events to the Event Hubs service. /// </summary> /// /// <param name="options">The set of options to consider when creating this batch.</param> /// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param> /// /// <returns>An <see cref="EventDataBatch" /> with the requested <paramref name="options"/>.</returns> /// /// <seealso cref="CreateBatchAsync(CreateBatchOptions, CancellationToken)" /> /// public virtual async ValueTask <EventDataBatch> CreateBatchAsync(CreateBatchOptions options, CancellationToken cancellationToken = default) { options = options?.Clone() ?? new CreateBatchOptions(); AssertSinglePartitionReference(options.PartitionId, options.PartitionKey); TransportEventBatch transportBatch = await EventHubProducer.CreateBatchAsync(options, cancellationToken).ConfigureAwait(false); return(new EventDataBatch(transportBatch, FullyQualifiedNamespace, EventHubName, options.ToSendOptions())); }
public async Task EventHubProducerAppliesDiagnosticIdToEventsOnBatchSend() { Activity activity = new Activity("SomeActivity").Start(); var eventHubName = "SomeName"; var endpoint = new Uri("amqp://some.endpoint.com/path"); var writtenEventsData = new List <EventData>(); var batchTransportMock = new Mock <TransportEventBatch>(); var transportMock = new Mock <TransportEventHubProducer>(); batchTransportMock .Setup(m => m.TryAdd(It.IsAny <EventData>())) .Returns <EventData>(e => { var hasSpace = writtenEventsData.Count <= 1; if (hasSpace) { writtenEventsData.Add(e); } return(hasSpace); }); transportMock .Setup(m => m.SendAsync(It.IsAny <IEnumerable <EventData> >(), It.IsAny <SendOptions>(), It.IsAny <CancellationToken>())) .Returns(Task.CompletedTask); transportMock .Setup(m => m.CreateBatchAsync(It.IsAny <BatchOptions>(), It.IsAny <CancellationToken>())) .Returns(new ValueTask <TransportEventBatch>(Task.FromResult(batchTransportMock.Object))); var producer = new EventHubProducer(transportMock.Object, endpoint, eventHubName, new EventHubProducerOptions(), Mock.Of <EventHubRetryPolicy>()); var eventData1 = new EventData(ReadOnlyMemory <byte> .Empty); var eventData2 = new EventData(ReadOnlyMemory <byte> .Empty); var eventData3 = new EventData(ReadOnlyMemory <byte> .Empty); EventDataBatch batch = await producer.CreateBatchAsync(); Assert.That(batch.TryAdd(eventData1), Is.True, "The first event should have been added to the batch."); Assert.That(batch.TryAdd(eventData2), Is.True, "The second event should have been added to the batch."); Assert.That(batch.TryAdd(eventData3), Is.False, "The third event should not have been added to the batch."); await producer.SendAsync(batch); activity.Stop(); Assert.That(writtenEventsData.Count, Is.EqualTo(2), "Each of the events in the batch should have been instrumented."); foreach (EventData eventData in writtenEventsData) { Assert.That(eventData.Properties.TryGetValue(DiagnosticProperty.DiagnosticIdAttribute, out object value), Is.True, "The events should have a diagnostic identifier property."); Assert.That(value, Is.EqualTo(activity.Id), "The diagnostics identifier should match the activity in the active scope."); } Assert.That(eventData3.Properties.ContainsKey(DiagnosticProperty.DiagnosticIdAttribute), Is.False, "Events that were not accepted into the batch should not have been instrumented."); }
public async Task CreateBatchSetsTheSendOptionsForTheEventBatch() { var batchOptions = new BatchOptions { PartitionKey = "Hi", MaximumizeInBytes = 9999 }; var transportProducer = new ObservableTransportProducerMock(); var producer = new EventHubProducer(transportProducer, new Uri("amqp://some.endpoint.com/path"), "dummy", new EventHubProducerOptions(), Mock.Of <EventHubRetryPolicy>()); EventDataBatch eventBatch = await producer.CreateBatchAsync(batchOptions); Assert.That(eventBatch.SendOptions, Is.SameAs(transportProducer.CreateBatchCalledWith), "The batch options should have used for the send options."); ; }
public void CreateBatchForASpecificPartitionDoesNotAllowAPartitionHashKey() { var batchOptions = new BatchOptions { PartitionKey = "testKey" }; var transportProducer = new ObservableTransportProducerMock(); var producer = new EventHubProducer(transportProducer, new Uri("amqp://some.endpoint.com/path"), "dummy", new EventHubProducerOptions { PartitionId = "1" }, Mock.Of <EventHubRetryPolicy>()); Assert.That(async() => await producer.CreateBatchAsync(batchOptions), Throws.InvalidOperationException); }
public async Task CreateBatchDefaultsBatchOptions() { var expectedOptions = new BatchOptions(); var transportProducer = new ObservableTransportProducerMock(); var producer = new EventHubProducer(transportProducer, new Uri("amqp://some.endpoint.com/path"), "dummy", new EventHubProducerOptions(), Mock.Of <EventHubRetryPolicy>()); await producer.CreateBatchAsync(); Assert.That(transportProducer.CreateBatchCalledWith, Is.Not.Null, "The batch creation should have passed options."); Assert.That(transportProducer.CreateBatchCalledWith, Is.Not.SameAs(expectedOptions), "The options should have been cloned."); Assert.That(transportProducer.CreateBatchCalledWith.PartitionKey, Is.EqualTo(expectedOptions.PartitionKey), "The partition key should match."); Assert.That(transportProducer.CreateBatchCalledWith.MaximumizeInBytes, Is.EqualTo(expectedOptions.MaximumizeInBytes), "The maximum size should match."); }
public async Task CreateBatchInvokesTheTransportProducer() { var batchOptions = new BatchOptions { PartitionKey = "Hi", MaximumizeInBytes = 9999 }; var transportProducer = new ObservableTransportProducerMock(); var producer = new EventHubProducer(transportProducer, "dummy", new EventHubProducerOptions(), Mock.Of <EventHubRetryPolicy>()); await producer.CreateBatchAsync(batchOptions); Assert.That(transportProducer.CreateBatchCalledWith, Is.Not.Null, "The batch creation should have passed options."); Assert.That(transportProducer.CreateBatchCalledWith, Is.Not.SameAs(batchOptions), "The options should have been cloned."); Assert.That(transportProducer.CreateBatchCalledWith.PartitionKey, Is.EqualTo(batchOptions.PartitionKey), "The partition key should match."); Assert.That(transportProducer.CreateBatchCalledWith.MaximumizeInBytes, Is.EqualTo(batchOptions.MaximumizeInBytes), "The maximum size should match."); }
public async Task EventHubProducerCreatesDiagnosticScopeOnBatchSend() { using var testListener = new ClientDiagnosticListener(DiagnosticSourceName); var eventHubName = "SomeName"; var endpoint = new Uri("amqp://endpoint"); var eventCount = 0; var batchTransportMock = new Mock <TransportEventBatch>(); batchTransportMock .Setup(m => m.TryAdd(It.IsAny <EventData>())) .Returns(() => { eventCount++; return(eventCount <= 3); }); var transportMock = new Mock <TransportEventHubProducer>(); transportMock .Setup(m => m.SendAsync(It.IsAny <IEnumerable <EventData> >(), It.IsAny <SendOptions>(), It.IsAny <CancellationToken>())) .Returns(Task.CompletedTask); transportMock .Setup(m => m.CreateBatchAsync(It.IsAny <BatchOptions>(), It.IsAny <CancellationToken>())) .Returns(new ValueTask <TransportEventBatch>(Task.FromResult(batchTransportMock.Object))); var producer = new EventHubProducer(transportMock.Object, endpoint, eventHubName, new EventHubProducerOptions(), Mock.Of <EventHubRetryPolicy>()); var eventData = new EventData(ReadOnlyMemory <byte> .Empty); EventDataBatch batch = await producer.CreateBatchAsync(); Assert.True(batch.TryAdd(eventData)); await producer.SendAsync(batch); ClientDiagnosticListener.ProducedDiagnosticScope sendScope = testListener.AssertScope(DiagnosticProperty.ProducerActivityName, new KeyValuePair <string, string>(DiagnosticProperty.TypeAttribute, DiagnosticProperty.EventHubProducerType), new KeyValuePair <string, string>(DiagnosticProperty.ServiceContextAttribute, DiagnosticProperty.EventHubsServiceContext), new KeyValuePair <string, string>(DiagnosticProperty.EventHubAttribute, eventHubName), new KeyValuePair <string, string>(DiagnosticProperty.EndpointAttribute, endpoint.ToString())); ClientDiagnosticListener.ProducedDiagnosticScope messageScope = testListener.AssertScope(DiagnosticProperty.EventActivityName); Assert.That(eventData.Properties[DiagnosticProperty.DiagnosticIdAttribute], Is.EqualTo(messageScope.Activity.Id), "The diagnostics identifier should match."); Assert.That(messageScope.Activity, Is.Not.SameAs(sendScope.Activity), "The activities should not be the same instance."); }
public async Task ProducerCanSendZeroLengthEventBatch() { await using (EventHubScope scope = await EventHubScope.CreateAsync(1)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var client = new EventHubClient(connectionString)) await using (EventHubProducer producer = client.CreateProducer()) { using EventDataBatch batch = await producer.CreateBatchAsync(); batch.TryAdd(new EventData(Array.Empty <byte>())); Assert.That(batch.Count, Is.EqualTo(1), "The batch should contain a single event."); Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing); } } }
public async Task ProducerCanSendAnEventBatch() { await using (EventHubScope scope = await EventHubScope.CreateAsync(1)) { var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName); await using (var client = new EventHubClient(connectionString)) await using (EventHubProducer producer = client.CreateProducer()) { using EventDataBatch batch = await producer.CreateBatchAsync(); batch.TryAdd(new EventData(Encoding.UTF8.GetBytes("This is a message"))); batch.TryAdd(new EventData(Encoding.UTF8.GetBytes("This is another message"))); batch.TryAdd(new EventData(Encoding.UTF8.GetBytes("So many messages"))); Assert.That(batch.Count, Is.EqualTo(3), "The batch should contain all 3 events."); Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing); } } }
/// <summary> /// Runs the sample using the specified Event Hubs connection information. /// </summary> /// /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param> /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param> /// public async Task RunAsync(string connectionString, string eventHubName) { // We will start by creating a client and a producer, each using their default set of options. await using (var client = new EventHubClient(connectionString, eventHubName)) await using (EventHubProducer producer = client.CreateProducer()) { // There is a limit to the size of an event or batch of events that can be published at once. This limit varies, and depends // on the underlying transport and protocol that the Event Hub producer is using. Because the size limit is based on the // size of an event or batch as it would be sent over the network, it is not simple to understand the size of an event as // it is being created. For this reason, producers creating events with a large body or batching a large number of events // may experience errors when they attempt to send. // // In order to avoid errors, support creation of known-good batches, and allow for producers with the need to predictably // control bandwidth use, the Event Hub producer allows creation of an Event Data Batch, which acts as a container for // a batch of events. Events can be added to the batch using a "TryAdd" pattern, allowing insight into when an event // or batch would be too large without triggering an exception. // // An Event Data Batch can be created using a partition key or without one, following the same rules as the Event Hub // producer with respect to partition routing. (see, Sample04_PublishEventsWithPartitionKey for more information.) // We will publish a batch of events based on simple sentences. var events = new EventData[] { new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")), new EventData(Encoding.UTF8.GetBytes("Goodbye, Event Hubs!")), new EventData(Encoding.UTF8.GetBytes("A third event, the plot thickens!")), new EventData(Encoding.UTF8.GetBytes("...and a fourth too.")) }; // When creating the batch, you may specify a custom set of options or allow the defaults to take precedence. // If a size limit is not specified, the maximum size allowed by the transport is used. In this example, we // will set a custom partition key and otherwise use the default options. BatchOptions options = new BatchOptions { PartitionKey = "This is a custom key!" }; EventDataBatch batch = await producer.CreateBatchAsync(options); foreach (EventData eventData in events) { if (!batch.TryAdd(eventData)) { throw new ApplicationException("The events will not fit in the same batch."); } } // When publishing a batch, no SendOptions may be specified. Those options are // all expressed upfront during batch creation and apply to all events in the batch. await producer.SendAsync(batch); Console.WriteLine("The event batch has been published."); } // At this point, our client and producer have passed their "using" scope and have safely been disposed of. We // have no further obligations. Console.WriteLine(); }