public async Task ProducerCanSendAnEventBatchUsingAPartitionHashKey()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
{
IEnumerable <EventData> events = Enumerable
.Range(0, 25)
.Select(index => new EventData(Encoding.UTF8.GetBytes(new string('X', index + 5))));
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var batchOptions = new BatchOptions {
PartitionKey = "some123key-!d"
};
await using (var client = new EventHubClient(connectionString))
await using (EventHubProducer producer = client.CreateProducer())
{
using EventDataBatch batch = await producer.CreateBatchAsync(batchOptions);
foreach (EventData eventData in events)
{
Assert.That(() => batch.TryAdd(eventData), Is.True, "An event was rejected by the batch; all events should be accepted.");
}
Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing);
}
}
}
public async Task ProducerCanSendLargeEventBatch()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString, new EventHubClientOptions {
RetryOptions = new RetryOptions {
TryTimeout = TimeSpan.FromMinutes(5)
}
}))
await using (EventHubProducer producer = client.CreateProducer())
{
using EventDataBatch batch = await producer.CreateBatchAsync();
// Actual limit is 1046520 for a single event.
batch.TryAdd(new EventData(new byte[100000 / 3]));
batch.TryAdd(new EventData(new byte[100000 / 3]));
batch.TryAdd(new EventData(new byte[100000 / 3]));
Assert.That(batch.Count, Is.EqualTo(3), "The batch should contain all 3 events.");
Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing);
}
}
}
/// <summary>
/// Creates a size-constraint batch to which <see cref="EventData" /> may be added using a try-based pattern. If an event would
/// exceed the maximum allowable size of the batch, the batch will not allow adding the event and signal that scenario using its
/// return value.
///
/// Because events that would violate the size constraint cannot be added, publishing a batch will not trigger an exception when
/// attempting to send the events to the Event Hubs service.
/// </summary>
///
/// <param name="options">The set of options to consider when creating this batch.</param>
/// <param name="cancellationToken">An optional <see cref="CancellationToken"/> instance to signal the request to cancel the operation.</param>
///
/// <returns>An <see cref="EventDataBatch" /> with the requested <paramref name="options"/>.</returns>
///
/// <seealso cref="CreateBatchAsync(CreateBatchOptions, CancellationToken)" />
///
public virtual async ValueTask <EventDataBatch> CreateBatchAsync(CreateBatchOptions options,
CancellationToken cancellationToken = default)
{
options = options?.Clone() ?? new CreateBatchOptions();
AssertSinglePartitionReference(options.PartitionId, options.PartitionKey);
TransportEventBatch transportBatch = await EventHubProducer.CreateBatchAsync(options, cancellationToken).ConfigureAwait(false);
return(new EventDataBatch(transportBatch, FullyQualifiedNamespace, EventHubName, options.ToSendOptions()));
}
public async Task EventHubProducerAppliesDiagnosticIdToEventsOnBatchSend()
{
Activity activity = new Activity("SomeActivity").Start();
var eventHubName = "SomeName";
var endpoint = new Uri("amqp://some.endpoint.com/path");
var writtenEventsData = new List <EventData>();
var batchTransportMock = new Mock <TransportEventBatch>();
var transportMock = new Mock <TransportEventHubProducer>();
batchTransportMock
.Setup(m => m.TryAdd(It.IsAny <EventData>()))
.Returns <EventData>(e =>
{
var hasSpace = writtenEventsData.Count <= 1;
if (hasSpace)
{
writtenEventsData.Add(e);
}
return(hasSpace);
});
transportMock
.Setup(m => m.SendAsync(It.IsAny <IEnumerable <EventData> >(), It.IsAny <SendOptions>(), It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask);
transportMock
.Setup(m => m.CreateBatchAsync(It.IsAny <BatchOptions>(), It.IsAny <CancellationToken>()))
.Returns(new ValueTask <TransportEventBatch>(Task.FromResult(batchTransportMock.Object)));
var producer = new EventHubProducer(transportMock.Object, endpoint, eventHubName, new EventHubProducerOptions(), Mock.Of <EventHubRetryPolicy>());
var eventData1 = new EventData(ReadOnlyMemory <byte> .Empty);
var eventData2 = new EventData(ReadOnlyMemory <byte> .Empty);
var eventData3 = new EventData(ReadOnlyMemory <byte> .Empty);
EventDataBatch batch = await producer.CreateBatchAsync();
Assert.That(batch.TryAdd(eventData1), Is.True, "The first event should have been added to the batch.");
Assert.That(batch.TryAdd(eventData2), Is.True, "The second event should have been added to the batch.");
Assert.That(batch.TryAdd(eventData3), Is.False, "The third event should not have been added to the batch.");
await producer.SendAsync(batch);
activity.Stop();
Assert.That(writtenEventsData.Count, Is.EqualTo(2), "Each of the events in the batch should have been instrumented.");
foreach (EventData eventData in writtenEventsData)
{
Assert.That(eventData.Properties.TryGetValue(DiagnosticProperty.DiagnosticIdAttribute, out object value), Is.True, "The events should have a diagnostic identifier property.");
Assert.That(value, Is.EqualTo(activity.Id), "The diagnostics identifier should match the activity in the active scope.");
}
Assert.That(eventData3.Properties.ContainsKey(DiagnosticProperty.DiagnosticIdAttribute), Is.False, "Events that were not accepted into the batch should not have been instrumented.");
}
public async Task CreateBatchSetsTheSendOptionsForTheEventBatch()
{
var batchOptions = new BatchOptions {
PartitionKey = "Hi", MaximumizeInBytes = 9999
};
var transportProducer = new ObservableTransportProducerMock();
var producer = new EventHubProducer(transportProducer, new Uri("amqp://some.endpoint.com/path"), "dummy", new EventHubProducerOptions(), Mock.Of <EventHubRetryPolicy>());
EventDataBatch eventBatch = await producer.CreateBatchAsync(batchOptions);
Assert.That(eventBatch.SendOptions, Is.SameAs(transportProducer.CreateBatchCalledWith), "The batch options should have used for the send options.");
;
}
public void CreateBatchForASpecificPartitionDoesNotAllowAPartitionHashKey()
{
var batchOptions = new BatchOptions {
PartitionKey = "testKey"
};
var transportProducer = new ObservableTransportProducerMock();
var producer = new EventHubProducer(transportProducer, new Uri("amqp://some.endpoint.com/path"), "dummy", new EventHubProducerOptions {
PartitionId = "1"
}, Mock.Of <EventHubRetryPolicy>());
Assert.That(async() => await producer.CreateBatchAsync(batchOptions), Throws.InvalidOperationException);
}
public async Task CreateBatchDefaultsBatchOptions()
{
var expectedOptions = new BatchOptions();
var transportProducer = new ObservableTransportProducerMock();
var producer = new EventHubProducer(transportProducer, new Uri("amqp://some.endpoint.com/path"), "dummy", new EventHubProducerOptions(), Mock.Of <EventHubRetryPolicy>());
await producer.CreateBatchAsync();
Assert.That(transportProducer.CreateBatchCalledWith, Is.Not.Null, "The batch creation should have passed options.");
Assert.That(transportProducer.CreateBatchCalledWith, Is.Not.SameAs(expectedOptions), "The options should have been cloned.");
Assert.That(transportProducer.CreateBatchCalledWith.PartitionKey, Is.EqualTo(expectedOptions.PartitionKey), "The partition key should match.");
Assert.That(transportProducer.CreateBatchCalledWith.MaximumizeInBytes, Is.EqualTo(expectedOptions.MaximumizeInBytes), "The maximum size should match.");
}
public async Task CreateBatchInvokesTheTransportProducer()
{
var batchOptions = new BatchOptions {
PartitionKey = "Hi", MaximumizeInBytes = 9999
};
var transportProducer = new ObservableTransportProducerMock();
var producer = new EventHubProducer(transportProducer, "dummy", new EventHubProducerOptions(), Mock.Of <EventHubRetryPolicy>());
await producer.CreateBatchAsync(batchOptions);
Assert.That(transportProducer.CreateBatchCalledWith, Is.Not.Null, "The batch creation should have passed options.");
Assert.That(transportProducer.CreateBatchCalledWith, Is.Not.SameAs(batchOptions), "The options should have been cloned.");
Assert.That(transportProducer.CreateBatchCalledWith.PartitionKey, Is.EqualTo(batchOptions.PartitionKey), "The partition key should match.");
Assert.That(transportProducer.CreateBatchCalledWith.MaximumizeInBytes, Is.EqualTo(batchOptions.MaximumizeInBytes), "The maximum size should match.");
}
public async Task EventHubProducerCreatesDiagnosticScopeOnBatchSend()
{
using var testListener = new ClientDiagnosticListener(DiagnosticSourceName);
var eventHubName = "SomeName";
var endpoint = new Uri("amqp://endpoint");
var eventCount = 0;
var batchTransportMock = new Mock <TransportEventBatch>();
batchTransportMock
.Setup(m => m.TryAdd(It.IsAny <EventData>()))
.Returns(() =>
{
eventCount++;
return(eventCount <= 3);
});
var transportMock = new Mock <TransportEventHubProducer>();
transportMock
.Setup(m => m.SendAsync(It.IsAny <IEnumerable <EventData> >(), It.IsAny <SendOptions>(), It.IsAny <CancellationToken>()))
.Returns(Task.CompletedTask);
transportMock
.Setup(m => m.CreateBatchAsync(It.IsAny <BatchOptions>(), It.IsAny <CancellationToken>()))
.Returns(new ValueTask <TransportEventBatch>(Task.FromResult(batchTransportMock.Object)));
var producer = new EventHubProducer(transportMock.Object, endpoint, eventHubName, new EventHubProducerOptions(), Mock.Of <EventHubRetryPolicy>());
var eventData = new EventData(ReadOnlyMemory <byte> .Empty);
EventDataBatch batch = await producer.CreateBatchAsync();
Assert.True(batch.TryAdd(eventData));
await producer.SendAsync(batch);
ClientDiagnosticListener.ProducedDiagnosticScope sendScope = testListener.AssertScope(DiagnosticProperty.ProducerActivityName,
new KeyValuePair <string, string>(DiagnosticProperty.TypeAttribute, DiagnosticProperty.EventHubProducerType),
new KeyValuePair <string, string>(DiagnosticProperty.ServiceContextAttribute, DiagnosticProperty.EventHubsServiceContext),
new KeyValuePair <string, string>(DiagnosticProperty.EventHubAttribute, eventHubName),
new KeyValuePair <string, string>(DiagnosticProperty.EndpointAttribute, endpoint.ToString()));
ClientDiagnosticListener.ProducedDiagnosticScope messageScope = testListener.AssertScope(DiagnosticProperty.EventActivityName);
Assert.That(eventData.Properties[DiagnosticProperty.DiagnosticIdAttribute], Is.EqualTo(messageScope.Activity.Id), "The diagnostics identifier should match.");
Assert.That(messageScope.Activity, Is.Not.SameAs(sendScope.Activity), "The activities should not be the same instance.");
}
public async Task ProducerCanSendZeroLengthEventBatch()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (EventHubProducer producer = client.CreateProducer())
{
using EventDataBatch batch = await producer.CreateBatchAsync();
batch.TryAdd(new EventData(Array.Empty <byte>()));
Assert.That(batch.Count, Is.EqualTo(1), "The batch should contain a single event.");
Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing);
}
}
}
public async Task ProducerCanSendAnEventBatch()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (EventHubProducer producer = client.CreateProducer())
{
using EventDataBatch batch = await producer.CreateBatchAsync();
batch.TryAdd(new EventData(Encoding.UTF8.GetBytes("This is a message")));
batch.TryAdd(new EventData(Encoding.UTF8.GetBytes("This is another message")));
batch.TryAdd(new EventData(Encoding.UTF8.GetBytes("So many messages")));
Assert.That(batch.Count, Is.EqualTo(3), "The batch should contain all 3 events.");
Assert.That(async() => await producer.SendAsync(batch), Throws.Nothing);
}
}
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// We will start by creating a client and a producer, each using their default set of options.
await using (var client = new EventHubClient(connectionString, eventHubName))
await using (EventHubProducer producer = client.CreateProducer())
{
// There is a limit to the size of an event or batch of events that can be published at once. This limit varies, and depends
// on the underlying transport and protocol that the Event Hub producer is using. Because the size limit is based on the
// size of an event or batch as it would be sent over the network, it is not simple to understand the size of an event as
// it is being created. For this reason, producers creating events with a large body or batching a large number of events
// may experience errors when they attempt to send.
//
// In order to avoid errors, support creation of known-good batches, and allow for producers with the need to predictably
// control bandwidth use, the Event Hub producer allows creation of an Event Data Batch, which acts as a container for
// a batch of events. Events can be added to the batch using a "TryAdd" pattern, allowing insight into when an event
// or batch would be too large without triggering an exception.
//
// An Event Data Batch can be created using a partition key or without one, following the same rules as the Event Hub
// producer with respect to partition routing. (see, Sample04_PublishEventsWithPartitionKey for more information.)
// We will publish a batch of events based on simple sentences.
var events = new EventData[]
{
new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")),
new EventData(Encoding.UTF8.GetBytes("Goodbye, Event Hubs!")),
new EventData(Encoding.UTF8.GetBytes("A third event, the plot thickens!")),
new EventData(Encoding.UTF8.GetBytes("...and a fourth too."))
};
// When creating the batch, you may specify a custom set of options or allow the defaults to take precedence.
// If a size limit is not specified, the maximum size allowed by the transport is used. In this example, we
// will set a custom partition key and otherwise use the default options.
BatchOptions options = new BatchOptions {
PartitionKey = "This is a custom key!"
};
EventDataBatch batch = await producer.CreateBatchAsync(options);
foreach (EventData eventData in events)
{
if (!batch.TryAdd(eventData))
{
throw new ApplicationException("The events will not fit in the same batch.");
}
}
// When publishing a batch, no SendOptions may be specified. Those options are
// all expressed upfront during batch creation and apply to all events in the batch.
await producer.SendAsync(batch);
Console.WriteLine("The event batch has been published.");
}
// At this point, our client and producer have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
