public async Task ClientCannotRetrieveMetadataWhenClosed(bool sync)
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var partition = (await client.GetPartitionIdsAsync()).First();
Assert.That(async() => await client.GetPropertiesAsync(), Throws.Nothing);
Assert.That(async() => await client.GetPartitionPropertiesAsync(partition), Throws.Nothing);
if (sync)
{
client.Close();
}
else
{
await client.CloseAsync();
}
await Task.Delay(TimeSpan.FromSeconds(5));
Assert.That(async() => await client.GetPartitionIdsAsync(), Throws.TypeOf <ObjectDisposedException>());
Assert.That(async() => await client.GetPropertiesAsync(), Throws.TypeOf <ObjectDisposedException>());
Assert.That(async() => await client.GetPartitionPropertiesAsync(partition), Throws.TypeOf <ObjectDisposedException>());
}
}
}
public async Task StopAsyncCallsPartitionProcessorCloseAsyncWithShutdownReason()
{
await using (var scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var closeCalls = new ConcurrentDictionary <string, int>();
var closeReasons = new ConcurrentDictionary <string, PartitionProcessorCloseReason>();
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onClose: (partitionContext, checkpointManager, reason) =>
{
closeCalls.AddOrUpdate(partitionContext.PartitionId, 1, (partitionId, value) => value + 1);
closeReasons[partitionContext.PartitionId] = reason;
}
);
hub.AddEventProcessors(1);
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// CloseAsync should have not been called when constructing the event processor or initializing the partition processors.
Assert.That(closeCalls.Keys, Is.Empty);
// Stop the event processors.
await hub.StopAllAsync();
// Validate results.
var partitionIds = await client.GetPartitionIdsAsync();
foreach (var partitionId in partitionIds)
{
Assert.That(closeCalls.TryGetValue(partitionId, out var calls), Is.True, $"{ partitionId }: CloseAsync should have been called.");
Assert.That(calls, Is.EqualTo(1), $"{ partitionId }: CloseAsync should have been called only once.");
Assert.That(closeReasons.TryGetValue(partitionId, out var reason), Is.True, $"{ partitionId }: close reason should have been set.");
Assert.That(reason, Is.EqualTo(PartitionProcessorCloseReason.Shutdown), $"{ partitionId }: unexpected close reason.");
}
Assert.That(closeCalls.Keys.Count, Is.EqualTo(partitionIds.Count()));
}
}
}
public async Task ProducerCanSendBatchToASpecificPartition()
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var partition = (await client.GetPartitionIdsAsync()).First();
var producerOptions = new EventHubProducerOptions {
PartitionId = partition
};
await using (var producer = client.CreateProducer(producerOptions))
{
var events = new[]
{
new EventData(Encoding.UTF8.GetBytes("This is a message")),
new EventData(Encoding.UTF8.GetBytes("This is another message")),
new EventData(Encoding.UTF8.GetBytes("Do we need more messages"))
};
Assert.That(async() => await producer.SendAsync(events), Throws.Nothing);
}
}
}
}
public async Task <int> OnExecute()
{
using var cts = new CancellationTokenSource();
Console.CancelKeyPress += (o, e) =>
{
Console.WriteLine("Shutting down...");
cts.Cancel();
};
var ct = cts.Token;
var tcs = new TaskCompletionSource <bool>();
await using var client = new EventHubClient(ConnectionString);
var partitions = await client.GetPartitionIdsAsync(ct);
Console.WriteLine($"Got {partitions.Length} partitions");
var tasks = new Task[partitions.Length];
for (var i = 0; i < partitions.Length; i++)
{
tasks[i] = RunForPartition(partitions[i], client, ct);
}
await Task.WhenAll(tasks);
return(0);
}
public async Task ClientCannotRetrieveMetadataWhenProxyIsInvalid()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var clientOptions = new EventHubClientOptions
{
Proxy = new WebProxy("http://1.2.3.4:9999"),
TransportType = TransportType.AmqpWebSockets,
RetryOptions = new RetryOptions {
TryTimeout = TimeSpan.FromMinutes(2)
}
};
await using (var client = new EventHubClient(connectionString))
await using (var invalidProxyClient = new EventHubClient(connectionString, clientOptions))
{
var partition = (await client.GetPartitionIdsAsync()).First();
Assert.That(async() => await invalidProxyClient.GetPartitionIdsAsync(), Throws.InstanceOf <WebSocketException>().Or.InstanceOf <TimeoutException>());
Assert.That(async() => await invalidProxyClient.GetPropertiesAsync(), Throws.InstanceOf <WebSocketException>().Or.InstanceOf <TimeoutException>());
Assert.That(async() => await invalidProxyClient.GetPartitionPropertiesAsync(partition), Throws.InstanceOf <WebSocketException>().Or.InstanceOf <TimeoutException>());
}
}
}
public async Task ReceiveCanReadOneEventBatch()
{
await using (var scope = await EventHubScope.CreateAsync(4))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var eventBatch = new[]
{
new EventData(Encoding.UTF8.GetBytes("One")),
new EventData(Encoding.UTF8.GetBytes("Two")),
new EventData(Encoding.UTF8.GetBytes("Three"))
};
await using (var client = new EventHubClient(connectionString))
{
var partition = (await client.GetPartitionIdsAsync()).First();
await using (var producer = client.CreateProducer(new EventHubProducerOptions {
PartitionId = partition
}))
await using (var consumer = client.CreateConsumer(partition, EventPosition.Latest))
{
// Initiate an operation to force the consumer to connect and set its position at the
// end of the event stream.
Assert.That(async() => await consumer.ReceiveAsync(1, TimeSpan.Zero), Throws.Nothing);
// Send the batch of events.
await producer.SendAsync(eventBatch);
// Recieve and validate the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < eventBatch.Length) && (++index < 5))
{
receivedEvents.AddRange(await consumer.ReceiveAsync(eventBatch.Length + 10, TimeSpan.FromMilliseconds(25)));
}
index = 0;
Assert.That(receivedEvents, Is.Not.Empty, "There should have been a set of events received.");
foreach (var receivedEvent in receivedEvents)
{
Assert.That(receivedEvent.IsEquivalentTo(eventBatch[index]), Is.True, $"The received event at index: { index } did not match the sent batch.");
++index;
}
Assert.That(index, Is.EqualTo(eventBatch.Length), "The number of received events did not match the batch size.");
}
}
}
}
public async Task StartAsyncCallsPartitionProcessorInitializeAsync()
{
await using (var scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var initializeCalls = new ConcurrentDictionary <string, int>();
// Create the event processor hub to manage our event processors.
var hub = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onInitialize: (partitionContext, checkpointManager) =>
initializeCalls.AddOrUpdate(partitionContext.PartitionId, 1, (partitionId, value) => value + 1)
);
hub.AddEventProcessors(1);
// InitializeAsync should have not been called when constructing the event processors.
Assert.That(initializeCalls.Keys, Is.Empty);
// Start the event processors.
await hub.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
// TODO: we'll probably need to extend this delay once load balancing is implemented.
await Task.Delay(5000);
// Validate results before calling stop. This way, we can make sure the initialize calls were
// triggered by start.
var partitionIds = await client.GetPartitionIdsAsync();
foreach (var partitionId in partitionIds)
{
Assert.That(initializeCalls.TryGetValue(partitionId, out var calls), Is.True, $"{ partitionId }: InitializeAsync should have been called.");
Assert.That(calls, Is.EqualTo(1), $"{ partitionId }: InitializeAsync should have been called only once.");
}
Assert.That(initializeCalls.Keys.Count, Is.EqualTo(partitionIds.Count()));
// Stop the event processors.
await hub.StopAllAsync();
}
}
}
private static async Task CreateSenderAndReceiver()
{
Console.Write("Creating the Sender and Receivers... ");
var partition = (await client.GetPartitionIdsAsync()).First();
var producerOptions = new EventHubProducerOptions
{
PartitionId = partition
};
sender = client.CreateProducer(producerOptions);
receiver = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partition, EventPosition.Latest);
Console.WriteLine("\tdone");
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// We will start by creating a client using its default set of options.
await using (var client = new EventHubClient(connectionString, eventHubName))
{
// Because partitions are owned by the Event Hubs service, it is not advised to assume that they have a stable
// and predictable set of identifiers. We'll inspect the Event Hub and select the first partition to use for
// publishing our event batch.
string[] partitionIds = await client.GetPartitionIdsAsync();
// In order to request that a producer be associated with a specific partition, it needs to be created with a custom
// set of options. Like the Event Hub client, there are options for a producer available to tune the behavior for
// operations that interact with the Event Hubs service.
//
// In our case, we will set the partition association but otherwise make use of the default options.
var producerOptions = new EventHubProducerOptions
{
PartitionId = partitionIds[0]
};
await using (var producer = client.CreateProducer(producerOptions))
{
// When an Event Hub producer is associated with any specific partition, it can publish events only to that partition.
// The producer has no ability to ask for the service to route events, including by using a partition key.
//
// If you attempt to use a partition key with an Event Hub producer that is associated with a partition, an exception
// will occur. Otherwise, publishing to a specific partition is exactly the same as other publishing scenarios.
// We will publish a small batch of events based on simple sentences.
var eventBatch = new EventData[]
{
new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")),
new EventData(Encoding.UTF8.GetBytes("Goodbye, Event Hubs!"))
};
await producer.SendAsync(eventBatch);
Console.WriteLine("The event batch has been published.");
}
}
// At this point, our client and producer have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
public async Task ConsumerWithNoOptionsCanReceive()
{
await using (var scope = await EventHubScope.CreateAsync(4))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var partition = (await client.GetPartitionIdsAsync()).First();
await using (var consumer = client.CreateConsumer(partition, EventPosition.Latest))
{
Assert.That(async() => await consumer.ReceiveAsync(1, TimeSpan.Zero), Throws.Nothing);
}
}
}
}
private static async Task CreateSenderAndReceiver()
{
Console.Write("Creating the Sender and Receivers... ");
var partition = (await client.GetPartitionIdsAsync()).First();
var senderOptions = new EventSenderOptions
{
PartitionId = partition
};
var receiverOptions = new EventReceiverOptions
{
BeginReceivingAt = EventPosition.NewEventsOnly
};
sender = client.CreateSender(senderOptions);
receiver = client.CreateReceiver(partition, receiverOptions);
Console.WriteLine("\tdone");
}
public async Task SendDoesNotUpdatePartitionPropertiesWhenSendingToDifferentPartition()
{
await using (var scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var partitionIds = await client.GetPartitionIdsAsync();
var events = new[] { new EventData(Encoding.UTF8.GetBytes("I should not update stuff")) };
await using (var producer0 = client.CreateProducer(new EventHubProducerOptions {
PartitionId = partitionIds[0]
}))
await using (var producer1 = client.CreateProducer(new EventHubProducerOptions {
PartitionId = partitionIds[1]
}))
{
// Sending events beforehand so the partition has some information
await producer0.SendAsync(events);
var oldPartitionProperties = await client.GetPartitionPropertiesAsync(partitionIds[0]);
Assert.That(oldPartitionProperties, Is.Not.Null, "A set of partition properties should have been returned.");
await producer1.SendAsync(events);
var newPartitionProperties = await client.GetPartitionPropertiesAsync(partitionIds[0]);
Assert.That(newPartitionProperties, Is.Not.Null, "A set of partition properties should have been returned.");
// All properties should remain the same
Assert.That(newPartitionProperties.Id, Is.EqualTo(oldPartitionProperties.Id));
Assert.That(newPartitionProperties.EventHubPath, Is.EqualTo(oldPartitionProperties.EventHubPath));
Assert.That(newPartitionProperties.BeginningSequenceNumber, Is.EqualTo(oldPartitionProperties.BeginningSequenceNumber));
Assert.That(newPartitionProperties.LastEnqueuedSequenceNumber, Is.EqualTo(oldPartitionProperties.LastEnqueuedSequenceNumber));
Assert.That(newPartitionProperties.LastEnqueuedOffset, Is.EqualTo(oldPartitionProperties.LastEnqueuedOffset));
}
}
}
}
public async Task SendUpdatesPartitionProperties()
{
await using (var scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var partition = (await client.GetPartitionIdsAsync()).First();
var events = new[] { new EventData(Encoding.UTF8.GetBytes("I should update stuff")) };
await using (var producer = client.CreateProducer(new EventHubProducerOptions {
PartitionId = partition
}))
{
// Sending events beforehand so the partition has some information
await producer.SendAsync(events);
var oldPartitionProperties = await client.GetPartitionPropertiesAsync(partition);
Assert.That(oldPartitionProperties, Is.Not.Null, "A set of partition properties should have been returned.");
await producer.SendAsync(events);
var newPartitionProperties = await client.GetPartitionPropertiesAsync(partition);
Assert.That(newPartitionProperties, Is.Not.Null, "A set of partition properties should have been returned.");
// The following properties should not have been altered
Assert.That(newPartitionProperties.Id, Is.EqualTo(oldPartitionProperties.Id));
Assert.That(newPartitionProperties.EventHubPath, Is.EqualTo(oldPartitionProperties.EventHubPath));
Assert.That(newPartitionProperties.BeginningSequenceNumber, Is.EqualTo(oldPartitionProperties.BeginningSequenceNumber));
// The following properties should have been updated
Assert.That(newPartitionProperties.LastEnqueuedSequenceNumber, Is.GreaterThan(oldPartitionProperties.LastEnqueuedSequenceNumber));
Assert.That(newPartitionProperties.LastEnqueuedOffset, Is.GreaterThan(oldPartitionProperties.LastEnqueuedOffset));
}
}
}
}
public async Task ClientPartitionIdsMatchPartitionProperties()
{
await using (var scope = await EventHubScope.CreateAsync(4))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var properties = await client.GetPropertiesAsync();
var partitions = await client.GetPartitionIdsAsync();
Assert.That(properties, Is.Not.Null, "A set of properties should have been returned.");
Assert.That(properties.PartitionIds, Is.Not.Null, "A set of partition identifiers for the properties should have been returned.");
Assert.That(partitions, Is.Not.Null, "A set of partition identifiers should have been returned.");
Assert.That(partitions, Is.EquivalentTo(properties.PartitionIds), "The partition identifiers returned directly should match those returned with properties.");
}
}
}
public async Task SenderCanSendToASpecificPartition()
{
await using (var scope = await EventHubScope.CreateAsync(4))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var partition = (await client.GetPartitionIdsAsync()).First();
var senderOptions = new EventSenderOptions {
PartitionId = partition
};
await using (var sender = client.CreateSender(senderOptions))
{
var events = new[] { new EventData(Encoding.UTF8.GetBytes("AWord")) };
Assert.That(async() => await sender.SendAsync(events), Throws.Nothing);
}
}
}
}
public async Task ReceiverWithOptionsCanReceive()
{
await using (var scope = await EventHubScope.CreateAsync(4))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var options = new EventReceiverOptions
{
ConsumerGroup = "$Default",
BeginReceivingAt = EventPosition.NewEventsOnly
};
await using (var client = new EventHubClient(connectionString))
{
var partition = (await client.GetPartitionIdsAsync()).First();
await using (var receiver = client.CreateReceiver(partition, options))
{
Assert.That(async() => await receiver.ReceiveAsync(1, TimeSpan.Zero), Throws.Nothing);
}
}
}
}
public async Task PartitionProcessorCanCreateACheckpointFromPartitionContext()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
// Send some events.
EventData lastEvent;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
var partitionId = (await client.GetPartitionIdsAsync()).First();
await using (EventHubProducer producer = client.CreateProducer())
await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest))
{
// Send a few events. We are only interested in the last one of them.
var dummyEventsCount = 10;
for (int i = 0; i < dummyEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
{
receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
lastEvent = receivedEvents.Last();
}
// Create a partition manager so we can retrieve the created checkpoint from it.
var partitionManager = new InMemoryPartitionManager();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
partitionManager,
onProcessEvents: (partitionContext, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Any())
{
partitionContext.UpdateCheckpointAsync(eventsList.Last());
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(client.FullyQualifiedNamespace, client.EventHubName, EventHubConsumer.DefaultConsumerGroupName);
Assert.That(ownershipEnumerable, Is.Not.Null);
Assert.That(ownershipEnumerable.Count, Is.EqualTo(1));
PartitionOwnership ownership = ownershipEnumerable.Single();
Assert.That(ownership.Offset.HasValue, Is.True);
Assert.That(ownership.Offset.Value, Is.EqualTo(lastEvent.Offset));
Assert.That(ownership.SequenceNumber.HasValue, Is.True);
Assert.That(ownership.SequenceNumber.Value, Is.EqualTo(lastEvent.SequenceNumber));
}
}
}
public async Task EventProcessorCanReceiveFromCheckpointedEventPosition()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
int receivedEventsCount = 0;
// Send some events.
var expectedEventsCount = 20;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
long?checkpointedSequenceNumber = default;
var partitionId = (await client.GetPartitionIdsAsync()).First();
await using (EventHubProducer producer = client.CreateProducer())
await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest))
{
// Send a few dummy events. We are not expecting to receive these.
var dummyEventsCount = 30;
for (int i = 0; i < dummyEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
{
receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
checkpointedSequenceNumber = receivedEvents.Last().SequenceNumber;
// Send the events we expect to receive.
for (int i = 0; i < expectedEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
}
// Create a partition manager and add an ownership with a checkpoint in it.
var partitionManager = new InMemoryPartitionManager();
await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
{
new PartitionOwnership(client.FullyQualifiedNamespace, client.EventHubName,
EventHubConsumer.DefaultConsumerGroupName, "ownerIdentifier", partitionId,
sequenceNumber: checkpointedSequenceNumber, lastModifiedTime: DateTimeOffset.UtcNow)
});
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
partitionManager,
onProcessEvents: (partitionContext, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
Interlocked.Add(ref receivedEventsCount, eventsList.Count);
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
}
}
}
public async Task PartitionProcessorProcessEventsAsyncReceivesAllEvents()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var allReceivedEvents = new ConcurrentDictionary <string, List <EventData> >();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
onProcessEvents: (partitionContext, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
allReceivedEvents.AddOrUpdate
(
partitionContext.PartitionId,
partitionId => eventsList,
(partitionId, list) =>
{
list.AddRange(eventsList);
return(list);
}
);
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Send some events.
var partitionIds = await client.GetPartitionIdsAsync();
var expectedEvents = new Dictionary <string, List <EventData> >();
foreach (var partitionId in partitionIds)
{
// Send a similar set of events for every partition.
expectedEvents[partitionId] = new List <EventData>
{
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: event processor tests are so long.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: there are so many of them.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: will they ever end?")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: let's add a few more messages.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: this is a monologue.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: loneliness is what I feel.")),
new EventData(Encoding.UTF8.GetBytes($"{ partitionId }: the end has come."))
};
await using (EventHubProducer producer = client.CreateProducer(new EventHubProducerOptions {
PartitionId = partitionId
}))
{
await producer.SendAsync(expectedEvents[partitionId]);
}
}
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results. Make sure we received every event in the correct partition processor,
// in the order they were sent.
foreach (var partitionId in partitionIds)
{
Assert.That(allReceivedEvents.TryGetValue(partitionId, out List <EventData> partitionReceivedEvents), Is.True, $"{ partitionId }: there should have been a set of events received.");
Assert.That(partitionReceivedEvents.Count, Is.EqualTo(expectedEvents[partitionId].Count), $"{ partitionId }: amount of received events should match.");
var index = 0;
foreach (EventData receivedEvent in partitionReceivedEvents)
{
Assert.That(receivedEvent.IsEquivalentTo(expectedEvents[partitionId][index]), Is.True, $"{ partitionId }: the received event at index { index } did not match the sent set of events.");
++index;
}
}
Assert.That(allReceivedEvents.Keys.Count, Is.EqualTo(partitionIds.Count()));
}
}
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// We will start by creating a client using its default set of options. It will be used by the event processor to
// communicate with the Azure Event Hubs service.
await using (var client = new EventHubClient(connectionString, eventHubName))
{
// An event processor is associated with a specific Event Hub and a consumer group. It receives events from
// multiple partitions in the Event Hub, passing them to the user for processing.
//
// A partition manager may create checkpoints and list/claim partition ownership. The user can implement their
// own partition manager by creating a subclass from the PartitionManager abstract class. Here we are creating
// a new instance of an InMemoryPartitionManager, provided by the Azure.Messaging.EventHubs.Processor namespace.
// This isn't relevant to understanding this sample, but is required by the event processor constructor.
PartitionManager partitionManager = new InMemoryPartitionManager();
// It's also possible to specify custom options upon event processor creation. We want to receive events from
// the latest available position so older events don't interfere with our sample. We also don't want to wait
// more than 1 second for every set of events.
EventProcessorOptions eventProcessorOptions = new EventProcessorOptions
{
InitialEventPosition = EventPosition.Latest,
MaximumReceiveWaitTime = TimeSpan.FromSeconds(1)
};
// Let's finally create our event processor. We're using the default consumer group that was created with the Event Hub.
var eventProcessor = new EventProcessor(EventHubConsumer.DefaultConsumerGroupName, client, partitionManager, eventProcessorOptions);
int totalEventsCount = 0;
int partitionsBeingProcessedCount = 0;
// TODO: explain callbacks setup once the public API is finished for the next preview.
eventProcessor.InitializeProcessingForPartitionAsync = (PartitionContext partitionContext) =>
{
// This is the last piece of code guaranteed to run before event processing, so all initialization
// must be done by the moment this method returns.
Interlocked.Increment(ref partitionsBeingProcessedCount);
Console.WriteLine($"\tPartition '{ partitionContext.PartitionId }': partition processing has started.");
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
eventProcessor.ProcessingForPartitionStoppedAsync = (PartitionContext partitionContext, PartitionProcessorCloseReason reason) =>
{
// The code to be run just before stopping processing events for a partition. This is the right place to dispose
// of objects that will no longer be used.
Interlocked.Decrement(ref partitionsBeingProcessedCount);
Console.WriteLine($"\tPartition '{ partitionContext.PartitionId }': partition processing has stopped. Reason: { reason }.");
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
eventProcessor.ProcessEventsAsync = (PartitionContext partitionContext, IEnumerable <EventData> events) =>
{
// Here the user can specify what to do with the events received from the event processor. We are counting how
// many events were received across all partitions so we can check whether all sent events were received.
//
// It's important to notice that this method is called even when no events are received after the maximum wait time, which
// can be specified by the user in the event processor options. In this case, the IEnumerable events is empty, but not null.
int eventsCount = events.Count();
if (eventsCount > 0)
{
Interlocked.Add(ref totalEventsCount, eventsCount);
Console.WriteLine($"\tPartition '{ partitionContext.PartitionId }': { eventsCount } event(s) received.");
}
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
eventProcessor.ProcessExceptionAsync = (PartitionContext partitionContext, Exception exception) =>
{
// All the unhandled exceptions encountered during the event processor execution are passed to this method so
// the user can decide how to handle them.
//
// This piece of code is not supposed to be reached by this sample. If the following message has been printed
// to the Console, then something unexpected has happened.
Console.WriteLine($"\tPartition '{ partitionContext.PartitionId }': an unhandled exception was encountered. This was not expected to happen.");
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
// Once started, the event processor will start to claim partitions and receive events from them.
Console.WriteLine("Starting the event processor.");
Console.WriteLine();
await eventProcessor.StartAsync();
Console.WriteLine("Event processor started.");
Console.WriteLine();
// Wait until the event processor has claimed ownership of all partitions in the Event Hub. This may take some time
// as there's a 10 seconds interval between claims. To be sure that we do not block forever in case the event processor
// fails, we will specify a fairly long time to wait and then cancel waiting.
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(400));
var partitionsCount = (await client.GetPartitionIdsAsync()).Length;
while (partitionsBeingProcessedCount < partitionsCount)
{
await Task.Delay(500, cancellationSource.Token);
}
// To test our event processor, we are publishing 10 sets of events to the Event Hub. Notice that we are not
// specifying a partition to send events to, so these sets may end up in different partitions.
EventData[] eventsToPublish = new EventData[]
{
new EventData(Encoding.UTF8.GetBytes("I am not the second event.")),
new EventData(Encoding.UTF8.GetBytes("I am not the first event."))
};
int amountOfSets = 10;
int expectedAmountOfEvents = amountOfSets * eventsToPublish.Length;
await using (EventHubProducer producer = client.CreateProducer())
{
Console.WriteLine();
Console.WriteLine("Sending events to the Event Hub.");
Console.WriteLine();
for (int i = 0; i < amountOfSets; i++)
{
await producer.SendAsync(eventsToPublish);
}
}
// Because there is some non-determinism in the messaging flow, the sent events may not be immediately
// available. For this reason, we wait 500 ms before resuming.
await Task.Delay(500);
// Once stopped, the event processor won't receive events anymore. In case there are still events being
// processed when the stop method is called, the processing will complete before the corresponding partition
// processor is closed.
Console.WriteLine();
Console.WriteLine("Stopping the event processor.");
Console.WriteLine();
await eventProcessor.StopAsync();
// Print out the amount of events that we received.
Console.WriteLine();
Console.WriteLine($"Amount of events received: { totalEventsCount }. Expected: { expectedAmountOfEvents }.");
}
// At this point, our client and producer have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// An Event Hub consumer is associated with a specific Event Hub partition and a consumer group. The consumer group is
// a label that identifies one or more consumers as a set. Often, consumer groups are named after the responsibility
// of the consumer in an application, such as "Telemetry" or "OrderProcessing". When an Event Hub is created, a default
// consumer group is created with it, called "$Default."
//
// Each consumer has a unique view of the events in the partition, meaning that events are available to all consumers
// and are not removed from the partition when a consumer reads them. This allows for different consumers to read and
// process events from the partition at different speeds and beginning with different events without interfering with
// one another.
//
// When events are published, they will continue to exist in the partition and be available for consuming until they
// reach an age where they are older than the retention period.
// (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events)
//
// Because events are not removed from the partition when consuming, a consumer must specify where in the partition it
// would like to begin reading events. For example, this may be starting from the very beginning of the stream, at an
// offset from the beginning, the next event available after a specific point in time, or at a specific event.
// We will start by creating a client using its default set of options.
await using (var client = new EventHubClient(connectionString, eventHubName))
{
// With our client, we can now inspect the partitions and find the identifier
// of the first.
string firstPartition = (await client.GetPartitionIdsAsync()).First();
// In this example, we will create our consumer for the first partition in the Event Hub, using the default consumer group
// that is created with an Event Hub. Our consumer will begin watching the partition at the very end, reading only new events
// that we will publish for it.
await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, firstPartition, EventPosition.Latest))
await using (EventHubProducer producer = client.CreateProducer(new EventHubProducerOptions {
PartitionId = firstPartition
}))
{
// Because our consumer is reading from the latest position, it won't see events that have previously
// been published. Before we can publish the events, we will need to ask the consumer to perform an operation,
// because it opens its connection only when it needs to. The first receive that we ask of it will not see
// any events, but will allow the consumer to start watching the partition.
//
// Because the maximum wait time is specivied as zero, this call will return immediately and will not
// have consumed any events.
await consumer.ReceiveAsync(1, TimeSpan.Zero);
// Now that the consumer is watching the partition, let's publish the event that we would like to
// receive.
await producer.SendAsync(new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")));
Console.WriteLine("The event batch has been published.");
// Because publishing and receving events is asynchronous, the events that we published may not
// be immediately available for our consumer to see.
//
// Each receive specifies the maximum amount of events that we would like in the batch and the maximum
// amount of time that we would like to wait for them. If there are enough events available to meet the
// requested amount, they'll be returned immediately. If not, the consumer will wait and collect events
// as they become available in an attempt to reach the requested amount. If the maximum time that we've
// allowed it to wait passes, the consumer will return the events that it has collected so far.
//
// Each Receive call may return between zero and the number of events that we requested, depending on the
// state of events in the partition. Likewise, it may return immediately or take up to the maximum wait
// time that we've allowed.
//
// We will ask for just our event, but allow a fairly long wait period to ensure that we're able to receive it.
// If you observe the time that the call takes, it is extremely likely that the request to recieve will complete
// long before the maximum wait time.
Stopwatch watch = Stopwatch.StartNew();
IEnumerable <EventData> receivedBatch = await consumer.ReceiveAsync(1, TimeSpan.FromSeconds(2.5));
watch.Stop();
// Print out the events that we received.
Console.WriteLine();
Console.WriteLine($"The following events were consumed in { watch.ElapsedMilliseconds } milliseconds:");
foreach (EventData eventData in receivedBatch)
{
// The body of our event was an encoded string; we'll recover the
// message by reversing the encoding process.
string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
Console.WriteLine($"\tMessage: \"{ message }\"");
}
}
}
// At this point, our client, consumer, and producer have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
public async Task ProducerSendsEventsInTheSameBatchToTheSamePartition()
{
var partitions = 10;
await using (var scope = await EventHubScope.CreateAsync(partitions))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (var producer = client.CreateProducer())
{
var eventBatch = Enumerable
.Range(0, 30)
.Select(index => new EventData(Encoding.UTF8.GetBytes("I'm getting used to this amount of messages")))
.ToList();
var partitionIds = await client.GetPartitionIdsAsync();
var partitionsCount = 0;
var receivedEventsCount = 0;
var consumers = new List <EventHubConsumer>();
try
{
for (var index = 0; index < partitions; index++)
{
consumers.Add(client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionIds[index], EventPosition.Latest));
// Initiate an operation to force the consumer to connect and set its position at the
// end of the event stream.
await consumers[index].ReceiveAsync(1, TimeSpan.Zero);
}
// Send the batch of events.
await producer.SendAsync(eventBatch);
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
foreach (var consumer in consumers)
{
var receivedEvents = new List <EventData>();
var index = 0;
while (++index < ReceiveRetryLimit)
{
receivedEvents.AddRange(await consumer.ReceiveAsync(eventBatch.Count + 10, TimeSpan.FromMilliseconds(25)));
}
if (receivedEvents.Count > 0)
{
partitionsCount++;
receivedEventsCount += receivedEvents.Count;
}
}
}
finally
{
foreach (var consumer in consumers)
{
consumer.Close();
}
}
Assert.That(partitionsCount, Is.EqualTo(1));
Assert.That(receivedEventsCount, Is.EqualTo(eventBatch.Count));
}
}
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// We will start by creating a client using its default set of options.
await using (var client = new EventHubClient(connectionString, eventHubName))
{
// With our client, we can now inspect the partitions and find the identifier
// of the first.
string firstPartition = (await client.GetPartitionIdsAsync()).First();
// In this example, we will create our consumer for the first partition in the Event Hub, using the default consumer group
// that is created with an Event Hub. Our consumer will begin watching the partition at the very end, reading only new events
// that we will publish for it.
await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, firstPartition, EventPosition.Latest))
await using (EventHubProducer producer = client.CreateProducer(new EventHubProducerOptions {
PartitionId = firstPartition
}))
{
// Because our consumer is reading from the latest position, it won't see events that have previously
// been published. Before we can publish the events, we will need to ask the consumer to perform an operation,
// because it opens its connection only when it needs to. The first receive that we ask of it will not see
// any events, but will allow the consumer to start watching the partition.
//
// Because the maximum wait time is specivied as zero, this call will return immediately and will not
// have consumed any events.
await consumer.ReceiveAsync(1, TimeSpan.Zero);
// Now that the consumer is watching the partition, let's publish a fair sized batch of events
// we would like for it to consume.
int eventBatchSize = 100;
EventData[] eventBatch = new EventData[eventBatchSize];
for (int index = 0; index < eventBatchSize; ++index)
{
eventBatch[index] = new EventData(Encoding.UTF8.GetBytes($"I am event #{ index }"));
}
await producer.SendAsync(eventBatch);
Console.WriteLine($"The event batch with { eventBatchSize } events has been published.");
// To allow for throughput in our application, we will process the published events by consuming them in
// small batches with a small wait time. This will allow us to receive and process more quickly than blocking
// to wait on a larger batch size.
//
// The values that are used in this example are intended just for illustration and not as guidance for a recommended
// set of defaults. In a real-world application, determining the right balance of batch size and wait time to achieve
// the desired performance will often vary depending on the application and event data being consumed. It is an area
// where some experimentation in the application context may prove helpful.
//
// Our example will attempt to read about 1/5 of the batch at a time, which should result in the need for 5 batches to
// be consumed. Because publishing and receving events is asynchronous, the events that we published may not be
// immediately available for our consumer to see. To compensate, we will allow for a small number of extra attempts beyond
// the expected 5 to to be sure that we don't stop reading before we receive all of our events.
var receivedEvents = new List <EventData>();
int consumeBatchSize = (int)Math.Floor(eventBatchSize / 5.0f);
int maximumAttempts = 15;
int attempts = 0;
while ((receivedEvents.Count < eventBatchSize) && (++attempts < maximumAttempts))
{
// Each receive, we ask for the maximum amount of events that we would like in the batch and
// specify the maximum amount of time that we would like to wait for them.
//
// The batch of events will be returned to us when either we have read our maximum number of events
// or when the time that we asked to wait has elapsed. This means that a batch we receive may have
// between zero and the maximum we asked for, depending on what was available in the partition.
//
// For this attempt, we will ask to receive our computed batch size (1/5 of published events) and wait, at most,
// 25 milliseconds to receive them. We'll then process them and if we haven't gotten all that we published, we'll
// make another attempt until either we have them or we have tried for our maximum number of attempts.
IEnumerable <EventData> receivedBatch = await consumer.ReceiveAsync(consumeBatchSize, TimeSpan.FromMilliseconds(25));
receivedEvents.AddRange(receivedBatch);
}
// Print out the events that we received.
Console.WriteLine();
Console.WriteLine($"Events Consumed: { receivedEvents.Count }");
foreach (EventData eventData in receivedEvents)
{
// The body of our event was an encoded string; we'll recover the
// message by reversing the encoding process.
string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
Console.WriteLine($"\tMessage: \"{ message }\"");
}
}
}
// At this point, our client, consumer, and producer have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// An Event Hub consumer is associated with a specific Event Hub partition and a consumer group. The consumer group is
// a label that identifies one or more consumers as a set. Often, consumer groups are named after the responsibility
// of the consumer in an application, such as "Telemetry" or "OrderProcessing". When an Event Hub is created, a default
// consumer group is created with it, called "$Default."
//
// Each consumer has a unique view of the events in the partition, meaning that events are available to all consumers
// and are not removed from the partition when a consumer reads them. This allows for different consumers to read and
// process events from the partition at different speeds and beginning with different events without interfering with
// one another.
//
// When events are published, they will continue to exist in the partition and be available for consuming until they
// reach an age where they are older than the retention period.
// (see: https://docs.microsoft.com/en-us/azure/event-hubs/event-hubs-faq#what-is-the-maximum-retention-period-for-events)
//
// Because events are not removed from the partition when consuming, a consumer must specify where in the partition it
// would like to begin reading events. For example, this may be starting from the very beginning of the stream, at an
// offset from the beginning, the next event available after a specific point in time, or at a specific event.
// We will start by creating a client using its default set of options.
await using (var client = new EventHubClient(connectionString, eventHubName))
{
// With our client, we can now inspect the partitions and find the identifier
// of the first.
string firstPartition = (await client.GetPartitionIdsAsync()).First();
// In this example, we will create our consumer for the first partition in the Event Hub, using the default consumer group
// that is created with an Event Hub. Our consumer will begin watching the partition at the very end, reading only new events
// that we will publish for it.
await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, firstPartition, EventPosition.Latest))
await using (EventHubProducer producer = client.CreateProducer(new EventHubProducerOptions {
PartitionId = firstPartition
}))
{
// Because our consumer is reading from the latest position, it won't see events that have previously
// been published. Before we can publish the events, we will need to ask the consumer to perform an operation,
// because it opens its connection only when it needs to. The first receive that we ask of it will not see
// any events, but will allow the consumer to start watching the partition.
//
// Because the maximum wait time is specified as zero, this call will return immediately and will not
// have consumed any events.
await consumer.ReceiveAsync(1, TimeSpan.Zero);
// Now that the consumer is watching the partition, let's publish the events that we would like to
// receive.
EventData[] eventsToPublish = new EventData[]
{
new EventData(Encoding.UTF8.GetBytes("Hello, Event Hubs!")),
new EventData(Encoding.UTF8.GetBytes("Goodbye, Event Hubs!"))
};
await producer.SendAsync(eventsToPublish);
Console.WriteLine("The event batch has been published.");
// Because publishing and receiving events is asynchronous, the events that we published may not be immediately
// available for our consumer to see. We will iterate over the available events in the partition, which should be
// just the events that we published.
//
// When a maximum wait time is specified, the iteration will ensure that it returns control after that time has elapsed,
// whether or not an event is available in the partition. If no event was available a null value will be emitted instead.
// This is intended to return control to the loop and avoid blocking for an indeterminate period of time to allow event
// processors to verify that the iterator is still consuming the partition and to make decisions on whether or not to continue
// if events are not arriving.
//
// For this example, we will specify a maximum wait time, and won't exit the loop until we've received at least one more
// event than we published, which is expected to be a null value triggered by exceeding the wait time.
//
// To be sure that we do not end up in an infinite loop, we will specify a fairly long time to allow processing to complete
// then cancel.
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
TimeSpan maximumWaitTime = TimeSpan.FromMilliseconds(250);
List <EventData> receivedEvents = new List <EventData>();
Stopwatch watch = Stopwatch.StartNew();
await foreach (EventData currentEvent in consumer.SubscribeToEvents(maximumWaitTime, cancellationSource.Token))
{
receivedEvents.Add(currentEvent);
if (receivedEvents.Count > eventsToPublish.Length)
{
watch.Stop();
break;
}
}
// Print out the events that we received.
Console.WriteLine();
Console.WriteLine($"The following events were consumed in { watch.ElapsedMilliseconds } milliseconds:");
foreach (EventData eventData in receivedEvents)
{
// The body of our event was an encoded string; we'll recover the
// message by reversing the encoding process.
string message = (eventData == null) ? "<< This was a null event >>" : Encoding.UTF8.GetString(eventData.Body.ToArray());
Console.WriteLine($"\tMessage: \"{ message }\"");
}
}
}
// At this point, our client, consumer, and producer have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// We will start by creating a client using its default set of options.
await using (var client = new EventHubClient(connectionString, eventHubName))
{
// With our client, we can now inspect the partitions and find the identifier of the first.
string firstPartition = (await client.GetPartitionIdsAsync()).First();
// In this example, we will make use of multiple consumers for the first partition in the Event Hub, using the default consumer group
// that is created with an Event Hub. Our initial consumer will begin watching the partition at the very end, reading only new events
// that we will publish for it.
await using (EventHubProducer producer = client.CreateProducer(new EventHubProducerOptions {
PartitionId = firstPartition
}))
{
// Each event that the initial consumer reads will have attributes set that describe the event's place in the
// partition, such as it's offset, sequence number, and the date/time that it was enqueued. These attributes can be
// used to create a new consumer that begins consuming at a known position.
//
// With Event Hubs, it is the responsibility of an application consuming events to keep track of those that it has processed,
// and to manage where in the partition the consumer begins reading events. This is done by using the position information to track
// state, commonly known as "creating a checkpoint."
//
// The goal is to preserve the position of an event in some form of durable state, such as writing it to a database, so that if the
// consuming application crashes or is otherwise restarted, it can retrieve that checkpoint information and use it to create a consumer that
// begins reading at the position where it left off.
//
// It is important to note that there is potential for a consumer to process an event and be unable to preserve the checkpoint. A well-designed
// consumer must be able to deal with processing the same event multiple times without it causing data corruption or otherwise creating issues.
// Event Hubs, like most event streaming systems, guarantees "at least once" delivery; even in cases where the consumer does not experience a restart,
// there is a small possibility that the service will return an event multiple times.
//
// In this example, we will publish a batch of events to be received with an initial consumer. The third event that is consumed will be captured
// and another consumer will use it's attributes to start reading the event that follows, consuming the same set of events that our initial consumer
// read, skipping over the first three.
int eventBatchSize = 50;
EventData thirdEvent;
await using (EventHubConsumer initialConsumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, firstPartition, EventPosition.Latest))
{
// The first receive that we ask of it will not see any events, but allows the consumer to start watching the partition. Because
// the maximum wait time is specified as zero, this call will return immediately and will not have consumed any events.
await initialConsumer.ReceiveAsync(1, TimeSpan.Zero);
// Now that the consumer is watching the partition, let's publish a batch of events.
EventData[] eventBatch = new EventData[eventBatchSize];
for (int index = 0; index < eventBatchSize; ++index)
{
eventBatch[index] = new EventData(Encoding.UTF8.GetBytes($"I am event #{ index }"));
}
await producer.SendAsync(eventBatch);
Console.WriteLine($"The event batch with { eventBatchSize } events has been published.");
// We will consume the events until all of the published events have been received.
var receivedEvents = new List <EventData>();
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
await foreach (EventData currentEvent in initialConsumer.SubscribeToEvents(cancellationSource.Token))
{
receivedEvents.Add(currentEvent);
if (receivedEvents.Count >= eventBatchSize)
{
break;
}
}
// Print out the events that we received.
Console.WriteLine();
Console.WriteLine($"The initial consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published. { eventBatchSize } were expected.");
foreach (EventData eventData in receivedEvents)
{
// The body of our event was an encoded string; we'll recover the
// message by reversing the encoding process.
string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
Console.WriteLine($"\tMessage: \"{ message }\"");
}
// Remember the third event that was consumed.
thirdEvent = receivedEvents[2];
}
// At this point, our initial consumer has passed its "using" scope and has been safely disposed of.
//
// Create a new consumer beginning using the third event as the last sequence number processed; this new consumer will begin reading at the next available
// sequence number, allowing it to read the set of published events beginning with the fourth one.
await using (EventHubConsumer newConsumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, firstPartition, EventPosition.FromSequenceNumber(thirdEvent.SequenceNumber.Value)))
{
// We will consume the events using the new consumer until all of the published events have been received.
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
int expectedCount = (eventBatchSize - 3);
var receivedEvents = new List <EventData>();
await foreach (EventData currentEvent in newConsumer.SubscribeToEvents(cancellationSource.Token))
{
receivedEvents.Add(currentEvent);
if (receivedEvents.Count >= expectedCount)
{
break;
}
}
// Print out the events that we received.
Console.WriteLine();
Console.WriteLine();
Console.WriteLine($"The new consumer processed { receivedEvents.Count } events of the { eventBatchSize } that were published. { expectedCount } were expected.");
foreach (EventData eventData in receivedEvents)
{
// The body of our event was an encoded string; we'll recover the
// message by reversing the encoding process.
string message = Encoding.UTF8.GetString(eventData.Body.ToArray());
Console.WriteLine($"\tMessage: \"{ message }\"");
}
}
}
}
// At this point, our client, all consumers, and producer have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
public async Task ProducerDoesNotSendToSpecificPartitionWhenPartitionIdIsNotSpecified(bool nullPartition)
{
var partitions = 10;
await using (var scope = await EventHubScope.CreateAsync(partitions))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
var producerOptions = new EventHubProducerOptions {
};
if (nullPartition)
{
producerOptions.PartitionId = null;
}
await using (var producer = client.CreateProducer(producerOptions))
{
var batches = 30;
var partitionIds = await client.GetPartitionIdsAsync();
var partitionsCount = 0;
var consumers = new List <EventHubConsumer>();
try
{
for (var index = 0; index < partitions; index++)
{
consumers.Add(client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionIds[index], EventPosition.Latest));
// Initiate an operation to force the consumer to connect and set its position at the
// end of the event stream.
await consumers[index].ReceiveAsync(1, TimeSpan.Zero);
}
// Send the batches of events.
for (var index = 0; index < batches; index++)
{
await producer.SendAsync(new EventData(Encoding.UTF8.GetBytes("It's not healthy to send so many messages")));
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
foreach (var consumer in consumers)
{
var receivedEvents = new List <EventData>();
var index = 0;
while (++index < ReceiveRetryLimit)
{
receivedEvents.AddRange(await consumer.ReceiveAsync(batches + 10, TimeSpan.FromMilliseconds(25)));
}
if (receivedEvents.Count > 0)
{
partitionsCount++;
}
}
}
finally
{
await Task.WhenAll(consumers.Select(consumer => consumer.CloseAsync()));
}
Assert.That(partitionsCount, Is.GreaterThan(1));
}
}
}
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// We will start by creating a client using its default set of options. It will be used by the event processor to
// communicate with the Azure Event Hubs service.
await using (var client = new EventHubClient(connectionString, eventHubName))
{
// An event processor is associated with a specific Event Hub and a consumer group. It receives events from
// multiple partitions in the Event Hub, passing them to the user for processing. It's worth mentioning that
// an event processor is a generic class, and it takes a partition processor as its underlying type.
//
// A partition processor is associated with a specific partition and is responsible for processing events when
// requested by the event processor. In order to use it as the event processor's underlying type, two conditions
// must be met:
//
// - It must be a class derived from BasePartitionProcessor.
//
// - It must have a parameterless constructor.
//
// We'll be using a SamplePartitionProcessor, whose implementation can be found at the end of this sample.
// A partition manager may create checkpoints and list/claim partition ownership. The user can implement their
// own partition manager by creating a subclass from the PartitionManager abstract class. Here we are creating
// a new instance of an InMemoryPartitionManager, provided by the Azure.Messaging.EventHubs.Processor namespace.
// This isn't relevant to understanding this sample, but is required by the event processor constructor.
PartitionManager partitionManager = new InMemoryPartitionManager();
// It's also possible to specify custom options upon event processor creation. We want to receive events from
// the latest available position so older events don't interfere with our sample. We also don't want to wait
// more than 1 second for every set of events.
EventProcessorOptions eventProcessorOptions = new EventProcessorOptions
{
InitialEventPosition = EventPosition.Latest,
MaximumReceiveWaitTime = TimeSpan.FromSeconds(1)
};
// Let's finally create our event processor. We're using the default consumer group that was created with the Event Hub.
var eventProcessor = new EventProcessor <SamplePartitionProcessor>(EventHubConsumer.DefaultConsumerGroupName, client, partitionManager, eventProcessorOptions);
// Once started, the event processor will start to claim partitions and receive events from them.
Console.WriteLine("Starting the event processor.");
Console.WriteLine();
await eventProcessor.StartAsync();
Console.WriteLine("Event processor started.");
Console.WriteLine();
// Wait until the event processor has claimed ownership of all partitions in the Event Hub. There should be a single
// active partition processor per owned partition. This may take some time as there's a 10 seconds interval between
// claims. To be sure that we do not block forever in case the event processor fails, we will specify a fairly long
// time to wait and then cancel waiting.
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(400));
var partitionsCount = (await client.GetPartitionIdsAsync()).Length;
while (SamplePartitionProcessor.ActiveInstancesCount < partitionsCount)
{
await Task.Delay(500, cancellationSource.Token);
}
// To test our event processor, we are publishing 10 sets of events to the Event Hub. Notice that we are not
// specifying a partition to send events to, so these sets may end up in different partitions.
EventData[] eventsToPublish = new EventData[]
{
new EventData(Encoding.UTF8.GetBytes("I am not the second event.")),
new EventData(Encoding.UTF8.GetBytes("I am not the first event."))
};
int amountOfSets = 10;
int expectedAmountOfEvents = amountOfSets * eventsToPublish.Length;
await using (EventHubProducer producer = client.CreateProducer())
{
Console.WriteLine("Sending events to the Event Hub.");
Console.WriteLine();
for (int i = 0; i < amountOfSets; i++)
{
await producer.SendAsync(eventsToPublish);
}
}
// Because there is some non-determinism in the messaging flow, the sent events may not be immediately
// available. For this reason, we wait 500 ms before resuming.
await Task.Delay(500);
// Once stopped, the event processor won't receive events anymore. In case there are still events being
// processed when the stop method is called, the processing will complete before the corresponding partition
// processor is closed.
Console.WriteLine();
Console.WriteLine("Stopping the event processor.");
Console.WriteLine();
await eventProcessor.StopAsync();
// Print out the amount of events that we received.
Console.WriteLine();
Console.WriteLine($"Amount of events received: { SamplePartitionProcessor.TotalEventsCount }. Expected: { expectedAmountOfEvents }.");
}
// At this point, our client and producer have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
public async Task ProducerSendsEventsWithTheSamePartitionHashKeyToTheSamePartition()
{
var partitions = 10;
var partitionKey = "some123key-!d";
await using (var scope = await EventHubScope.CreateAsync(partitions))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
await using (var producer = client.CreateProducer())
{
var batches = 5;
var partitionIds = await client.GetPartitionIdsAsync();
var partitionsCount = 0;
var receivedEventsCount = 0;
var consumers = new List <EventHubConsumer>();
try
{
for (var index = 0; index < partitions; index++)
{
consumers.Add(client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionIds[index], EventPosition.Latest));
// Initiate an operation to force the consumer to connect and set its position at the
// end of the event stream.
await consumers[index].ReceiveAsync(1, TimeSpan.Zero);
}
// Send the batches of events.
var batchOptions = new SendOptions {
PartitionKey = partitionKey
};
for (var index = 0; index < batches; index++)
{
await producer.SendAsync(new EventData(Encoding.UTF8.GetBytes($"Just a few messages ({ index })")), batchOptions);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
foreach (var consumer in consumers)
{
var receivedEvents = new List <EventData>();
var index = 0;
while (++index < ReceiveRetryLimit)
{
receivedEvents.AddRange(await consumer.ReceiveAsync(batches + 10, TimeSpan.FromMilliseconds(25)));
}
if (receivedEvents.Count > 0)
{
partitionsCount++;
receivedEventsCount += receivedEvents.Count;
foreach (var receivedEvent in receivedEvents)
{
Assert.That(receivedEvent.PartitionKey, Is.EqualTo(partitionKey));
}
}
}
}
finally
{
foreach (var consumer in consumers)
{
consumer.Close();
}
}
Assert.That(partitionsCount, Is.EqualTo(1));
Assert.That(receivedEventsCount, Is.EqualTo(batches));
}
}
}
public async Task ReceiveCanReadMultipleEventBatches()
{
await using (var scope = await EventHubScope.CreateAsync(4))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var eventBatch = new[]
{
new EventData(Encoding.UTF8.GetBytes("One")),
new EventData(Encoding.UTF8.GetBytes("Two")),
new EventData(Encoding.UTF8.GetBytes("Three")),
new EventData(Encoding.UTF8.GetBytes("Four")),
new EventData(Encoding.UTF8.GetBytes("Five")),
new EventData(Encoding.UTF8.GetBytes("Six")),
new EventData(Encoding.UTF8.GetBytes("Seven")),
new EventData(Encoding.UTF8.GetBytes("Eight")),
new EventData(Encoding.UTF8.GetBytes("Nine")),
new EventData(Encoding.UTF8.GetBytes("Ten")),
new EventData(Encoding.UTF8.GetBytes("Eleven")),
new EventData(Encoding.UTF8.GetBytes("Twelve")),
new EventData(Encoding.UTF8.GetBytes("Thirteen")),
new EventData(Encoding.UTF8.GetBytes("Fourteen")),
new EventData(Encoding.UTF8.GetBytes("Fifteen"))
};
var receiverOptions = new EventReceiverOptions
{
BeginReceivingAt = EventPosition.NewEventsOnly
};
await using (var client = new EventHubClient(connectionString))
{
var partition = (await client.GetPartitionIdsAsync()).First();
await using (var sender = client.CreateSender(new EventSenderOptions {
PartitionId = partition
}))
await using (var receiver = client.CreateReceiver(partition, receiverOptions))
{
// Initiate an operation to force the receiver to connect and set its position at the
// end of the event stream.
Assert.That(async() => await receiver.ReceiveAsync(1, TimeSpan.Zero), Throws.Nothing);
// Send the batch of events, receive and validate them.
await sender.SendAsync(eventBatch);
// Recieve and validate the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
var batchNumber = 1;
var batchSize = (eventBatch.Length / 3);
while ((receivedEvents.Count < eventBatch.Length) && (++index < eventBatch.Length + 3))
{
var currentReceiveBatch = await receiver.ReceiveAsync(batchSize, TimeSpan.FromMilliseconds(25));
receivedEvents.AddRange(currentReceiveBatch);
Assert.That(currentReceiveBatch, Is.Not.Empty, $"There should have been a set of events received for batch number: { batchNumber }.");
++batchNumber;
}
index = 0;
foreach (var receivedEvent in receivedEvents)
{
Assert.That(receivedEvent.IsEquivalentTo(eventBatch[index]), Is.True, $"The received event at index: { index } did not match the sent batch.");
++index;
}
Assert.That(index, Is.EqualTo(eventBatch.Length), "The number of received events did not match the batch size.");
}
}
}
}
