public async Task ClaimOwnershipAsyncCanClaimMultipleOwnership()
{
var partitionManager = new InMemoryPartitionManager();
var ownershipList = new List <PartitionOwnership>();
var ownershipCount = 5;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new PartitionOwnership
(
"eventHubName",
"consumerGroup",
"ownerIdentifier",
$"partitionId { i }"
));
}
await partitionManager.ClaimOwnershipAsync(ownershipList);
var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(ownershipCount));
Assert.That(storedOwnership.OrderBy(ownership => ownership.PartitionId).SequenceEqual(ownershipList), Is.True);
}
static async Task Consume(string connectionString, string consumerGroup, string eventHubName)
{
await using (var client = new EventHubClient(connectionString, eventHubName))
{
Func <PartitionContext, CheckpointManager, IPartitionProcessor> partitionProcessorFactory =
(partitionContext, checkpointManager) => new TemperatureProcessor(partitionContext.PartitionId);
var partitionManager = new InMemoryPartitionManager();
var eventProcessorOptions = new EventProcessorOptions
{
InitialEventPosition = EventPosition.Latest,
MaximumReceiveWaitTime = TimeSpan.FromSeconds(1)
};
var eventProcessor = new EventProcessor(
consumerGroup,
client,
partitionProcessorFactory,
partitionManager,
eventProcessorOptions);
await eventProcessor.StartAsync();
Console.WriteLine("Receiving...");
Console.ReadLine();
}
}
public async Task CheckpointUpdateUpdatesOwnershipInformation()
{
var partitionManager = new InMemoryPartitionManager();
var originalOwnership = new PartitionOwnership
("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", offset: 1, sequenceNumber: 2, lastModifiedTime: DateTimeOffset.UtcNow.Subtract(TimeSpan.FromMinutes(1)));
await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
{
originalOwnership
});
// ETag must have been set by the partition manager.
var originalLastModifiedTime = originalOwnership.LastModifiedTime;
var originalETag = originalOwnership.ETag;
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", 10, 20));
// Make sure the ownership has changed, even though the instance should be the same.
var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(1));
Assert.That(storedOwnership.Single(), Is.EqualTo(originalOwnership));
Assert.That(originalOwnership.Offset, Is.EqualTo(10));
Assert.That(originalOwnership.SequenceNumber, Is.EqualTo(20));
Assert.That(originalOwnership.LastModifiedTime, Is.GreaterThan(originalLastModifiedTime));
Assert.That(originalOwnership.ETag, Is.Not.EqualTo(originalETag));
}
public async Task CheckpointUpdateFailsWhenOwnerChanges()
{
var partitionManager = new InMemoryPartitionManager();
var originalOwnership = new PartitionOwnership
("eventHubName", "consumerGroup", "ownerIdentifier1", "partitionId", offset: 1, sequenceNumber: 2, lastModifiedTime: DateTimeOffset.UtcNow);
await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
{
originalOwnership
});
// ETag must have been set by the partition manager.
var originalLastModifiedTime = originalOwnership.LastModifiedTime;
var originalETag = originalOwnership.ETag;
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("eventHubName", "consumerGroup", "ownerIdentifier2", "partitionId", 10, 20));
// Make sure the ownership hasn't changed.
var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(1));
Assert.That(storedOwnership.Single(), Is.EqualTo(originalOwnership));
Assert.That(originalOwnership.OwnerIdentifier, Is.EqualTo("ownerIdentifier1"));
Assert.That(originalOwnership.Offset, Is.EqualTo(1));
Assert.That(originalOwnership.SequenceNumber, Is.EqualTo(2));
Assert.That(originalOwnership.LastModifiedTime, Is.EqualTo(originalLastModifiedTime));
Assert.That(originalOwnership.ETag, Is.EqualTo(originalETag));
}
public async Task ListOwnershipAsyncReturnsEmptyIEnumerableWhenThereAreNoOwnership()
{
var partitionManager = new InMemoryPartitionManager();
var ownership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup");
Assert.That(ownership, Is.Not.Null.And.Empty);
}
public async Task CheckpointUpdateFailsWhenAssociatedOwnershipDoesNotExist()
{
var partitionManager = new InMemoryPartitionManager();
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", offset : 10, sequenceNumber : 20));
var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership, Is.Empty);
}
public async Task ClaimOwnershipAsyncReturnsOnlyTheSuccessfullyClaimedOwnership()
{
var partitionManager = new InMemoryPartitionManager();
var ownershipList = new List <PartitionOwnership>();
var ownershipCount = 5;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
$"partitionId { i }"
));
}
await partitionManager.ClaimOwnershipAsync(ownershipList);
// The ETags must have been set by the partition manager.
var eTags = ownershipList.Select(ownership => ownership.ETag).ToList();
ownershipList.Clear();
// Use a valid eTag when 'i' is odd. This way, we can expect 'ownershipCount / 2' successful
// claims (rounded down).
var expectedClaimedCount = ownershipCount / 2;
for (int i = 0; i < ownershipCount; i++)
{
ownershipList.Add(
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
$"partitionId { i }",
offset: i,
eTag: i % 2 == 1 ? eTags[i] : null
));
}
IEnumerable <PartitionOwnership> claimedOwnership = await partitionManager.ClaimOwnershipAsync(ownershipList);
Assert.That(claimedOwnership, Is.Not.Null);
Assert.That(claimedOwnership.Count, Is.EqualTo(expectedClaimedCount));
Assert.That(claimedOwnership.OrderBy(ownership => ownership.Offset).SequenceEqual(ownershipList.Where(ownership => ownership.Offset % 2 == 1)), Is.True);
}
public async Task OwnershipClaimDoesNotInterfereWithOtherNamespaces()
{
var partitionManager = new InMemoryPartitionManager();
var ownershipList = new List <PartitionOwnership>();
var firstOwnership =
new PartitionOwnership
(
"namespace1",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId"
);
ownershipList.Add(firstOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
// ETag must have been set by the partition manager.
var eTag = firstOwnership.ETag;
ownershipList.Clear();
var secondOwnership =
new PartitionOwnership
(
"namespace2",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId",
eTag: eTag
);
ownershipList.Add(secondOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
IEnumerable <PartitionOwnership> storedOwnership1 = await partitionManager.ListOwnershipAsync("namespace1", "eventHubName", "consumerGroup");
IEnumerable <PartitionOwnership> storedOwnership2 = await partitionManager.ListOwnershipAsync("namespace2", "eventHubName", "consumerGroup");
Assert.That(storedOwnership1, Is.Not.Null);
Assert.That(storedOwnership1.Count, Is.EqualTo(1));
Assert.That(storedOwnership1.Single(), Is.EqualTo(firstOwnership));
Assert.That(storedOwnership2, Is.Not.Null);
Assert.That(storedOwnership2.Count, Is.EqualTo(1));
Assert.That(storedOwnership2.Single(), Is.EqualTo(secondOwnership));
}
public async Task OwnershipClaimSucceedsWhenETagIsValid()
{
var partitionManager = new InMemoryPartitionManager();
var ownershipList = new List <PartitionOwnership>();
var firstOwnership =
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId",
offset: 1
);
ownershipList.Add(firstOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
// ETag must have been set by the partition manager.
var eTag = firstOwnership.ETag;
ownershipList.Clear();
var secondOwnership =
new PartitionOwnership
(
"namespace",
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId",
offset: 2,
eTag: eTag
);
ownershipList.Add(secondOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(1));
Assert.That(storedOwnership.Single(), Is.EqualTo(secondOwnership));
}
public async Task OwnershipClaimFailsWhenETagIsInvalid(string eTag)
{
var partitionManager = new InMemoryPartitionManager();
var ownershipList = new List <PartitionOwnership>();
var firstOwnership =
new PartitionOwnership
(
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId",
offset: 1
);
ownershipList.Add(firstOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
ownershipList.Clear();
var secondOwnership =
new PartitionOwnership
(
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId",
offset: 2,
eTag: eTag
);
ownershipList.Add(secondOwnership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(1));
Assert.That(storedOwnership.Single(), Is.EqualTo(firstOwnership));
}
public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions()
{
var partitionManager = new InMemoryPartitionManager();
var ownership1 = new PartitionOwnership
("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId1", offset: 1);
var ownership2 = new PartitionOwnership
("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId2", offset: 1);
await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
{
ownership1,
ownership2
});
await partitionManager.UpdateCheckpointAsync(new Checkpoint
("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId1", 10, 20));
Assert.That(ownership1.Offset, Is.EqualTo(10));
Assert.That(ownership2.Offset, Is.EqualTo(1));
}
public async Task FirstOwnershipClaimSucceeds()
{
var partitionManager = new InMemoryPartitionManager();
var ownershipList = new List <PartitionOwnership>();
var ownership =
new PartitionOwnership
(
"eventHubName",
"consumerGroup",
"ownerIdentifier",
"partitionId"
);
ownershipList.Add(ownership);
await partitionManager.ClaimOwnershipAsync(ownershipList);
var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup");
Assert.That(storedOwnership, Is.Not.Null);
Assert.That(storedOwnership.Count, Is.EqualTo(1));
Assert.That(storedOwnership.Single(), Is.EqualTo(ownership));
}
/// <summary>
/// Initializes a new instance of the <see cref="EventProcessorManager"/> class.
/// </summary>
///
/// <param name="consumerGroup">The name of the consumer group the event processors are associated with. Events are read in the context of this group.</param>
/// <param name="client">The client used to interact with the Azure Event Hubs service.</param>
/// <param name="options">The set of options to use for the event processors.</param>
/// <param name="onInitialize">A callback action to be called on <see cref="PartitionProcessor.InitializeAsync" />.</param>
/// <param name="onClose">A callback action to be called on <see cref="PartitionProcessor.CloseAsync" />.</param>
/// <param name="onProcessEvents">A callback action to be called on <see cref="PartitionProcessor.ProcessEventsAsync" />.</param>
/// <param name="onProcessError">A callback action to be called on <see cref="PartitionProcessor.ProcessErrorAsync" />.</param>
///
public EventProcessorManager(string consumerGroup,
EventHubClient client,
EventProcessorOptions options = null,
Action <PartitionContext, CheckpointManager> onInitialize = null,
Action <PartitionContext, CheckpointManager, PartitionProcessorCloseReason> onClose = null,
Action <PartitionContext, CheckpointManager, IEnumerable <EventData>, CancellationToken> onProcessEvents = null,
Action <PartitionContext, CheckpointManager, Exception, CancellationToken> onProcessError = null)
{
ConsumerGroup = consumerGroup;
InnerClient = client;
PartitionProcessorFactory = (partitionContext, checkpointManager) =>
new PartitionProcessor
(
partitionContext,
checkpointManager,
onInitialize,
onClose,
onProcessEvents,
onProcessError
);
InnerPartitionManager = new InMemoryPartitionManager();
// In case it has not been specified, set the maximum receive wait time to 2 seconds because the default
// value (1 minute) would take too much time.
Options = options?.Clone() ?? new EventProcessorOptions();
if (Options.MaximumReceiveWaitTime == null)
{
Options.MaximumReceiveWaitTime = TimeSpan.FromSeconds(2);
}
EventProcessors = new List <EventProcessor>();
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// An event processor is associated with a specific Event Hub and a consumer group. It receives events from
// multiple partitions in the Event Hub, passing them to a handler delegate for processing using code that you
// provide.
//
// These handler delegates are invoked for each event or error that occurs during operation of the event processor. For
// a given partition, only a single event will be dispatched for processing at a time so that the order of events within a
// partition is preserved. Partitions, however, are processed concurrently. As a result, your handlers are potentially processing
// multiple events or errors at any given time.
// A partition manager may create checkpoints and list/claim partition ownership. A developer may implement their
// own partition manager by creating a subclass from the PartitionManager abstract class. Here we are creating
// a new instance of an InMemoryPartitionManager, provided by the Azure.Messaging.EventHubs.Processor namespace.
// This isn't relevant to understanding this sample, but is required by the event processor constructor.
var partitionManager = new InMemoryPartitionManager();
// It's also possible to specify custom options upon event processor creation. We don't want to wait
// more than 1 second for every set of events.
var eventProcessorOptions = new EventProcessorClientOptions
{
MaximumReceiveWaitTime = TimeSpan.FromSeconds(1)
};
// Let's finally create our event processor. We're using the default consumer group that was created with the Event Hub.
await using (var eventProcessor = new EventProcessorClient(EventHubConsumerClient.DefaultConsumerGroupName, partitionManager, connectionString, eventHubName, eventProcessorOptions))
{
int totalEventsCount = 0;
int partitionsBeingProcessedCount = 0;
// TODO: explain callbacks setup once the public API is finished for the next preview.
eventProcessor.InitializeProcessingForPartitionAsync = (initializationContext) =>
{
// This is the last piece of code guaranteed to run before event processing, so all initialization
// must be done by the moment this method returns.
// We want to receive events from the latest available position so older events don't interfere with our sample.
initializationContext.DefaultStartingPosition = EventPosition.Latest;
Interlocked.Increment(ref partitionsBeingProcessedCount);
Console.WriteLine($"\tPartition '{ initializationContext.Context.PartitionId }': partition processing has started.");
// This method is asynchronous, which means it's expected to return a Task.
return(new ValueTask());
};
eventProcessor.ProcessingForPartitionStoppedAsync = (stopContext) =>
{
// The code to be run just before stopping processing events for a partition. This is the right place to dispose
// of objects that will no longer be used.
Interlocked.Decrement(ref partitionsBeingProcessedCount);
Console.WriteLine($"\tPartition '{ stopContext.Context.PartitionId }': partition processing has stopped. Reason: { stopContext.Reason }.");
// This method is asynchronous, which means it's expected to return a Task.
return(new ValueTask());
};
eventProcessor.ProcessEventAsync = (processorEvent) =>
{
// Here the user can specify what to do with the event received from the event processor. We are counting how
// many events were received across all partitions so we can check whether all sent events were received.
//
// It's important to notice that this method is called even when no event is received after the maximum wait time, which
// can be specified by the user in the event processor options. In this case, the received event is null.
if (processorEvent.Data != null)
{
Interlocked.Increment(ref totalEventsCount);
Console.WriteLine($"\tPartition '{ processorEvent.Context.PartitionId }': event received.");
}
// This method is asynchronous, which means it's expected to return a Task.
return(new ValueTask());
};
eventProcessor.ProcessExceptionAsync = (errorContext) =>
{
// Any exception which occurs as a result of the event processor itself will be passed to
// this delegate so it may be handled. The processor will continue to process events if
// it is able to unless this handler explicitly requests that it stop doing so.
//
// It is important to note that this does not include exceptions during event processing; those
// are considered responsibility of the developer implementing the event processing handler. It
// is, therefore, highly encouraged that best practices for exception handling practices are
// followed with that delegate.
//
// This piece of code is not supposed to be reached by this sample. If the following message has been printed
// to the Console, then something unexpected has happened.
Console.WriteLine($"\tPartition '{ errorContext.PartitionId }': an unhandled exception was encountered. This was not expected to happen.");
// This method is asynchronous, which means it's expected to return a Task.
return(new ValueTask());
};
// Once started, the event processor will start to claim partitions and receive events from them.
Console.WriteLine("Starting the event processor.");
Console.WriteLine();
await eventProcessor.StartAsync();
Console.WriteLine("Event processor started.");
Console.WriteLine();
// Wait until the event processor has claimed ownership of all partitions in the Event Hub. This may take some time
// as there's a 10 seconds interval between claims. To be sure that we do not block forever in case the event processor
// fails, we will specify a fairly long time to wait and then cancel waiting.
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(400));
await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName))
{
var partitionsCount = (await producerClient.GetPartitionIdsAsync()).Length;
while (partitionsBeingProcessedCount < partitionsCount)
{
await Task.Delay(500, cancellationSource.Token);
}
// To test our event processor, we are publishing 10 sets of events to the Event Hub. Notice that we are not
// specifying a partition to send events to, so these sets may end up in different partitions.
int amountOfSets = 10;
int eventsPerSet = 2;
int expectedAmountOfEvents = amountOfSets * eventsPerSet;
Console.WriteLine();
Console.WriteLine("Sending events to the Event Hub.");
Console.WriteLine();
for (int i = 0; i < amountOfSets; i++)
{
using EventDataBatch eventBatch = await producerClient.CreateBatchAsync();
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I am not the second event.")));
eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I am not the first event.")));
await producerClient.SendAsync(eventBatch);
}
// Because there is some non-determinism in the messaging flow, the sent events may not be immediately
// available. For this reason, we wait 500 ms before resuming.
await Task.Delay(500);
// Once stopped, the event processor won't receive events anymore. In case there are still events being
// processed when the stop method is called, the processing will complete before the corresponding partition
// processor is closed.
Console.WriteLine();
Console.WriteLine("Stopping the event processor.");
Console.WriteLine();
await eventProcessor.StopAsync();
// Print out the amount of events that we received.
Console.WriteLine();
Console.WriteLine($"Amount of events received: { totalEventsCount }. Expected: { expectedAmountOfEvents }.");
}
}
// At this point, our clients have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
public async Task PartitionProcessorCanCreateACheckpointFromPartitionContext()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
// Send some events.
EventData lastEvent;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
var partitionId = (await client.GetPartitionIdsAsync()).First();
await using (EventHubProducer producer = client.CreateProducer())
await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest))
{
// Send a few events. We are only interested in the last one of them.
var dummyEventsCount = 10;
for (int i = 0; i < dummyEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
{
receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
lastEvent = receivedEvents.Last();
}
// Create a partition manager so we can retrieve the created checkpoint from it.
var partitionManager = new InMemoryPartitionManager();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
partitionManager,
onProcessEvents: (partitionContext, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Any())
{
partitionContext.UpdateCheckpointAsync(eventsList.Last());
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(client.FullyQualifiedNamespace, client.EventHubName, EventHubConsumer.DefaultConsumerGroupName);
Assert.That(ownershipEnumerable, Is.Not.Null);
Assert.That(ownershipEnumerable.Count, Is.EqualTo(1));
PartitionOwnership ownership = ownershipEnumerable.Single();
Assert.That(ownership.Offset.HasValue, Is.True);
Assert.That(ownership.Offset.Value, Is.EqualTo(lastEvent.Offset));
Assert.That(ownership.SequenceNumber.HasValue, Is.True);
Assert.That(ownership.SequenceNumber.Value, Is.EqualTo(lastEvent.SequenceNumber));
}
}
}
public async Task EventProcessorCanReceiveFromCheckpointedEventPosition()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
await using (var client = new EventHubClient(connectionString))
{
int receivedEventsCount = 0;
// Send some events.
var expectedEventsCount = 20;
var dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
long?checkpointedSequenceNumber = default;
var partitionId = (await client.GetPartitionIdsAsync()).First();
await using (EventHubProducer producer = client.CreateProducer())
await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest))
{
// Send a few dummy events. We are not expecting to receive these.
var dummyEventsCount = 30;
for (int i = 0; i < dummyEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
// Receive the events; because there is some non-determinism in the messaging flow, the
// sent events may not be immediately available. Allow for a small number of attempts to receive, in order
// to account for availability delays.
var receivedEvents = new List <EventData>();
var index = 0;
while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
{
receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
}
Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));
checkpointedSequenceNumber = receivedEvents.Last().SequenceNumber;
// Send the events we expect to receive.
for (int i = 0; i < expectedEventsCount; i++)
{
await producer.SendAsync(dummyEvent);
}
}
// Create a partition manager and add an ownership with a checkpoint in it.
var partitionManager = new InMemoryPartitionManager();
await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
{
new PartitionOwnership(client.FullyQualifiedNamespace, client.EventHubName,
EventHubConsumer.DefaultConsumerGroupName, "ownerIdentifier", partitionId,
sequenceNumber: checkpointedSequenceNumber, lastModifiedTime: DateTimeOffset.UtcNow)
});
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
EventHubConsumer.DefaultConsumerGroupName,
client,
partitionManager,
onProcessEvents: (partitionContext, events, cancellationToken) =>
{
// Make it a list so we can safely enumerate it.
var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());
if (eventsList.Count > 0)
{
Interlocked.Add(ref receivedEventsCount, eventsList.Count);
}
}
);
eventProcessorManager.AddEventProcessors(1);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize and receive events.
await eventProcessorManager.WaitStabilization();
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
// Validate results.
Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
}
}
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// We will start by creating a client using its default set of options. It will be used by the event processor to
// communicate with the Azure Event Hubs service.
await using (var client = new EventHubClient(connectionString, eventHubName))
{
// An event processor is associated with a specific Event Hub and a consumer group. It receives events from
// multiple partitions in the Event Hub, passing them to the user for processing.
//
// A partition manager may create checkpoints and list/claim partition ownership. The user can implement their
// own partition manager by creating a subclass from the PartitionManager abstract class. Here we are creating
// a new instance of an InMemoryPartitionManager, provided by the Azure.Messaging.EventHubs.Processor namespace.
// This isn't relevant to understanding this sample, but is required by the event processor constructor.
PartitionManager partitionManager = new InMemoryPartitionManager();
// It's also possible to specify custom options upon event processor creation. We want to receive events from
// the latest available position so older events don't interfere with our sample. We also don't want to wait
// more than 1 second for every set of events.
EventProcessorOptions eventProcessorOptions = new EventProcessorOptions
{
InitialEventPosition = EventPosition.Latest,
MaximumReceiveWaitTime = TimeSpan.FromSeconds(1)
};
// Let's finally create our event processor. We're using the default consumer group that was created with the Event Hub.
var eventProcessor = new EventProcessor(EventHubConsumer.DefaultConsumerGroupName, client, partitionManager, eventProcessorOptions);
int totalEventsCount = 0;
int partitionsBeingProcessedCount = 0;
// TODO: explain callbacks setup once the public API is finished for the next preview.
eventProcessor.InitializeProcessingForPartitionAsync = (PartitionContext partitionContext) =>
{
// This is the last piece of code guaranteed to run before event processing, so all initialization
// must be done by the moment this method returns.
Interlocked.Increment(ref partitionsBeingProcessedCount);
Console.WriteLine($"\tPartition '{ partitionContext.PartitionId }': partition processing has started.");
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
eventProcessor.ProcessingForPartitionStoppedAsync = (PartitionContext partitionContext, PartitionProcessorCloseReason reason) =>
{
// The code to be run just before stopping processing events for a partition. This is the right place to dispose
// of objects that will no longer be used.
Interlocked.Decrement(ref partitionsBeingProcessedCount);
Console.WriteLine($"\tPartition '{ partitionContext.PartitionId }': partition processing has stopped. Reason: { reason }.");
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
eventProcessor.ProcessEventsAsync = (PartitionContext partitionContext, IEnumerable <EventData> events) =>
{
// Here the user can specify what to do with the events received from the event processor. We are counting how
// many events were received across all partitions so we can check whether all sent events were received.
//
// It's important to notice that this method is called even when no events are received after the maximum wait time, which
// can be specified by the user in the event processor options. In this case, the IEnumerable events is empty, but not null.
int eventsCount = events.Count();
if (eventsCount > 0)
{
Interlocked.Add(ref totalEventsCount, eventsCount);
Console.WriteLine($"\tPartition '{ partitionContext.PartitionId }': { eventsCount } event(s) received.");
}
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
eventProcessor.ProcessExceptionAsync = (PartitionContext partitionContext, Exception exception) =>
{
// All the unhandled exceptions encountered during the event processor execution are passed to this method so
// the user can decide how to handle them.
//
// This piece of code is not supposed to be reached by this sample. If the following message has been printed
// to the Console, then something unexpected has happened.
Console.WriteLine($"\tPartition '{ partitionContext.PartitionId }': an unhandled exception was encountered. This was not expected to happen.");
// This method is asynchronous, which means it's expected to return a Task.
return(Task.CompletedTask);
};
// Once started, the event processor will start to claim partitions and receive events from them.
Console.WriteLine("Starting the event processor.");
Console.WriteLine();
await eventProcessor.StartAsync();
Console.WriteLine("Event processor started.");
Console.WriteLine();
// Wait until the event processor has claimed ownership of all partitions in the Event Hub. This may take some time
// as there's a 10 seconds interval between claims. To be sure that we do not block forever in case the event processor
// fails, we will specify a fairly long time to wait and then cancel waiting.
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(400));
var partitionsCount = (await client.GetPartitionIdsAsync()).Length;
while (partitionsBeingProcessedCount < partitionsCount)
{
await Task.Delay(500, cancellationSource.Token);
}
// To test our event processor, we are publishing 10 sets of events to the Event Hub. Notice that we are not
// specifying a partition to send events to, so these sets may end up in different partitions.
EventData[] eventsToPublish = new EventData[]
{
new EventData(Encoding.UTF8.GetBytes("I am not the second event.")),
new EventData(Encoding.UTF8.GetBytes("I am not the first event."))
};
int amountOfSets = 10;
int expectedAmountOfEvents = amountOfSets * eventsToPublish.Length;
await using (EventHubProducer producer = client.CreateProducer())
{
Console.WriteLine();
Console.WriteLine("Sending events to the Event Hub.");
Console.WriteLine();
for (int i = 0; i < amountOfSets; i++)
{
await producer.SendAsync(eventsToPublish);
}
}
// Because there is some non-determinism in the messaging flow, the sent events may not be immediately
// available. For this reason, we wait 500 ms before resuming.
await Task.Delay(500);
// Once stopped, the event processor won't receive events anymore. In case there are still events being
// processed when the stop method is called, the processing will complete before the corresponding partition
// processor is closed.
Console.WriteLine();
Console.WriteLine("Stopping the event processor.");
Console.WriteLine();
await eventProcessor.StopAsync();
// Print out the amount of events that we received.
Console.WriteLine();
Console.WriteLine($"Amount of events received: { totalEventsCount }. Expected: { expectedAmountOfEvents }.");
}
// At this point, our client and producer have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// We will start by creating a client using its default set of options. It will be used by the event processor to
// communicate with the Azure Event Hubs service.
await using (var client = new EventHubClient(connectionString, eventHubName))
{
// An event processor is associated with a specific Event Hub and a consumer group. It receives events from
// multiple partitions in the Event Hub, passing them to the user for processing. It's worth mentioning that
// an event processor is a generic class, and it takes a partition processor as its underlying type.
//
// A partition processor is associated with a specific partition and is responsible for processing events when
// requested by the event processor. In order to use it as the event processor's underlying type, two conditions
// must be met:
//
// - It must be a class derived from BasePartitionProcessor.
//
// - It must have a parameterless constructor.
//
// We'll be using a SamplePartitionProcessor, whose implementation can be found at the end of this sample.
// A partition manager may create checkpoints and list/claim partition ownership. The user can implement their
// own partition manager by creating a subclass from the PartitionManager abstract class. Here we are creating
// a new instance of an InMemoryPartitionManager, provided by the Azure.Messaging.EventHubs.Processor namespace.
// This isn't relevant to understanding this sample, but is required by the event processor constructor.
PartitionManager partitionManager = new InMemoryPartitionManager();
// It's also possible to specify custom options upon event processor creation. We want to receive events from
// the latest available position so older events don't interfere with our sample. We also don't want to wait
// more than 1 second for every set of events.
EventProcessorOptions eventProcessorOptions = new EventProcessorOptions
{
InitialEventPosition = EventPosition.Latest,
MaximumReceiveWaitTime = TimeSpan.FromSeconds(1)
};
// Let's finally create our event processor. We're using the default consumer group that was created with the Event Hub.
var eventProcessor = new EventProcessor <SamplePartitionProcessor>(EventHubConsumer.DefaultConsumerGroupName, client, partitionManager, eventProcessorOptions);
// Once started, the event processor will start to claim partitions and receive events from them.
Console.WriteLine("Starting the event processor.");
Console.WriteLine();
await eventProcessor.StartAsync();
Console.WriteLine("Event processor started.");
Console.WriteLine();
// Wait until the event processor has claimed ownership of all partitions in the Event Hub. There should be a single
// active partition processor per owned partition. This may take some time as there's a 10 seconds interval between
// claims. To be sure that we do not block forever in case the event processor fails, we will specify a fairly long
// time to wait and then cancel waiting.
CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(400));
var partitionsCount = (await client.GetPartitionIdsAsync()).Length;
while (SamplePartitionProcessor.ActiveInstancesCount < partitionsCount)
{
await Task.Delay(500, cancellationSource.Token);
}
// To test our event processor, we are publishing 10 sets of events to the Event Hub. Notice that we are not
// specifying a partition to send events to, so these sets may end up in different partitions.
EventData[] eventsToPublish = new EventData[]
{
new EventData(Encoding.UTF8.GetBytes("I am not the second event.")),
new EventData(Encoding.UTF8.GetBytes("I am not the first event."))
};
int amountOfSets = 10;
int expectedAmountOfEvents = amountOfSets * eventsToPublish.Length;
await using (EventHubProducer producer = client.CreateProducer())
{
Console.WriteLine("Sending events to the Event Hub.");
Console.WriteLine();
for (int i = 0; i < amountOfSets; i++)
{
await producer.SendAsync(eventsToPublish);
}
}
// Because there is some non-determinism in the messaging flow, the sent events may not be immediately
// available. For this reason, we wait 500 ms before resuming.
await Task.Delay(500);
// Once stopped, the event processor won't receive events anymore. In case there are still events being
// processed when the stop method is called, the processing will complete before the corresponding partition
// processor is closed.
Console.WriteLine();
Console.WriteLine("Stopping the event processor.");
Console.WriteLine();
await eventProcessor.StopAsync();
// Print out the amount of events that we received.
Console.WriteLine();
Console.WriteLine($"Amount of events received: { SamplePartitionProcessor.TotalEventsCount }. Expected: { expectedAmountOfEvents }.");
}
// At this point, our client and producer have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}
/// <summary>
/// Runs the sample using the specified Event Hubs connection information.
/// </summary>
///
/// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
///
public async Task RunAsync(string connectionString,
string eventHubName)
{
// We will start by creating a client using its default set of options. It will be used by the event processor to
// communicate with the Azure Event Hubs service.
await using (var client = new EventHubClient(connectionString, eventHubName))
{
// An event processor is associated with a specific Event Hub and a consumer group. It receives events from
// all partitions in the Event Hub, passing them to the user for processing.
//
// A partition processor is associated with a specific partition and is responsible for processing events when
// requested by the event processor. An instance is provided to the event processor when requested from a
// factory function, and takes the form of a class which implements the IPartitionProcessor interface.
//
// The factory function is provided to the event processor when it is created. The factory is responsible for
// creating a partition processor based on two arguments:
//
// A partition context: contains information about the partition the partition processor will be processing
// events from. In this sample, we are only interested in its partition id.
//
// A checkpoint manager: responsible for the creation of checkpoints. It's not used in this sample.
//
// We'll be using a SamplePartitionProcessor, whose implementation can be found at the end of this sample. Its
// constructor takes the associated partition id so it can provide useful log messages.
Func <PartitionContext, CheckpointManager, IPartitionProcessor> partitionProcessorFactory =
(partitionContext, checkpointManager) => new SamplePartitionProcessor(partitionContext.PartitionId);
// A partition manager may create checkpoints and list/claim partition ownership. The user can implement their
// own partition manager by creating a subclass from the PartitionManager abstract class. Here we are creating
// a new instance of an InMemoryPartitionManager, provided by the Azure.Messaging.EventHubs.Processor namespace.
// This isn't relevant to understanding this sample, but is required by the event processor constructor.
PartitionManager partitionManager = new InMemoryPartitionManager();
// It's also possible to specify custom options upon event processor creation. We want to receive events from
// the latest available position so older events don't interfere with our sample. We also don't want to wait
// more than 1 second for every set of events.
EventProcessorOptions eventProcessorOptions = new EventProcessorOptions
{
InitialEventPosition = EventPosition.Latest,
MaximumReceiveWaitTime = TimeSpan.FromSeconds(1)
};
// Let's finally create our event processor. We're using the default consumer group that was created with the Event Hub.
EventProcessor eventProcessor = new EventProcessor(EventHubConsumer.DefaultConsumerGroupName, client, partitionProcessorFactory, partitionManager, eventProcessorOptions);
// Once started, the event processor will start to receive events from all partitions.
Console.WriteLine("Starting the event processor.");
Console.WriteLine();
await eventProcessor.StartAsync();
Console.WriteLine();
Console.WriteLine("Event processor started.");
// To test our event processor, we are publishing 10 sets of events to the Event Hub. Notice that we are not
// specifying a partition to send events to, so these sets may end up in different partitions.
EventData[] eventsToPublish = new EventData[]
{
new EventData(Encoding.UTF8.GetBytes("I am not the second event.")),
new EventData(Encoding.UTF8.GetBytes("I am not the first event."))
};
int amountOfSets = 10;
int expectedAmountOfEvents = amountOfSets * eventsToPublish.Length;
await using (EventHubProducer producer = client.CreateProducer())
{
Console.WriteLine();
Console.WriteLine("Sending events to the Event Hub.");
Console.WriteLine();
for (int i = 0; i < amountOfSets; i++)
{
await producer.SendAsync(eventsToPublish);
}
}
// Because there is some non-determinism in the messaging flow, the sent events may not be immediately
// available. For this reason, we wait 500 ms before resuming.
await Task.Delay(500);
// Once stopped, the event processor won't receive events anymore. In case there are still events being
// processed when the stop method is called, the processing will complete before the corresponding partition
// processor is closed.
Console.WriteLine();
Console.WriteLine("Stopping the event processor.");
Console.WriteLine();
await eventProcessor.StopAsync();
// Print out the amount of events that we received.
Console.WriteLine();
Console.WriteLine($"Amount of events received: { SamplePartitionProcessor.TotalEventsCount }. Expected: { expectedAmountOfEvents }.");
}
// At this point, our client and producer have passed their "using" scope and have safely been disposed of. We
// have no further obligations.
Console.WriteLine();
}