public async Task ClaimOwnershipAsyncCanClaimMultipleOwnership()
        {
            var partitionManager = new InMemoryPartitionManager();
            var ownershipList    = new List <PartitionOwnership>();
            var ownershipCount   = 5;

            for (int i = 0; i < ownershipCount; i++)
            {
                ownershipList.Add(
                    new PartitionOwnership
                    (
                        "eventHubName",
                        "consumerGroup",
                        "ownerIdentifier",
                        $"partitionId { i }"
                    ));
            }

            await partitionManager.ClaimOwnershipAsync(ownershipList);

            var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup");

            Assert.That(storedOwnership, Is.Not.Null);
            Assert.That(storedOwnership.Count, Is.EqualTo(ownershipCount));
            Assert.That(storedOwnership.OrderBy(ownership => ownership.PartitionId).SequenceEqual(ownershipList), Is.True);
        }
Exemple #2
0
        static async Task Consume(string connectionString, string consumerGroup, string eventHubName)
        {
            await using (var client = new EventHubClient(connectionString, eventHubName))
            {
                Func <PartitionContext, CheckpointManager, IPartitionProcessor> partitionProcessorFactory =
                    (partitionContext, checkpointManager) => new TemperatureProcessor(partitionContext.PartitionId);

                var partitionManager = new InMemoryPartitionManager();

                var eventProcessorOptions = new EventProcessorOptions
                {
                    InitialEventPosition   = EventPosition.Latest,
                    MaximumReceiveWaitTime = TimeSpan.FromSeconds(1)
                };

                var eventProcessor = new EventProcessor(
                    consumerGroup,
                    client,
                    partitionProcessorFactory,
                    partitionManager,
                    eventProcessorOptions);

                await eventProcessor.StartAsync();

                Console.WriteLine("Receiving...");
                Console.ReadLine();
            }
        }
        public async Task CheckpointUpdateUpdatesOwnershipInformation()
        {
            var partitionManager  = new InMemoryPartitionManager();
            var originalOwnership = new PartitionOwnership
                                        ("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", offset: 1, sequenceNumber: 2, lastModifiedTime: DateTimeOffset.UtcNow.Subtract(TimeSpan.FromMinutes(1)));

            await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
            {
                originalOwnership
            });

            // ETag must have been set by the partition manager.

            var originalLastModifiedTime = originalOwnership.LastModifiedTime;
            var originalETag             = originalOwnership.ETag;

            await partitionManager.UpdateCheckpointAsync(new Checkpoint
                                                         ("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", 10, 20));

            // Make sure the ownership has changed, even though the instance should be the same.

            var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup");

            Assert.That(storedOwnership, Is.Not.Null);
            Assert.That(storedOwnership.Count, Is.EqualTo(1));
            Assert.That(storedOwnership.Single(), Is.EqualTo(originalOwnership));

            Assert.That(originalOwnership.Offset, Is.EqualTo(10));
            Assert.That(originalOwnership.SequenceNumber, Is.EqualTo(20));
            Assert.That(originalOwnership.LastModifiedTime, Is.GreaterThan(originalLastModifiedTime));
            Assert.That(originalOwnership.ETag, Is.Not.EqualTo(originalETag));
        }
        public async Task CheckpointUpdateFailsWhenOwnerChanges()
        {
            var partitionManager  = new InMemoryPartitionManager();
            var originalOwnership = new PartitionOwnership
                                        ("eventHubName", "consumerGroup", "ownerIdentifier1", "partitionId", offset: 1, sequenceNumber: 2, lastModifiedTime: DateTimeOffset.UtcNow);

            await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
            {
                originalOwnership
            });

            // ETag must have been set by the partition manager.

            var originalLastModifiedTime = originalOwnership.LastModifiedTime;
            var originalETag             = originalOwnership.ETag;

            await partitionManager.UpdateCheckpointAsync(new Checkpoint
                                                         ("eventHubName", "consumerGroup", "ownerIdentifier2", "partitionId", 10, 20));

            // Make sure the ownership hasn't changed.

            var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup");

            Assert.That(storedOwnership, Is.Not.Null);
            Assert.That(storedOwnership.Count, Is.EqualTo(1));
            Assert.That(storedOwnership.Single(), Is.EqualTo(originalOwnership));

            Assert.That(originalOwnership.OwnerIdentifier, Is.EqualTo("ownerIdentifier1"));
            Assert.That(originalOwnership.Offset, Is.EqualTo(1));
            Assert.That(originalOwnership.SequenceNumber, Is.EqualTo(2));
            Assert.That(originalOwnership.LastModifiedTime, Is.EqualTo(originalLastModifiedTime));
            Assert.That(originalOwnership.ETag, Is.EqualTo(originalETag));
        }
        public async Task ListOwnershipAsyncReturnsEmptyIEnumerableWhenThereAreNoOwnership()
        {
            var partitionManager = new InMemoryPartitionManager();
            var ownership        = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup");

            Assert.That(ownership, Is.Not.Null.And.Empty);
        }
        public async Task CheckpointUpdateFailsWhenAssociatedOwnershipDoesNotExist()
        {
            var partitionManager = new InMemoryPartitionManager();

            await partitionManager.UpdateCheckpointAsync(new Checkpoint
                                                         ("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId", offset : 10, sequenceNumber : 20));

            var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup");

            Assert.That(storedOwnership, Is.Not.Null);
            Assert.That(storedOwnership, Is.Empty);
        }
        public async Task ClaimOwnershipAsyncReturnsOnlyTheSuccessfullyClaimedOwnership()
        {
            var partitionManager = new InMemoryPartitionManager();
            var ownershipList    = new List <PartitionOwnership>();
            var ownershipCount   = 5;

            for (int i = 0; i < ownershipCount; i++)
            {
                ownershipList.Add(
                    new PartitionOwnership
                    (
                        "namespace",
                        "eventHubName",
                        "consumerGroup",
                        "ownerIdentifier",
                        $"partitionId { i }"
                    ));
            }

            await partitionManager.ClaimOwnershipAsync(ownershipList);

            // The ETags must have been set by the partition manager.

            var eTags = ownershipList.Select(ownership => ownership.ETag).ToList();

            ownershipList.Clear();

            // Use a valid eTag when 'i' is odd.  This way, we can expect 'ownershipCount / 2' successful
            // claims (rounded down).

            var expectedClaimedCount = ownershipCount / 2;

            for (int i = 0; i < ownershipCount; i++)
            {
                ownershipList.Add(
                    new PartitionOwnership
                    (
                        "namespace",
                        "eventHubName",
                        "consumerGroup",
                        "ownerIdentifier",
                        $"partitionId { i }",
                        offset: i,
                        eTag: i % 2 == 1 ? eTags[i] : null
                    ));
            }

            IEnumerable <PartitionOwnership> claimedOwnership = await partitionManager.ClaimOwnershipAsync(ownershipList);

            Assert.That(claimedOwnership, Is.Not.Null);
            Assert.That(claimedOwnership.Count, Is.EqualTo(expectedClaimedCount));
            Assert.That(claimedOwnership.OrderBy(ownership => ownership.Offset).SequenceEqual(ownershipList.Where(ownership => ownership.Offset % 2 == 1)), Is.True);
        }
        public async Task OwnershipClaimDoesNotInterfereWithOtherNamespaces()
        {
            var partitionManager = new InMemoryPartitionManager();
            var ownershipList    = new List <PartitionOwnership>();
            var firstOwnership   =
                new PartitionOwnership
                (
                    "namespace1",
                    "eventHubName",
                    "consumerGroup",
                    "ownerIdentifier",
                    "partitionId"
                );

            ownershipList.Add(firstOwnership);

            await partitionManager.ClaimOwnershipAsync(ownershipList);

            // ETag must have been set by the partition manager.

            var eTag = firstOwnership.ETag;

            ownershipList.Clear();

            var secondOwnership =
                new PartitionOwnership
                (
                    "namespace2",
                    "eventHubName",
                    "consumerGroup",
                    "ownerIdentifier",
                    "partitionId",
                    eTag: eTag
                );

            ownershipList.Add(secondOwnership);

            await partitionManager.ClaimOwnershipAsync(ownershipList);

            IEnumerable <PartitionOwnership> storedOwnership1 = await partitionManager.ListOwnershipAsync("namespace1", "eventHubName", "consumerGroup");

            IEnumerable <PartitionOwnership> storedOwnership2 = await partitionManager.ListOwnershipAsync("namespace2", "eventHubName", "consumerGroup");

            Assert.That(storedOwnership1, Is.Not.Null);
            Assert.That(storedOwnership1.Count, Is.EqualTo(1));
            Assert.That(storedOwnership1.Single(), Is.EqualTo(firstOwnership));

            Assert.That(storedOwnership2, Is.Not.Null);
            Assert.That(storedOwnership2.Count, Is.EqualTo(1));
            Assert.That(storedOwnership2.Single(), Is.EqualTo(secondOwnership));
        }
        public async Task OwnershipClaimSucceedsWhenETagIsValid()
        {
            var partitionManager = new InMemoryPartitionManager();
            var ownershipList    = new List <PartitionOwnership>();
            var firstOwnership   =
                new PartitionOwnership
                (
                    "namespace",
                    "eventHubName",
                    "consumerGroup",
                    "ownerIdentifier",
                    "partitionId",
                    offset: 1
                );

            ownershipList.Add(firstOwnership);

            await partitionManager.ClaimOwnershipAsync(ownershipList);

            // ETag must have been set by the partition manager.

            var eTag = firstOwnership.ETag;

            ownershipList.Clear();

            var secondOwnership =
                new PartitionOwnership
                (
                    "namespace",
                    "eventHubName",
                    "consumerGroup",
                    "ownerIdentifier",
                    "partitionId",
                    offset: 2,
                    eTag: eTag
                );

            ownershipList.Add(secondOwnership);

            await partitionManager.ClaimOwnershipAsync(ownershipList);

            IEnumerable <PartitionOwnership> storedOwnership = await partitionManager.ListOwnershipAsync("namespace", "eventHubName", "consumerGroup");

            Assert.That(storedOwnership, Is.Not.Null);
            Assert.That(storedOwnership.Count, Is.EqualTo(1));
            Assert.That(storedOwnership.Single(), Is.EqualTo(secondOwnership));
        }
        public async Task OwnershipClaimFailsWhenETagIsInvalid(string eTag)
        {
            var partitionManager = new InMemoryPartitionManager();
            var ownershipList    = new List <PartitionOwnership>();
            var firstOwnership   =
                new PartitionOwnership
                (
                    "eventHubName",
                    "consumerGroup",
                    "ownerIdentifier",
                    "partitionId",
                    offset: 1
                );

            ownershipList.Add(firstOwnership);

            await partitionManager.ClaimOwnershipAsync(ownershipList);

            ownershipList.Clear();

            var secondOwnership =
                new PartitionOwnership
                (
                    "eventHubName",
                    "consumerGroup",
                    "ownerIdentifier",
                    "partitionId",
                    offset: 2,
                    eTag: eTag
                );

            ownershipList.Add(secondOwnership);

            await partitionManager.ClaimOwnershipAsync(ownershipList);

            var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup");

            Assert.That(storedOwnership, Is.Not.Null);
            Assert.That(storedOwnership.Count, Is.EqualTo(1));
            Assert.That(storedOwnership.Single(), Is.EqualTo(firstOwnership));
        }
        public async Task CheckpointUpdateDoesNotInterfereWithOtherPartitions()
        {
            var partitionManager = new InMemoryPartitionManager();

            var ownership1 = new PartitionOwnership
                                 ("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId1", offset: 1);
            var ownership2 = new PartitionOwnership
                                 ("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId2", offset: 1);

            await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
            {
                ownership1,
                ownership2
            });

            await partitionManager.UpdateCheckpointAsync(new Checkpoint
                                                         ("eventHubName", "consumerGroup", "ownerIdentifier", "partitionId1", 10, 20));

            Assert.That(ownership1.Offset, Is.EqualTo(10));
            Assert.That(ownership2.Offset, Is.EqualTo(1));
        }
        public async Task FirstOwnershipClaimSucceeds()
        {
            var partitionManager = new InMemoryPartitionManager();
            var ownershipList    = new List <PartitionOwnership>();
            var ownership        =
                new PartitionOwnership
                (
                    "eventHubName",
                    "consumerGroup",
                    "ownerIdentifier",
                    "partitionId"
                );

            ownershipList.Add(ownership);

            await partitionManager.ClaimOwnershipAsync(ownershipList);

            var storedOwnership = await partitionManager.ListOwnershipAsync("eventHubName", "consumerGroup");

            Assert.That(storedOwnership, Is.Not.Null);
            Assert.That(storedOwnership.Count, Is.EqualTo(1));
            Assert.That(storedOwnership.Single(), Is.EqualTo(ownership));
        }
Exemple #13
0
        /// <summary>
        ///   Initializes a new instance of the <see cref="EventProcessorManager"/> class.
        /// </summary>
        ///
        /// <param name="consumerGroup">The name of the consumer group the event processors are associated with.  Events are read in the context of this group.</param>
        /// <param name="client">The client used to interact with the Azure Event Hubs service.</param>
        /// <param name="options">The set of options to use for the event processors.</param>
        /// <param name="onInitialize">A callback action to be called on <see cref="PartitionProcessor.InitializeAsync" />.</param>
        /// <param name="onClose">A callback action to be called on <see cref="PartitionProcessor.CloseAsync" />.</param>
        /// <param name="onProcessEvents">A callback action to be called on <see cref="PartitionProcessor.ProcessEventsAsync" />.</param>
        /// <param name="onProcessError">A callback action to be called on <see cref="PartitionProcessor.ProcessErrorAsync" />.</param>
        ///
        public EventProcessorManager(string consumerGroup,
                                     EventHubClient client,
                                     EventProcessorOptions options = null,
                                     Action <PartitionContext, CheckpointManager> onInitialize = null,
                                     Action <PartitionContext, CheckpointManager, PartitionProcessorCloseReason> onClose = null,
                                     Action <PartitionContext, CheckpointManager, IEnumerable <EventData>, CancellationToken> onProcessEvents = null,
                                     Action <PartitionContext, CheckpointManager, Exception, CancellationToken> onProcessError = null)
        {
            ConsumerGroup = consumerGroup;
            InnerClient   = client;

            PartitionProcessorFactory = (partitionContext, checkpointManager) =>
                                        new PartitionProcessor
                                        (
                partitionContext,
                checkpointManager,
                onInitialize,
                onClose,
                onProcessEvents,
                onProcessError
                                        );

            InnerPartitionManager = new InMemoryPartitionManager();

            // In case it has not been specified, set the maximum receive wait time to 2 seconds because the default
            // value (1 minute) would take too much time.

            Options = options?.Clone() ?? new EventProcessorOptions();

            if (Options.MaximumReceiveWaitTime == null)
            {
                Options.MaximumReceiveWaitTime = TimeSpan.FromSeconds(2);
            }

            EventProcessors = new List <EventProcessor>();
        }
Exemple #14
0
        /// <summary>
        ///   Runs the sample using the specified Event Hubs connection information.
        /// </summary>
        ///
        /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
        /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
        ///
        public async Task RunAsync(string connectionString,
                                   string eventHubName)
        {
            // An event processor is associated with a specific Event Hub and a consumer group.  It receives events from
            // multiple partitions in the Event Hub, passing them to a handler delegate for processing using code that you
            // provide.
            //
            // These handler delegates are invoked for each event or error that occurs during operation of the event processor.  For
            // a given partition, only a single event will be dispatched for processing at a time so that the order of events within a
            // partition is preserved.  Partitions, however, are processed concurrently.  As a result, your handlers are potentially processing
            // multiple events or errors at any given time.

            // A partition manager may create checkpoints and list/claim partition ownership.  A developer may implement their
            // own partition manager by creating a subclass from the PartitionManager abstract class.  Here we are creating
            // a new instance of an InMemoryPartitionManager, provided by the Azure.Messaging.EventHubs.Processor namespace.
            // This isn't relevant to understanding this sample, but is required by the event processor constructor.

            var partitionManager = new InMemoryPartitionManager();

            // It's also possible to specify custom options upon event processor creation.  We don't want to wait
            // more than 1 second for every set of events.

            var eventProcessorOptions = new EventProcessorClientOptions
            {
                MaximumReceiveWaitTime = TimeSpan.FromSeconds(1)
            };

            // Let's finally create our event processor.  We're using the default consumer group that was created with the Event Hub.

            await using (var eventProcessor = new EventProcessorClient(EventHubConsumerClient.DefaultConsumerGroupName, partitionManager, connectionString, eventHubName, eventProcessorOptions))
            {
                int totalEventsCount = 0;
                int partitionsBeingProcessedCount = 0;

                // TODO: explain callbacks setup once the public API is finished for the next preview.

                eventProcessor.InitializeProcessingForPartitionAsync = (initializationContext) =>
                {
                    // This is the last piece of code guaranteed to run before event processing, so all initialization
                    // must be done by the moment this method returns.

                    // We want to receive events from the latest available position so older events don't interfere with our sample.

                    initializationContext.DefaultStartingPosition = EventPosition.Latest;

                    Interlocked.Increment(ref partitionsBeingProcessedCount);

                    Console.WriteLine($"\tPartition '{ initializationContext.Context.PartitionId }': partition processing has started.");

                    // This method is asynchronous, which means it's expected to return a Task.

                    return(new ValueTask());
                };

                eventProcessor.ProcessingForPartitionStoppedAsync = (stopContext) =>
                {
                    // The code to be run just before stopping processing events for a partition.  This is the right place to dispose
                    // of objects that will no longer be used.

                    Interlocked.Decrement(ref partitionsBeingProcessedCount);

                    Console.WriteLine($"\tPartition '{ stopContext.Context.PartitionId }': partition processing has stopped. Reason: { stopContext.Reason }.");

                    // This method is asynchronous, which means it's expected to return a Task.

                    return(new ValueTask());
                };

                eventProcessor.ProcessEventAsync = (processorEvent) =>
                {
                    // Here the user can specify what to do with the event received from the event processor.  We are counting how
                    // many events were received across all partitions so we can check whether all sent events were received.
                    //
                    // It's important to notice that this method is called even when no event is received after the maximum wait time, which
                    // can be specified by the user in the event processor options.  In this case, the received event is null.

                    if (processorEvent.Data != null)
                    {
                        Interlocked.Increment(ref totalEventsCount);
                        Console.WriteLine($"\tPartition '{ processorEvent.Context.PartitionId }': event received.");
                    }

                    // This method is asynchronous, which means it's expected to return a Task.

                    return(new ValueTask());
                };

                eventProcessor.ProcessExceptionAsync = (errorContext) =>
                {
                    // Any exception which occurs as a result of the event processor itself will be passed to
                    // this delegate so it may be handled.  The processor will continue to process events if
                    // it is able to unless this handler explicitly requests that it stop doing so.
                    //
                    // It is important to note that this does not include exceptions during event processing; those
                    // are considered responsibility of the developer implementing the event processing handler.  It
                    // is, therefore, highly encouraged that best practices for exception handling practices are
                    // followed with that delegate.
                    //
                    // This piece of code is not supposed to be reached by this sample.  If the following message has been printed
                    // to the Console, then something unexpected has happened.

                    Console.WriteLine($"\tPartition '{ errorContext.PartitionId }': an unhandled exception was encountered. This was not expected to happen.");

                    // This method is asynchronous, which means it's expected to return a Task.

                    return(new ValueTask());
                };

                // Once started, the event processor will start to claim partitions and receive events from them.

                Console.WriteLine("Starting the event processor.");
                Console.WriteLine();

                await eventProcessor.StartAsync();

                Console.WriteLine("Event processor started.");
                Console.WriteLine();

                // Wait until the event processor has claimed ownership of all partitions in the Event Hub.  This may take some time
                // as there's a 10 seconds interval between claims.  To be sure that we do not block forever in case the event processor
                // fails, we will specify a fairly long time to wait and then cancel waiting.

                CancellationTokenSource cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(TimeSpan.FromSeconds(400));

                await using (var producerClient = new EventHubProducerClient(connectionString, eventHubName))
                {
                    var partitionsCount = (await producerClient.GetPartitionIdsAsync()).Length;

                    while (partitionsBeingProcessedCount < partitionsCount)
                    {
                        await Task.Delay(500, cancellationSource.Token);
                    }

                    // To test our event processor, we are publishing 10 sets of events to the Event Hub.  Notice that we are not
                    // specifying a partition to send events to, so these sets may end up in different partitions.

                    int amountOfSets           = 10;
                    int eventsPerSet           = 2;
                    int expectedAmountOfEvents = amountOfSets * eventsPerSet;

                    Console.WriteLine();
                    Console.WriteLine("Sending events to the Event Hub.");
                    Console.WriteLine();

                    for (int i = 0; i < amountOfSets; i++)
                    {
                        using EventDataBatch eventBatch = await producerClient.CreateBatchAsync();

                        eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I am not the second event.")));
                        eventBatch.TryAdd(new EventData(Encoding.UTF8.GetBytes("I am not the first event.")));

                        await producerClient.SendAsync(eventBatch);
                    }

                    // Because there is some non-determinism in the messaging flow, the sent events may not be immediately
                    // available.  For this reason, we wait 500 ms before resuming.

                    await Task.Delay(500);

                    // Once stopped, the event processor won't receive events anymore.  In case there are still events being
                    // processed when the stop method is called, the processing will complete before the corresponding partition
                    // processor is closed.

                    Console.WriteLine();
                    Console.WriteLine("Stopping the event processor.");
                    Console.WriteLine();

                    await eventProcessor.StopAsync();

                    // Print out the amount of events that we received.

                    Console.WriteLine();
                    Console.WriteLine($"Amount of events received: { totalEventsCount }. Expected: { expectedAmountOfEvents }.");
                }
            }

            // At this point, our clients have passed their "using" scope and have safely been disposed of.  We
            // have no further obligations.

            Console.WriteLine();
        }
Exemple #15
0
        public async Task PartitionProcessorCanCreateACheckpointFromPartitionContext()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                {
                    // Send some events.

                    EventData lastEvent;
                    var       dummyEvent = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));

                    var partitionId = (await client.GetPartitionIdsAsync()).First();

                    await using (EventHubProducer producer = client.CreateProducer())
                        await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest))
                        {
                            // Send a few events.  We are only interested in the last one of them.

                            var dummyEventsCount = 10;

                            for (int i = 0; i < dummyEventsCount; i++)
                            {
                                await producer.SendAsync(dummyEvent);
                            }

                            // Receive the events; because there is some non-determinism in the messaging flow, the
                            // sent events may not be immediately available.  Allow for a small number of attempts to receive, in order
                            // to account for availability delays.

                            var receivedEvents = new List <EventData>();
                            var index          = 0;

                            while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
                            {
                                receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
                            }

                            Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));

                            lastEvent = receivedEvents.Last();
                        }

                    // Create a partition manager so we can retrieve the created checkpoint from it.

                    var partitionManager = new InMemoryPartitionManager();

                    // Create the event processor manager to manage our event processors.

                    var eventProcessorManager = new EventProcessorManager
                                                (
                        EventHubConsumer.DefaultConsumerGroupName,
                        client,
                        partitionManager,
                        onProcessEvents: (partitionContext, events, cancellationToken) =>
                    {
                        // Make it a list so we can safely enumerate it.

                        var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());

                        if (eventsList.Any())
                        {
                            partitionContext.UpdateCheckpointAsync(eventsList.Last());
                        }
                    }
                                                );

                    eventProcessorManager.AddEventProcessors(1);

                    // Start the event processors.

                    await eventProcessorManager.StartAllAsync();

                    // Make sure the event processors have enough time to stabilize and receive events.

                    await eventProcessorManager.WaitStabilization();

                    // Stop the event processors.

                    await eventProcessorManager.StopAllAsync();

                    // Validate results.

                    IEnumerable <PartitionOwnership> ownershipEnumerable = await partitionManager.ListOwnershipAsync(client.FullyQualifiedNamespace, client.EventHubName, EventHubConsumer.DefaultConsumerGroupName);

                    Assert.That(ownershipEnumerable, Is.Not.Null);
                    Assert.That(ownershipEnumerable.Count, Is.EqualTo(1));

                    PartitionOwnership ownership = ownershipEnumerable.Single();

                    Assert.That(ownership.Offset.HasValue, Is.True);
                    Assert.That(ownership.Offset.Value, Is.EqualTo(lastEvent.Offset));

                    Assert.That(ownership.SequenceNumber.HasValue, Is.True);
                    Assert.That(ownership.SequenceNumber.Value, Is.EqualTo(lastEvent.SequenceNumber));
                }
            }
        }
Exemple #16
0
        public async Task EventProcessorCanReceiveFromCheckpointedEventPosition()
        {
            await using (EventHubScope scope = await EventHubScope.CreateAsync(1))
            {
                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);

                await using (var client = new EventHubClient(connectionString))
                {
                    int receivedEventsCount = 0;

                    // Send some events.

                    var  expectedEventsCount        = 20;
                    var  dummyEvent                 = new EventData(Encoding.UTF8.GetBytes("I'm dummy."));
                    long?checkpointedSequenceNumber = default;

                    var partitionId = (await client.GetPartitionIdsAsync()).First();

                    await using (EventHubProducer producer = client.CreateProducer())
                        await using (EventHubConsumer consumer = client.CreateConsumer(EventHubConsumer.DefaultConsumerGroupName, partitionId, EventPosition.Earliest))
                        {
                            // Send a few dummy events.  We are not expecting to receive these.

                            var dummyEventsCount = 30;

                            for (int i = 0; i < dummyEventsCount; i++)
                            {
                                await producer.SendAsync(dummyEvent);
                            }

                            // Receive the events; because there is some non-determinism in the messaging flow, the
                            // sent events may not be immediately available.  Allow for a small number of attempts to receive, in order
                            // to account for availability delays.

                            var receivedEvents = new List <EventData>();
                            var index          = 0;

                            while ((receivedEvents.Count < dummyEventsCount) && (++index < ReceiveRetryLimit))
                            {
                                receivedEvents.AddRange(await consumer.ReceiveAsync(dummyEventsCount + 10, TimeSpan.FromMilliseconds(25)));
                            }

                            Assert.That(receivedEvents.Count, Is.EqualTo(dummyEventsCount));

                            checkpointedSequenceNumber = receivedEvents.Last().SequenceNumber;

                            // Send the events we expect to receive.

                            for (int i = 0; i < expectedEventsCount; i++)
                            {
                                await producer.SendAsync(dummyEvent);
                            }
                        }

                    // Create a partition manager and add an ownership with a checkpoint in it.

                    var partitionManager = new InMemoryPartitionManager();

                    await partitionManager.ClaimOwnershipAsync(new List <PartitionOwnership>()
                    {
                        new PartitionOwnership(client.FullyQualifiedNamespace, client.EventHubName,
                                               EventHubConsumer.DefaultConsumerGroupName, "ownerIdentifier", partitionId,
                                               sequenceNumber: checkpointedSequenceNumber, lastModifiedTime: DateTimeOffset.UtcNow)
                    });

                    // Create the event processor manager to manage our event processors.

                    var eventProcessorManager = new EventProcessorManager
                                                (
                        EventHubConsumer.DefaultConsumerGroupName,
                        client,
                        partitionManager,
                        onProcessEvents: (partitionContext, events, cancellationToken) =>
                    {
                        // Make it a list so we can safely enumerate it.

                        var eventsList = new List <EventData>(events ?? Enumerable.Empty <EventData>());

                        if (eventsList.Count > 0)
                        {
                            Interlocked.Add(ref receivedEventsCount, eventsList.Count);
                        }
                    }
                                                );

                    eventProcessorManager.AddEventProcessors(1);

                    // Start the event processors.

                    await eventProcessorManager.StartAllAsync();

                    // Make sure the event processors have enough time to stabilize and receive events.

                    await eventProcessorManager.WaitStabilization();

                    // Stop the event processors.

                    await eventProcessorManager.StopAllAsync();

                    // Validate results.

                    Assert.That(receivedEventsCount, Is.EqualTo(expectedEventsCount));
                }
            }
        }
        /// <summary>
        ///   Runs the sample using the specified Event Hubs connection information.
        /// </summary>
        ///
        /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
        /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
        ///
        public async Task RunAsync(string connectionString,
                                   string eventHubName)
        {
            // We will start by creating a client using its default set of options.  It will be used by the event processor to
            // communicate with the Azure Event Hubs service.

            await using (var client = new EventHubClient(connectionString, eventHubName))
            {
                // An event processor is associated with a specific Event Hub and a consumer group.  It receives events from
                // multiple partitions in the Event Hub, passing them to the user for processing.
                //
                // A partition manager may create checkpoints and list/claim partition ownership.  The user can implement their
                // own partition manager by creating a subclass from the PartitionManager abstract class.  Here we are creating
                // a new instance of an InMemoryPartitionManager, provided by the Azure.Messaging.EventHubs.Processor namespace.
                // This isn't relevant to understanding this sample, but is required by the event processor constructor.

                PartitionManager partitionManager = new InMemoryPartitionManager();

                // It's also possible to specify custom options upon event processor creation.  We want to receive events from
                // the latest available position so older events don't interfere with our sample.  We also don't want to wait
                // more than 1 second for every set of events.

                EventProcessorOptions eventProcessorOptions = new EventProcessorOptions
                {
                    InitialEventPosition   = EventPosition.Latest,
                    MaximumReceiveWaitTime = TimeSpan.FromSeconds(1)
                };

                // Let's finally create our event processor.  We're using the default consumer group that was created with the Event Hub.

                var eventProcessor = new EventProcessor(EventHubConsumer.DefaultConsumerGroupName, client, partitionManager, eventProcessorOptions);

                int totalEventsCount = 0;
                int partitionsBeingProcessedCount = 0;

                // TODO: explain callbacks setup once the public API is finished for the next preview.

                eventProcessor.InitializeProcessingForPartitionAsync = (PartitionContext partitionContext) =>
                {
                    // This is the last piece of code guaranteed to run before event processing, so all initialization
                    // must be done by the moment this method returns.

                    Interlocked.Increment(ref partitionsBeingProcessedCount);

                    Console.WriteLine($"\tPartition '{ partitionContext.PartitionId }': partition processing has started.");

                    // This method is asynchronous, which means it's expected to return a Task.

                    return(Task.CompletedTask);
                };

                eventProcessor.ProcessingForPartitionStoppedAsync = (PartitionContext partitionContext, PartitionProcessorCloseReason reason) =>
                {
                    // The code to be run just before stopping processing events for a partition.  This is the right place to dispose
                    // of objects that will no longer be used.

                    Interlocked.Decrement(ref partitionsBeingProcessedCount);

                    Console.WriteLine($"\tPartition '{ partitionContext.PartitionId }': partition processing has stopped. Reason: { reason }.");

                    // This method is asynchronous, which means it's expected to return a Task.

                    return(Task.CompletedTask);
                };

                eventProcessor.ProcessEventsAsync = (PartitionContext partitionContext, IEnumerable <EventData> events) =>
                {
                    // Here the user can specify what to do with the events received from the event processor.  We are counting how
                    // many events were received across all partitions so we can check whether all sent events were received.
                    //
                    // It's important to notice that this method is called even when no events are received after the maximum wait time, which
                    // can be specified by the user in the event processor options.  In this case, the IEnumerable events is empty, but not null.

                    int eventsCount = events.Count();

                    if (eventsCount > 0)
                    {
                        Interlocked.Add(ref totalEventsCount, eventsCount);
                        Console.WriteLine($"\tPartition '{ partitionContext.PartitionId }': { eventsCount } event(s) received.");
                    }

                    // This method is asynchronous, which means it's expected to return a Task.

                    return(Task.CompletedTask);
                };

                eventProcessor.ProcessExceptionAsync = (PartitionContext partitionContext, Exception exception) =>
                {
                    // All the unhandled exceptions encountered during the event processor execution are passed to this method so
                    // the user can decide how to handle them.
                    //
                    // This piece of code is not supposed to be reached by this sample.  If the following message has been printed
                    // to the Console, then something unexpected has happened.

                    Console.WriteLine($"\tPartition '{ partitionContext.PartitionId }': an unhandled exception was encountered. This was not expected to happen.");

                    // This method is asynchronous, which means it's expected to return a Task.

                    return(Task.CompletedTask);
                };

                // Once started, the event processor will start to claim partitions and receive events from them.

                Console.WriteLine("Starting the event processor.");
                Console.WriteLine();

                await eventProcessor.StartAsync();

                Console.WriteLine("Event processor started.");
                Console.WriteLine();

                // Wait until the event processor has claimed ownership of all partitions in the Event Hub.  This may take some time
                // as there's a 10 seconds interval between claims.  To be sure that we do not block forever in case the event processor
                // fails, we will specify a fairly long time to wait and then cancel waiting.

                CancellationTokenSource cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(TimeSpan.FromSeconds(400));

                var partitionsCount = (await client.GetPartitionIdsAsync()).Length;

                while (partitionsBeingProcessedCount < partitionsCount)
                {
                    await Task.Delay(500, cancellationSource.Token);
                }

                // To test our event processor, we are publishing 10 sets of events to the Event Hub.  Notice that we are not
                // specifying a partition to send events to, so these sets may end up in different partitions.

                EventData[] eventsToPublish = new EventData[]
                {
                    new EventData(Encoding.UTF8.GetBytes("I am not the second event.")),
                    new EventData(Encoding.UTF8.GetBytes("I am not the first event."))
                };

                int amountOfSets           = 10;
                int expectedAmountOfEvents = amountOfSets * eventsToPublish.Length;

                await using (EventHubProducer producer = client.CreateProducer())
                {
                    Console.WriteLine();
                    Console.WriteLine("Sending events to the Event Hub.");
                    Console.WriteLine();

                    for (int i = 0; i < amountOfSets; i++)
                    {
                        await producer.SendAsync(eventsToPublish);
                    }
                }

                // Because there is some non-determinism in the messaging flow, the sent events may not be immediately
                // available.  For this reason, we wait 500 ms before resuming.

                await Task.Delay(500);

                // Once stopped, the event processor won't receive events anymore.  In case there are still events being
                // processed when the stop method is called, the processing will complete before the corresponding partition
                // processor is closed.

                Console.WriteLine();
                Console.WriteLine("Stopping the event processor.");
                Console.WriteLine();

                await eventProcessor.StopAsync();

                // Print out the amount of events that we received.

                Console.WriteLine();
                Console.WriteLine($"Amount of events received: { totalEventsCount }. Expected: { expectedAmountOfEvents }.");
            }

            // At this point, our client and producer have passed their "using" scope and have safely been disposed of.  We
            // have no further obligations.

            Console.WriteLine();
        }
        /// <summary>
        ///   Runs the sample using the specified Event Hubs connection information.
        /// </summary>
        ///
        /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
        /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
        ///
        public async Task RunAsync(string connectionString,
                                   string eventHubName)
        {
            // We will start by creating a client using its default set of options.  It will be used by the event processor to
            // communicate with the Azure Event Hubs service.

            await using (var client = new EventHubClient(connectionString, eventHubName))
            {
                // An event processor is associated with a specific Event Hub and a consumer group.  It receives events from
                // multiple partitions in the Event Hub, passing them to the user for processing.  It's worth mentioning that
                // an event processor is a generic class, and it takes a partition processor as its underlying type.
                //
                // A partition processor is associated with a specific partition and is responsible for processing events when
                // requested by the event processor.  In order to use it as the event processor's underlying type, two conditions
                // must be met:
                //
                //     - It must be a class derived from BasePartitionProcessor.
                //
                //     - It must have a parameterless constructor.
                //
                // We'll be using a SamplePartitionProcessor, whose implementation can be found at the end of this sample.

                // A partition manager may create checkpoints and list/claim partition ownership.  The user can implement their
                // own partition manager by creating a subclass from the PartitionManager abstract class.  Here we are creating
                // a new instance of an InMemoryPartitionManager, provided by the Azure.Messaging.EventHubs.Processor namespace.
                // This isn't relevant to understanding this sample, but is required by the event processor constructor.

                PartitionManager partitionManager = new InMemoryPartitionManager();

                // It's also possible to specify custom options upon event processor creation.  We want to receive events from
                // the latest available position so older events don't interfere with our sample.  We also don't want to wait
                // more than 1 second for every set of events.

                EventProcessorOptions eventProcessorOptions = new EventProcessorOptions
                {
                    InitialEventPosition   = EventPosition.Latest,
                    MaximumReceiveWaitTime = TimeSpan.FromSeconds(1)
                };

                // Let's finally create our event processor.  We're using the default consumer group that was created with the Event Hub.

                var eventProcessor = new EventProcessor <SamplePartitionProcessor>(EventHubConsumer.DefaultConsumerGroupName, client, partitionManager, eventProcessorOptions);

                // Once started, the event processor will start to claim partitions and receive events from them.

                Console.WriteLine("Starting the event processor.");
                Console.WriteLine();

                await eventProcessor.StartAsync();

                Console.WriteLine("Event processor started.");
                Console.WriteLine();

                // Wait until the event processor has claimed ownership of all partitions in the Event Hub.  There should be a single
                // active partition processor per owned partition.  This may take some time as there's a 10 seconds interval between
                // claims.  To be sure that we do not block forever in case the event processor fails, we will specify a fairly long
                // time to wait and then cancel waiting.

                CancellationTokenSource cancellationSource = new CancellationTokenSource();
                cancellationSource.CancelAfter(TimeSpan.FromSeconds(400));

                var partitionsCount = (await client.GetPartitionIdsAsync()).Length;

                while (SamplePartitionProcessor.ActiveInstancesCount < partitionsCount)
                {
                    await Task.Delay(500, cancellationSource.Token);
                }

                // To test our event processor, we are publishing 10 sets of events to the Event Hub.  Notice that we are not
                // specifying a partition to send events to, so these sets may end up in different partitions.

                EventData[] eventsToPublish = new EventData[]
                {
                    new EventData(Encoding.UTF8.GetBytes("I am not the second event.")),
                    new EventData(Encoding.UTF8.GetBytes("I am not the first event."))
                };

                int amountOfSets           = 10;
                int expectedAmountOfEvents = amountOfSets * eventsToPublish.Length;

                await using (EventHubProducer producer = client.CreateProducer())
                {
                    Console.WriteLine("Sending events to the Event Hub.");
                    Console.WriteLine();

                    for (int i = 0; i < amountOfSets; i++)
                    {
                        await producer.SendAsync(eventsToPublish);
                    }
                }

                // Because there is some non-determinism in the messaging flow, the sent events may not be immediately
                // available.  For this reason, we wait 500 ms before resuming.

                await Task.Delay(500);

                // Once stopped, the event processor won't receive events anymore.  In case there are still events being
                // processed when the stop method is called, the processing will complete before the corresponding partition
                // processor is closed.

                Console.WriteLine();
                Console.WriteLine("Stopping the event processor.");
                Console.WriteLine();

                await eventProcessor.StopAsync();

                // Print out the amount of events that we received.

                Console.WriteLine();
                Console.WriteLine($"Amount of events received: { SamplePartitionProcessor.TotalEventsCount }. Expected: { expectedAmountOfEvents }.");
            }

            // At this point, our client and producer have passed their "using" scope and have safely been disposed of.  We
            // have no further obligations.

            Console.WriteLine();
        }
Exemple #19
0
        /// <summary>
        ///   Runs the sample using the specified Event Hubs connection information.
        /// </summary>
        ///
        /// <param name="connectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
        /// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that she sample should run against.</param>
        ///
        public async Task RunAsync(string connectionString,
                                   string eventHubName)
        {
            // We will start by creating a client using its default set of options.  It will be used by the event processor to
            // communicate with the Azure Event Hubs service.

            await using (var client = new EventHubClient(connectionString, eventHubName))
            {
                // An event processor is associated with a specific Event Hub and a consumer group.  It receives events from
                // all partitions in the Event Hub, passing them to the user for processing.
                //
                // A partition processor is associated with a specific partition and is responsible for processing events when
                // requested by the event processor.  An instance is provided to the event processor when requested from a
                // factory function, and takes the form of a class which implements the IPartitionProcessor interface.
                //
                // The factory function is provided to the event processor when it is created.  The factory is responsible for
                // creating a partition processor based on two arguments:
                //
                //   A partition context: contains information about the partition the partition processor will be processing
                //   events from.  In this sample, we are only interested in its partition id.
                //
                //   A checkpoint manager: responsible for the creation of checkpoints.  It's not used in this sample.
                //
                // We'll be using a SamplePartitionProcessor, whose implementation can be found at the end of this sample.  Its
                // constructor takes the associated partition id so it can provide useful log messages.

                Func <PartitionContext, CheckpointManager, IPartitionProcessor> partitionProcessorFactory =
                    (partitionContext, checkpointManager) => new SamplePartitionProcessor(partitionContext.PartitionId);

                // A partition manager may create checkpoints and list/claim partition ownership.  The user can implement their
                // own partition manager by creating a subclass from the PartitionManager abstract class.  Here we are creating
                // a new instance of an InMemoryPartitionManager, provided by the Azure.Messaging.EventHubs.Processor namespace.
                // This isn't relevant to understanding this sample, but is required by the event processor constructor.

                PartitionManager partitionManager = new InMemoryPartitionManager();

                // It's also possible to specify custom options upon event processor creation.  We want to receive events from
                // the latest available position so older events don't interfere with our sample.  We also don't want to wait
                // more than 1 second for every set of events.

                EventProcessorOptions eventProcessorOptions = new EventProcessorOptions
                {
                    InitialEventPosition   = EventPosition.Latest,
                    MaximumReceiveWaitTime = TimeSpan.FromSeconds(1)
                };

                // Let's finally create our event processor.  We're using the default consumer group that was created with the Event Hub.

                EventProcessor eventProcessor = new EventProcessor(EventHubConsumer.DefaultConsumerGroupName, client, partitionProcessorFactory, partitionManager, eventProcessorOptions);

                // Once started, the event processor will start to receive events from all partitions.

                Console.WriteLine("Starting the event processor.");
                Console.WriteLine();

                await eventProcessor.StartAsync();

                Console.WriteLine();
                Console.WriteLine("Event processor started.");

                // To test our event processor, we are publishing 10 sets of events to the Event Hub.  Notice that we are not
                // specifying a partition to send events to, so these sets may end up in different partitions.

                EventData[] eventsToPublish = new EventData[]
                {
                    new EventData(Encoding.UTF8.GetBytes("I am not the second event.")),
                    new EventData(Encoding.UTF8.GetBytes("I am not the first event."))
                };

                int amountOfSets           = 10;
                int expectedAmountOfEvents = amountOfSets * eventsToPublish.Length;

                await using (EventHubProducer producer = client.CreateProducer())
                {
                    Console.WriteLine();
                    Console.WriteLine("Sending events to the Event Hub.");
                    Console.WriteLine();

                    for (int i = 0; i < amountOfSets; i++)
                    {
                        await producer.SendAsync(eventsToPublish);
                    }
                }

                // Because there is some non-determinism in the messaging flow, the sent events may not be immediately
                // available.  For this reason, we wait 500 ms before resuming.

                await Task.Delay(500);

                // Once stopped, the event processor won't receive events anymore.  In case there are still events being
                // processed when the stop method is called, the processing will complete before the corresponding partition
                // processor is closed.

                Console.WriteLine();
                Console.WriteLine("Stopping the event processor.");
                Console.WriteLine();

                await eventProcessor.StopAsync();

                // Print out the amount of events that we received.

                Console.WriteLine();
                Console.WriteLine($"Amount of events received: { SamplePartitionProcessor.TotalEventsCount }. Expected: { expectedAmountOfEvents }.");
            }

            // At this point, our client and producer have passed their "using" scope and have safely been disposed of.  We
            // have no further obligations.

            Console.WriteLine();
        }