Exemplo n.º 1
0
        public async Task <List <EventData> > GetMessagesForDevice(string deviceId, DateTime startTime, int maxPerPartition = 100, int waitTimeSecs = 5)
        {
            var messages = new List <EventData>();

            EventHubClient    eventHubClient    = EventHubClient.CreateFromConnectionString(this.eventHubConnectionString);
            PartitionReceiver partitionReceiver = eventHubClient.CreateReceiver(
                EventHubConsumerGroup,
                EventHubPartitionKeyResolver.ResolveToPartition(deviceId, (await eventHubClient.GetRuntimeInformationAsync()).PartitionCount),
                EventPosition.FromEnqueuedTime(startTime));

            // Retry a few times due to weird behavior with ReceiveAsync() not returning all messages available
            for (int i = 0; i < 3; i++)
            {
                IEnumerable <EventData> events = await partitionReceiver.ReceiveAsync(maxPerPartition, TimeSpan.FromSeconds(waitTimeSecs));

                if (events != null)
                {
                    messages.AddRange(events);
                }

                if (i < 3)
                {
                    await Task.Delay(TimeSpan.FromSeconds(5));
                }
            }

            await partitionReceiver.CloseAsync();

            await eventHubClient.CloseAsync();

            return(messages);
        }
        /// <summary>
        ///   Performs the tasks needed to initialize and set up the environment for an instance
        ///   of the test scenario.  When multiple instances are run in parallel, setup will be
        ///   run once for each prior to its execution.
        /// </summary>
        ///
        public async override Task SetupAsync()
        {
            await base.SetupAsync();

            // Attempt to take a consumer group from the available set; to ensure that the
            // test scenario can support the requested level of parallelism without violating
            // the concurrent reader limits of a consumer group, the default consumer group
            // should not be used.

            if (!ConsumerGroups.TryDequeue(out var consumerGroup))
            {
                throw new InvalidOperationException("Unable to reserve a consumer group to read from.");
            }

            _receiver = new PartitionReceiver(
                consumerGroup,
                PartitionId,
                EventPosition.Earliest,
                TestEnvironment.EventHubsConnectionString,
                Scope.EventHubName);

            // Force the connection and link creation by reading a single event.

            await _receiver.ReceiveBatchAsync(1).ConfigureAwait(false);
        }
Exemplo n.º 3
0
        public void ReadLastEnqueuedEventInformationPopulatesFromTheLastReceivedEvent()
        {
            var lastEvent = new EventData
                            (
                eventBody: Array.Empty <byte>(),
                lastPartitionSequenceNumber: 12345,
                lastPartitionOffset: 89101,
                lastPartitionEnqueuedTime: DateTimeOffset.Parse("2015-10-27T00:00:00Z"),
                lastPartitionInformationRetrievalTime: DateTimeOffset.Parse("2012-03-04T08:49:00Z")
                            );

            var eventHub      = "someHub";
            var partition     = "PART";
            var transportMock = new ObservableTransportConsumerMock {
                LastReceivedEvent = lastEvent
            };
            var receiver = new PartitionReceiver("group", partition, eventHub, true, TimeSpan.Zero, transportMock);
            var metrics  = receiver.ReadLastEnqueuedEventInformation();

            Assert.That(metrics.EventHubName, Is.EqualTo(eventHub), "The Event Hub name should match.");
            Assert.That(metrics.PartitionId, Is.EqualTo(partition), "The partition id should match.");
            Assert.That(metrics.LastEnqueuedSequenceNumber, Is.EqualTo(lastEvent.LastPartitionSequenceNumber), "The sequence number should match.");
            Assert.That(metrics.LastEnqueuedOffset, Is.EqualTo(lastEvent.LastPartitionOffset), "The offset should match.");
            Assert.That(metrics.LastEnqueuedTime, Is.EqualTo(lastEvent.LastPartitionEnqueuedTime), "The enqueue time should match.");
            Assert.That(metrics.InformationReceived, Is.EqualTo(lastEvent.LastPartitionInformationRetrievalTime), "The retrieval time should match.");
        }
Exemplo n.º 4
0
        private static async Task ReceiveMessagesFromPartition(string partition, CancellationToken ct)
        {
            receiver = eventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, partition, EventPosition.FromEnqueuedTime(DateTime.Now));

            Console.WriteLine($"Connected to partition {receiver.PartitionId}, consumer group {receiver.ConsumerGroupName}");

            while (true)
            {
                if (ct.IsCancellationRequested)
                {
                    return;
                }

                // Receive a maximum of 100 messages in this call to ReceiveAsync
                var ehEvents = await receiver.ReceiveAsync(100);

                // ReceiveAsync can return null if there are no messages
                if (ehEvents != null)
                {
                    // Since ReceiveAsync can return more than a single event you will need a loop to process
                    foreach (var ehEvent in ehEvents)
                    {
                        // Decode the byte array segment
                        var message = UnicodeEncoding.UTF8.GetString(ehEvent.Body.Array);
                        Console.WriteLine($"Received. '{message}'");
                    }
                }
            }
        }
        public async Task PartitionReceiverCannotRetrievePartitionPropertiesWhenConnectionIsClosed()
        {
            var partitionCount = 1;

            await using (EventHubScope scope = await EventHubScope.CreateAsync(partitionCount))
            {
                var cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(20));

                var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
                var connection       = new EventHubConnection(connectionString);

                var partitionId = default(string);

                await using (var producer = new EventHubProducerClient(connection))
                {
                    partitionId = (await producer.GetPartitionIdsAsync(cancellationSource.Token)).First();
                }

                await using (var receiver = new PartitionReceiver(EventHubConsumerClient.DefaultConsumerGroupName, partitionId, EventPosition.Earliest, connection))
                {
                    Assert.That(async() => await receiver.GetPartitionPropertiesAsync(cancellationSource.Token), Throws.Nothing);

                    await connection.CloseAsync(cancellationSource.Token);

                    Assert.That(async() => await receiver.GetPartitionPropertiesAsync(cancellationSource.Token), Throws.InstanceOf <EventHubsException>().And.Property(nameof(EventHubsException.Reason)).EqualTo(EventHubsException.FailureReason.ClientClosed));
                    Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The cancellation token should not have been signaled.");
                }
            }
        }
Exemplo n.º 6
0
        public async Task <IList <EventData> > GetMessagesFromAllPartitions(DateTime startTime, int maxPerPartition = 10, int waitTimeSecs = 5)
        {
            var messages = new List <EventData>();
            EventHubRuntimeInformation rtInfo = await this.eventHubClient.GetRuntimeInformationAsync();

            foreach (string partition in rtInfo.PartitionIds)
            {
                PartitionReceiver partitionReceiver = this.eventHubClient.CreateReceiver(
                    PartitionReceiver.DefaultConsumerGroupName,
                    partition,
                    startTime);

                // Retry a few times to make sure we get all expected messages.
                for (int i = 0; i < 3; i++)
                {
                    IEnumerable <EventData> events = await partitionReceiver.ReceiveAsync(maxPerPartition, TimeSpan.FromSeconds(waitTimeSecs));

                    if (events != null)
                    {
                        messages.AddRange(events);
                    }

                    if (i < 3)
                    {
                        await Task.Delay(TimeSpan.FromSeconds(5));
                    }
                }
                await partitionReceiver.CloseAsync();
            }
            return(messages);
        }
        async Task InvokeOnNull()
        {
            PartitionReceiver partitionReceiver = this.EventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, "0", EventPosition.FromEnd());

            try
            {
                EventWaitHandle nullReceivedEvent = new EventWaitHandle(false, EventResetMode.ManualReset);
                var             handler           = new TestPartitionReceiveHandler();

                handler.EventsReceived += (s, eventDatas) =>
                {
                    if (eventDatas == null)
                    {
                        TestUtility.Log("Received null.");
                        nullReceivedEvent.Set();
                    }
                };

                partitionReceiver.SetReceiveHandler(handler, true);

                if (!nullReceivedEvent.WaitOne(TimeSpan.FromSeconds(120)))
                {
                    throw new InvalidOperationException("Did not receive null.");
                }
            }
            finally
            {
                // Unregister handler.
                partitionReceiver.SetReceiveHandler(null);

                // Close clients.
                await partitionReceiver.CloseAsync();
            }
        }
Exemplo n.º 8
0
        /// <summary>
        ///   Performs the tasks needed to initialize and set up the environment for an instance
        ///   of the test scenario.  When multiple instances are run in parallel, setup will be
        ///   run once for each prior to its execution.
        /// </summary>
        ///
        public async override Task SetupAsync()
        {
            await base.SetupAsync().ConfigureAwait(false);

            // Attempt to take a consumer group from the available set; to ensure that the
            // test scenario can support the requested level of parallelism without violating
            // the concurrent reader limits of a consumer group, the default consumer group
            // should not be used.

            if (!s_consumerGroups.TryDequeue(out var consumerGroup))
            {
                throw new InvalidOperationException("Unable to reserve a consumer group to read from.");
            }

            _client      = EventHubClient.CreateFromConnectionString(TestUtility.BuildEventHubsConnectionString(s_scope.EventHubName));
            _partitionId = (await _client.GetRuntimeInformationAsync().ConfigureAwait(false)).PartitionIds[0];

            _receiver = _client.CreateReceiver(
                consumerGroup,
                _partitionId,
                EventPosition.FromStart());

            // Force the connection and link creation by reading a single event.

            await _receiver.ReceiveAsync(1).ConfigureAwait(false);
        }
        public static async Task <EventHubTestListener> CreateListenerPal(string deviceName)
        {
            PartitionReceiver receiver = null;
            Stopwatch         sw       = new Stopwatch();

            sw.Start();

            var builder = new EventHubsConnectionStringBuilder(Configuration.IoTHub.EventHubString)
            {
                EntityPath = Configuration.IoTHub.EventHubCompatibleName
            };

            EventHubClient eventHubClient          = EventHubClient.CreateFromConnectionString(builder.ToString());
            var            eventRuntimeInformation = await eventHubClient.GetRuntimeInformationAsync().ConfigureAwait(false);

            var    eventHubPartitionsCount = eventRuntimeInformation.PartitionCount;
            string partition         = EventHubPartitionKeyResolver.ResolveToPartition(deviceName, eventHubPartitionsCount);
            string consumerGroupName = Configuration.IoTHub.EventHubConsumerGroup;

            while (receiver == null && sw.Elapsed.TotalMinutes < MaximumWaitTimeInMinutes)
            {
                try
                {
                    receiver = eventHubClient.CreateReceiver(consumerGroupName, partition, DateTime.Now.AddMinutes(-5));
                }
                catch (QuotaExceededException ex)
                {
                    s_log.WriteLine($"{nameof(EventHubTestListener)}.{nameof(CreateListener)}: Cannot create receiver: {ex}");
                }
            }

            sw.Stop();

            return(new EventHubTestListener(receiver));
        }
Exemplo n.º 10
0
        async Task OpenClientsAsync() // throws EventHubsException, IOException, InterruptedException, ExecutionException
        {
            // Create new clients
            EventPosition eventPosition = await this.PartitionContext.GetInitialOffsetAsync().ConfigureAwait(false);

            long epoch = this.Lease.Epoch;

            ProcessorEventSource.Log.PartitionPumpCreateClientsStart(this.Host.HostName, this.PartitionContext.PartitionId, epoch,
                                                                     $"Offset:{eventPosition.Offset}, SequenceNumber:{eventPosition.SequenceNumber}, DateTime:{eventPosition.EnqueuedTimeUtc}");
            this.eventHubClient          = this.Host.CreateEventHubClient();
            this.eventHubClient.WebProxy = this.Host.EventProcessorOptions.WebProxy;

            var receiverOptions = new ReceiverOptions()
            {
                // Enable receiver metrics?
                EnableReceiverRuntimeMetric = this.Host.EventProcessorOptions.EnableReceiverRuntimeMetric,

                // Use host name as the identifier for debugging purpose
                // Shorten host name if name is longer than max allowed lenght.
                Identifier = this.Host.HostName.Length > ClientConstants.MaxReceiverIdentifierLength ?
                             this.Host.HostName.Substring(0, ClientConstants.MaxReceiverIdentifierLength) : this.Host.HostName
            };

            // Create new receiver and set options
            this.partitionReceiver = this.eventHubClient.CreateEpochReceiver(
                this.PartitionContext.ConsumerGroupName,
                this.PartitionContext.PartitionId,
                eventPosition,
                epoch,
                receiverOptions);

            this.partitionReceiver.PrefetchCount = this.Host.EventProcessorOptions.PrefetchCount;

            ProcessorEventSource.Log.PartitionPumpCreateClientsStop(this.Host.HostName, this.PartitionContext.PartitionId);
        }
Exemplo n.º 11
0
        public async Task SmallReceiveTimeout()
        {
            var maxClients = 4;

            // Issue receives with 1 second so that some of the Receive calls will timeout while creating AMQP link.
            // Even those Receive calls should return NULL instead of bubbling the exception up.
            var receiveTimeoutInSeconds = 1;

            var tasks = Enumerable.Range(0, maxClients)
                        .Select(async i =>
            {
                PartitionReceiver receiver = null;

                try
                {
                    TestUtility.Log($"Testing with {receiveTimeoutInSeconds} seconds on client {i}.");

                    // Start receiving from a future time so that Receive call won't be able to fetch any events.
                    var ehClient = EventHubClient.CreateFromConnectionString(TestUtility.EventHubsConnectionString);
                    receiver     = ehClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, "0", EventPosition.FromEnqueuedTime(DateTime.UtcNow.AddMinutes(1)));
                    var ed       = await receiver.ReceiveAsync(1, TimeSpan.FromSeconds(receiveTimeoutInSeconds));
                    if (ed == null)
                    {
                        TestUtility.Log($"Received NULL from client {i}");
                    }
                }
                finally
                {
                    await receiver.CloseAsync();
                }
            });

            await Task.WhenAll(tasks);
        }
Exemplo n.º 12
0
        public async Task ReceiveEventsAsync(
            string deviceId,
            Func <EventData, bool> onEventReceived,
            CancellationToken token)
        {
            EventHubClient    client    = this.EventHubClient;
            int               count     = (await client.GetRuntimeInformationAsync()).PartitionCount;
            string            partition = EventHubPartitionKeyResolver.ResolveToPartition(deviceId, count);
            PartitionReceiver receiver  = client.CreateReceiver("$Default", partition, EventPosition.FromEnd());

            var result = new TaskCompletionSource <bool>();

            using (token.Register(() => result.TrySetCanceled()))
            {
                receiver.SetReceiveHandler(
                    new PartitionReceiveHandler(
                        data =>
                {
                    bool done = onEventReceived(data);
                    if (done)
                    {
                        result.TrySetResult(true);
                    }

                    return(done);
                }));

                await result.Task;
            }

            await receiver.CloseAsync();
        }
Exemplo n.º 13
0
        async Task OpenClientsAsync() // throws EventHubsException, IOException, InterruptedException, ExecutionException
        {
            // Create new clients
            EventPosition eventPosition = await this.PartitionContext.GetInitialOffsetAsync().ConfigureAwait(false);

            long epoch = this.Lease.Epoch;

            ProcessorEventSource.Log.PartitionPumpCreateClientsStart(this.Host.HostName, this.PartitionContext.PartitionId, epoch,
                                                                     $"Offset:{eventPosition.Offset}, SequenceNumber:{eventPosition.SequenceNumber}, DateTime:{eventPosition.EnqueuedTimeUtc}");
            this.eventHubClient          = this.Host.CreateEventHubClient();
            this.eventHubClient.WebProxy = this.Host.EventProcessorOptions.WebProxy;

            var receiverOptions = new ReceiverOptions()
            {
                // Enable receiver metrics?
                EnableReceiverRuntimeMetric = this.Host.EventProcessorOptions.EnableReceiverRuntimeMetric
            };

            // Create new receiver and set options
            this.partitionReceiver = this.eventHubClient.CreateEpochReceiver(
                this.PartitionContext.ConsumerGroupName,
                this.PartitionContext.PartitionId,
                eventPosition,
                epoch,
                receiverOptions);

            this.partitionReceiver.PrefetchCount = this.Host.EventProcessorOptions.PrefetchCount;

            ProcessorEventSource.Log.PartitionPumpCreateClientsStop(this.Host.HostName, this.PartitionContext.PartitionId);
        }
Exemplo n.º 14
0
        async Task NonexistentEntity()
        {
            // Rebuild connection string with a nonexistent entity.
            var csb = new EventHubsConnectionStringBuilder(TestUtility.EventHubsConnectionString);

            csb.EntityPath = Guid.NewGuid().ToString();
            var ehClient = EventHubClient.CreateFromConnectionString(csb.ToString());

            // GetRuntimeInformationAsync on a nonexistent entity.
            await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
            {
                TestUtility.Log("Getting entity information from a nonexistent entity.");
                await ehClient.GetRuntimeInformationAsync();
                throw new InvalidOperationException("GetRuntimeInformation call should have failed");
            });

            // GetPartitionRuntimeInformationAsync on a nonexistent entity.
            await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
            {
                TestUtility.Log("Getting partition information from a nonexistent entity.");
                await ehClient.GetPartitionRuntimeInformationAsync("0");
                throw new InvalidOperationException("GetPartitionRuntimeInformation call should have failed");
            });

            // Try sending.
            PartitionSender sender = null;
            await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
            {
                TestUtility.Log("Sending an event to nonexistent entity.");
                sender = ehClient.CreatePartitionSender("0");
                await sender.SendAsync(new EventData(Encoding.UTF8.GetBytes("this send should fail.")));
                throw new InvalidOperationException("Send call should have failed");
            });

            await sender.CloseAsync();

            // Try receiving.
            PartitionReceiver receiver = null;
            await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
            {
                TestUtility.Log("Receiving from nonexistent entity.");
                receiver = ehClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, "0", EventPosition.FromStart());
                await receiver.ReceiveAsync(1);
                throw new InvalidOperationException("Receive call should have failed");
            });

            await receiver.CloseAsync();

            // Try receiving on an nonexistent consumer group.
            ehClient = EventHubClient.CreateFromConnectionString(TestUtility.EventHubsConnectionString);
            await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
            {
                TestUtility.Log("Receiving from nonexistent consumer group.");
                receiver = ehClient.CreateReceiver(Guid.NewGuid().ToString(), "0", EventPosition.FromStart());
                await receiver.ReceiveAsync(1);
                throw new InvalidOperationException("Receive call should have failed");
            });

            await receiver.CloseAsync();
        }
        private async Task <PartitionReceiver> CreateEventHubReceiver(string deviceName)
        {
            PartitionReceiver eventHubReceiver = null;
            Stopwatch         sw = new Stopwatch();

            sw.Start();

            var builder = new EventHubsConnectionStringBuilder(Configuration.IoTHub.EventHubString)
            {
                EntityPath = Configuration.IoTHub.EventHubCompatibleName
            };

            EventHubClient eventHubClient          = EventHubClient.CreateFromConnectionString(builder.ToString());
            var            eventRuntimeInformation = await eventHubClient.GetRuntimeInformationAsync().ConfigureAwait(false);

            var    eventHubPartitionsCount = eventRuntimeInformation.PartitionCount;
            string partition         = EventHubPartitionKeyResolver.ResolveToPartition(deviceName, eventHubPartitionsCount);
            string consumerGroupName = Configuration.IoTHub.EventHubConsumerGroup;

            while (eventHubReceiver == null && sw.Elapsed.Minutes < 1)
            {
                try
                {
                    eventHubReceiver = eventHubClient.CreateReceiver(consumerGroupName, partition, DateTime.Now.AddMinutes(-5));
                }
                catch (QuotaExceededException ex)
                {
                    Debug.WriteLine(ex);
                }
            }

            sw.Stop();

            return(eventHubReceiver);
        }
        async Task CleanUpClientsAsync() // swallows all exceptions
        {
            if (this.partitionReceiver != null)
            {
                // Taking the lock means that there is no ProcessEventsAsync call in progress.
                Task closeTask;
                using (await this.ProcessingAsyncLock.LockAsync().ConfigureAwait(false))
                {
                    // Calling PartitionReceiver.CloseAsync will gracefully close the IPartitionReceiveHandler we have installed.
                    ProcessorEventSource.Log.PartitionPumpInfo(this.Host.Id, this.PartitionContext.PartitionId, "Closing PartitionReceiver");
                    closeTask = this.partitionReceiver.CloseAsync();
                }

                await closeTask.ConfigureAwait(false);

                this.partitionReceiver = null;
            }

            if (this.eventHubClient != null)
            {
                ProcessorEventSource.Log.PartitionPumpInfo(this.Host.Id, this.PartitionContext.PartitionId, "Closing EventHubClient");
                await this.eventHubClient.CloseAsync().ConfigureAwait(false);

                this.eventHubClient = null;
            }
        }
Exemplo n.º 17
0
        private static async Task <IEventHubReceiver> CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, Logger logger, ITelemetryProducer telemetryProducer)
        {
            bool offsetInclusive         = true;
            var  connectionStringBuilder = new EventHubsConnectionStringBuilder(partitionSettings.Hub.ConnectionString)
            {
                EntityPath = partitionSettings.Hub.Path
            };
            EventHubClient client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString());

            // if we have a starting offset or if we're not configured to start reading from utc now, read from offset
            if (!partitionSettings.Hub.StartFromNow ||
                offset != EventHubConstants.StartOfStream)
            {
                logger.Info("Starting to read from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
            }
            else
            {
                // to start reading from most recent data, we get the latest offset from the partition.
                EventHubPartitionRuntimeInformation partitionInfo =
                    await client.GetPartitionRuntimeInformationAsync(partitionSettings.Partition);

                offset          = partitionInfo.LastEnqueuedOffset;
                offsetInclusive = false;
                logger.Info("Starting to read latest messages from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
            }

            PartitionReceiver receiver = client.CreateReceiver(partitionSettings.Hub.ConsumerGroup, partitionSettings.Partition, offset, offsetInclusive);

            if (partitionSettings.Hub.PrefetchCount.HasValue)
            {
                receiver.PrefetchCount = partitionSettings.Hub.PrefetchCount.Value;
            }

            return(new EventHubReceiverProxy(receiver));
        }
Exemplo n.º 18
0
        public void ConnectionConstructorSetsTheDefaultMaximumReceiveWaitTime()
        {
            var expected = TimeSpan.FromMilliseconds(270);
            var receiver = new PartitionReceiver("group", "id", "name", false, expected, Mock.Of <TransportConsumer>());

            Assert.That(GetDefaultMaximumReceiveWaitTime(receiver), Is.EqualTo(expected));
        }
        async Task ReceiveTimeout()
        {
            var testValues = new[] { 10, 30, 120 };

            PartitionReceiver receiver = null;

            try
            {
                foreach (var receiveTimeoutInSeconds in testValues)
                {
                    Log($"Testing with {receiveTimeoutInSeconds} seconds.");

                    // Start receiving from a future time so that Receive call won't be able to fetch any events.
                    receiver = this.EventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, "0", DateTime.UtcNow.AddMinutes(1));

                    var startTime = DateTime.Now;
                    await receiver.ReceiveAsync(1, TimeSpan.FromSeconds(receiveTimeoutInSeconds));

                    // Receive call should have waited more than receive timeout.
                    // Give 100 milliseconds of buffer.
                    var diff = DateTime.Now.Subtract(startTime).TotalSeconds;
                    Assert.True(diff >= receiveTimeoutInSeconds - 0.1, $"Hit timeout {diff} seconds into Receive call while testing {receiveTimeoutInSeconds} seconds timeout.");

                    // Timeout should not be late more than 5 seconds.
                    // This is just a logical buffer for timeout behavior validation.
                    Assert.True(diff < receiveTimeoutInSeconds + 5, $"Hit timeout {diff} seconds into Receive call while testing {receiveTimeoutInSeconds} seconds timeout.");
                }
            }
            finally
            {
                await receiver.CloseAsync();
            }
        }
Exemplo n.º 20
0
        public void ConnectionConstructorSetsTrackingLastEnqueuedEvent()
        {
            var expected = true;
            var receiver = new PartitionReceiver("group", "id", "name", expected, TimeSpan.Zero, Mock.Of <TransportConsumer>());

            Assert.That(GetTrackingLastEnqueuedEvent(receiver), Is.EqualTo(expected));
        }
        public async Task ReadPartitionWithReceiver()
        {
            #region Snippet:EventHubs_Sample05_ReadPartitionWithReceiver

            var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
            var eventHubName     = "<< NAME OF THE EVENT HUB >>";
            var consumerGroup    = EventHubConsumerClient.DefaultConsumerGroupName;
            /*@@*/
            /*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
            /*@@*/ eventHubName     = _scope.EventHubName;
            /*@@*/ consumerGroup    = _availableConsumerGroups.Dequeue();

            using CancellationTokenSource cancellationSource = new CancellationTokenSource();
            cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));

            string firstPartition;

            await using (var producer = new EventHubProducerClient(connectionString, eventHubName))
            {
                firstPartition = (await producer.GetPartitionIdsAsync()).First();
            }

            var receiver = new PartitionReceiver(
                consumerGroup,
                firstPartition,
                EventPosition.Earliest,
                connectionString,
                eventHubName);

            try
            {
                while (!cancellationSource.IsCancellationRequested)
                {
                    int      batchSize = 50;
                    TimeSpan waitTime  = TimeSpan.FromSeconds(1);

                    IEnumerable <EventData> eventBatch = await receiver.ReceiveBatchAsync(
                        batchSize,
                        waitTime,
                        cancellationSource.Token);

                    foreach (EventData eventData in eventBatch)
                    {
                        byte[] eventBodyBytes = eventData.EventBody.ToArray();
                        Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { firstPartition }");
                    }
                }
            }
            catch (TaskCanceledException)
            {
                // This is expected if the cancellation token is
                // signaled.
            }
            finally
            {
                await receiver.CloseAsync();
            }

            #endregion
        }
        public static void CreateListenerPalAndReceiveMessages()
        {
            var builder = new EventHubsConnectionStringBuilder(Configuration.IoTHub.EventHubString)
            {
                EntityPath = Configuration.IoTHub.EventHubCompatibleName
            };

            EventHubClient eventHubClient          = EventHubClient.CreateFromConnectionString(builder.ToString());
            var            eventRuntimeInformation = eventHubClient.GetRuntimeInformationAsync().Result;
            var            eventHubPartitionsCount = eventRuntimeInformation.PartitionCount;
            string         consumerGroupName       = Configuration.IoTHub.EventHubConsumerGroup;

            foreach (string partitionId in eventRuntimeInformation.PartitionIds)
            {
                try
                {
                    PartitionReceiver receiver = eventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, partitionId, DateTime.Now.AddMinutes(-LookbackTimeInMinutes));
                    s_log.WriteLine($"EventHub receiver created for partition {partitionId}, listening from {LookbackTimeInMinutes}");

                    new Task(async() =>
                    {
                        while (true)
                        {
                            IEnumerable <EventData> eventDatas = await receiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(OperationTimeoutInSeconds)).ConfigureAwait(false);
                            ProcessEventData(eventDatas);
                        }
                    }).Start();
                }
                catch (EventHubsException ex)
                {
                    s_log.WriteLine($"{nameof(EventHubTestListener)}.{nameof(CreateListenerPalAndReceiveMessages)}: Cannot create receiver for partitionID {partitionId}: {ex}");
                }
            }
        }
Exemplo n.º 23
0
        protected async Task VerifyDataOnIoTHub(string moduleId)
        {
            Console.WriteLine($"Verifying data on IoTHub from {moduleId}");

            // First Verify if module is already running.
            await this.bootstrapper.VerifyModuleIsRunning(moduleId);

            var builder = new EventHubsConnectionStringBuilder(this.eventhubCompatibleEndpointWithEntityPath)
            {
                TransportType = this.eventHubClientTransportType
            };

            Console.WriteLine($"Receiving events from device '{this.context.DeviceId}' on Event Hub '{builder.EntityPath}'");

            EventHubClient eventHubClient =
                EventHubClient.CreateFromConnectionString(builder.ToString());

            this.proxy.ForEach(p => eventHubClient.WebProxy = p);

            PartitionReceiver eventHubReceiver = eventHubClient.CreateReceiver(
                "$Default",
                EventHubPartitionKeyResolver.ResolveToPartition(
                    this.context.DeviceId,
                    (await eventHubClient.GetRuntimeInformationAsync()).PartitionCount),
                EventPosition.FromEnd());

            // TODO: [Improvement] should verify test results without using event hub, which introduce latency.
            var result = new TaskCompletionSource <bool>();

            using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(20))) // This long timeout is needed in case event hub is slow to process messages
            {
                using (cts.Token.Register(() => result.TrySetCanceled()))
                {
                    eventHubReceiver.SetReceiveHandler(
                        new PartitionReceiveHandler(
                            eventData =>
                    {
                        eventData.SystemProperties.TryGetValue("iothub-connection-device-id", out object devId);
                        eventData.SystemProperties.TryGetValue("iothub-connection-module-id", out object modId);

                        if (devId != null && devId.ToString().Equals(this.context.DeviceId) &&
                            modId != null && modId.ToString().Equals(moduleId))
                        {
                            result.TrySetResult(true);
                            return(true);
                        }

                        return(false);
                    }));

                    await result.Task;
                }
            }

            Console.WriteLine("VerifyDataOnIoTHub completed.");
            await eventHubReceiver.CloseAsync();

            await eventHubClient.CloseAsync();
        }
Exemplo n.º 24
0
        public void CloseClosesTheTransportConsumer()
        {
            var transportConsumer = new ObservableTransportConsumerMock();
            var receiver          = new PartitionReceiver("group", "0", "hub", true, TimeSpan.Zero, transportConsumer);

            receiver.Close();
            Assert.That(transportConsumer.WasCloseCalled, Is.True);
        }
Exemplo n.º 25
0
        private async Task ReceiverPumpAsync(PartitionReceiver receiver, Func <IReadOnlyCollection <QueueMessage>, Task> onMessage, int maxBatchSize, CancellationToken cancellationToken)
        {
            while (true)
            {
                try
                {
                    IEnumerable <EventData> events = await receiver.ReceiveAsync(maxBatchSize, _waitTime);

                    if (events != null && !cancellationToken.IsCancellationRequested)
                    {
                        List <QueueMessage> qms = events.Select(ed => Converter.ToQueueMessage(ed, receiver.PartitionId)).ToList();
                        await onMessage(qms);

                        QueueMessage lastMessage = qms.LastOrDefault();

                        //save state
                        if (lastMessage != null)
                        {
                            const string sequenceNumberPropertyName = "x-opt-sequence-number";

                            if (lastMessage.Properties.TryGetValue(sequenceNumberPropertyName, out string sequenceNumber))
                            {
                                long?sequenceNumberLong = null;
                                if (long.TryParse(sequenceNumber, out long seqenceNumberNonNullable))
                                {
                                    sequenceNumberLong = seqenceNumberNonNullable;
                                }

                                await _state.SetPartitionStateAsync(receiver.PartitionId, sequenceNumberLong);
                            }
                        }
                    }

                    if (cancellationToken.IsCancellationRequested)
                    {
                        await receiver.CloseAsync();

                        return;
                    }
                }
                catch (ArgumentException ex)
                {
                    Console.WriteLine("failed with message: '{0}', clearing partition state.", ex);

                    await _state.SetPartitionStateAsync(receiver.PartitionId, EventPosition.FromStart().SequenceNumber);
                }
                catch (OperationCanceledException)
                {
                    return;
                }
                catch (Exception ex)
                {
                    Console.WriteLine("receiver stopped: {0}", ex);

                    return;
                }
            }
        }
Exemplo n.º 26
0
 internal AzureEncryptedEventHubReader(
     PartitionReceiver reader,
     Func <PartitionReceiver, Task <PartitionReceiver> > receiverRefresh,
     Func <IEnumerable <EventMessage>, string, Task> onMessagesReceived,
     byte[] encryptionKey)
     : base(reader, receiverRefresh, onMessagesReceived)
 {
     _encryptionKey = encryptionKey;
 }
Exemplo n.º 27
0
        public void ReceiveAsyncValidatesTheMaximumWaitTime(int timeSpanDelta)
        {
            var transportConsumer = new ObservableTransportConsumerMock();
            var receiver          = new PartitionReceiver("group", "0", "hub", true, TimeSpan.Zero, transportConsumer);
            var expectedWaitTime  = TimeSpan.FromMilliseconds(timeSpanDelta);

            using var cancellation = new CancellationTokenSource();
            Assert.That(async() => await receiver.ReceiveAsync(32, expectedWaitTime, cancellation.Token), Throws.InstanceOf <ArgumentException>());
        }
Exemplo n.º 28
0
        public void ResumeFrom(long position)
        {
            EventPosition eventPosition = (position == -1) ?
                                          EventPosition.FromStart() : EventPosition.FromSequenceNumber(position);

            var _eventHubClient = GetEventHubClient(processId);

            _eventHubReceiver = _eventHubClient.CreateReceiver("$Default", (processId / 8).ToString(), eventPosition);
        }
Exemplo n.º 29
0
        protected async Task VerifyDataOnIoTHubAsync()
        {
            // Leaf device without parent not expected to send messages
            if (!this.edgeDeviceId.HasValue)
            {
                return;
            }

            var builder = new EventHubsConnectionStringBuilder(this.eventhubCompatibleEndpointWithEntityPath)
            {
                TransportType = this.eventHubClientTransportType
            };

            Console.WriteLine($"Receiving events from device '{this.context.Device.Id}' on Event Hub '{builder.EntityPath}'");

            EventHubClient eventHubClient =
                EventHubClient.CreateFromConnectionString(builder.ToString());

            this.proxy.ForEach(p => eventHubClient.WebProxy = p);

            PartitionReceiver eventHubReceiver = eventHubClient.CreateReceiver(
                "$Default",
                EventHubPartitionKeyResolver.ResolveToPartition(
                    this.context.Device.Id,
                    (await eventHubClient.GetRuntimeInformationAsync()).PartitionCount),
                EventPosition.FromEnqueuedTime(DateTime.Now.AddMinutes(-5)));

            var result = new TaskCompletionSource <bool>();

            using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(3)))
            {
                using (cts.Token.Register(() => result.TrySetCanceled()))
                {
                    eventHubReceiver.SetReceiveHandler(
                        new PartitionReceiveHandler(
                            eventData =>
                    {
                        eventData.SystemProperties.TryGetValue("iothub-connection-device-id", out var devId);

                        if (devId != null && devId.ToString().Equals(this.context.Device.Id, StringComparison.Ordinal) &&
                            Encoding.UTF8.GetString(eventData.Body).Contains(this.context.MessageGuid, StringComparison.Ordinal))
                        {
                            result.TrySetResult(true);
                            return(true);
                        }

                        return(false);
                    }));

                    await result.Task;
                }
            }

            await eventHubReceiver.CloseAsync();

            await eventHubClient.CloseAsync();
        }
Exemplo n.º 30
0
        async Task PartitionReceiverSetReceiveHandler()
        {
            Log("Receiving Events via PartitionReceiver.SetReceiveHandler()");
            string            partitionId       = "1";
            PartitionReceiver partitionReceiver = this.EventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, partitionId, DateTime.UtcNow.AddMinutes(-10));
            PartitionSender   partitionSender   = this.EventHubClient.CreatePartitionSender(partitionId);

            try
            {
                string uniqueEventId = Guid.NewGuid().ToString();
                Log($"Sending an event to Partition {partitionId} with custom property EventId {uniqueEventId}");
                var sendEvent = new EventData(Encoding.UTF8.GetBytes("Hello EventHub!"));
                sendEvent.Properties = new Dictionary <string, object> {
                    ["EventId"] = uniqueEventId
                };
                await partitionSender.SendAsync(sendEvent);

                EventWaitHandle dataReceivedEvent = new EventWaitHandle(false, EventResetMode.ManualReset);
                var             handler           = new TestPartitionReceiveHandler();
                handler.ErrorReceived  += (s, e) => Log($"TestPartitionReceiveHandler.ProcessError {e.GetType().Name}: {e.Message}");
                handler.EventsReceived += (s, eventDatas) =>
                {
                    int count = eventDatas != null?eventDatas.Count() : 0;

                    Log($"Received {count} event(s):");

                    foreach (var eventData in eventDatas)
                    {
                        object objectValue;
                        if (eventData.Properties != null && eventData.Properties.TryGetValue("EventId", out objectValue))
                        {
                            Log($"Received message with EventId {objectValue}");
                            string receivedId = objectValue.ToString();
                            if (receivedId == uniqueEventId)
                            {
                                Log("Success");
                                dataReceivedEvent.Set();
                                break;
                            }
                        }
                    }
                };

                partitionReceiver.SetReceiveHandler(handler);

                if (!dataReceivedEvent.WaitOne(TimeSpan.FromSeconds(20)))
                {
                    throw new InvalidOperationException("Data Received Event was not signaled.");
                }
            }
            finally
            {
                await partitionSender.CloseAsync();

                await partitionReceiver.CloseAsync();
            }
        }