async Task InvokeAfterReceiveTimeoutFalse() { const int ReceiveTimeoutInSeconds = 15; Log("Calling RegisterEventProcessorAsync with InvokeProcessorAfterReceiveTimeout=false"); var eventProcessorHost = new EventProcessorHost( string.Empty, PartitionReceiver.DefaultConsumerGroupName, this.EventHubConnectionString, this.StorageConnectionString, this.LeaseContainerName); var processorOptions = new EventProcessorOptions { ReceiveTimeout = TimeSpan.FromSeconds(ReceiveTimeoutInSeconds), InvokeProcessorAfterReceiveTimeout = false, MaxBatchSize = 100 }; var emptyBatchReceiveEvent = new AsyncAutoResetEvent(false); var processorFactory = new TestEventProcessorFactory(); processorFactory.OnCreateProcessor += (f, createArgs) => { var processor = createArgs.Item2; string partitionId = createArgs.Item1.PartitionId; processor.OnProcessEvents += (_, eventsArgs) => { int eventCount = eventsArgs.Item2 != null?eventsArgs.Item2.events.Count() : 0; Log($"Partition {partitionId} TestEventProcessor processing {eventCount} event(s)"); if (eventCount == 0) { emptyBatchReceiveEvent.Set(); } }; }; await eventProcessorHost.RegisterEventProcessorFactoryAsync(processorFactory, processorOptions); try { Log("Verifying no empty batches arrive..."); bool waitSucceeded = await emptyBatchReceiveEvent.WaitAsync(TimeSpan.FromSeconds(ReceiveTimeoutInSeconds * 2)); Assert.False(waitSucceeded, "No empty batch should have been received!"); } finally { Log("Calling UnregisterEventProcessorAsync"); await eventProcessorHost.UnregisterEventProcessorAsync(); } }
async Task <Dictionary <string, List <EventData> > > RunGenericScenario(EventProcessorHost eventProcessorHost, EventProcessorOptions epo = null, int totalNumberOfEventsToSend = 1, bool checkPointLastEvent = true, bool checkPointBatch = false) { var receivedEvents = new ConcurrentDictionary <string, List <EventData> >(); var lastReceivedAt = DateTime.Now; if (epo == null) { epo = new EventProcessorOptions { ReceiveTimeout = TimeSpan.FromSeconds(15), MaxBatchSize = 100 }; } try { Log($"Calling RegisterEventProcessorAsync"); var processorFactory = new TestEventProcessorFactory(); processorFactory.OnCreateProcessor += (f, createArgs) => { var processor = createArgs.Item2; string partitionId = createArgs.Item1.PartitionId; string hostName = createArgs.Item1.Owner; processor.OnOpen += (_, partitionContext) => Log($"{hostName} > Partition {partitionId} TestEventProcessor opened"); processor.OnClose += (_, closeArgs) => Log($"{hostName} > Partition {partitionId} TestEventProcessor closing: {closeArgs.Item2}"); processor.OnProcessError += (_, errorArgs) => Log($"{hostName} > Partition {partitionId} TestEventProcessor process error {errorArgs.Item2.Message}"); processor.OnProcessEvents += (_, eventsArgs) => { int eventCount = eventsArgs.Item2 != null?eventsArgs.Item2.events.Count() : 0; Log($"{hostName} > Partition {partitionId} TestEventProcessor processing {eventCount} event(s)"); if (eventCount > 0) { List <EventData> events; receivedEvents.TryGetValue(partitionId, out events); if (events == null) { events = new List <EventData>(); } events.AddRange(eventsArgs.Item2.events); receivedEvents[partitionId] = events; lastReceivedAt = DateTime.Now; } eventsArgs.Item2.checkPointLastEvent = checkPointLastEvent; eventsArgs.Item2.checkPointBatch = checkPointBatch; }; }; await eventProcessorHost.RegisterEventProcessorFactoryAsync(processorFactory, epo); Log($"Sending {totalNumberOfEventsToSend} event(s) to each partition"); var sendTasks = new List <Task>(); foreach (var partitionId in PartitionIds) { for (int i = 0; i < totalNumberOfEventsToSend; i++) { sendTasks.Add(this.SendToPartitionAsync(partitionId, $"{partitionId} event.", this.ConnectionStringBuilder.ToString())); } } await Task.WhenAll(sendTasks); // Wait until all partitions are silent, i.e. no more events to receive. while (lastReceivedAt > DateTime.Now.AddSeconds(-30)) { await Task.Delay(1000); } Log("Verifying at least an event was received by each partition"); foreach (var partitionId in PartitionIds) { Assert.True(receivedEvents.ContainsKey(partitionId), $"Partition {partitionId} didn't receive any message!"); } Log("Success"); } finally { Log("Calling UnregisterEventProcessorAsync"); await eventProcessorHost.UnregisterEventProcessorAsync(); } return(receivedEvents.ToDictionary(kvp => kvp.Key, kvp => kvp.Value)); }
async Task MultipleConsumerGroups() { var customConsumerGroupName = "cgroup1"; // Generate a new lease container name that will be used through out the test. string leaseContainerName = Guid.NewGuid().ToString(); var consumerGroupNames = new[] { PartitionReceiver.DefaultConsumerGroupName, customConsumerGroupName }; var processorOptions = new EventProcessorOptions { ReceiveTimeout = TimeSpan.FromSeconds(15), MaxBatchSize = 100 }; var processorFactory = new TestEventProcessorFactory(); var partitionReceiveEvents = new ConcurrentDictionary <string, AsyncAutoResetEvent>(); var hosts = new List <EventProcessorHost>(); // Confirm that custom consumer group exists before starting hosts. try { // Create a receiver on the consumer group and try to receive. // Receive call will fail if consumer group is missing. var ehClient = EventHubClient.CreateFromConnectionString(this.EventHubConnectionString); var receiver = ehClient.CreateReceiver(customConsumerGroupName, this.PartitionIds.First(), PartitionReceiver.StartOfStream); await receiver.ReceiveAsync(1, TimeSpan.FromSeconds(5)); } catch (MessagingEntityNotFoundException) { throw new Exception($"Cunsumer group {customConsumerGroupName} cannot be found. MultipleConsumerGroups unit test requires consumer group '{customConsumerGroupName}' to be created before running the test."); } processorFactory.OnCreateProcessor += (f, createArgs) => { var processor = createArgs.Item2; string partitionId = createArgs.Item1.PartitionId; string hostName = createArgs.Item1.Owner; string consumerGroupName = createArgs.Item1.ConsumerGroupName; processor.OnOpen += (_, partitionContext) => Log($"{hostName} > {consumerGroupName} > Partition {partitionId} TestEventProcessor opened"); processor.OnClose += (_, closeArgs) => Log($"{hostName} > {consumerGroupName} > Partition {partitionId} TestEventProcessor closing: {closeArgs.Item2}"); processor.OnProcessError += (_, errorArgs) => Log($"{hostName} > {consumerGroupName} > Partition {partitionId} TestEventProcessor process error {errorArgs.Item2.Message}"); processor.OnProcessEvents += (_, eventsArgs) => { int eventCount = eventsArgs.Item2 != null?eventsArgs.Item2.events.Count() : 0; Log($"{hostName} > {consumerGroupName} > Partition {partitionId} TestEventProcessor processing {eventCount} event(s)"); if (eventCount > 0) { var receivedEvent = partitionReceiveEvents[consumerGroupName + "-" + partitionId]; receivedEvent.Set(); } }; }; try { // Register a new host for each consumer group. foreach (var consumerGroupName in consumerGroupNames) { var eventProcessorHost = new EventProcessorHost( string.Empty, consumerGroupName, this.EventHubConnectionString, this.StorageConnectionString, leaseContainerName); Log($"Calling RegisterEventProcessorAsync on consumer group {consumerGroupName}"); foreach (var partitionId in PartitionIds) { partitionReceiveEvents[consumerGroupName + "-" + partitionId] = new AsyncAutoResetEvent(false); } await eventProcessorHost.RegisterEventProcessorFactoryAsync(processorFactory, processorOptions); hosts.Add(eventProcessorHost); } Log("Sending an event to each partition"); var sendTasks = new List <Task>(); foreach (var partitionId in PartitionIds) { sendTasks.Add(this.SendToPartitionAsync(partitionId, $"{partitionId} event.", this.ConnectionStringBuilder.ToString())); } await Task.WhenAll(sendTasks); Log("Verifying an event was received by each partition for each consumer group"); foreach (var consumerGroupName in consumerGroupNames) { foreach (var partitionId in PartitionIds) { var receivedEvent = partitionReceiveEvents[consumerGroupName + "-" + partitionId]; bool partitionReceivedMessage = await receivedEvent.WaitAsync(TimeSpan.FromSeconds(30)); Assert.True(partitionReceivedMessage, $"ConsumerGroup {consumerGroupName} > Partition {partitionId} didn't receive any message!"); } } Log("Success"); } finally { Log("Calling UnregisterEventProcessorAsync on both hosts."); foreach (var eph in hosts) { await eph.UnregisterEventProcessorAsync(); } } }
async Task InvokeAfterReceiveTimeoutTrue() { const int ReceiveTimeoutInSeconds = 15; Log("Testing EventProcessorHost with InvokeProcessorAfterReceiveTimeout=true"); var emptyBatchReceiveEvents = new ConcurrentDictionary <string, AsyncAutoResetEvent>(); foreach (var partitionId in PartitionIds) { emptyBatchReceiveEvents[partitionId] = new AsyncAutoResetEvent(false); } var eventProcessorHost = new EventProcessorHost( string.Empty, PartitionReceiver.DefaultConsumerGroupName, this.EventHubConnectionString, this.StorageConnectionString, this.LeaseContainerName); var processorOptions = new EventProcessorOptions { ReceiveTimeout = TimeSpan.FromSeconds(ReceiveTimeoutInSeconds), InvokeProcessorAfterReceiveTimeout = true, MaxBatchSize = 100 }; var processorFactory = new TestEventProcessorFactory(); processorFactory.OnCreateProcessor += (f, createArgs) => { var processor = createArgs.Item2; string partitionId = createArgs.Item1.PartitionId; processor.OnOpen += (_, partitionContext) => Log($"Partition {partitionId} TestEventProcessor opened"); processor.OnProcessEvents += (_, eventsArgs) => { int eventCount = eventsArgs.Item2.events != null?eventsArgs.Item2.events.Count() : 0; Log($"Partition {partitionId} TestEventProcessor processing {eventCount} event(s)"); if (eventCount == 0) { var emptyBatchReceiveEvent = emptyBatchReceiveEvents[partitionId]; emptyBatchReceiveEvent.Set(); } }; }; await eventProcessorHost.RegisterEventProcessorFactoryAsync(processorFactory, processorOptions); try { Log("Waiting for each partition to receive an empty batch of events..."); foreach (var partitionId in PartitionIds) { var emptyBatchReceiveEvent = emptyBatchReceiveEvents[partitionId]; bool emptyBatchReceived = await emptyBatchReceiveEvent.WaitAsync(TimeSpan.FromSeconds(ReceiveTimeoutInSeconds * 2)); Assert.True(emptyBatchReceived, $"Partition {partitionId} didn't receive an empty batch!"); } } finally { Log("Calling UnregisterEventProcessorAsync"); await eventProcessorHost.UnregisterEventProcessorAsync(); } }
async Task MultipleProcessorHosts() { Log("Testing with 2 EventProcessorHost instances"); var partitionReceiveEvents = new ConcurrentDictionary <string, AsyncAutoResetEvent>(); foreach (var partitionId in PartitionIds) { partitionReceiveEvents[partitionId] = new AsyncAutoResetEvent(false); } int hostCount = 2; var hosts = new List <EventProcessorHost>(); try { for (int i = 0; i < hostCount; i++) { Log("Creating EventProcessorHost"); var eventProcessorHost = new EventProcessorHost( string.Empty, // Passing empty as entity path here rsince path is already in EH connection string. PartitionReceiver.DefaultConsumerGroupName, this.EventHubConnectionString, this.StorageConnectionString, this.LeaseContainerName); hosts.Add(eventProcessorHost); Log($"Calling RegisterEventProcessorAsync"); var processorOptions = new EventProcessorOptions { ReceiveTimeout = TimeSpan.FromSeconds(10), InvokeProcessorAfterReceiveTimeout = true, MaxBatchSize = 100 }; var processorFactory = new TestEventProcessorFactory(); processorFactory.OnCreateProcessor += (f, createArgs) => { var processor = createArgs.Item2; string partitionId = createArgs.Item1.PartitionId; string hostName = createArgs.Item1.Owner; processor.OnOpen += (_, partitionContext) => Log($"{hostName} > Partition {partitionId} TestEventProcessor opened"); processor.OnClose += (_, closeArgs) => Log($"{hostName} > Partition {partitionId} TestEventProcessor closing: {closeArgs.Item2}"); processor.OnProcessError += (_, errorArgs) => Log($"{hostName} > Partition {partitionId} TestEventProcessor process error {errorArgs.Item2.Message}"); processor.OnProcessEvents += (_, eventsArgs) => { int eventCount = eventsArgs.Item2 != null?eventsArgs.Item2.events.Count() : 0; Log($"{hostName} > Partition {partitionId} TestEventProcessor processing {eventCount} event(s)"); if (eventCount > 0) { var receivedEvent = partitionReceiveEvents[partitionId]; receivedEvent.Set(); } }; }; await eventProcessorHost.RegisterEventProcessorFactoryAsync(processorFactory, processorOptions); } Log("Waiting for partition ownership to settle..."); await Task.Delay(TimeSpan.FromSeconds(30)); Log("Sending an event to each partition"); var sendTasks = new List <Task>(); foreach (var partitionId in PartitionIds) { sendTasks.Add(this.SendToPartitionAsync(partitionId, $"{partitionId} event.", this.ConnectionStringBuilder.ToString())); } await Task.WhenAll(sendTasks); Log("Verifying an event was received by each partition"); foreach (var partitionId in PartitionIds) { var receivedEvent = partitionReceiveEvents[partitionId]; bool partitionReceivedMessage = await receivedEvent.WaitAsync(TimeSpan.FromSeconds(30)); Assert.True(partitionReceivedMessage, $"Partition {partitionId} didn't receive any message!"); } } finally { var shutdownTasks = new List <Task>(); foreach (var host in hosts) { Log($"Host {host} Calling UnregisterEventProcessorAsync."); shutdownTasks.Add(host.UnregisterEventProcessorAsync()); } await Task.WhenAll(shutdownTasks); } }
async Task <GenericScenarioResult> RunGenericScenario(EventProcessorHost eventProcessorHost, EventProcessorOptions epo = null, int totalNumberOfEventsToSend = 1, bool checkpointLastEvent = true, bool checkpointBatch = false, bool checkpoingEveryEvent = false) { var runResult = new GenericScenarioResult(); var lastReceivedAt = DateTime.Now; if (epo == null) { epo = new EventProcessorOptions { ReceiveTimeout = TimeSpan.FromSeconds(15), MaxBatchSize = 100 }; epo.SetExceptionHandler(TestEventProcessorFactory.ErrorNotificationHandler); } try { Log($"Calling RegisterEventProcessorAsync"); var processorFactory = new TestEventProcessorFactory(); processorFactory.OnCreateProcessor += (f, createArgs) => { var processor = createArgs.Item2; string partitionId = createArgs.Item1.PartitionId; string hostName = createArgs.Item1.Owner; processor.OnOpen += (_, partitionContext) => Log($"{hostName} > Partition {partitionId} TestEventProcessor opened"); processor.OnClose += (_, closeArgs) => Log($"{hostName} > Partition {partitionId} TestEventProcessor closing: {closeArgs.Item2}"); processor.OnProcessError += (_, errorArgs) => { Log($"{hostName} > Partition {partitionId} TestEventProcessor process error {errorArgs.Item2.Message}"); Interlocked.Increment(ref runResult.NumberOfFailures); }; processor.OnProcessEvents += (_, eventsArgs) => { int eventCount = eventsArgs.Item2.events != null?eventsArgs.Item2.events.Count() : 0; Log($"{hostName} > Partition {partitionId} TestEventProcessor processing {eventCount} event(s)"); if (eventCount > 0) { lastReceivedAt = DateTime.Now; runResult.AddEvents(partitionId, eventsArgs.Item2.events); foreach (var e in eventsArgs.Item2.events) { // Checkpoint every event received? if (checkpoingEveryEvent) { eventsArgs.Item1.CheckpointAsync(e).Wait(); } } } eventsArgs.Item2.checkPointLastEvent = checkpointLastEvent; eventsArgs.Item2.checkPointBatch = checkpointBatch; }; }; await eventProcessorHost.RegisterEventProcessorFactoryAsync(processorFactory, epo); // Wait 5 seconds to avoid races in scenarios like EndOfStream. await Task.Delay(5000); Log($"Sending {totalNumberOfEventsToSend} event(s) to each partition"); var sendTasks = new List <Task>(); foreach (var partitionId in PartitionIds) { for (int i = 0; i < totalNumberOfEventsToSend; i++) { sendTasks.Add(this.SendToPartitionAsync(partitionId, $"{partitionId} event.", this.ConnectionStringBuilder.ToString())); } } await Task.WhenAll(sendTasks); // Wait until all partitions are silent, i.e. no more events to receive. while (lastReceivedAt > DateTime.Now.AddSeconds(-30)) { await Task.Delay(1000); } Log($"Verifying at least {totalNumberOfEventsToSend} event(s) was received by each partition"); foreach (var partitionId in PartitionIds) { Assert.True(runResult.ReceivedEvents.ContainsKey(partitionId) && runResult.ReceivedEvents[partitionId].Count >= totalNumberOfEventsToSend, $"Partition {partitionId} didn't receive expected number of messages. Expected {totalNumberOfEventsToSend}, received {runResult.ReceivedEvents[partitionId].Count}."); } Log("Success"); } finally { Log("Calling UnregisterEventProcessorAsync"); await eventProcessorHost.UnregisterEventProcessorAsync(); } return(runResult); }
async Task HostShouldRecoverAfterReceiverDisconnection() { // We will target one partition and do validation on it. var targetPartition = this.PartitionIds.First(); int targetPartitionOpens = 0; int targetPartitionCloses = 0; int targetPartitionErrors = 0; PartitionReceiver externalReceiver = null; var eventProcessorHost = new EventProcessorHost( "ephhost", string.Empty, PartitionReceiver.DefaultConsumerGroupName, this.EventHubConnectionString, this.StorageConnectionString, Guid.NewGuid().ToString()); try { var processorFactory = new TestEventProcessorFactory(); processorFactory.OnCreateProcessor += (f, createArgs) => { var processor = createArgs.Item2; string partitionId = createArgs.Item1.PartitionId; string hostName = createArgs.Item1.Owner; processor.OnOpen += (_, partitionContext) => { Log($"{hostName} > Partition {partitionId} TestEventProcessor opened"); if (partitionId == targetPartition) { Interlocked.Increment(ref targetPartitionOpens); } }; processor.OnClose += (_, closeArgs) => { Log($"{hostName} > Partition {partitionId} TestEventProcessor closing: {closeArgs.Item2}"); if (partitionId == targetPartition && closeArgs.Item2 == CloseReason.Shutdown) { Interlocked.Increment(ref targetPartitionCloses); } }; processor.OnProcessError += (_, errorArgs) => { Log($"{hostName} > Partition {partitionId} TestEventProcessor process error {errorArgs.Item2.Message}"); if (partitionId == targetPartition && errorArgs.Item2 is ReceiverDisconnectedException) { Interlocked.Increment(ref targetPartitionErrors); } }; }; await eventProcessorHost.RegisterEventProcessorFactoryAsync(processorFactory); // Wait 15 seconds then create a new epoch receiver. // This will trigger ReceiverDisconnectedExcetion in the host. await Task.Delay(15000); Log("Creating a new receiver with epoch 2. This will trigger ReceiverDisconnectedException in the host."); var ehClient = EventHubClient.CreateFromConnectionString(this.EventHubConnectionString); externalReceiver = ehClient.CreateEpochReceiver(PartitionReceiver.DefaultConsumerGroupName, targetPartition, PartitionReceiver.StartOfStream, 2); await externalReceiver.ReceiveAsync(100, TimeSpan.FromSeconds(5)); // Give another 1 minute for host to recover then do the validatins. await Task.Delay(60000); Log("Verifying that host was able to receive ReceiverDisconnectedException"); Assert.True(targetPartitionErrors == 1, $"Host received {targetPartitionErrors} ReceiverDisconnectedExceptions!"); Log("Verifying that host was able to reopen the partition"); Assert.True(targetPartitionOpens == 2, $"Host opened target partition {targetPartitionOpens} times!"); Log("Verifying that host notified by close"); Assert.True(targetPartitionCloses == 1, $"Host closed target partition {targetPartitionCloses} times!"); } finally { Log("Calling UnregisterEventProcessorAsync"); await eventProcessorHost.UnregisterEventProcessorAsync(); if (externalReceiver != null) { await externalReceiver.CloseAsync(); } } }
async Task HostShouldRecoverWhenProcessEventsAsyncThrows() { var lastReceivedAt = DateTime.Now; var lastReceivedAtLock = new object(); var poisonMessageReceived = false; var poisonMessageProperty = "poison"; var processorFactory = new TestEventProcessorFactory(); var receivedEventCounts = new ConcurrentDictionary <string, int>(); var eventProcessorHost = new EventProcessorHost( null, PartitionReceiver.DefaultConsumerGroupName, this.EventHubConnectionString, this.StorageConnectionString, this.LeaseContainerName); processorFactory.OnCreateProcessor += (f, createArgs) => { var processor = createArgs.Item2; string partitionId = createArgs.Item1.PartitionId; string hostName = createArgs.Item1.Owner; string consumerGroupName = createArgs.Item1.ConsumerGroupName; processor.OnOpen += (_, partitionContext) => Log($"{hostName} > {consumerGroupName} > Partition {partitionId} TestEventProcessor opened"); processor.OnClose += (_, closeArgs) => Log($"{hostName} > {consumerGroupName} > Partition {partitionId} TestEventProcessor closing: {closeArgs.Item2}"); processor.OnProcessError += (_, errorArgs) => { Log($"{hostName} > {consumerGroupName} > Partition {partitionId} TestEventProcessor process error {errorArgs.Item2.Message}"); // Throw once more here depending on where we are at exception sequence. if (errorArgs.Item2.Message.Contains("ExceptionSequence1")) { throw new Exception("ExceptionSequence2"); } }; processor.OnProcessEvents += (_, eventsArgs) => { int eventCount = eventsArgs.Item2.events != null?eventsArgs.Item2.events.Count() : 0; Log($"{hostName} > {consumerGroupName} > Partition {partitionId} TestEventProcessor processing {eventCount} event(s)"); if (eventCount > 0) { lock (lastReceivedAtLock) { lastReceivedAt = DateTime.Now; } foreach (var e in eventsArgs.Item2.events) { // If this is poisoned event then throw. if (!poisonMessageReceived && e.Properties.ContainsKey(poisonMessageProperty)) { poisonMessageReceived = true; Log($"Received poisoned message from partition {partitionId}"); throw new Exception("ExceptionSequence1"); } // Track received events so we can validate at the end. if (!receivedEventCounts.ContainsKey(partitionId)) { receivedEventCounts[partitionId] = 0; } receivedEventCounts[partitionId]++; } } }; }; try { Log("Registering processorFactory..."); var epo = new EventProcessorOptions() { MaxBatchSize = 100 }; await eventProcessorHost.RegisterEventProcessorFactoryAsync(processorFactory, epo); Log("Waiting for partition ownership to settle..."); await Task.Delay(TimeSpan.FromSeconds(5)); // Send first set of messages. Log("Sending an event to each partition as the first set of messages."); var sendTasks = new List <Task>(); foreach (var partitionId in PartitionIds) { sendTasks.Add(this.SendToPartitionAsync(partitionId, $"{partitionId} event.", this.ConnectionStringBuilder.ToString())); } await Task.WhenAll(sendTasks); // Now send 1 poisoned message. This will fail one of the partition pumps. Log($"Sending a poison event to partition {PartitionIds.First()}"); var client = EventHubClient.CreateFromConnectionString(this.EventHubConnectionString); var pSender = client.CreatePartitionSender(PartitionIds.First()); var ed = new EventData(Encoding.UTF8.GetBytes("This is poison message")); ed.Properties[poisonMessageProperty] = true; await pSender.SendAsync(ed); // Wait sometime. The host should fail and then recever during this time. await Task.Delay(30000); // Send second set of messages. Log("Sending an event to each partition as the second set of messages."); sendTasks.Clear(); foreach (var partitionId in PartitionIds) { sendTasks.Add(this.SendToPartitionAsync(partitionId, $"{partitionId} event.", this.ConnectionStringBuilder.ToString())); } await Task.WhenAll(sendTasks); Log("Waiting until hosts are idle, i.e. no more messages to receive."); while (lastReceivedAt > DateTime.Now.AddSeconds(-60)) { await Task.Delay(1000); } Log("Verifying poison message was received"); Assert.True(poisonMessageReceived, "Didn't receive poison message!"); Log("Verifying received events by each partition"); foreach (var partitionId in PartitionIds) { if (!receivedEventCounts.ContainsKey(partitionId)) { throw new Exception($"Partition {partitionId} didn't receive any messages!"); } var receivedEventCount = receivedEventCounts[partitionId]; Assert.True(receivedEventCount >= 2, $"Partition {partitionId} received {receivedEventCount} where as at least 2 expected!"); } } finally { Log("Calling UnregisterEventProcessorAsync."); await eventProcessorHost.UnregisterEventProcessorAsync(); } }