public async Task <List <EventData> > GetMessagesForDevice(string deviceId, DateTime startTime, int maxPerPartition = 100, int waitTimeSecs = 5)
{
var messages = new List <EventData>();
EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(this.eventHubConnectionString);
PartitionReceiver partitionReceiver = eventHubClient.CreateReceiver(
EventHubConsumerGroup,
EventHubPartitionKeyResolver.ResolveToPartition(deviceId, (await eventHubClient.GetRuntimeInformationAsync()).PartitionCount),
EventPosition.FromEnqueuedTime(startTime));
// Retry a few times due to weird behavior with ReceiveAsync() not returning all messages available
for (int i = 0; i < 3; i++)
{
IEnumerable <EventData> events = await partitionReceiver.ReceiveAsync(maxPerPartition, TimeSpan.FromSeconds(waitTimeSecs));
if (events != null)
{
messages.AddRange(events);
}
if (i < 3)
{
await Task.Delay(TimeSpan.FromSeconds(5));
}
}
await partitionReceiver.CloseAsync();
await eventHubClient.CloseAsync();
return(messages);
}
/// <summary>
/// Performs the tasks needed to initialize and set up the environment for an instance
/// of the test scenario. When multiple instances are run in parallel, setup will be
/// run once for each prior to its execution.
/// </summary>
///
public async override Task SetupAsync()
{
await base.SetupAsync();
// Attempt to take a consumer group from the available set; to ensure that the
// test scenario can support the requested level of parallelism without violating
// the concurrent reader limits of a consumer group, the default consumer group
// should not be used.
if (!ConsumerGroups.TryDequeue(out var consumerGroup))
{
throw new InvalidOperationException("Unable to reserve a consumer group to read from.");
}
_receiver = new PartitionReceiver(
consumerGroup,
PartitionId,
EventPosition.Earliest,
TestEnvironment.EventHubsConnectionString,
Scope.EventHubName);
// Force the connection and link creation by reading a single event.
await _receiver.ReceiveBatchAsync(1).ConfigureAwait(false);
}
public void ReadLastEnqueuedEventInformationPopulatesFromTheLastReceivedEvent()
{
var lastEvent = new EventData
(
eventBody: Array.Empty <byte>(),
lastPartitionSequenceNumber: 12345,
lastPartitionOffset: 89101,
lastPartitionEnqueuedTime: DateTimeOffset.Parse("2015-10-27T00:00:00Z"),
lastPartitionInformationRetrievalTime: DateTimeOffset.Parse("2012-03-04T08:49:00Z")
);
var eventHub = "someHub";
var partition = "PART";
var transportMock = new ObservableTransportConsumerMock {
LastReceivedEvent = lastEvent
};
var receiver = new PartitionReceiver("group", partition, eventHub, true, TimeSpan.Zero, transportMock);
var metrics = receiver.ReadLastEnqueuedEventInformation();
Assert.That(metrics.EventHubName, Is.EqualTo(eventHub), "The Event Hub name should match.");
Assert.That(metrics.PartitionId, Is.EqualTo(partition), "The partition id should match.");
Assert.That(metrics.LastEnqueuedSequenceNumber, Is.EqualTo(lastEvent.LastPartitionSequenceNumber), "The sequence number should match.");
Assert.That(metrics.LastEnqueuedOffset, Is.EqualTo(lastEvent.LastPartitionOffset), "The offset should match.");
Assert.That(metrics.LastEnqueuedTime, Is.EqualTo(lastEvent.LastPartitionEnqueuedTime), "The enqueue time should match.");
Assert.That(metrics.InformationReceived, Is.EqualTo(lastEvent.LastPartitionInformationRetrievalTime), "The retrieval time should match.");
}
private static async Task ReceiveMessagesFromPartition(string partition, CancellationToken ct)
{
receiver = eventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, partition, EventPosition.FromEnqueuedTime(DateTime.Now));
Console.WriteLine($"Connected to partition {receiver.PartitionId}, consumer group {receiver.ConsumerGroupName}");
while (true)
{
if (ct.IsCancellationRequested)
{
return;
}
// Receive a maximum of 100 messages in this call to ReceiveAsync
var ehEvents = await receiver.ReceiveAsync(100);
// ReceiveAsync can return null if there are no messages
if (ehEvents != null)
{
// Since ReceiveAsync can return more than a single event you will need a loop to process
foreach (var ehEvent in ehEvents)
{
// Decode the byte array segment
var message = UnicodeEncoding.UTF8.GetString(ehEvent.Body.Array);
Console.WriteLine($"Received. '{message}'");
}
}
}
}
public async Task PartitionReceiverCannotRetrievePartitionPropertiesWhenConnectionIsClosed()
{
var partitionCount = 1;
await using (EventHubScope scope = await EventHubScope.CreateAsync(partitionCount))
{
var cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(20));
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var connection = new EventHubConnection(connectionString);
var partitionId = default(string);
await using (var producer = new EventHubProducerClient(connection))
{
partitionId = (await producer.GetPartitionIdsAsync(cancellationSource.Token)).First();
}
await using (var receiver = new PartitionReceiver(EventHubConsumerClient.DefaultConsumerGroupName, partitionId, EventPosition.Earliest, connection))
{
Assert.That(async() => await receiver.GetPartitionPropertiesAsync(cancellationSource.Token), Throws.Nothing);
await connection.CloseAsync(cancellationSource.Token);
Assert.That(async() => await receiver.GetPartitionPropertiesAsync(cancellationSource.Token), Throws.InstanceOf <EventHubsException>().And.Property(nameof(EventHubsException.Reason)).EqualTo(EventHubsException.FailureReason.ClientClosed));
Assert.That(cancellationSource.IsCancellationRequested, Is.False, "The cancellation token should not have been signaled.");
}
}
}
public async Task <IList <EventData> > GetMessagesFromAllPartitions(DateTime startTime, int maxPerPartition = 10, int waitTimeSecs = 5)
{
var messages = new List <EventData>();
EventHubRuntimeInformation rtInfo = await this.eventHubClient.GetRuntimeInformationAsync();
foreach (string partition in rtInfo.PartitionIds)
{
PartitionReceiver partitionReceiver = this.eventHubClient.CreateReceiver(
PartitionReceiver.DefaultConsumerGroupName,
partition,
startTime);
// Retry a few times to make sure we get all expected messages.
for (int i = 0; i < 3; i++)
{
IEnumerable <EventData> events = await partitionReceiver.ReceiveAsync(maxPerPartition, TimeSpan.FromSeconds(waitTimeSecs));
if (events != null)
{
messages.AddRange(events);
}
if (i < 3)
{
await Task.Delay(TimeSpan.FromSeconds(5));
}
}
await partitionReceiver.CloseAsync();
}
return(messages);
}
async Task InvokeOnNull()
{
PartitionReceiver partitionReceiver = this.EventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, "0", EventPosition.FromEnd());
try
{
EventWaitHandle nullReceivedEvent = new EventWaitHandle(false, EventResetMode.ManualReset);
var handler = new TestPartitionReceiveHandler();
handler.EventsReceived += (s, eventDatas) =>
{
if (eventDatas == null)
{
TestUtility.Log("Received null.");
nullReceivedEvent.Set();
}
};
partitionReceiver.SetReceiveHandler(handler, true);
if (!nullReceivedEvent.WaitOne(TimeSpan.FromSeconds(120)))
{
throw new InvalidOperationException("Did not receive null.");
}
}
finally
{
// Unregister handler.
partitionReceiver.SetReceiveHandler(null);
// Close clients.
await partitionReceiver.CloseAsync();
}
}
/// <summary>
/// Performs the tasks needed to initialize and set up the environment for an instance
/// of the test scenario. When multiple instances are run in parallel, setup will be
/// run once for each prior to its execution.
/// </summary>
///
public async override Task SetupAsync()
{
await base.SetupAsync().ConfigureAwait(false);
// Attempt to take a consumer group from the available set; to ensure that the
// test scenario can support the requested level of parallelism without violating
// the concurrent reader limits of a consumer group, the default consumer group
// should not be used.
if (!s_consumerGroups.TryDequeue(out var consumerGroup))
{
throw new InvalidOperationException("Unable to reserve a consumer group to read from.");
}
_client = EventHubClient.CreateFromConnectionString(TestUtility.BuildEventHubsConnectionString(s_scope.EventHubName));
_partitionId = (await _client.GetRuntimeInformationAsync().ConfigureAwait(false)).PartitionIds[0];
_receiver = _client.CreateReceiver(
consumerGroup,
_partitionId,
EventPosition.FromStart());
// Force the connection and link creation by reading a single event.
await _receiver.ReceiveAsync(1).ConfigureAwait(false);
}
public static async Task <EventHubTestListener> CreateListenerPal(string deviceName)
{
PartitionReceiver receiver = null;
Stopwatch sw = new Stopwatch();
sw.Start();
var builder = new EventHubsConnectionStringBuilder(Configuration.IoTHub.EventHubString)
{
EntityPath = Configuration.IoTHub.EventHubCompatibleName
};
EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(builder.ToString());
var eventRuntimeInformation = await eventHubClient.GetRuntimeInformationAsync().ConfigureAwait(false);
var eventHubPartitionsCount = eventRuntimeInformation.PartitionCount;
string partition = EventHubPartitionKeyResolver.ResolveToPartition(deviceName, eventHubPartitionsCount);
string consumerGroupName = Configuration.IoTHub.EventHubConsumerGroup;
while (receiver == null && sw.Elapsed.TotalMinutes < MaximumWaitTimeInMinutes)
{
try
{
receiver = eventHubClient.CreateReceiver(consumerGroupName, partition, DateTime.Now.AddMinutes(-5));
}
catch (QuotaExceededException ex)
{
s_log.WriteLine($"{nameof(EventHubTestListener)}.{nameof(CreateListener)}: Cannot create receiver: {ex}");
}
}
sw.Stop();
return(new EventHubTestListener(receiver));
}
async Task OpenClientsAsync() // throws EventHubsException, IOException, InterruptedException, ExecutionException
{
// Create new clients
EventPosition eventPosition = await this.PartitionContext.GetInitialOffsetAsync().ConfigureAwait(false);
long epoch = this.Lease.Epoch;
ProcessorEventSource.Log.PartitionPumpCreateClientsStart(this.Host.HostName, this.PartitionContext.PartitionId, epoch,
$"Offset:{eventPosition.Offset}, SequenceNumber:{eventPosition.SequenceNumber}, DateTime:{eventPosition.EnqueuedTimeUtc}");
this.eventHubClient = this.Host.CreateEventHubClient();
this.eventHubClient.WebProxy = this.Host.EventProcessorOptions.WebProxy;
var receiverOptions = new ReceiverOptions()
{
// Enable receiver metrics?
EnableReceiverRuntimeMetric = this.Host.EventProcessorOptions.EnableReceiverRuntimeMetric,
// Use host name as the identifier for debugging purpose
// Shorten host name if name is longer than max allowed lenght.
Identifier = this.Host.HostName.Length > ClientConstants.MaxReceiverIdentifierLength ?
this.Host.HostName.Substring(0, ClientConstants.MaxReceiverIdentifierLength) : this.Host.HostName
};
// Create new receiver and set options
this.partitionReceiver = this.eventHubClient.CreateEpochReceiver(
this.PartitionContext.ConsumerGroupName,
this.PartitionContext.PartitionId,
eventPosition,
epoch,
receiverOptions);
this.partitionReceiver.PrefetchCount = this.Host.EventProcessorOptions.PrefetchCount;
ProcessorEventSource.Log.PartitionPumpCreateClientsStop(this.Host.HostName, this.PartitionContext.PartitionId);
}
public async Task SmallReceiveTimeout()
{
var maxClients = 4;
// Issue receives with 1 second so that some of the Receive calls will timeout while creating AMQP link.
// Even those Receive calls should return NULL instead of bubbling the exception up.
var receiveTimeoutInSeconds = 1;
var tasks = Enumerable.Range(0, maxClients)
.Select(async i =>
{
PartitionReceiver receiver = null;
try
{
TestUtility.Log($"Testing with {receiveTimeoutInSeconds} seconds on client {i}.");
// Start receiving from a future time so that Receive call won't be able to fetch any events.
var ehClient = EventHubClient.CreateFromConnectionString(TestUtility.EventHubsConnectionString);
receiver = ehClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, "0", EventPosition.FromEnqueuedTime(DateTime.UtcNow.AddMinutes(1)));
var ed = await receiver.ReceiveAsync(1, TimeSpan.FromSeconds(receiveTimeoutInSeconds));
if (ed == null)
{
TestUtility.Log($"Received NULL from client {i}");
}
}
finally
{
await receiver.CloseAsync();
}
});
await Task.WhenAll(tasks);
}
public async Task ReceiveEventsAsync(
string deviceId,
Func <EventData, bool> onEventReceived,
CancellationToken token)
{
EventHubClient client = this.EventHubClient;
int count = (await client.GetRuntimeInformationAsync()).PartitionCount;
string partition = EventHubPartitionKeyResolver.ResolveToPartition(deviceId, count);
PartitionReceiver receiver = client.CreateReceiver("$Default", partition, EventPosition.FromEnd());
var result = new TaskCompletionSource <bool>();
using (token.Register(() => result.TrySetCanceled()))
{
receiver.SetReceiveHandler(
new PartitionReceiveHandler(
data =>
{
bool done = onEventReceived(data);
if (done)
{
result.TrySetResult(true);
}
return(done);
}));
await result.Task;
}
await receiver.CloseAsync();
}
async Task OpenClientsAsync() // throws EventHubsException, IOException, InterruptedException, ExecutionException
{
// Create new clients
EventPosition eventPosition = await this.PartitionContext.GetInitialOffsetAsync().ConfigureAwait(false);
long epoch = this.Lease.Epoch;
ProcessorEventSource.Log.PartitionPumpCreateClientsStart(this.Host.HostName, this.PartitionContext.PartitionId, epoch,
$"Offset:{eventPosition.Offset}, SequenceNumber:{eventPosition.SequenceNumber}, DateTime:{eventPosition.EnqueuedTimeUtc}");
this.eventHubClient = this.Host.CreateEventHubClient();
this.eventHubClient.WebProxy = this.Host.EventProcessorOptions.WebProxy;
var receiverOptions = new ReceiverOptions()
{
// Enable receiver metrics?
EnableReceiverRuntimeMetric = this.Host.EventProcessorOptions.EnableReceiverRuntimeMetric
};
// Create new receiver and set options
this.partitionReceiver = this.eventHubClient.CreateEpochReceiver(
this.PartitionContext.ConsumerGroupName,
this.PartitionContext.PartitionId,
eventPosition,
epoch,
receiverOptions);
this.partitionReceiver.PrefetchCount = this.Host.EventProcessorOptions.PrefetchCount;
ProcessorEventSource.Log.PartitionPumpCreateClientsStop(this.Host.HostName, this.PartitionContext.PartitionId);
}
async Task NonexistentEntity()
{
// Rebuild connection string with a nonexistent entity.
var csb = new EventHubsConnectionStringBuilder(TestUtility.EventHubsConnectionString);
csb.EntityPath = Guid.NewGuid().ToString();
var ehClient = EventHubClient.CreateFromConnectionString(csb.ToString());
// GetRuntimeInformationAsync on a nonexistent entity.
await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
{
TestUtility.Log("Getting entity information from a nonexistent entity.");
await ehClient.GetRuntimeInformationAsync();
throw new InvalidOperationException("GetRuntimeInformation call should have failed");
});
// GetPartitionRuntimeInformationAsync on a nonexistent entity.
await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
{
TestUtility.Log("Getting partition information from a nonexistent entity.");
await ehClient.GetPartitionRuntimeInformationAsync("0");
throw new InvalidOperationException("GetPartitionRuntimeInformation call should have failed");
});
// Try sending.
PartitionSender sender = null;
await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
{
TestUtility.Log("Sending an event to nonexistent entity.");
sender = ehClient.CreatePartitionSender("0");
await sender.SendAsync(new EventData(Encoding.UTF8.GetBytes("this send should fail.")));
throw new InvalidOperationException("Send call should have failed");
});
await sender.CloseAsync();
// Try receiving.
PartitionReceiver receiver = null;
await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
{
TestUtility.Log("Receiving from nonexistent entity.");
receiver = ehClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, "0", EventPosition.FromStart());
await receiver.ReceiveAsync(1);
throw new InvalidOperationException("Receive call should have failed");
});
await receiver.CloseAsync();
// Try receiving on an nonexistent consumer group.
ehClient = EventHubClient.CreateFromConnectionString(TestUtility.EventHubsConnectionString);
await Assert.ThrowsAsync <MessagingEntityNotFoundException>(async() =>
{
TestUtility.Log("Receiving from nonexistent consumer group.");
receiver = ehClient.CreateReceiver(Guid.NewGuid().ToString(), "0", EventPosition.FromStart());
await receiver.ReceiveAsync(1);
throw new InvalidOperationException("Receive call should have failed");
});
await receiver.CloseAsync();
}
private async Task <PartitionReceiver> CreateEventHubReceiver(string deviceName)
{
PartitionReceiver eventHubReceiver = null;
Stopwatch sw = new Stopwatch();
sw.Start();
var builder = new EventHubsConnectionStringBuilder(Configuration.IoTHub.EventHubString)
{
EntityPath = Configuration.IoTHub.EventHubCompatibleName
};
EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(builder.ToString());
var eventRuntimeInformation = await eventHubClient.GetRuntimeInformationAsync().ConfigureAwait(false);
var eventHubPartitionsCount = eventRuntimeInformation.PartitionCount;
string partition = EventHubPartitionKeyResolver.ResolveToPartition(deviceName, eventHubPartitionsCount);
string consumerGroupName = Configuration.IoTHub.EventHubConsumerGroup;
while (eventHubReceiver == null && sw.Elapsed.Minutes < 1)
{
try
{
eventHubReceiver = eventHubClient.CreateReceiver(consumerGroupName, partition, DateTime.Now.AddMinutes(-5));
}
catch (QuotaExceededException ex)
{
Debug.WriteLine(ex);
}
}
sw.Stop();
return(eventHubReceiver);
}
async Task CleanUpClientsAsync() // swallows all exceptions
{
if (this.partitionReceiver != null)
{
// Taking the lock means that there is no ProcessEventsAsync call in progress.
Task closeTask;
using (await this.ProcessingAsyncLock.LockAsync().ConfigureAwait(false))
{
// Calling PartitionReceiver.CloseAsync will gracefully close the IPartitionReceiveHandler we have installed.
ProcessorEventSource.Log.PartitionPumpInfo(this.Host.Id, this.PartitionContext.PartitionId, "Closing PartitionReceiver");
closeTask = this.partitionReceiver.CloseAsync();
}
await closeTask.ConfigureAwait(false);
this.partitionReceiver = null;
}
if (this.eventHubClient != null)
{
ProcessorEventSource.Log.PartitionPumpInfo(this.Host.Id, this.PartitionContext.PartitionId, "Closing EventHubClient");
await this.eventHubClient.CloseAsync().ConfigureAwait(false);
this.eventHubClient = null;
}
}
private static async Task <IEventHubReceiver> CreateReceiver(EventHubPartitionSettings partitionSettings, string offset, Logger logger, ITelemetryProducer telemetryProducer)
{
bool offsetInclusive = true;
var connectionStringBuilder = new EventHubsConnectionStringBuilder(partitionSettings.Hub.ConnectionString)
{
EntityPath = partitionSettings.Hub.Path
};
EventHubClient client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString());
// if we have a starting offset or if we're not configured to start reading from utc now, read from offset
if (!partitionSettings.Hub.StartFromNow ||
offset != EventHubConstants.StartOfStream)
{
logger.Info("Starting to read from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
}
else
{
// to start reading from most recent data, we get the latest offset from the partition.
EventHubPartitionRuntimeInformation partitionInfo =
await client.GetPartitionRuntimeInformationAsync(partitionSettings.Partition);
offset = partitionInfo.LastEnqueuedOffset;
offsetInclusive = false;
logger.Info("Starting to read latest messages from EventHub partition {0}-{1} at offset {2}", partitionSettings.Hub.Path, partitionSettings.Partition, offset);
}
PartitionReceiver receiver = client.CreateReceiver(partitionSettings.Hub.ConsumerGroup, partitionSettings.Partition, offset, offsetInclusive);
if (partitionSettings.Hub.PrefetchCount.HasValue)
{
receiver.PrefetchCount = partitionSettings.Hub.PrefetchCount.Value;
}
return(new EventHubReceiverProxy(receiver));
}
public void ConnectionConstructorSetsTheDefaultMaximumReceiveWaitTime()
{
var expected = TimeSpan.FromMilliseconds(270);
var receiver = new PartitionReceiver("group", "id", "name", false, expected, Mock.Of <TransportConsumer>());
Assert.That(GetDefaultMaximumReceiveWaitTime(receiver), Is.EqualTo(expected));
}
async Task ReceiveTimeout()
{
var testValues = new[] { 10, 30, 120 };
PartitionReceiver receiver = null;
try
{
foreach (var receiveTimeoutInSeconds in testValues)
{
Log($"Testing with {receiveTimeoutInSeconds} seconds.");
// Start receiving from a future time so that Receive call won't be able to fetch any events.
receiver = this.EventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, "0", DateTime.UtcNow.AddMinutes(1));
var startTime = DateTime.Now;
await receiver.ReceiveAsync(1, TimeSpan.FromSeconds(receiveTimeoutInSeconds));
// Receive call should have waited more than receive timeout.
// Give 100 milliseconds of buffer.
var diff = DateTime.Now.Subtract(startTime).TotalSeconds;
Assert.True(diff >= receiveTimeoutInSeconds - 0.1, $"Hit timeout {diff} seconds into Receive call while testing {receiveTimeoutInSeconds} seconds timeout.");
// Timeout should not be late more than 5 seconds.
// This is just a logical buffer for timeout behavior validation.
Assert.True(diff < receiveTimeoutInSeconds + 5, $"Hit timeout {diff} seconds into Receive call while testing {receiveTimeoutInSeconds} seconds timeout.");
}
}
finally
{
await receiver.CloseAsync();
}
}
public void ConnectionConstructorSetsTrackingLastEnqueuedEvent()
{
var expected = true;
var receiver = new PartitionReceiver("group", "id", "name", expected, TimeSpan.Zero, Mock.Of <TransportConsumer>());
Assert.That(GetTrackingLastEnqueuedEvent(receiver), Is.EqualTo(expected));
}
public async Task ReadPartitionWithReceiver()
{
#region Snippet:EventHubs_Sample05_ReadPartitionWithReceiver
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
/*@@*/
/*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = _scope.EventHubName;
/*@@*/ consumerGroup = _availableConsumerGroups.Dequeue();
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
string firstPartition;
await using (var producer = new EventHubProducerClient(connectionString, eventHubName))
{
firstPartition = (await producer.GetPartitionIdsAsync()).First();
}
var receiver = new PartitionReceiver(
consumerGroup,
firstPartition,
EventPosition.Earliest,
connectionString,
eventHubName);
try
{
while (!cancellationSource.IsCancellationRequested)
{
int batchSize = 50;
TimeSpan waitTime = TimeSpan.FromSeconds(1);
IEnumerable <EventData> eventBatch = await receiver.ReceiveBatchAsync(
batchSize,
waitTime,
cancellationSource.Token);
foreach (EventData eventData in eventBatch)
{
byte[] eventBodyBytes = eventData.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { firstPartition }");
}
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await receiver.CloseAsync();
}
#endregion
}
public static void CreateListenerPalAndReceiveMessages()
{
var builder = new EventHubsConnectionStringBuilder(Configuration.IoTHub.EventHubString)
{
EntityPath = Configuration.IoTHub.EventHubCompatibleName
};
EventHubClient eventHubClient = EventHubClient.CreateFromConnectionString(builder.ToString());
var eventRuntimeInformation = eventHubClient.GetRuntimeInformationAsync().Result;
var eventHubPartitionsCount = eventRuntimeInformation.PartitionCount;
string consumerGroupName = Configuration.IoTHub.EventHubConsumerGroup;
foreach (string partitionId in eventRuntimeInformation.PartitionIds)
{
try
{
PartitionReceiver receiver = eventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, partitionId, DateTime.Now.AddMinutes(-LookbackTimeInMinutes));
s_log.WriteLine($"EventHub receiver created for partition {partitionId}, listening from {LookbackTimeInMinutes}");
new Task(async() =>
{
while (true)
{
IEnumerable <EventData> eventDatas = await receiver.ReceiveAsync(int.MaxValue, TimeSpan.FromSeconds(OperationTimeoutInSeconds)).ConfigureAwait(false);
ProcessEventData(eventDatas);
}
}).Start();
}
catch (EventHubsException ex)
{
s_log.WriteLine($"{nameof(EventHubTestListener)}.{nameof(CreateListenerPalAndReceiveMessages)}: Cannot create receiver for partitionID {partitionId}: {ex}");
}
}
}
protected async Task VerifyDataOnIoTHub(string moduleId)
{
Console.WriteLine($"Verifying data on IoTHub from {moduleId}");
// First Verify if module is already running.
await this.bootstrapper.VerifyModuleIsRunning(moduleId);
var builder = new EventHubsConnectionStringBuilder(this.eventhubCompatibleEndpointWithEntityPath)
{
TransportType = this.eventHubClientTransportType
};
Console.WriteLine($"Receiving events from device '{this.context.DeviceId}' on Event Hub '{builder.EntityPath}'");
EventHubClient eventHubClient =
EventHubClient.CreateFromConnectionString(builder.ToString());
this.proxy.ForEach(p => eventHubClient.WebProxy = p);
PartitionReceiver eventHubReceiver = eventHubClient.CreateReceiver(
"$Default",
EventHubPartitionKeyResolver.ResolveToPartition(
this.context.DeviceId,
(await eventHubClient.GetRuntimeInformationAsync()).PartitionCount),
EventPosition.FromEnd());
// TODO: [Improvement] should verify test results without using event hub, which introduce latency.
var result = new TaskCompletionSource <bool>();
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(20))) // This long timeout is needed in case event hub is slow to process messages
{
using (cts.Token.Register(() => result.TrySetCanceled()))
{
eventHubReceiver.SetReceiveHandler(
new PartitionReceiveHandler(
eventData =>
{
eventData.SystemProperties.TryGetValue("iothub-connection-device-id", out object devId);
eventData.SystemProperties.TryGetValue("iothub-connection-module-id", out object modId);
if (devId != null && devId.ToString().Equals(this.context.DeviceId) &&
modId != null && modId.ToString().Equals(moduleId))
{
result.TrySetResult(true);
return(true);
}
return(false);
}));
await result.Task;
}
}
Console.WriteLine("VerifyDataOnIoTHub completed.");
await eventHubReceiver.CloseAsync();
await eventHubClient.CloseAsync();
}
public void CloseClosesTheTransportConsumer()
{
var transportConsumer = new ObservableTransportConsumerMock();
var receiver = new PartitionReceiver("group", "0", "hub", true, TimeSpan.Zero, transportConsumer);
receiver.Close();
Assert.That(transportConsumer.WasCloseCalled, Is.True);
}
private async Task ReceiverPumpAsync(PartitionReceiver receiver, Func <IReadOnlyCollection <QueueMessage>, Task> onMessage, int maxBatchSize, CancellationToken cancellationToken)
{
while (true)
{
try
{
IEnumerable <EventData> events = await receiver.ReceiveAsync(maxBatchSize, _waitTime);
if (events != null && !cancellationToken.IsCancellationRequested)
{
List <QueueMessage> qms = events.Select(ed => Converter.ToQueueMessage(ed, receiver.PartitionId)).ToList();
await onMessage(qms);
QueueMessage lastMessage = qms.LastOrDefault();
//save state
if (lastMessage != null)
{
const string sequenceNumberPropertyName = "x-opt-sequence-number";
if (lastMessage.Properties.TryGetValue(sequenceNumberPropertyName, out string sequenceNumber))
{
long?sequenceNumberLong = null;
if (long.TryParse(sequenceNumber, out long seqenceNumberNonNullable))
{
sequenceNumberLong = seqenceNumberNonNullable;
}
await _state.SetPartitionStateAsync(receiver.PartitionId, sequenceNumberLong);
}
}
}
if (cancellationToken.IsCancellationRequested)
{
await receiver.CloseAsync();
return;
}
}
catch (ArgumentException ex)
{
Console.WriteLine("failed with message: '{0}', clearing partition state.", ex);
await _state.SetPartitionStateAsync(receiver.PartitionId, EventPosition.FromStart().SequenceNumber);
}
catch (OperationCanceledException)
{
return;
}
catch (Exception ex)
{
Console.WriteLine("receiver stopped: {0}", ex);
return;
}
}
}
internal AzureEncryptedEventHubReader(
PartitionReceiver reader,
Func <PartitionReceiver, Task <PartitionReceiver> > receiverRefresh,
Func <IEnumerable <EventMessage>, string, Task> onMessagesReceived,
byte[] encryptionKey)
: base(reader, receiverRefresh, onMessagesReceived)
{
_encryptionKey = encryptionKey;
}
public void ReceiveAsyncValidatesTheMaximumWaitTime(int timeSpanDelta)
{
var transportConsumer = new ObservableTransportConsumerMock();
var receiver = new PartitionReceiver("group", "0", "hub", true, TimeSpan.Zero, transportConsumer);
var expectedWaitTime = TimeSpan.FromMilliseconds(timeSpanDelta);
using var cancellation = new CancellationTokenSource();
Assert.That(async() => await receiver.ReceiveAsync(32, expectedWaitTime, cancellation.Token), Throws.InstanceOf <ArgumentException>());
}
public void ResumeFrom(long position)
{
EventPosition eventPosition = (position == -1) ?
EventPosition.FromStart() : EventPosition.FromSequenceNumber(position);
var _eventHubClient = GetEventHubClient(processId);
_eventHubReceiver = _eventHubClient.CreateReceiver("$Default", (processId / 8).ToString(), eventPosition);
}
protected async Task VerifyDataOnIoTHubAsync()
{
// Leaf device without parent not expected to send messages
if (!this.edgeDeviceId.HasValue)
{
return;
}
var builder = new EventHubsConnectionStringBuilder(this.eventhubCompatibleEndpointWithEntityPath)
{
TransportType = this.eventHubClientTransportType
};
Console.WriteLine($"Receiving events from device '{this.context.Device.Id}' on Event Hub '{builder.EntityPath}'");
EventHubClient eventHubClient =
EventHubClient.CreateFromConnectionString(builder.ToString());
this.proxy.ForEach(p => eventHubClient.WebProxy = p);
PartitionReceiver eventHubReceiver = eventHubClient.CreateReceiver(
"$Default",
EventHubPartitionKeyResolver.ResolveToPartition(
this.context.Device.Id,
(await eventHubClient.GetRuntimeInformationAsync()).PartitionCount),
EventPosition.FromEnqueuedTime(DateTime.Now.AddMinutes(-5)));
var result = new TaskCompletionSource <bool>();
using (var cts = new CancellationTokenSource(TimeSpan.FromMinutes(3)))
{
using (cts.Token.Register(() => result.TrySetCanceled()))
{
eventHubReceiver.SetReceiveHandler(
new PartitionReceiveHandler(
eventData =>
{
eventData.SystemProperties.TryGetValue("iothub-connection-device-id", out var devId);
if (devId != null && devId.ToString().Equals(this.context.Device.Id, StringComparison.Ordinal) &&
Encoding.UTF8.GetString(eventData.Body).Contains(this.context.MessageGuid, StringComparison.Ordinal))
{
result.TrySetResult(true);
return(true);
}
return(false);
}));
await result.Task;
}
}
await eventHubReceiver.CloseAsync();
await eventHubClient.CloseAsync();
}
async Task PartitionReceiverSetReceiveHandler()
{
Log("Receiving Events via PartitionReceiver.SetReceiveHandler()");
string partitionId = "1";
PartitionReceiver partitionReceiver = this.EventHubClient.CreateReceiver(PartitionReceiver.DefaultConsumerGroupName, partitionId, DateTime.UtcNow.AddMinutes(-10));
PartitionSender partitionSender = this.EventHubClient.CreatePartitionSender(partitionId);
try
{
string uniqueEventId = Guid.NewGuid().ToString();
Log($"Sending an event to Partition {partitionId} with custom property EventId {uniqueEventId}");
var sendEvent = new EventData(Encoding.UTF8.GetBytes("Hello EventHub!"));
sendEvent.Properties = new Dictionary <string, object> {
["EventId"] = uniqueEventId
};
await partitionSender.SendAsync(sendEvent);
EventWaitHandle dataReceivedEvent = new EventWaitHandle(false, EventResetMode.ManualReset);
var handler = new TestPartitionReceiveHandler();
handler.ErrorReceived += (s, e) => Log($"TestPartitionReceiveHandler.ProcessError {e.GetType().Name}: {e.Message}");
handler.EventsReceived += (s, eventDatas) =>
{
int count = eventDatas != null?eventDatas.Count() : 0;
Log($"Received {count} event(s):");
foreach (var eventData in eventDatas)
{
object objectValue;
if (eventData.Properties != null && eventData.Properties.TryGetValue("EventId", out objectValue))
{
Log($"Received message with EventId {objectValue}");
string receivedId = objectValue.ToString();
if (receivedId == uniqueEventId)
{
Log("Success");
dataReceivedEvent.Set();
break;
}
}
}
};
partitionReceiver.SetReceiveHandler(handler);
if (!dataReceivedEvent.WaitOne(TimeSpan.FromSeconds(20)))
{
throw new InvalidOperationException("Data Received Event was not signaled.");
}
}
finally
{
await partitionSender.CloseAsync();
await partitionReceiver.CloseAsync();
}
}