/// <summary>
/// using pre-configured spn to access key vault, then retrieve sas/conn string for storage
/// </summary>
/// <returns></returns>
private bool TryCreateClientFromKeyVault()
{
_logger.LogInformation("trying to access hub from kv...");
try
{
var authBuilder = new AadTokenProvider(_aadSettings);
Task <string> AuthCallback(string authority, string resource, string scope)
{
return(authBuilder.GetAccessTokenAsync(resource));
}
var kvClient = new KeyVaultClient(AuthCallback);
var connStrSecret = kvClient
.GetSecretAsync(_vaultSettings.VaultUrl, _hubSettings.ConnectionStringSecretName).Result;
Consumer = new EventHubConsumerClient(_consumerGroupName, connStrSecret.Value, _hubSettings.HubName);
var partitionKeys = Consumer.GetPartitionIdsAsync().GetAwaiter().GetResult();
_logger.LogInformation(
$"Succeed to access hub using conn str from kv, partition keys: {string.Join(",", partitionKeys)}");
return(true);
}
catch (Exception ex)
{
_logger.LogError(ex, "faield to access hub from kv...");
return(false);
}
}
public async Task ReceiveAsync(Action <Message <TKey, TValue> > action)
{
//TODO: For production make use of processors
// https://devblogs.microsoft.com/azure-sdk/eventhubs-clients/#choosing-a-client-from-azure-messaging-eventhubs
await using var consumer = new EventHubConsumerClient(_consumerGroup, _connectionString, _hubName);
try
{
while (true)
{
// To ensure that we do not wait for an indeterminate length of time, we'll
// stop reading after we receive five events. For a fresh Event Hub, those
// will be the first five that we had published. We'll also ask for
// cancellation after 90 seconds, just to be safe.
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(1000);
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsAsync(cancellationSource.Token))
{
action(JsonConvert.DeserializeObject <Message <TKey, TValue> >(partitionEvent.Data.EventBody.ToString()));
}
}
}
finally
{
await consumer.CloseAsync();
}
}
/// <summary>
/// Starts the partition pump. In case it's already running, nothing happens.
/// </summary>
///
/// <returns>A task to be resolved on when the operation has completed.</returns>
///
public async Task StartAsync()
{
if (RunningTask == null)
{
await RunningTaskSemaphore.WaitAsync().ConfigureAwait(false);
try
{
if (RunningTask == null)
{
// We expect the token source to be null, but we are playing safe.
RunningTaskTokenSource?.Cancel();
RunningTaskTokenSource = new CancellationTokenSource();
InnerConsumer = new EventHubConsumerClient(ConsumerGroup, Context.PartitionId, StartingPosition, Connection);
RunningTask = RunAsync(RunningTaskTokenSource.Token);
}
}
finally
{
RunningTaskSemaphore.Release();
}
}
}
public static IServiceCollection AddEventsService(this IServiceCollection services)
{
var serviceProvider = services.BuildServiceProvider();
var eventHubConfiguration = serviceProvider.GetRequiredService <IEventsServiceConfiguration>();
services.TryAddSingleton(implementationFactory =>
{
var eventHubProducerClient = new EventHubProducerClient(eventHubConfiguration.ListenAndSendConnectionString,
eventHubConfiguration.EventHubName);
return(eventHubProducerClient);
});
services.TryAddSingleton(implementationFactory =>
{
var eventHubConsumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName,
eventHubConfiguration.ListenAndSendConnectionString,
eventHubConfiguration.EventHubName);
return(eventHubConsumerClient);
});
services.TryAddSingleton <IEventsReceiverService, EventsReceiverService>();
services.TryAddSingleton <IEventsSenderService, EventsSenderService>();
services.TryAddSingleton <IReceivedEventsProcessor, ReceivedEventsProcessor>();
services.TryAddSingleton <EventsBackgroundService>();
return(services);
}
public async Task ReceiveEventsFromDeviceAsync(CancellationToken ct)
{
await using var consumer = new EventHubConsumerClient(
EventHubConsumerClient.DefaultConsumerGroupName,
_eventHubConnectionString,
_eventHubName);
try
{
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsAsync(ct))
{
object idString;
partitionEvent.Data.Properties.TryGetValue("user-id", out idString);
if (idString != null)
{
var id = Guid.Parse(idString.ToString());
if (_eventQueues.ContainsKey(id))
{
_eventQueues[id].Add(partitionEvent.Data);
}
}
}
}
catch (TaskCanceledException)
{
}
}
public async Task Read()
{
try
{
#region Snippet:EventHubs_ReadMe_Read
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
/*@@*/
/*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = _scope.EventHubName;
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
await using (var consumer = new EventHubConsumerClient(consumerGroup, connectionString, eventHubName))
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(45));
await foreach (PartitionEvent receivedEvent in consumer.ReadEventsAsync(cancellationSource.Token))
{
// At this point, the loop will wait for events to be available in the Event Hub. When an event
// is available, the loop will iterate with the event that was received. Because we did not
// specify a maximum wait time, the loop will wait forever unless cancellation is requested using
// the cancellation token.
}
}
#endregion
}
catch (TaskCanceledException)
{
// Expected
}
}
public async Task InspectPartition()
{
#region Snippet:EventHubs_Sample03_InspectPartition
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
/*@@*/
/*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = _scope.EventHubName;
var consumer = new EventHubConsumerClient(consumerGroup, connectionString, eventHubName);
try
{
string[] partitions = await consumer.GetPartitionIdsAsync();
string firstPartition = partitions.FirstOrDefault();
PartitionProperties partitionProperties = await consumer.GetPartitionPropertiesAsync(firstPartition);
Debug.WriteLine($"Partition: { partitionProperties.Id }");
Debug.WriteLine($"\tThe partition contains no events: { partitionProperties.IsEmpty }");
Debug.WriteLine($"\tThe first sequence number is: { partitionProperties.BeginningSequenceNumber }");
Debug.WriteLine($"\tThe last sequence number is: { partitionProperties.LastEnqueuedSequenceNumber }");
Debug.WriteLine($"\tThe last offset is: { partitionProperties.LastEnqueuedOffset }");
Debug.WriteLine($"\tThe last enqueued time is: { partitionProperties.LastEnqueuedTime }, in UTC.");
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
/// <summary>
/// Performs the tasks needed to initialize and set up the environment for an instance
/// of the test scenario. When multiple instances are run in parallel, setup will be
/// run once for each prior to its execution.
/// </summary>
///
public async override Task SetupAsync()
{
await base.SetupAsync();
// Attempt to take a consumer group from the available set; to ensure that the
// test scenario can support the requested level of parallelism without violating
// the concurrent reader limits of a consumer group, the default consumer group
// should not be used.
if (!ConsumerGroups.TryDequeue(out var consumerGroup))
{
throw new InvalidOperationException("Unable to reserve a consumer group to read from.");
}
_consumer = new EventHubConsumerClient(consumerGroup, TestEnvironment.EventHubsConnectionString, Scope.EventHubName);
// In order to allow reading across multiple iterations, capture an enumerator. Without using a consistent
// enumerator, a new AMQP link would be created and the position reset each time RunAsync was invoked.
_readEnumerator = _consumer
.ReadEventsFromPartitionAsync(PartitionId, EventPosition.Earliest)
.ConfigureAwait(false)
.GetAsyncEnumerator();
// Force the connection and link creation by reading a single event.
if (!(await _readEnumerator.MoveNextAsync()))
{
throw new InvalidOperationException("Unable to read from the partition.");
}
}
public static async Task ReceiveMessagesFromDeviceAsync(CancellationToken cancelToken)
{
try
{
string eventHubConnectionString = await IotHubConnection.GetEventHubsConnectionStringAsync(iotHubConnectionString);
await using var consumerClient = new EventHubConsumerClient(
EventHubConsumerClient.DefaultConsumerGroupName,
eventHubConnectionString);
await foreach (PartitionEvent partitionEvent in consumerClient.ReadEventsAsync(cancelToken))
{
if (partitionEvent.Data == null)
{
continue;
}
string data = Encoding.UTF8.GetString(partitionEvent.Data.Body.ToArray());
Console.WriteLine($"Message received. Partition: {partitionEvent.Partition.PartitionId} Data: '{data}'");
}
}
catch (TaskCanceledException) { } // do nothing
catch (Exception ex)
{
Console.WriteLine($"Error reading event: {ex}");
}
}
public async Task ConfigureConsumerRetryByProperty()
{
#region Snippet:EventHubs_Sample02_ConsumerRetryByProperty
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
/*@@*/
/*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = _scope.EventHubName;
var consumerOptions = new EventHubConsumerClientOptions();
consumerOptions.RetryOptions.Mode = EventHubsRetryMode.Fixed;
consumerOptions.RetryOptions.MaximumRetries = 5;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName,
consumerOptions);
#endregion
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(EventHubsTestEnvironment.Instance.TestExecutionTimeLimit);
await consumer.CloseAsync(cancellationSource.Token).IgnoreExceptions();
}
public async Task ConfigureConsumerRetryWithFullOptions()
{
#region Snippet:EventHubs_Sample02_ConsumerRetryWithFullOptions
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
/*@@*/
/*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = _scope.EventHubName;
var consumerOptions = new EventHubConsumerClientOptions
{
RetryOptions = new EventHubsRetryOptions
{
Mode = EventHubsRetryMode.Exponential,
MaximumRetries = 5,
Delay = TimeSpan.FromMilliseconds(800),
MaximumDelay = TimeSpan.FromSeconds(10)
}
};
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName,
consumerOptions);
#endregion
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(EventHubsTestEnvironment.Instance.TestExecutionTimeLimit);
await consumer.CloseAsync(cancellationSource.Token).IgnoreExceptions();
}
private static TimeSpan maxWaitTimeForFirstEvent = new TimeSpan(0, 0, 10); // 10 seconds
/// <summary>
/// Saves all data from all partitions of an eventhub consumer group
/// </summary>
/// <param name="ehNsConnectionString">Connection string for the EventHub Namespace</param>
/// <param name="ehName">Name of the EventHub</param>
/// <param name="ehConsumerGroup">The consumer group of the EventHub</param>
/// <param name="outputFilename">The x dimension size to crop the picture. The default is 0 indicating no cropping is required.</param>
static async Task Main(string ehNsConnectionString, string ehName, string ehConsumerGroup, FileInfo outputFilename)
{
Console.WriteLine($"EH-Name {ehName}");
Console.WriteLine($"EH-Consumer Group {ehConsumerGroup}");
EventHubConsumerClient ehClient = new EventHubConsumerClient(ehConsumerGroup, ehNsConnectionString, ehName);
var ehPartitions = await ehClient.GetPartitionIdsAsync();
Console.WriteLine($"Reading data from {ehPartitions.Length} partitions");
// configure data retrieval
var readOptions = new ReadEventOptions();
readOptions.MaximumWaitTime = maxWaitTimeForFirstEvent;
var ehEvents = ehClient.ReadEventsAsync(readOptions);
// delete target file if it exists
if (outputFilename.Exists)
{
outputFilename.Delete();
}
// grab data
using (var outStream = new FileStream(outputFilename.FullName, FileMode.Append)) {
await foreach (PartitionEvent ehEvent in ehEvents)
{
byte[] data = ehEvent.Data.Body.ToArray();
Console.WriteLine($"Retrieved {data.Length} bytes");
outStream.Write(data, 0, data.Length);
}
}
}
public async Task ReadAllPartitionsWaitTime()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadAllPartitionsWaitTime
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
int loopTicks = 0;
int maximumTicks = 10;
var options = new ReadEventOptions
{
MaximumWaitTime = TimeSpan.FromSeconds(1)
};
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsAsync(options))
{
if (partitionEvent.Data != null)
{
string readFromPartition = partitionEvent.Partition.PartitionId;
byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
}
else
{
Debug.WriteLine("Wait time elapsed; no event was available.");
}
loopTicks++;
if (loopTicks >= maximumTicks)
{
break;
}
}
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
public async Task ReadPartitionTrackLastEnqueued()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadPartitionTrackLastEnqueued
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
/*@@*/
/*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = scope.EventHubName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
EventPosition startingPosition = EventPosition.Earliest;
var options = new ReadEventOptions
{
TrackLastEnqueuedEventProperties = true
};
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
firstPartition,
startingPosition,
options,
cancellationSource.Token))
{
LastEnqueuedEventProperties properties =
partitionEvent.Partition.ReadLastEnqueuedEventProperties();
Debug.WriteLine($"Partition: { partitionEvent.Partition.PartitionId }");
Debug.WriteLine($"\tThe last sequence number is: { properties.SequenceNumber }");
Debug.WriteLine($"\tThe last offset is: { properties.Offset }");
Debug.WriteLine($"\tThe last enqueued time is: { properties.EnqueuedTime }, in UTC.");
Debug.WriteLine($"\tThe information was updated at: { properties.LastReceivedTime }, in UTC.");
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
public EventsReceiverService(IEventsServiceConfiguration eventHubServiceConfiguration,
EventHubConsumerClient client,
ILogger <EventsReceiverService> logger)
{
_eventHubServiceConfiguration = eventHubServiceConfiguration;
_client = client;
_logger = logger;
}
private static void SetupClients(string eventHubCompatibleConnectionString, string eventHubName,
string serviceClientConnectionString)
{
_eventHubConsumerClient = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName,
eventHubCompatibleConnectionString, eventHubName);
_serviceClient = ServiceClient.CreateFromConnectionString(serviceClientConnectionString);
_registryManager = RegistryManager.CreateFromConnectionString(serviceClientConnectionString);
}
private async Task BackgroundReceive(string connectionString, string eventHubName, string partitionId, CancellationToken cancellationToken)
{
var reportTasks = new List <Task>();
EventPosition eventPosition;
if (LastReceivedSequenceNumber.TryGetValue(partitionId, out long sequenceNumber))
{
eventPosition = EventPosition.FromSequenceNumber(sequenceNumber, false);
}
else
{
eventPosition = EventPosition.Latest;
}
await using (var consumerClient = new EventHubConsumerClient("$Default", connectionString, eventHubName))
{
Interlocked.Decrement(ref consumersToConnect);
await foreach (var receivedEvent in consumerClient.ReadEventsFromPartitionAsync(partitionId, eventPosition, new ReadEventOptions {
MaximumWaitTime = TimeSpan.FromSeconds(5)
}))
{
if (receivedEvent.Data != null)
{
var key = Encoding.UTF8.GetString(receivedEvent.Data.Body.ToArray());
if (MissingEvents.TryRemove(key, out var expectedEvent))
{
if (HaveSameProperties(expectedEvent, receivedEvent.Data))
{
Interlocked.Increment(ref successfullyReceivedEventsCount);
}
else
{
reportTasks.Add(ReportCorruptedPropertiesEvent(partitionId, expectedEvent, receivedEvent.Data));
}
}
else
{
reportTasks.Add(ReportCorruptedBodyEvent(partitionId, receivedEvent.Data));
}
LastReceivedSequenceNumber[partitionId] = receivedEvent.Data.SequenceNumber;
}
if (cancellationToken.IsCancellationRequested)
{
break;
}
}
}
await Task.WhenAll(reportTasks);
}
public async Task ReadAllPartitions()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadAllPartitions
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(45));
int eventsRead = 0;
int maximumEvents = 3;
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsAsync(cancellationSource.Token))
{
string readFromPartition = partitionEvent.Partition.PartitionId;
byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
eventsRead++;
if (eventsRead >= maximumEvents)
{
break;
}
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
// Asynchronously create a PartitionReceiver for a partition and then start
// reading any messages sent from the simulated client.
private static async Task ReceiveMessagesFromDeviceAsync(CancellationToken cancellationToken)
{
// If you chose to copy the "Event Hub-compatible endpoint" from the "Built-in endpoints" section
// of your IoT Hub instance in the Azure portal, you can set the connection string to that value
// directly and remove the call to "BuildEventHubsConnectionString".
string connectionString = BuildEventHubsConnectionString(EventHubsCompatibleEndpoint, IotHubSasKeyName, IotHubSasKey);
// Create the consumer using the default consumer group using a direct connection to the service.
// Information on using the client with a proxy can be found in the README for this quick start, here:
// https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Quickstarts/read-d2c-messages/README.md#websocket-and-proxy-support
//
await using EventHubConsumerClient consumer = new EventHubConsumerClient(EventHubConsumerClient.DefaultConsumerGroupName, connectionString, EventHubName);
Console.WriteLine("Listening for messages on all partitions");
try
{
// Begin reading events for all partitions, starting with the first event in each partition and waiting indefinitely for
// events to become available. Reading can be canceled by breaking out of the loop when an event is processed or by
// signaling the cancellation token.
//
// The "ReadEventsAsync" method on the consumer is a good starting point for consuming events for prototypes
// and samples. For real-world production scenarios, it is strongly recommended that you consider using the
// "EventProcessorClient" from the "Azure.Messaging.EventHubs.Processor" package.
//
// More information on the "EventProcessorClient" and its benefits can be found here:
// https://github.com/Azure/azure-sdk-for-net/blob/master/sdk/eventhub/Azure.Messaging.EventHubs.Processor/README.md
//
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsAsync(cancellationToken))
{
Console.WriteLine("Message received on partition {0}:", partitionEvent.Partition.PartitionId);
string data = Encoding.UTF8.GetString(partitionEvent.Data.Body.ToArray());
Console.WriteLine("\t{0}:", data);
Console.WriteLine("Application properties (set by device):");
foreach (var prop in partitionEvent.Data.Properties)
{
Console.WriteLine("\t{0}: {1}", prop.Key, prop.Value);
}
Console.WriteLine("System properties (set by IoT Hub):");
foreach (var prop in partitionEvent.Data.SystemProperties)
{
Console.WriteLine("\t{0}: {1}", prop.Key, prop.Value);
}
}
}
catch (TaskCanceledException)
{
// This is expected when the token is signaled; it should not be considered an
// error in this scenario.
}
}
public IoTHubReaderService(
IAzureClientFactory <EventHubConsumerClient> producerFactory,
ILogger <IoTHubReaderService> logger,
IConfiguration configuration)
{
_logger = logger;
_resultsClient = producerFactory.CreateClient("IoTHub");
readEventsCanseler.Token.Register(() => {
_logger.LogInformation("ReadEventsFromPartitionAsync cansel");
});
}
private static async Task MainAsync(string[] args)
{
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
var producer = new EventHubProducerClient(EventHubConnectionString, EventHubName);
var consumer = new EventHubConsumerClient(consumerGroup, EventHubConnectionString, EventHubName);
await SendMessagesToEventHub(producer, consumer, 10);
Console.WriteLine("Press ENTER to exit.");
Console.ReadLine();
}
public static async Task Main(string[] args)
{
ConsoleHelper.WriteColorMessage("Cheese Cave Operator\n", ConsoleColor.Yellow);
// Combine the values into a Connection String
var connectionString = $"Endpoint={eventHubsCompatibleEndpoint};" +
$"SharedAccessKeyName={iotHubSasKeyName};" +
$"SharedAccessKey={iotHubSasKey};" +
$"EntityPath={eventHubsCompatiblePath}";
// Assigns the value "$Default"
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
// The EventHubConsumerClient class is used to consume events from
// an Event Hub.
consumer = new EventHubConsumerClient(
consumerGroup,
connectionString);
// An array of partition IDs is stored in d2cPartitions variable,
// where it will be shortly used to create a list of tasks that will
// receive messages from each partition.
var d2cPartitions = await consumer.GetPartitionIdsAsync();
// UNCOMMENT device twin management below here
// // A registry manager is used to access the digital twins.
// registryManager = RegistryManager
// .CreateFromConnectionString(serviceConnectionString);
// await SetTwinProperties();
// // Create a ServiceClient to communicate with service-facing endpoint
// // on your hub.
// serviceClient = ServiceClient
// .CreateFromConnectionString(serviceConnectionString);
// // Invokes a Direct Method on the device
// await InvokeMethod();
// Create receivers to listen for messages.
// As messages sent from devices to an IoT Hub may be handled by any
// of the partitions, the app has to retrieve messages from each.
// The next section of code creates a list of asynchronous tasks -
// each task will receive messages from a specific partition.
var tasks = new List <Task>();
foreach (string partition in d2cPartitions)
{
tasks.Add(ReceiveMessagesFromDeviceAsync(partition));
}
// The final line will wait for all tasks to complete - as each task
// is going to be in an infinite loop, this line prevents the
// application from exiting.
Task.WaitAll(tasks.ToArray());
}
public async Task StartAsyncCallsPartitionProcessorInitializeAsync()
{
await using (EventHubScope scope = await EventHubScope.CreateAsync(2))
{
var connectionString = TestEnvironment.BuildConnectionStringForEventHub(scope.EventHubName);
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
await using (var consumer = new EventHubConsumerClient(consumerGroup, connectionString))
{
var initializeCalls = new ConcurrentDictionary <string, int>();
// Create the event processor manager to manage our event processors.
var eventProcessorManager = new EventProcessorManager
(
consumerGroup,
connectionString,
onInitialize: eventArgs =>
initializeCalls.AddOrUpdate(eventArgs.PartitionId, 1, (partitionId, value) => value + 1)
);
eventProcessorManager.AddEventProcessors(1);
// InitializeAsync should have not been called when constructing the event processors.
Assert.That(initializeCalls.Keys, Is.Empty);
// Start the event processors.
await eventProcessorManager.StartAllAsync();
// Make sure the event processors have enough time to stabilize.
await eventProcessorManager.WaitStabilization();
// Validate results before calling stop. This way, we can make sure the initialize calls were
// triggered by start.
var partitionIds = await consumer.GetPartitionIdsAsync();
foreach (var partitionId in partitionIds)
{
Assert.That(initializeCalls.TryGetValue(partitionId, out var calls), Is.True, $"{ partitionId }: InitializeAsync should have been called.");
Assert.That(calls, Is.EqualTo(1), $"{ partitionId }: InitializeAsync should have been called only once.");
}
Assert.That(initializeCalls.Keys.Count, Is.EqualTo(partitionIds.Count()));
// Stop the event processors.
await eventProcessorManager.StopAllAsync();
}
}
}
public async Task WatchMessagesWithLimit(int limit, int messageTimeout, string consumerGroup, string connectionString, string eventHubName)
{
string consumerGroupWithDefault = EventHubConsumerClient.DefaultConsumerGroupName;
if (consumerGroup != null)
{
consumerGroupWithDefault = consumerGroup;
}
await using (var consumer = new EventHubConsumerClient(consumerGroupWithDefault, connectionString, eventHubName))
{
EventPosition startingPosition = EventPosition.Latest;
using var cancellationSource = new System.Threading.CancellationTokenSource();
int maxWaitTime = messageTimeout == 0 ? 30 : messageTimeout;
cancellationSource.CancelAfter(TimeSpan.FromSeconds(maxWaitTime));
string[] partitionIds = await consumer.GetPartitionIdsAsync();
var partitions = new IAsyncEnumerable <PartitionEvent> [partitionIds.Length];
for (int i = 0; i < partitionIds.Length; i++)
{
partitions[i] = consumer.ReadEventsFromPartitionAsync(partitionIds[i], startingPosition, cancellationSource.Token);
}
var mergedPartitions = AsyncEnumerable.Merge <PartitionEvent>(partitions);
var maxMessages = Math.Max(1, limit);
try
{
Console.WriteLine("Waiting for messages..");
int messageCount = 0;
await foreach (var pe in mergedPartitions.Take <PartitionEvent>(maxMessages))
{
//Console.WriteLine($"Event received on partition {pe.Partition.PartitionId} with body {Encoding.UTF8.GetString(pe.Data.Body.ToArray())}");
DisplayMessage(pe);
messageCount = messageCount + 1;
if (messageCount >= maxMessages)
{
Console.WriteLine($"Total messages received: {messageCount}");
break;
}
}
}
catch (Exception ex)
{
Console.WriteLine($"{ex}");
}
}
}
public async Task ReadPartitionFromSequence()
{
await using var scope = await EventHubScope.CreateAsync(1);
#region Snippet:EventHubs_Sample05_ReadPartitionFromSequence
#if SNIPPET
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
#else
var connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
var eventHubName = scope.EventHubName;
#endif
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
PartitionProperties properties = await consumer.GetPartitionPropertiesAsync(firstPartition, cancellationSource.Token);
EventPosition startingPosition = EventPosition.FromSequenceNumber(properties.LastEnqueuedSequenceNumber);
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
firstPartition,
startingPosition,
cancellationSource.Token))
{
string readFromPartition = partitionEvent.Partition.PartitionId;
byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
public IEventHub GetClientEventsEventHub()
{
var consumer = new EventHubConsumerClient(
_clientEventsOptions.EventProcessingConsumerGroup,
_clientEventsOptions.ClientEventsListenerEventHubConnectionString);
var producer = new EventHubProducerClient(
_clientEventsOptions.ClientEventsSenderEventHubConnectionString,
_clientEventsOptions.EventHubName);
return(new AzureEventHub(consumer, producer));
}
// Asynchronously create a PartitionReceiver for a partition and then start
// reading any messages sent from the simulated client.
private static async Task ReceiveMessagesFromDeviceAsync(CancellationToken ct)
{
string connectionString = _parameters.GetEventHubConnectionString();
// Create the consumer using the default consumer group using a direct connection to the service.
// Information on using the client with a proxy can be found in the README for this quick start, here:
// https://github.com/Azure-Samples/azure-iot-samples-csharp/tree/master/iot-hub/Quickstarts/ReadD2cMessages/README.md#websocket-and-proxy-support
await using var consumer = new EventHubConsumerClient(
EventHubConsumerClient.DefaultConsumerGroupName,
connectionString,
_parameters.EventHubName);
Console.WriteLine("Listening for messages on all partitions.");
try
{
// Begin reading events for all partitions, starting with the first event in each partition and waiting indefinitely for
// events to become available. Reading can be canceled by breaking out of the loop when an event is processed or by
// signaling the cancellation token.
//
// The "ReadEventsAsync" method on the consumer is a good starting point for consuming events for prototypes
// and samples. For real-world production scenarios, it is strongly recommended that you consider using the
// "EventProcessorClient" from the "Azure.Messaging.EventHubs.Processor" package.
//
// More information on the "EventProcessorClient" and its benefits can be found here:
// https://github.com/Azure/azure-sdk-for-net/blob/master/sdk/eventhub/Azure.Messaging.EventHubs.Processor/README.md
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsAsync(ct))
{
Console.WriteLine($"\nMessage received on partition {partitionEvent.Partition.PartitionId}:");
string data = Encoding.UTF8.GetString(partitionEvent.Data.Body.ToArray());
Console.WriteLine($"\tMessage body: {data}");
Console.WriteLine("\tApplication properties (set by device):");
foreach (KeyValuePair <string, object> prop in partitionEvent.Data.Properties)
{
Console.WriteLine($"\t\t{prop.Key}: {prop.Value}");
}
Console.WriteLine("\tSystem properties (set by IoT Hub):");
foreach (KeyValuePair <string, object> prop in partitionEvent.Data.SystemProperties)
{
Console.WriteLine($"\t\t{prop.Key}: {prop.Value}");
}
}
}
catch (TaskCanceledException)
{
// This is expected when the token is signaled; it should not be considered an
// error in this scenario.
}
}
static private async Task GetEvents()
{
EventHubConsumerClient client = new EventHubConsumerClient("$Default", connstring, hubname);
var cancellation = new CancellationToken();
Console.WriteLine("Getting the events");
await foreach (PartitionEvent Allevent in client.ReadEventsAsync(cancellation))
{
EventData event_data = Allevent.Data;
Console.WriteLine(Encoding.UTF8.GetString(event_data.Body.ToArray()));
}
}
public async Task ReadPartitionFromDate()
{
#region Snippet:EventHubs_Sample05_ReadPartitionFromDate
var connectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
/*@@*/
/*@@*/ connectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = _scope.EventHubName;
/*@@*/ consumerGroup = _availableConsumerGroups.Dequeue();
var consumer = new EventHubConsumerClient(
consumerGroup,
connectionString,
eventHubName);
try
{
using CancellationTokenSource cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
DateTimeOffset oneHourAgo = DateTimeOffset.UtcNow.Subtract(TimeSpan.FromHours(1));
EventPosition startingPosition = EventPosition.FromEnqueuedTime(oneHourAgo);
string firstPartition = (await consumer.GetPartitionIdsAsync(cancellationSource.Token)).First();
await foreach (PartitionEvent partitionEvent in consumer.ReadEventsFromPartitionAsync(
firstPartition,
startingPosition,
cancellationSource.Token))
{
string readFromPartition = partitionEvent.Partition.PartitionId;
byte[] eventBodyBytes = partitionEvent.Data.EventBody.ToArray();
Debug.WriteLine($"Read event of length { eventBodyBytes.Length } from { readFromPartition }");
}
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
await consumer.CloseAsync();
}
#endregion
}
public async Task EventHub_InitialOffsetFromEnqueuedTime()
{
// Mark the time now and send a message which should be the only one that is picked up when we run the actual test host
var producer = new EventHubProducerClient(EventHubsTestEnvironment.Instance.EventHubsConnectionString, _eventHubScope.EventHubName);
for (int i = 0; i < 3; i++)
{
// send one at a time so they will have slightly different enqueued times
await producer.SendAsync(new EventData[] { new EventData(new BinaryData("data")) });
await Task.Delay(1000);
}
var consumer = new EventHubConsumerClient(
EventHubConsumerClient.DefaultConsumerGroupName,
EventHubsTestEnvironment.Instance.EventHubsConnectionString,
_eventHubScope.EventHubName);
var events = consumer.ReadEventsAsync();
_initialOffsetEnqueuedTimeUTC = DateTime.UtcNow;
await foreach (PartitionEvent evt in events)
{
// use the timestamp from the first event for our FromEnqueuedTime
_initialOffsetEnqueuedTimeUTC = evt.Data.EnqueuedTime;
break;
}
var initialOffsetOptions = new InitialOffsetOptions();
var(jobHost, host) = BuildHost <EventHubTestInitialOffsetFromEnqueuedTimeJobs>(
builder =>
{
builder.ConfigureServices(services =>
{
services.Configure <EventHubOptions>(options =>
{
options.InitialOffsetOptions.Type = "FromEnqueuedTime";
// for some reason, this doesn't seem to work if including milliseconds in the format
options.InitialOffsetOptions.EnqueuedTimeUTC = _initialOffsetEnqueuedTimeUTC.ToString("yyyy-MM-ddTHH:mm:ssZ");
});
});
ConfigureTestEventHub(builder);
});
using (jobHost)
{
bool result = _eventWait.WaitOne(Timeout);
Assert.True(result);
}
}
