/// <summary>
/// Initializes a new instance of the <see cref="ShortWaitTimeMock"/> class.
/// </summary>
///
/// <param name="consumerGroup">The name of the consumer group this event processor is associated with. Events are read in the context of this group.</param>
/// <param name="partitionManager">Interacts with the storage system with responsibility for creation of checkpoints and for ownership claim.</param>
/// <param name="connection">The client used to interact with the Azure Event Hubs service.</param>
/// <param name="options">The set of options to use for this event processor.</param>
///
public ShortWaitTimeMock(string consumerGroup,
PartitionManager partitionManager,
EventHubConnection connection,
EventProcessorClientOptions options) : base(consumerGroup, partitionManager, connection, options)
{
}
public async Task ProcessWithHeartbeat()
{
await using var eventHubScope = await EventHubScope.CreateAsync(1);
await using var storageScope = await StorageScope.CreateAsync();
#region Snippet:EventHubs_Processor_Sample04_ProcessWithHeartbeat
var storageConnectionString = "<< CONNECTION STRING FOR THE STORAGE ACCOUNT >>";
var blobContainerName = "<< NAME OF THE BLOB CONTAINER >>";
/*@@*/
/*@@*/ storageConnectionString = StorageTestEnvironment.Instance.StorageConnectionString;
/*@@*/ blobContainerName = storageScope.ContainerName;
var eventHubsConnectionString = "<< CONNECTION STRING FOR THE EVENT HUBS NAMESPACE >>";
var eventHubName = "<< NAME OF THE EVENT HUB >>";
var consumerGroup = "<< NAME OF THE EVENT HUB CONSUMER GROUP >>";
/*@@*/
/*@@*/ eventHubsConnectionString = EventHubsTestEnvironment.Instance.EventHubsConnectionString;
/*@@*/ eventHubName = eventHubScope.EventHubName;
/*@@*/ consumerGroup = eventHubScope.ConsumerGroups.First();
var storageClient = new BlobContainerClient(
storageConnectionString,
blobContainerName);
var processorOptions = new EventProcessorClientOptions
{
MaximumWaitTime = TimeSpan.FromMilliseconds(250)
};
var processor = new EventProcessorClient(
storageClient,
consumerGroup,
eventHubsConnectionString,
eventHubName,
processorOptions);
async Task processEventHandler(ProcessEventArgs args)
{
try
{
if (args.HasEvent)
{
await Application.ProcessEventAndCheckpointAsync(
args.Data,
args.Partition,
args.CancellationToken);
}
await Application.SendHeartbeatAsync(args.CancellationToken);
}
catch (Exception ex)
{
Application.HandleProcessingException(args, ex);
}
}
try
{
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(30));
// The error handler is not relevant for this sample; for
// illustration, it is delegating the implementation to the
// host application.
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += Application.ProcessorErrorHandler;
try
{
await processor.StartProcessingAsync(cancellationSource.Token);
await Task.Delay(Timeout.Infinite, cancellationSource.Token);
}
catch (TaskCanceledException)
{
// This is expected if the cancellation token is
// signaled.
}
finally
{
// This may take slightly longer than the length of
// time defined as part of the MaximumWaitTime configured
// for the processor; in this example, 250 milliseconds.
await processor.StopProcessingAsync();
}
}
catch
{
// If this block is invoked, then something external to the
// processor was the source of the exception.
}
finally
{
// It is encouraged that you unregister your handlers when you have
// finished using the Event Processor to ensure proper cleanup.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= Application.ProcessorErrorHandler;
}
#endregion
}
public ReadableOptionsMock(string consumerGroup,
PartitionManager partitionManager,
string connectionString,
EventProcessorClientOptions options = default) : base(consumerGroup, partitionManager, connectionString, options)
{
}
/// <summary>
/// Initializes a new instance of the <see cref="MockEventProcessorClient" /> class.
/// </summary>
///
/// <param name="storageManager">The client responsible for interaction with durable storage, responsible for persisting checkpoints and load-balancing state.</param>
/// <param name="consumerGroup">The name of the consumer group this processor is associated with. Events are read in the context of this group.</param>
/// <param name="fullyQualifiedNamespace">The fully qualified Event Hubs namespace to connect to. This is likely to be similar to <c>{yournamespace}.servicebus.windows.net</c>.</param>
/// <param name="eventHubName">The name of the specific Event Hub to associate the processor with.</param>
/// <param name="connectionFactory">A factory used to provide new <see cref="EventHubConnection" /> instances.</param>
/// <param name="clientOptions">The set of options to use for this processor.</param>
/// <param name="fakePartitionProcessing"><c>true</c> if <see cref="RunPartitionProcessingAsync" /> should be overridden; otherwise, <c>false</c>.</param>
/// <param name="numberOfPartitions">The amount of partitions the associated Event Hub has.</param>
/// <param name="loadBalancer">The <see cref="PartitionLoadBalancer" /> used to manage partition load balance operations.</param>
///
internal MockEventProcessorClient(PartitionManager storageManager,
string consumerGroup = "consumerGroup",
string fullyQualifiedNamespace = "somehost.com",
string eventHubName = "somehub",
Func <EventHubConnection> connectionFactory = default,
EventProcessorClientOptions clientOptions = default,
bool fakePartitionProcessing = true,
int numberOfPartitions = 3,
PartitionLoadBalancer loadBalancer = default) : base(storageManager, consumerGroup, fullyQualifiedNamespace, eventHubName, connectionFactory, clientOptions, loadBalancer)
{
StorageManager = storageManager;
var partitionIds = Enumerable
.Range(1, numberOfPartitions)
.Select(p => p.ToString())
.ToArray();
foreach (var partitionId in partitionIds)
{
EventPipeline[partitionId] = new ConcurrentQueue <EventData>();
}
FakeRunPartitionProcessingAsync = fakePartitionProcessing;
ProcessErrorAsync += eventArgs =>
{
Exception[] newException = new Exception[] { eventArgs.Exception };
ExceptionCalls.AddOrUpdate(
eventArgs.PartitionId,
newException,
(partitionId, value) => value.Concat(newException).ToArray());
return(Task.CompletedTask);
};
ProcessEventAsync += eventArgs =>
{
EventData[] newEvent = new EventData[] { eventArgs.Data };
ProcessEventCalls.AddOrUpdate(
eventArgs.Partition.PartitionId,
newEvent,
(partitionId, value) => value.Concat(newEvent).ToArray());
return(Task.CompletedTask);
};
PartitionInitializingAsync += eventArgs =>
{
InitializeCalls.AddOrUpdate(eventArgs.PartitionId, 1, (partitionId, value) => value + 1);
return(Task.CompletedTask);
};
PartitionClosingAsync += eventArgs =>
{
CloseCalls.AddOrUpdate(eventArgs.PartitionId, 1, (partitionId, value) => value + 1);
StopReasons[eventArgs.PartitionId] = eventArgs.Reason;
return(Task.CompletedTask);
};
}
/// <summary>
/// Runs the sample using the specified Event Hubs and Azure storage connection information.
/// </summary>
///
/// <param name="eventHubsConnectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints and state should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints and state should be persisted.</param>
///
public async Task RunAsync(string eventHubsConnectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// The Event Processor client offers additional options on creation, allowing you to control different aspects of its behavior
// should your scenario have needs that differ from the common defaults. If you choose not to provide these options, the default behaviors
// suitable to most scenarios are used.
//
// The processor allows you to customize how it interacts with the Event Hubs service, such as by influencing how it connects
// to the service by specifying the transport that communication should use and whether a proxy should be used for network communications. Please
// note that a proxy is only supported when using WebSockets as a transport; it isn't compatible with raw TCP or other transport channels.
//
// This sample will customize the transport for the connection, using WebSockets and will adjust some of the retry-related values for
// illustration.
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName);
var processorOptions = new EventProcessorClientOptions
{
ConnectionOptions = new EventHubConnectionOptions
{
TransportType = EventHubsTransportType.AmqpWebSockets,
Proxy = (IWebProxy)null
},
RetryOptions = new EventHubsRetryOptions
{
MaximumRetries = 5,
TryTimeout = TimeSpan.FromMinutes(1)
}
};
EventProcessorClient processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName, processorOptions);
// Once the client has been created, it may be used normally. For completeness, we'll
// mirror the minimal skeleton from our first sample, starting and stopping the processor
// without performing any processing.
Task processEventHandler(ProcessEventArgs eventArgs) => Task.CompletedTask;
Task processErrorHandler(ProcessErrorEventArgs eventArgs) => Task.CompletedTask;
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
await processor.StartProcessingAsync();
await Task.Delay(TimeSpan.FromSeconds(1));
await processor.StopProcessingAsync();
}
finally
{
// It is encouraged that you unregister your handlers when you have finished
// using the Event Processor to ensure proper cleanup. This is especially
// important when using lambda expressions or handlers in any form that may
// contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
// The Event Processor client has been stopped and is not explicitly disposable; there
// is nothing further that we need to do for cleanup.
Console.WriteLine();
}
/// <summary>
/// Runs the sample using the specified Event Hubs and Azure storage connection information.
/// </summary>
///
/// <param name="eventHubsConnectionString">The connection string for the Event Hubs namespace that the sample should target.</param>
/// <param name="eventHubName">The name of the Event Hub, sometimes known as its path, that the sample should run against.</param>
/// <param name="blobStorageConnectionString">The connection string for the storage account where checkpoints and state should be persisted.</param>
/// <param name="blobContainerName">The name of the blob storage container where checkpoints and state should be persisted.</param>
///
public async Task RunAsync(string eventHubsConnectionString,
string eventHubName,
string blobStorageConnectionString,
string blobContainerName)
{
// In this example, our Event Processor client will be configured to use a maximum wait time which is considered when the processor is reading events.
// When events are available, the Event Processor client will pass them to the ProcessEvent handler to be processed. When no events are available in any
// partition for longer than the specified maximum wait interval, the processor will invoke the ProcessEvent handler with a set of arguments that do not
// include an event. This allows your handler code to receive control and perform actions such as sending a heartbeat, emitting telemetry, or another
// operation specific to your application needs.
//
// For our processor, we'll specify a small maximum wait time value as part of the options.
EventProcessorClientOptions clientOptions = new EventProcessorClientOptions
{
MaximumWaitTime = TimeSpan.FromMilliseconds(150)
};
string consumerGroup = EventHubConsumerClient.DefaultConsumerGroupName;
BlobContainerClient storageClient = new BlobContainerClient(blobStorageConnectionString, blobContainerName);
EventProcessorClient processor = new EventProcessorClient(storageClient, consumerGroup, eventHubsConnectionString, eventHubName, clientOptions);
// For this example, we'll create a simple event handler that writes to the
// console each time it was invoked.
int eventIndex = 0;
async Task processEventHandler(ProcessEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return;
}
try
{
// The "HasEvent" property of the arguments will be set if an event was available from the
// Event Hubs service. If so, the argument properties for the event is populated and checkpoints
// may be created.
//
// If the "HasEvent" property is unset, the event will be empty and checkpoints may not be created.
if (eventArgs.HasEvent)
{
Console.WriteLine($"Event Received: { Encoding.UTF8.GetString(eventArgs.Data.EventBody.ToBytes().ToArray()) }");
}
// Simulate sending a heartbeat using a simple helper that writes a status to the
// console.
await SendHeartbeatAsync();
++eventIndex;
}
catch (Exception ex)
{
Console.WriteLine();
Console.WriteLine($"An error was observed while processing events. Message: { ex.Message }");
Console.WriteLine();
}
};
// For this example, exceptions will just be logged to the console.
Task processErrorHandler(ProcessErrorEventArgs eventArgs)
{
if (eventArgs.CancellationToken.IsCancellationRequested)
{
return(Task.CompletedTask);
}
Console.WriteLine();
Console.WriteLine("===============================");
Console.WriteLine($"The error handler was invoked during the operation: { eventArgs.Operation ?? "Unknown" }, for Exception: { eventArgs.Exception.Message }");
Console.WriteLine("===============================");
Console.WriteLine();
return(Task.CompletedTask);
}
processor.ProcessEventAsync += processEventHandler;
processor.ProcessErrorAsync += processErrorHandler;
try
{
Console.WriteLine("Starting the Event Processor client...");
Console.WriteLine();
eventIndex = 0;
await processor.StartProcessingAsync();
using var cancellationSource = new CancellationTokenSource();
cancellationSource.CancelAfter(TimeSpan.FromSeconds(90));
// We'll publish a batch of events for our processor to receive. We'll split the events into a couple of batches to
// increase the chance they'll be spread around to different partitions and introduce a delay between batches to
// allow for the handler to be invoked without an available event interleaved.
var expectedEvents = new List <EventData>()
{
new EventData(Encoding.UTF8.GetBytes("First Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, First Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Second Batch")),
new EventData(Encoding.UTF8.GetBytes("First Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Second Event, Third Batch")),
new EventData(Encoding.UTF8.GetBytes("Third Event, Third Batch"))
};
int sentIndex = 0;
int numberOfBatches = 3;
int eventsPerBatch = (expectedEvents.Count / numberOfBatches);
await using (var producer = new EventHubProducerClient(eventHubsConnectionString, eventHubName))
{
while (sentIndex < expectedEvents.Count)
{
using EventDataBatch eventBatch = await producer.CreateBatchAsync();
for (int index = 0; index < eventsPerBatch; ++index)
{
eventBatch.TryAdd(expectedEvents[sentIndex]);
++sentIndex;
}
await producer.SendAsync(eventBatch);
await Task.Delay(250, cancellationSource.Token);
}
}
// We'll allow the Event Processor client to read and dispatch the events that we published, along with
// ensuring a few invocations with no event. Note that, due to non-determinism in the timing, we may or may
// not see all of the events from our batches read.
while ((!cancellationSource.IsCancellationRequested) && (eventIndex <= expectedEvents.Count + 5))
{
await Task.Delay(TimeSpan.FromMilliseconds(250));
}
// Once we arrive at this point, either cancellation was requested or we have processed all of our events. In
// both cases, we'll want to shut down the processor.
Console.WriteLine();
Console.WriteLine("Stopping the processor...");
await processor.StopProcessingAsync();
}
finally
{
// It is encouraged that you unregister your handlers when you have finished
// using the Event Processor to ensure proper cleanup. This is especially
// important when using lambda expressions or handlers in any form that may
// contain closure scopes or hold other references.
processor.ProcessEventAsync -= processEventHandler;
processor.ProcessErrorAsync -= processErrorHandler;
}
// The Event Processor client has been stopped and is not explicitly disposable; there
// is nothing further that we need to do for cleanup.
Console.WriteLine();
}
public async Task Start(CancellationToken cancellationToken)
{
while (!cancellationToken.IsCancellationRequested)
{
var options = new EventProcessorClientOptions
{
RetryOptions = new EventHubsRetryOptions
{
TryTimeout = Configuration.ReadTimeout
}
};
var storageClient = default(BlobContainerClient);
var processor = default(EventProcessorClient);
try
{
storageClient = new BlobContainerClient(Configuration.StorageConnectionString, Configuration.BlobContainer);
processor = new EventProcessorClient(storageClient, EventHubConsumerClient.DefaultConsumerGroupName, Configuration.EventHubsConnectionString, Configuration.EventHub, options);
processor.ProcessEventAsync += ProcessEventHandler;
processor.ProcessErrorAsync += ProcessErrorHandler;
await processor.StartProcessingAsync(cancellationToken).ConfigureAwait(false);
await Task.Delay(Timeout.Infinite, cancellationToken).ConfigureAwait(false);
}
catch (TaskCanceledException)
{
// No action needed.
}
catch (Exception ex) when
(ex is OutOfMemoryException ||
ex is StackOverflowException ||
ex is ThreadAbortException)
{
throw;
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.ProcessorRestarted);
ErrorsObserved.Add(ex);
}
finally
{
// Because publishing was canceled at the same time as the processor,
// wait a short bit to allow for time processing the newly published events.
await Task.Delay(TimeSpan.FromMinutes(5)).ConfigureAwait(false);
// Constrain stopping the processor, just in case it has issues. It should not be allowed
// to hang, it should be abandoned so that processing can restart.
using var cancellationSource = new CancellationTokenSource(TimeSpan.FromSeconds(15));
try
{
if (processor != null)
{
await processor.StopProcessingAsync(cancellationSource.Token).ConfigureAwait(false);
}
}
catch (Exception ex)
{
Interlocked.Increment(ref Metrics.ProcessorRestarted);
ErrorsObserved.Add(ex);
}
processor.ProcessEventAsync -= ProcessEventHandler;
processor.ProcessErrorAsync -= ProcessErrorHandler;
}
}
}