public async Task OpenAsync(IEventHubPartitionContext partitionContext)
{
Lazy <EventProcessorEventArgs> lazyEventArgs =
new Lazy <EventProcessorEventArgs>(
() =>
new EventProcessorEventArgs(
EventProcessorEventType.Opened,
partitionContext));
await this.InvokeEventHandler(this.OnOpened, lazyEventArgs);
}
public EventProcessorEventArgs(
EventProcessorEventType eventType,
IEventHubPartitionContext partitionContext,
EventProcessorCloseReason?closeReason = null,
Exception exception = null,
IEnumerable <IEventData> dataCollection = null)
{
this.EventType = eventType;
this.PartitionContext = partitionContext;
this.CloseReason = closeReason;
this.Exception = exception;
this.DataCollection = dataCollection;
}
public async Task ProcessEventsAsync(
IEventHubPartitionContext partitionContext,
IEnumerable <IEventData> messages)
{
Lazy <EventProcessorEventArgs> lazyEventArgs =
new Lazy <EventProcessorEventArgs>(
() =>
new EventProcessorEventArgs(
EventProcessorEventType.DataReceived,
partitionContext,
dataCollection: messages));
await this.InvokeEventHandler(this.OnDataReceived, lazyEventArgs);
}
public async Task ProcessErrorAsync(
IEventHubPartitionContext partitionContext,
Exception error)
{
Lazy <EventProcessorEventArgs> lazyEventArgs =
new Lazy <EventProcessorEventArgs>(
() =>
new EventProcessorEventArgs(
EventProcessorEventType.ErrorOccurred,
partitionContext,
exception: error));
await this.InvokeEventHandler(this.OnErrorOccurred, lazyEventArgs);
}
public async Task CloseAsync(
IEventHubPartitionContext partitionContext,
EventProcessorCloseReason closeReason)
{
Lazy <EventProcessorEventArgs> lazyEventArgs =
new Lazy <EventProcessorEventArgs>(
() =>
new EventProcessorEventArgs(
EventProcessorEventType.Closed,
partitionContext,
closeReason: closeReason));
await this.InvokeEventHandler(this.OnClosed, lazyEventArgs);
}
protected override async Task InternalRunAsync()
{
IEventHubClient eventHubClient =
new EventHubClient(
this.EventHubConnectionString,
this.EventHubEntityPath);
this.PrintObjectInfo(eventHubClient);
IEventHubRuntimeInformation runtimeInformation =
await eventHubClient.GetEventHubRuntimeInformationAsync()
.ConfigureAwait(false);
this.PrintObjectInfo(runtimeInformation);
IEnumerable <IEventHubPartitionRuntimeInformation> partionInfos =
runtimeInformation.PartitionIds
.Select(
pid => eventHubClient
.GetPartitionRuntimeInformationAsync(pid)
.ConfigureAwait(false).GetAwaiter().GetResult())
.ToArray();
this.PrintObjectInfo(partionInfos);
int batchCount = 500;
int batchSize = 100;
for (int batchIdx = 0; batchIdx < batchCount; ++batchIdx)
{
await eventHubClient
.BatchSendAsync(
new int[batchSize].Select(
(v, i) => $"Data {batchIdx}-{i} at {DateTime.UtcNow:o} dummy: {new string('$', 256)}"))
.ConfigureAwait(false);
}
Console.WriteLine($"Total sent: {batchCount * batchSize}");
IEventProcessor eventProcessor =
new EventProcessor(
this.EventHubConsumerGroup,
this.StorageAccountConnectionString,
this.StorageAccountContainer)
{
Options = new EventProcessorOptions
{
MaxBatchSize = 50,
PrefetchCount = 500
}
};
int totalReceived = 0;
Dictionary <string, List <IEventData> > bufferDic =
new Dictionary <string, List <IEventData> >();
Dictionary <string, DateTimeOffset> lastDumpTimeDic =
new Dictionary <string, DateTimeOffset>();
Dictionary <string, IEventHubPartitionContext> partitionContextDic =
new Dictionary <string, IEventHubPartitionContext>();
CancellationTokenSource cancellationTokenSource =
new CancellationTokenSource();
TaskFactory taskFactory =
new TaskFactory(cancellationTokenSource.Token);
List <Task> tasksList = new List <Task>();
DateTimeOffset initialTime =
DateTimeOffset.FromUnixTimeSeconds(
DateTimeOffset.UtcNow.ToUnixTimeSeconds());
foreach (var partitionInfo in partionInfos)
{
string partitionId = partitionInfo.PartitionId;
bufferDic[partitionId] = new List <IEventData>();
lastDumpTimeDic[partitionId] =
initialTime.AddSeconds(
int.Parse(partitionId) % partionInfos.Count());
Console.WriteLine($"last dump time for partition {partitionId}: {lastDumpTimeDic[partitionId]}");
Task task = taskFactory.StartNew(
async() =>
{
while (true)
{
DateTimeOffset checkTime =
DateTimeOffset.FromUnixTimeSeconds(
DateTimeOffset.UtcNow.ToUnixTimeSeconds());
List <IEventData> buffer = bufferDic[partitionId];
if (cancellationTokenSource.Token.IsCancellationRequested)
{
if (buffer.Any())
{
Console.WriteLine($"Dump {buffer.Count} messages from partition {partitionId} at {checkTime:o}.");
totalReceived += buffer.Count();
buffer.Clear();
}
break;
}
DateTimeOffset lastDumpTime =
lastDumpTimeDic[partitionId];
double secondsSinceLastDump =
checkTime.Subtract(lastDumpTime).TotalSeconds;
if (buffer.Any() &&
secondsSinceLastDump >= partionInfos.Count() &&
(secondsSinceLastDump % partionInfos.Count()) == 0)
{
IEventData checkpointData = null;
lock (buffer)
{
Console.WriteLine($"Dump {buffer.Count} messages from partition {partitionId} at {checkTime:o}.");
totalReceived += buffer.Count();
checkpointData =
buffer
.OrderByDescending(e => e.SequenceNumber)
.FirstOrDefault();
buffer.Clear();
}
if (checkpointData != null)
{
IEventHubPartitionContext partitionContext =
partitionContextDic[partitionId];
await partitionContext
.CheckpointAsync(checkpointData).ConfigureAwait(false);
}
lastDumpTimeDic[partitionId] = checkTime;
// Console.WriteLine($"last dump time for partition {partitionId}: {lastDumpTimeDic[partitionId]}");
}
else
{
Thread.Sleep(TimeSpan.FromSeconds(1));
}
}
});
tasksList.Add(task);
}
Task eventProcessorHandler(object obj, EventProcessorEventArgs args)
{
// Console.WriteLine($"[Partition - {args.PartitionContext.PartitionId}] [{args.EventType}]");
// this.PrintObjectInfo(args);
if (args.EventType == EventProcessorEventType.DataReceived)
{
string partitionId = args.PartitionContext.PartitionId;
partitionContextDic[partitionId] = args.PartitionContext;
List <IEventData> buffer = bufferDic[partitionId];
lock (buffer)
{
Console.WriteLine($"Buffered {args.DataCollection.Count()} messages from partition {partitionId} at {DateTimeOffset.UtcNow:o}.");
buffer.AddRange(args.DataCollection);
}
}
return(Task.CompletedTask);
}
eventProcessor.OnClosed += eventProcessorHandler;
eventProcessor.OnOpened += eventProcessorHandler;
eventProcessor.OnErrorOccurred += eventProcessorHandler;
eventProcessor.OnDataReceived += eventProcessorHandler;
IEventProcessorRegistration registration =
await eventHubClient.RegisterAsync(eventProcessor)
.ConfigureAwait(false);
char c;
do
{
Console.WriteLine("Receiving event data, press 'q' to exit.");
c = Console.ReadKey(true).KeyChar;
} while(c != 'q');
cancellationTokenSource.Cancel();
Task.WaitAll(tasksList.ToArray());
Console.WriteLine($"Total received: {totalReceived}");
await registration.UnregisterAsync().ConfigureAwait(false);
}
