public async Task RegisterHubAsync()
{
if (_isRegistered)
{
return;
}
if (_processorHost == null)
{
_logger.LogInformation("Create event processor host");
_processorHost = _dataEventProcessorFactory.CreateEventProcessorHost();
}
_dataEventProcessorFactory.SetCallback(HandleProcessEventsAsync);
var options = new EventProcessorOptions();
options.SetExceptionHandler(e =>
{
_logger.LogWarning(e.Exception, "Error received from event processor. Action: {0} Hostname: {1} PartitionId: {2}", e.Action, e.Hostname, e.PartitionId);
});
_logger.LogInformation("Connect to azure event hub");
await _processorHost.RegisterEventProcessorFactoryAsync(_dataEventProcessorFactory, options).ConfigureAwait(false);
_isRegistered = true;
}
private async Task RunAsync()
{
Console.WriteLine("Registering EventProcessor...");
var eventProcessorHost = new EventProcessorHost(
EhEntityPath,
PartitionReceiver.DefaultConsumerGroupName,
EhConnectionString,
StorageConnectionString,
StorageContainerName);
var eventProcessorOptions = new EventProcessorOptions();
eventProcessorOptions.SetExceptionHandler(e => {
Console.WriteLine("Error: " + e.Exception);
});
// Registers the Event Processor Host and starts receiving messages
await eventProcessorHost.RegisterEventProcessorAsync <SimpleEventProcessor>(eventProcessorOptions);
Console.WriteLine("Receiving. Press ENTER to stop worker.");
Console.ReadLine();
// Disposes of the Event Processor Host
await eventProcessorHost.UnregisterEventProcessorAsync();
}
static void Main(string[] args)
{
string processorHostName = Guid.NewGuid().ToString();
var Options = new EventProcessorOptions()
{
MaxBatchSize = 1,
};
Options.SetExceptionHandler((ex) =>
{
System.Diagnostics.Debug.WriteLine($"Exception : {ex}");
});
var eventHubCS = "event hub connection string";
var storageCS = "storage connection string";
var containerName = "test";
var eventHubname = "test2";
EventProcessorHost eventProcessorHost = new EventProcessorHost(eventHubname, "$Default", eventHubCS, storageCS, containerName);
eventProcessorHost.RegisterEventProcessorAsync <MyEventProcessor>(Options).Wait();
while (true)
{
//do nothing
}
}
/// <summary>
/// Constructs a new instance.
/// </summary>
public EventHubConfiguration()
{
// Our default options will delegate to our own exception
// logger. Customers can override this completely by setting their
// own EventProcessorOptions instance.
EventProcessorOptions = EventProcessorOptions.DefaultOptions;
EventProcessorOptions.MaxBatchSize = 64;
EventProcessorOptions.PrefetchCount = EventProcessorOptions.MaxBatchSize * 4;
EventProcessorOptions.SetExceptionHandler(ExceptionReceivedHandler);
}
private void CreateProcessorHostConnection()
{
ProcessorHost = new EventProcessorHost(
EventHubEntityPath,
PartitionReceiver.DefaultConsumerGroupName,
EventHubConnectionString,
StorageConnectionString,
StorageContainerName
);
ProcessorOptions = new EventProcessorOptions();
ProcessorOptions.SetExceptionHandler(LogProcessorErrors);
}
private async Task ConnectToIotHubAsync(CancellationToken ct)
{
EventProcessorHost eventProcessorHost;
// Get configuration settings
string iotHubTelemetryConsumerGroup = ConfigurationProvider.GetConfigurationSettingValue("IotHubTelemetryConsumerGroup");
string iotHubEventHubName = ConfigurationProvider.GetConfigurationSettingValue("IotHubEventHubName");
string iotHubEventHubEndpointIotHubOwnerConnectionString = ConfigurationProvider.GetConfigurationSettingValue("IotHubEventHubEndpointIotHubOwnerConnectionString");
string solutionStorageAccountConnectionString = ConfigurationProvider.GetConfigurationSettingValue("SolutionStorageAccountConnectionString");
// Initialize EventProcessorHost.
Trace.TraceInformation("Creating EventProcessorHost for IoTHub: {0}, ConsumerGroup: {1}, ConnectionString: {2}, StorageConnectionString: {3}",
iotHubEventHubName, iotHubTelemetryConsumerGroup, iotHubEventHubEndpointIotHubOwnerConnectionString, solutionStorageAccountConnectionString);
string StorageContainerName = "telemetrycheckpoints";
eventProcessorHost = new EventProcessorHost(
iotHubEventHubName,
iotHubTelemetryConsumerGroup,
iotHubEventHubEndpointIotHubOwnerConnectionString,
solutionStorageAccountConnectionString,
StorageContainerName);
// Registers the Event Processor Host and starts receiving messages.
EventProcessorOptions options = new EventProcessorOptions();
options.InitialOffsetProvider = ((partitionId) => DateTime.UtcNow);
options.SetExceptionHandler(EventProcessorHostExceptionHandler);
try
{
await eventProcessorHost.RegisterEventProcessorAsync <MessageProcessor>(options);
Trace.TraceInformation($"EventProcessor successfully registered");
}
catch (Exception e)
{
Trace.TraceInformation($"Exception during register EventProcessorHost '{e.Message}'");
}
// Wait till shutdown.
while (true)
{
if (ct.IsCancellationRequested)
{
Trace.TraceInformation($"Application is shutting down. Unregistering EventProcessorHost...");
await eventProcessorHost.UnregisterEventProcessorAsync();
return;
}
await Task.Delay(1000);
}
}
/// <summary>
/// Returns the event process options
/// </summary>
private EventProcessorOptions GetEventProcessorOptions(int offsetInSeconds)
{
var options = new EventProcessorOptions()
{
MaxBatchSize = 500,
PrefetchCount = 500,
ReceiveTimeout = TimeSpan.FromSeconds(20),
InitialOffsetProvider = (partitionId) => EventPosition.FromEnqueuedTime(DateTime.UtcNow.AddSeconds(-offsetInSeconds)),
};
options.SetExceptionHandler(new Action <ExceptionReceivedEventArgs>(EventHub_ExceptionReceived));
return(options);
}
public async Task StartAsync(CancellationToken cancellationToken)
{
_options.SetExceptionHandler(args =>
{
_logger.TrackException(args.Exception, new
{
Partition = args.PartitionId,
});
});
await _host.RegisterEventProcessorFactoryAsync(_factory, _options);
_logger.TrackEvent("MessageEventProcessorServiceStarted");
}
public static async Task Main(string[] args)
{
IConfigurationBuilder configBuilder = SetupConfig();
var config = configBuilder.Build();
LoggerFactory loggerFactory = SetupLoggerFactory();
_logger = loggerFactory.CreateLogger("VacancyAnalyticsWebJob");
var vacancyEventStoreConnString = config.GetConnectionString(VacancyEventStoreConnStringKey);
var vacancyEventHubConnString = config.GetConnectionString(VacancyEventHubConnStringKey);
var queueStorageConnString = config.GetConnectionString(QueueStorageConnStringKey);
var eventStoreWriter = new VacancyEventStoreWriter(vacancyEventStoreConnString, loggerFactory.CreateLogger <VacancyEventStoreWriter>());
var factory = new EventProcessorFactory(loggerFactory.CreateLogger <VacancyEventProcessor>(), eventStoreWriter);
var epHostName = IsDevelopment ? HostNamePrefix : GetUniqueEventHostProcessorName();
var eventProcessorHost = new EventProcessorHost(
EventHubName,
PartitionReceiver.DefaultConsumerGroupName,
vacancyEventHubConnString,
queueStorageConnString,
epHostName);
var opts = new EventProcessorOptions
{
PrefetchCount = 512,
MaxBatchSize = 256
};
try
{
opts.SetExceptionHandler(HandleEventProcessorException);
await eventProcessorHost.RegisterEventProcessorFactoryAsync(factory, opts);
Console.WriteLine("Receiving. Press enter key to stop worker job.");
}
catch (Exception ex)
{
_logger.LogError(ex, ex.Message);
}
Console.ReadLine();
await eventProcessorHost.UnregisterEventProcessorAsync();
}
public void Initialize(ExtensionConfigContext context)
{
if (context == null)
{
throw new ArgumentNullException(nameof(context));
}
EventProcessorOptions options = _options.GetOptions();
options.SetExceptionHandler(ExceptionReceivedHandler);
// apply at config level (batchCheckpointFrequency)
// TODO: Revisit this... All configurable options should move to a proper Options type.
_configuration.ConfigurationSection.Bind(_options);
context
.AddConverter <string, EventData>(ConvertString2EventData)
.AddConverter <EventData, string>(ConvertEventData2String)
.AddConverter <byte[], EventData>(ConvertBytes2EventData)
.AddConverter <EventData, byte[]>(ConvertEventData2Bytes)
.AddOpenConverter <OpenType.Poco, EventData>(ConvertPocoToEventData);
// register our trigger binding provider
var triggerBindingProvider = new EventHubTriggerAttributeBindingProvider(_nameResolver, _converterManager, _options, _loggerFactory);
context.AddBindingRule <EventHubTriggerAttribute>()
.BindToTrigger(triggerBindingProvider);
// register our binding provider
context.AddBindingRule <EventHubAttribute>()
.BindToCollector(BuildFromAttribute);
ExceptionHandler = (e =>
{
LogExceptionReceivedEvent(e, _loggerFactory);
});
}
async Task <GenericScenarioResult> RunGenericScenario(EventProcessorHost eventProcessorHost,
EventProcessorOptions epo = null, int totalNumberOfEventsToSend = 1, bool checkpointLastEvent = true,
bool checkpointBatch = false, bool checkpoingEveryEvent = false)
{
var runResult = new GenericScenarioResult();
var lastReceivedAt = DateTime.Now;
if (epo == null)
{
epo = new EventProcessorOptions
{
ReceiveTimeout = TimeSpan.FromSeconds(15),
MaxBatchSize = 100
};
epo.SetExceptionHandler(TestEventProcessorFactory.ErrorNotificationHandler);
}
try
{
TestUtility.Log($"Calling RegisterEventProcessorAsync");
var processorFactory = new TestEventProcessorFactory();
processorFactory.OnCreateProcessor += (f, createArgs) =>
{
var processor = createArgs.Item2;
string partitionId = createArgs.Item1.PartitionId;
string hostName = createArgs.Item1.Owner;
processor.OnOpen += (_, partitionContext) => TestUtility.Log($"{hostName} > Partition {partitionId} TestEventProcessor opened");
processor.OnClose += (_, closeArgs) => TestUtility.Log($"{hostName} > Partition {partitionId} TestEventProcessor closing: {closeArgs.Item2}");
processor.OnProcessError += (_, errorArgs) =>
{
TestUtility.Log($"{hostName} > Partition {partitionId} TestEventProcessor process error {errorArgs.Item2.Message}");
Interlocked.Increment(ref runResult.NumberOfFailures);
};
processor.OnProcessEvents += (_, eventsArgs) =>
{
int eventCount = eventsArgs.Item2.events != null?eventsArgs.Item2.events.Count() : 0;
TestUtility.Log($"{hostName} > Partition {partitionId} TestEventProcessor processing {eventCount} event(s)");
if (eventCount > 0)
{
lastReceivedAt = DateTime.Now;
runResult.AddEvents(partitionId, eventsArgs.Item2.events);
foreach (var e in eventsArgs.Item2.events)
{
// Checkpoint every event received?
if (checkpoingEveryEvent)
{
eventsArgs.Item1.CheckpointAsync(e).Wait();
}
}
}
eventsArgs.Item2.checkPointLastEvent = checkpointLastEvent;
eventsArgs.Item2.checkPointBatch = checkpointBatch;
};
};
await eventProcessorHost.RegisterEventProcessorFactoryAsync(processorFactory, epo);
// Wait 5 seconds to avoid races in scenarios like EndOfStream.
await Task.Delay(5000);
TestUtility.Log($"Sending {totalNumberOfEventsToSend} event(s) to each partition");
var ehClient = EventHubClient.CreateFromConnectionString(TestUtility.EventHubsConnectionString);
var sendTasks = new List <Task>();
foreach (var partitionId in PartitionIds)
{
sendTasks.Add(TestUtility.SendToPartitionAsync(ehClient, partitionId, $"{partitionId} event.", totalNumberOfEventsToSend));
}
await Task.WhenAll(sendTasks);
// Wait until all partitions are silent, i.e. no more events to receive.
while (lastReceivedAt > DateTime.Now.AddSeconds(-30))
{
await Task.Delay(1000);
}
TestUtility.Log($"Verifying at least {totalNumberOfEventsToSend} event(s) was received by each partition");
foreach (var partitionId in PartitionIds)
{
Assert.True(runResult.ReceivedEvents.ContainsKey(partitionId) &&
runResult.ReceivedEvents[partitionId].Count >= totalNumberOfEventsToSend,
$"Partition {partitionId} didn't receive expected number of messages. Expected {totalNumberOfEventsToSend}, received {runResult.ReceivedEvents[partitionId].Count}.");
}
TestUtility.Log("Success");
}
finally
{
TestUtility.Log("Calling UnregisterEventProcessorAsync");
await eventProcessorHost.UnregisterEventProcessorAsync();
}
return(runResult);
}
private async Task ConnectToIotHubAsync(CancellationToken ct)
{
EventProcessorHost eventProcessorHost;
EventProcessorHost eventProcessorHost2;
// Get configuration settings
string iotHubTelemetryConsumerGroup = ConfigurationProvider.GetConfigurationSettingValue("IotHubTelemetryConsumerGroup");
string iotHubEventHubName = ConfigurationProvider.GetConfigurationSettingValue("IotHubEventHubName");
string iotHubEventHubEndpointIotHubOwnerConnectionString = ConfigurationProvider.GetConfigurationSettingValue("IotHubEventHubEndpointIotHubOwnerConnectionString");
string solutionStorageAccountConnectionString = ConfigurationProvider.GetConfigurationSettingValue("SolutionStorageAccountConnectionString");
//eventhub 합침
string EventHubConnectionString = "Endpoint=sb://king012101man.servicebus.windows.net/;SharedAccessKeyName=RootManageSharedAccessKey;SharedAccessKey=7dm6eJ3uPa2HEZufnENhXabjHltsKkycwYdy8Mp0+Ns=";
string EventHubName = "kingsman03";
//이름2로 바꿈.
string StorageContainerName2 = "containerking06";
string StorageAccountName = "king012101man";
string StorageAccountKey = "jY67taiqQoDM8L5Od7vVR8N3Ki+ONj2T+xTbBMjxXrQVFXMNt801H9Kxg2XSmjv/pHEeWqqmpdJzzaKEAi39mA==";
string StorageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", StorageAccountName, StorageAccountKey);
// Initialize EventProcessorHost.
Trace.TraceInformation("Creating EventProcessorHost for IoTHub: {0}, ConsumerGroup: {1}, ConnectionString: {2}, StorageConnectionString: {3}",
iotHubEventHubName, iotHubTelemetryConsumerGroup, iotHubEventHubEndpointIotHubOwnerConnectionString, solutionStorageAccountConnectionString);
string StorageContainerName = "telemetrycheckpoints";
eventProcessorHost = new EventProcessorHost(
iotHubEventHubName,
iotHubTelemetryConsumerGroup,
iotHubEventHubEndpointIotHubOwnerConnectionString,
solutionStorageAccountConnectionString,
StorageContainerName2);
//eventhub 합침
eventProcessorHost2 = new EventProcessorHost(
EventHubName,
PartitionReceiver.DefaultConsumerGroupName,
EventHubConnectionString,
StorageConnectionString,
StorageContainerName);
// Registers the Event Processor Host and starts receiving messages.
EventProcessorOptions options = new EventProcessorOptions();
options.InitialOffsetProvider = ((partitionId) => DateTime.UtcNow);
options.SetExceptionHandler(EventProcessorHostExceptionHandler);
EventProcessorOptions options2 = new EventProcessorOptions();
options2.InitialOffsetProvider = ((partitionId) => DateTime.UtcNow);
options2.SetExceptionHandler(EventProcessorHostExceptionHandler);
try
{ //eventhub 합침.
await eventProcessorHost2.RegisterEventProcessorAsync <SimpleEventProcessor>(options2);
await eventProcessorHost.RegisterEventProcessorAsync <MessageProcessor>(options);
Trace.TraceInformation($"EventProcessor successfully registered");
}
catch (Exception e)
{
Trace.TraceInformation($"Exception during register EventProcessorHost '{e.Message}'");
}
// Wait till shutdown.
while (true)
{
if (ct.IsCancellationRequested)
{
Trace.TraceInformation($"Application is shutting down. Unregistering EventProcessorHost...");
await eventProcessorHost2.UnregisterEventProcessorAsync();
await eventProcessorHost.UnregisterEventProcessorAsync();
return;
}
await Task.Delay(2000);
}
}
public async Task Start()
{
_isRunning = true;
ILeaseManager leaseManager = null;
ICheckpointManager checkpointManager = null;
switch (Settings.LeaseType)
{
case LeaseType.CloudSqlStorage:
SqlLeaseManager <PrototypeSqlLease> sqlLeaseManager =
new SqlLeaseManager <PrototypeSqlLease>(Name,
Settings.LeaseSqlConnectionString,
x => new PrototypeSqlLease(x),
x => new PrototypeSqlLease(x))
{
LeaseDuration = Settings.ExpireLeaseEvery,
};
leaseManager = sqlLeaseManager;
checkpointManager = sqlLeaseManager;
break;
case LeaseType.LocalDiskStorage:
DiskLeaseManager diskLeaseManager = new DiskLeaseManager(Settings.EventHubName, Settings.EventHubConsumerGroup, Name)
{
LeaseDuration = Settings.ExpireLeaseEvery,
};
leaseManager = diskLeaseManager;
checkpointManager = diskLeaseManager;
break;
}
if (leaseManager != null && checkpointManager != null)
{
// Custom construction that assigns a custom lease and checkpoint manager.
_eventProcessorHost = new EventProcessorHost(Name,
Settings.EventHubName,
Settings.EventHubConsumerGroup,
Settings.EventHubConnectionString,
checkpointManager,
leaseManager);
}
else
{
// The default construction uses Azure Storage lease and checkpoint management.
_eventProcessorHost = new EventProcessorHost(Name,
Settings.EventHubName,
Settings.EventHubConsumerGroup,
Settings.EventHubConnectionString,
Settings.LeaseConnectionString,
Settings.EventHubName);
}
_eventProcessorHost.PartitionManagerOptions = new PartitionManagerOptions
{
RenewInterval = Settings.RenewLeaseEvery,
LeaseDuration = Settings.ExpireLeaseEvery,
};
EventProcessorOptions options = new EventProcessorOptions
{
MaxBatchSize = Settings.ReadBatchSize,
PrefetchCount = Settings.ReadPrefetchCount,
ReceiveTimeout = Settings.ReadReceiveTimeout,
EnableReceiverRuntimeMetric = Settings.ReadEnableReceiverRuntimeMetric,
InvokeProcessorAfterReceiveTimeout = false,
};
options.SetExceptionHandler(UnhandledExceptionHandler);
await _eventProcessorHost.RegisterEventProcessorAsync <EventHubsReader>(options).ConfigureAwait(false);
}
async Task <Dictionary <string, List <EventData> > > RunGenericScenario(EventProcessorHost eventProcessorHost,
EventProcessorOptions epo = null, int totalNumberOfEventsToSend = 1, bool checkPointLastEvent = true,
bool checkPointBatch = false)
{
var receivedEvents = new ConcurrentDictionary <string, List <EventData> >();
var lastReceivedAt = DateTime.Now;
if (epo == null)
{
epo = new EventProcessorOptions
{
ReceiveTimeout = TimeSpan.FromSeconds(15),
MaxBatchSize = 100
};
epo.SetExceptionHandler(TestEventProcessorFactory.ErrorNotificationHandler);
}
try
{
Log($"Calling RegisterEventProcessorAsync");
var processorFactory = new TestEventProcessorFactory();
processorFactory.OnCreateProcessor += (f, createArgs) =>
{
var processor = createArgs.Item2;
string partitionId = createArgs.Item1.PartitionId;
string hostName = createArgs.Item1.Owner;
processor.OnOpen += (_, partitionContext) => Log($"{hostName} > Partition {partitionId} TestEventProcessor opened");
processor.OnClose += (_, closeArgs) => Log($"{hostName} > Partition {partitionId} TestEventProcessor closing: {closeArgs.Item2}");
processor.OnProcessError += (_, errorArgs) => Log($"{hostName} > Partition {partitionId} TestEventProcessor process error {errorArgs.Item2.Message}");
processor.OnProcessEvents += (_, eventsArgs) =>
{
int eventCount = eventsArgs.Item2.events != null?eventsArgs.Item2.events.Count() : 0;
Log($"{hostName} > Partition {partitionId} TestEventProcessor processing {eventCount} event(s)");
if (eventCount > 0)
{
List <EventData> events;
receivedEvents.TryGetValue(partitionId, out events);
if (events == null)
{
events = new List <EventData>();
}
events.AddRange(eventsArgs.Item2.events);
receivedEvents[partitionId] = events;
lastReceivedAt = DateTime.Now;
}
eventsArgs.Item2.checkPointLastEvent = checkPointLastEvent;
eventsArgs.Item2.checkPointBatch = checkPointBatch;
};
};
await eventProcessorHost.RegisterEventProcessorFactoryAsync(processorFactory, epo);
// Wait 5 seconds to avoid races in scenarios like EndOfStream.
await Task.Delay(5000);
Log($"Sending {totalNumberOfEventsToSend} event(s) to each partition");
var sendTasks = new List <Task>();
foreach (var partitionId in PartitionIds)
{
for (int i = 0; i < totalNumberOfEventsToSend; i++)
{
sendTasks.Add(this.SendToPartitionAsync(partitionId, $"{partitionId} event.", this.ConnectionStringBuilder.ToString()));
}
}
await Task.WhenAll(sendTasks);
// Wait until all partitions are silent, i.e. no more events to receive.
while (lastReceivedAt > DateTime.Now.AddSeconds(-30))
{
await Task.Delay(1000);
}
Log("Verifying at least an event was received by each partition");
foreach (var partitionId in PartitionIds)
{
Assert.True(receivedEvents.ContainsKey(partitionId), $"Partition {partitionId} didn't receive any message!");
}
Log("Success");
}
finally
{
Log("Calling UnregisterEventProcessorAsync");
await eventProcessorHost.UnregisterEventProcessorAsync();
}
return(receivedEvents.ToDictionary(kvp => kvp.Key, kvp => kvp.Value));
}