static void Main(string[] args)
{
string eventHubConnectionString = Environment.GetEnvironmentVariable("APIMEVENTS-EVENTHUB-CONNECTIONSTRING", EnvironmentVariableTarget.Process);
string eventHubName = Environment.GetEnvironmentVariable("APIMEVENTS-EVENTHUB-NAME", EnvironmentVariableTarget.Process);
string storageAccountName = Environment.GetEnvironmentVariable("APIMEVENTS-STORAGEACCOUNT-NAME", EnvironmentVariableTarget.Process);
string storageAccountKey = Environment.GetEnvironmentVariable("APIMEVENTS-STORAGEACCOUNT-KEY", EnvironmentVariableTarget.Process);
string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
storageAccountName, storageAccountKey);
string eventProcessorHostName = Guid.NewGuid().ToString();
var eventProcessorHost = new EventProcessorHost(
eventProcessorHostName,
eventHubName,
EventHubConsumerGroup.DefaultGroupName,
eventHubConnectionString,
storageConnectionString);
var logger = new ConsoleLogger(LogLevel.Debug);
logger.LogDebug("Registering EventProcessor...");
var httpMessageProcessor = new RunscopeHttpMessageProcessor(new HttpClient(), logger);
eventProcessorHost.RegisterEventProcessorFactoryAsync(
new ApimHttpEventProcessorFactory(httpMessageProcessor, logger));
Console.WriteLine("Receiving. Press enter key to stop worker.");
Console.ReadLine();
eventProcessorHost.UnregisterEventProcessorAsync().Wait();
}
public static async Task<ColdStorageCoordinator> CreateAsync(string hostName, Configuration configuration)
{
ColdStorageEventSource.Log.InitializingEventHubListener(configuration.EventHubName, configuration.ConsumerGroupName);
var storageAccount = CloudStorageAccount.Parse(configuration.BlobWriterStorageAccount);
Func<string, IBlobWriter> blobWriterFactory =
partitionId =>
new RollingBlobWriter.RollingBlobWriter(new PartitionAndDateNamingStrategy(partitionId, configuration.BlobPrefix),
storageAccount,
configuration.ContainerName,
configuration.RollSizeForBlobWriterMb);
var ns = NamespaceManager.CreateFromConnectionString(configuration.EventHubConnectionString);
try
{
await ns.GetConsumerGroupAsync(configuration.EventHubName, configuration.ConsumerGroupName);
}
catch (Exception e)
{
ColdStorageEventSource.Log.InvalidEventHubConsumerGroupName(e, configuration.EventHubName, configuration.ConsumerGroupName);
throw;
}
ColdStorageEventSource.Log.ConsumerGroupFound(configuration.EventHubName, configuration.ConsumerGroupName);
var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, configuration.EventHubName);
var factory = new ColdStorageEventProcessorFactory(
blobWriterFactory,
configuration.CircuitBreakerWarningLevel,
configuration.CircuitBreakerTripLevel,
configuration.CircuitBreakerStallInterval,
configuration.CircuitBreakerLogCooldownInterval,
eventHubId
);
var options = new EventProcessorOptions()
{
MaxBatchSize = configuration.MaxBatchSize,
PrefetchCount = configuration.PreFetchCount,
ReceiveTimeOut = configuration.ReceiveTimeout,
InvokeProcessorAfterReceiveTimeout = true
};
options.ExceptionReceived +=
(s, e) => ColdStorageEventSource.Log.ErrorProcessingMessage(e.Exception, e.Action);
var host = new EventProcessorHost(
hostName,
consumerGroupName: configuration.ConsumerGroupName,
eventHubPath: configuration.EventHubName,
eventHubConnectionString: configuration.EventHubConnectionString,
storageConnectionString: configuration.CheckpointStorageAccount);
await host.RegisterEventProcessorFactoryAsync(factory, options);
return new ColdStorageCoordinator(host);
}
private static async Task MainAsync(string[] args)
{
//carica le impostazioni in appsettings
var builder = new ConfigurationBuilder()
.SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile(
"appsettings.json",
optional: true,
reloadOnChange: true);
var configuration = builder.Build();
Console.WriteLine("Registering EventProcessor...");
var eventProcessorHost = new EventProcessorHost(
configuration["ConnectionStrings:EventHubPath"],
PartitionReceiver.DefaultConsumerGroupName,
configuration["ConnectionStrings:EventHubCS"],
configuration["ConnectionStrings:StorageAccountCS"],
configuration["ConnectionStrings:StorageContainer"]);
// Registers the Event Processor Host and starts receiving messages
await eventProcessorHost.RegisterEventProcessorFactoryAsync(
new TeleEventProcessorFactory(configuration["ConnectionStrings:DefaultConnection"]));
Console.WriteLine("Receiving. Press ENTER to stop worker.");
Console.ReadLine();
// Disposes of the Event Processor Host
await eventProcessorHost.UnregisterEventProcessorAsync();
}
private static async Task StartHost()
{
//Event Hub
var eventHubConnectionString = GetEventHubConnectionString();
var storageConnectionString = GetStorageConnectionString();
string hostname = Guid.NewGuid().ToString();
host = new EventProcessorHost(
hostname,
Properties.Settings.Default.EventHubName,
EventHubConsumerGroup.DefaultGroupName,
eventHubConnectionString,
storageConnectionString, Properties.Settings.Default.EventHubName.ToLowerInvariant());
factory = new HeartbeatEventProcessorFactory(hostname);
try
{
Console.WriteLine("{0} > Registering host: {1}", DateTime.Now.ToString(), hostname);
var options = new EventProcessorOptions();
options.ExceptionReceived += OptionsOnExceptionReceived;
await host.RegisterEventProcessorFactoryAsync(factory);
}
catch (Exception exception)
{
Console.ForegroundColor = ConsoleColor.Red;
Console.WriteLine("{0} > Exception: {1}", DateTime.Now.ToString(), exception.Message);
Console.ResetColor();
}
}
protected override async Task ExecuteAsync(CancellationToken cancellationToken)
{
string hubNamespace = _demoSettings.EventHubNamespace;
var endpoint = new Uri($"sb://{hubNamespace}.servicebus.windows.net/");
string hubName = _demoSettings.EventHubName;
var storageAccount = CloudStorageAccount.Parse(_demoSettings.EventHubStorageConnectionString);
var host = new EventProcessorHost(
endpoint,
hubName,
"$Default",
new ManagedIdentityEventHubsTokenProvider(_demoSettings.ManagedIdentityTenantId),
storageAccount,
_demoSettings.EventHubStorageContainerName);
try
{
await host.RegisterEventProcessorFactoryAsync(this);
await Task.Delay(-1, cancellationToken);
}
finally
{
await host.UnregisterEventProcessorAsync();
}
}
public async Task Start()
{
_eventProcessorHost = new EventProcessorHost(
_settings.EntityPath,
PartitionReceiver.DefaultConsumerGroupName,
_settings.ConnectionString,
string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", _settings.StorageAccountName, _settings.StorageAccountKey),
_settings.StorageContainerName);
EventProcessorFactory <T> eventProcessorFactory = new EventProcessorFactory <T>();
await _eventProcessorHost.RegisterEventProcessorFactoryAsync(eventProcessorFactory);
eventProcessorFactory.Processor.OnEventClose += (context, reason) => {
OnEventClose?.Invoke(context, reason);
};
eventProcessorFactory.Processor.OnEventOpen += (context) => {
OnEventOpen?.Invoke(context);
};
eventProcessorFactory.Processor.OnEventError += (context, error) => {
OnEventError?.Invoke(context, error);
};
eventProcessorFactory.Processor.OnProcessEvents += (context, eventMessages) => {
OnProcessEvents?.Invoke(context, eventMessages);
};
}
public static async Task Start()
{
var eventHubConnectionString = GetEventHubConnectionString();
var storageConnectionString = ConfigManager.Config.TableConnectionString;
var eventHubName = ConfigManager.Config.EventHubName;
// here it's using eventhub as lease name. but it can be specified as any you want.
// if the host is having same lease name, it will be shared between hosts.
// by default it is using eventhub name as lease name.
host = new EventProcessorHost(
hostName,
eventHubName,
consumerGroupName,
eventHubConnectionString,
storageConnectionString,
eventHubName.ToLowerInvariant());
factory = new EventProcessorFactory(hostName);
try
{
Trace.WriteLine(string.Format("{0} > Registering host: {1}", DateTime.Now.ToString(), hostName), "Information");
var options = new EventProcessorOptions();
options.ExceptionReceived += OptionsOnExceptionReceived;
await host.RegisterEventProcessorFactoryAsync(factory);
}
catch (Exception exception)
{
Trace.WriteLine(string.Format("{0} > Exception: {1}", DateTime.Now.ToString(), exception.Message), "Error");
Console.ResetColor();
}
}
static void Main(string[] args)
{
var logger = new ConsoleLogger(LogLevel.Debug);
logger.LogInfo("STARTING Moesif API Management Event Processor. Reading environment variables");
// Load configuration paramaters from Environment
string eventHubConnectionString = ParamConfig.loadNonEmpty(AzureAppParamNames.EVENTHUB_CONN_STRING);
string eventHubName = ParamConfig.loadNonEmpty(AzureAppParamNames.EVENTHUB_NAME);
string storageAccountName = ParamConfig.loadNonEmpty(AzureAppParamNames.STORAGEACCOUNT_NAME);
string storageAccountKey = ParamConfig.loadNonEmpty(AzureAppParamNames.STORAGEACCOUNT_KEY);
// This App utilizes the "$Default" Eventhub consumer group
// In future, we should make this configurable via environment variables
string eventHubConsumerGroupName = EventHubConsumerGroup.DefaultGroupName;
// Create connection string for azure storage account to store checkpoints
string storageConnectionString = makeStorageAccountConnString(storageAccountName,
storageAccountKey);
string eventProcessorHostName = Guid.NewGuid().ToString();
var eventProcessorHost = new EventProcessorHost(
eventProcessorHostName,
eventHubName,
eventHubConsumerGroupName,
eventHubConnectionString,
storageConnectionString);
logger.LogDebug("Registering EventProcessor...");
var httpMessageProcessor = new MoesifHttpMessageProcessor(logger);
eventProcessorHost.RegisterEventProcessorFactoryAsync(
new ApimHttpEventProcessorFactory(httpMessageProcessor, logger));
logger.LogInfo("Process is running. Press enter key to end...");
Console.ReadLine();
logger.LogInfo("STOPPING Moesif API Management Event Processor");
eventProcessorHost.UnregisterEventProcessorAsync().Wait();
}
static async Task MainAsync(string[] args)
{
string eventHubPath = args[0]; // same as EntityPath
string ehConnectionString = args[1];
string leaseContainerName = args[2];
string storageAccountName = args[3];
string storageAccountKey = args[4];
var eventProcessorHost = new EventProcessorHost(
eventHubPath,
PartitionReceiver.DefaultConsumerGroupName,
ehConnectionString,
string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", storageAccountName, storageAccountKey),
leaseContainerName);
// Registers the Event Processor Host and starts receiving messages
await eventProcessorHost.RegisterEventProcessorFactoryAsync(new SimpleEventProcessorFactory("hello"), new EventProcessorOptions
{
InitialOffsetProvider = (partitionId) => DateTime.UtcNow
});
Console.WriteLine("Receiving. Press ENTER to stop worker.");
Console.ReadLine();
// Disposes of the Event Processor Host
await eventProcessorHost.UnregisterEventProcessorAsync();
}
static void Main(string[] args)
{
string eventHubConnectionString = Environment.GetEnvironmentVariable("APIMEVENTS-EVENTHUB-CONNECTIONSTRING", EnvironmentVariableTarget.Process);
string eventHubName = Environment.GetEnvironmentVariable("APIMEVENTS-EVENTHUB-NAME", EnvironmentVariableTarget.Process);
string storageAccountName = Environment.GetEnvironmentVariable("APIMEVENTS-STORAGEACCOUNT-NAME", EnvironmentVariableTarget.Process);
string storageAccountKey = Environment.GetEnvironmentVariable("APIMEVENTS-STORAGEACCOUNT-KEY", EnvironmentVariableTarget.Process);
string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
storageAccountName, storageAccountKey);
string eventProcessorHostName = Guid.NewGuid().ToString();
var eventProcessorHost = new EventProcessorHost(
eventProcessorHostName,
eventHubName,
EventHubConsumerGroup.DefaultGroupName,
eventHubConnectionString,
storageConnectionString);
var logger = new ConsoleLogger(LogLevel.Debug);
logger.LogDebug("Registering EventProcessor...");
var httpMessageProcessor = new RunscopeHttpMessageProcessor(new HttpClient(), logger);
eventProcessorHost.RegisterEventProcessorFactoryAsync(
new ApimHttpEventProcessorFactory(httpMessageProcessor, logger));
Console.WriteLine("Receiving. Press enter key to stop worker.");
Console.ReadLine();
eventProcessorHost.UnregisterEventProcessorAsync().Wait();
}
public static async Task Start(string hostName, string eventHubName, string consumerGroupName, string containerName, string blobPrefix,
string serviceBusConnectionString, string storageConnectionString)
{
// Create Consumer Group if it doesn't exist
if (consumerGroupName != null)
{
NamespaceManager manager = NamespaceManager.CreateFromConnectionString(serviceBusConnectionString);
ConsumerGroupDescription description = new ConsumerGroupDescription(eventHubName, consumerGroupName);
manager.CreateConsumerGroupIfNotExists(description);
}
// Create the Event Processor Host
var host = new EventProcessorHost(
hostName,
eventHubName,
consumerGroupName == null ? EventHubConsumerGroup.DefaultGroupName : consumerGroupName,
serviceBusConnectionString,
storageConnectionString,
eventHubName);
// Create the Factory
var factory = new BlobEventProcessorHostFactory(storageConnectionString, containerName, blobPrefix);
// Register the Factory
await host.RegisterEventProcessorFactoryAsync(factory);
}
public EventHubConsumer(ILogger logger, IConfigurationSection config, MessageHandler handler)
{
if (logger == null)
{
throw new ArgumentNullException(nameof(logger));
}
_logger = logger;
var options = new EventHubConsumerConfig();
config.Bind(options);
_eventHubName = options.TopicName;
logger.LogInformation("Starting consumer");
var connectionStringBuilder = new EventHubsConnectionStringBuilder($"Endpoint=sb://{options.Namespace}.servicebus.windows.net/;SharedAccessKeyName={options.AuthKeyName};SharedAccessKey={options.AuthKey}")
{
EntityPath = options.TopicName
};
var eventProcessorHostName = Guid.NewGuid().ToString();
_eventProcessorHost = new EventProcessorHost(
eventProcessorHostName,
options.TopicName,
options.Group,
connectionStringBuilder.ToString(),
$"DefaultEndpointsProtocol=https;AccountName={options.StorageAccountName};AccountKey={options.StorageAccountKey};EndpointSuffix=core.windows.net",
options.StorageContainerName);
_eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventHubEventProcessorFactory(logger, handler)).Wait();
}
static async Task Main(string[] args)
{
var builder = new ConfigurationBuilder().SetBasePath(Directory.GetCurrentDirectory())
.AddJsonFile("appSettings.json", optional: true, reloadOnChange: true);
var configuration = builder.Build();
var host = new EventProcessorHost(
configuration["eventHubPath"],
configuration["eventHubConsumerGroup"],
configuration["eventHubConnectionString"],
configuration["StorageConnectionString"],
configuration["leaseContainerName"]
);
var factory = new IotHubProcessorFactory(
configuration["StorageConnectionString"]);
await host.RegisterEventProcessorFactoryAsync(factory);
Console.WriteLine("IotHubEventProcessor is running...");
Console.WriteLine("");
Console.WriteLine("");
Console.WriteLine("ManulaConnectedCiotola");
Console.WriteLine("");
Console.WriteLine("");
Console.ReadLine();
}
private async static void Run(TypeContainer tc)
{
string eventHubConfigFile = _settings.EventHubConfigFile;
var eventHubConfig = ConfigUtility.ReadConfig<Dictionary<string, string>>(eventHubConfigFile);
string eventHubName = eventHubConfig["EventHubName"];
string eventHubConnectionString = eventHubConfig["EventHubConnectionString"];
string azureStorageConnectionString = eventHubConfig["AzureStorageConnectionString"];
string eventProcessorHostName = "IoTEventHubTSDBWriter";
try
{
EventProcessorHost host = new EventProcessorHost(
eventProcessorHostName,
eventHubName,
EventHubConsumerGroup.DefaultGroupName,
eventHubConnectionString,
azureStorageConnectionString,
eventHubName.ToLowerInvariant());
IEventProcessorFactory factory = new TsdbEventProcessorFactory();
IInitializer factoryInitializer = factory as IInitializer;
factoryInitializer.Initialize(_settings);
var options = new EventProcessorOptions();
options.ExceptionReceived += Options_ExceptionReceived;
await host.RegisterEventProcessorFactoryAsync(factory, options);
}
catch (Exception ex)
{
Debug.WriteLine(ex.Message);
}
}
private static async Task MainAsync(string[] args)
{
Console.WriteLine("Registering EventProcessor....");
var eventProcessorHost = new EventProcessorHost(
EventHubName,
PartitionReceiver.DefaultConsumerGroupName,
EventHubConnectionString,
StorageConnectionString,
StorageContainerName
);
// Create an instance of IEventProcessorFactory with dependencies (connection string in this case) to inject into host
IEventProcessorFactory processorFactory = new ProcessorFactory("some connection string");
// Register the Event Processor host and starts receiving messages
await eventProcessorHost.RegisterEventProcessorFactoryAsync(processorFactory);
Console.WriteLine("Receiving. Press Enter to stop worker");
Console.ReadLine();
//Dispose of the Event Processor Host
await eventProcessorHost.UnregisterEventProcessorAsync();
}
private static async Task CheckpointLatest(EventHubRuntimeInformation runtimeInformation)
{
var leasedPartitionIds = new HashSet <string>();
var stopwatch = new Stopwatch();
EventProcessorHost processorHost = GetEventProcessorHost();
var factory = new CheckpointerFactory(() => new Checkpointer
{
OnOpen = partitionContext => leasedPartitionIds.Add(partitionContext.PartitionId),
OnCheckpoint = eventData => stopwatch.Restart()
});
await processorHost.RegisterEventProcessorFactoryAsync(factory);
do
{
await Task.Delay(10);
}while (leasedPartitionIds.Count < runtimeInformation.PartitionCount);
stopwatch.Start();
do
{
await Task.Delay(10);
}while (stopwatch.Elapsed.TotalSeconds < 1.0);
stopwatch.Stop();
await processorHost.UnregisterEventProcessorAsync();
}
public AzureEventHubListener(SynchronizedInputWriter synchronizedInputWriter, ILogger logger,
string eventHubConnectionString, string eventHubName, string consumerGroupName,
string storageConnectionString, string storageContainerName)
{
_eventProcessorHost = new EventProcessorHost(
eventHubName,
consumerGroupName,
eventHubConnectionString,
storageConnectionString,
storageContainerName)
{
PartitionManagerOptions = new PartitionManagerOptions()
{
RenewInterval = TimeSpan.FromSeconds(10),
LeaseDuration = TimeSpan.FromSeconds(20),
}
};
var eventProcessorOptions = new EventProcessorOptions()
{
InvokeProcessorAfterReceiveTimeout = false,
EnableReceiverRuntimeMetric = false,
MaxBatchSize = 100,
ReceiveTimeout = TimeSpan.FromSeconds(25),
};
// Registers ClefEventProcessor in running EventProcessorHost instance
var factory = new InputEventProcessorFactory <ClefEventProcessor>(synchronizedInputWriter, logger);
_eventProcessorHost.RegisterEventProcessorFactoryAsync(factory, eventProcessorOptions);
}
protected override async Task RunAsync(CancellationToken cancellationToken)
{
var configurationPackage = Context.CodePackageActivationContext.GetConfigurationPackageObject("Config");
var connectionString = configurationPackage.Settings.Sections["ConnectionString"].Parameters["SFTestDBConnection"].Value;
var serviceBusConnection = configurationPackage.Settings.Sections["ConnectionString"]
.Parameters["SFTestServiceBus"].Value;
var eventProcessorHost = new EventProcessorHost(
configurationPackage.Settings.Sections["ConnectionString"].Parameters["EntityPath"].Value,
PartitionReceiver.DefaultConsumerGroupName,
configurationPackage.Settings.Sections["ConnectionString"].Parameters["SFTestEventHub"].Value,
configurationPackage.Settings.Sections["ConnectionString"].Parameters["SFTestDB"].Value,
LeaseConnection);
try
{
await eventProcessorHost.RegisterEventProcessorFactoryAsync(new A(connectionString, serviceBusConnection, Context), new EventProcessorOptions { MaxBatchSize = 50 });
cancellationToken.Register(async() => await eventProcessorHost.UnregisterEventProcessorAsync());
}
catch (Exception e)
{
}
finally
{
await base.RunAsync(cancellationToken);
}
}
private static void ProcessEventHub(TextWriter log)
{
string eventHubConnectionString = ConfigurationManager.AppSettings["eventHubConnectionString"];
string eventHubName = ConfigurationManager.AppSettings["eventHubName"];
string consumerGroup = ConfigurationManager.AppSettings["consumerGroup"];
string storageAccountName = ConfigurationManager.AppSettings["storageAccountName"];
string storageAccountKey = ConfigurationManager.AppSettings["storageAccountKey"];
bool newEventsOnly = ConfigurationManager.AppSettings["newEventsOnly"] == "true";
string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}",
storageAccountName, storageAccountKey);
ManualResetEvent oSignalProcessEnd = new ManualResetEvent(false);
string eventProcessorHostName = Guid.NewGuid().ToString();
EventProcessorHost eventProcessorHost = newEventsOnly ?
new EventProcessorHost(eventProcessorHostName, eventHubName, consumerGroup,
eventHubConnectionString, storageConnectionString, eventProcessorHostName) :
new EventProcessorHost(eventProcessorHostName, eventHubName, consumerGroup,
eventHubConnectionString, storageConnectionString);
log.WriteLine("Registering EventProcessor...");
var options = newEventsOnly ?
new EventProcessorOptions {
InitialOffsetProvider = (partitionId) => DateTime.UtcNow
} :
new EventProcessorOptions();
options.ExceptionReceived += (sender, e) => ProcessEventHubException(sender, e, log, oSignalProcessEnd);
eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventProcessorFactory(log), options).Wait();
oSignalProcessEnd.WaitOne();
eventProcessorHost.UnregisterEventProcessorAsync().Wait();
}
public async void ReceiveOffers()
{
var eventProcessorFactory = new Fac(OnOfferAdded);
await eventProcessorHost.RegisterEventProcessorFactoryAsync(eventProcessorFactory);
//await eventProcessorHost.RegisterEventProcessorAsync<OfferEventProcessor>();
}
public async Task StartAsync(CancellationToken cancellationToken)
{
eventProcessorHost = new EventProcessorHost(eventProcessorHostConfig.HostName,
eventProcessorHostConfig.EventHubPath,
eventProcessorHostConfig.ConsumerGroupName,
eventProcessorHostConfig.EventHubConnectionString,
eventProcessorHostConfig.StorageConnectionString,
eventProcessorHostConfig.LeaseContainerName);
if (eventProcessorOptions == null)
{
await eventProcessorHost.RegisterEventProcessorFactoryAsync(iotHubEventProcessorFactory);
}
else
{
await eventProcessorHost.RegisterEventProcessorFactoryAsync(iotHubEventProcessorFactory, eventProcessorOptions);
}
}
public override async Task ConnectHandlerAsync(string queueName,
Func <IEventMessage, CancellationToken, Task> messageReceived, CancellationToken cancellationToken)
{
_eventProcessors.Add(queueName, new AzureEventHubProcessorFactory(messageReceived, cancellationToken));
await _eventHost.RegisterEventProcessorFactoryAsync(
new AzureEventHubProcessorFactory(messageReceived, cancellationToken),
EventProcessorOptions.DefaultOptions);
}
private async Task StartListenerAsync(EventProcessorHost eventProcessorHost, CancellationToken cancellationToken)
{
_logger.LogInformation("StartListenerAsync: MetricsIngestor/Starting");
// start the event listener
await eventProcessorHost.RegisterEventProcessorFactoryAsync(new IngestorEventProcessorFactory(cancellationToken, _logger), GetEventProcessorOptions(60));
_serviceState = ServiceState.Running;
_logger.LogInformation("StartListenerAsync: MetricsIngestor/Started");
}
protected override async Task ExecuteAsync(CancellationToken stoppingToken)
{
await _eventProcessor.RegisterEventProcessorFactoryAsync(new BaseballEventProcessorFactory(_hub, _config));
while (!stoppingToken.IsCancellationRequested)
{
await Task.Delay(500);
}
await _eventProcessor.UnregisterEventProcessorAsync();
}
public async Task ReceiveAsync(Action <T> action)
{
_eventProcessorHost = new EventProcessorHost(
_eventHubName,
PartitionReceiver.DefaultConsumerGroupName,
_eventHubConnectionString,
_storageConnectionString,
_storageContainerName);
await _eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventProcessorFactory <T>(action));
}
protected EventHubConsumer(EventHubMessageBus messageBus, ITopicGroupConsumerSettings consumerSettings, Func <EventHubConsumer, EventProcessor> eventProcessorFactory)
{
MessageBus = messageBus;
EventProcessorFactory = eventProcessorFactory;
Log.InfoFormat("Creating EventProcessorHost for Topic: {0}, Group: {1}", consumerSettings.Topic, consumerSettings.Group);
EventProcessorHost = MessageBus.EventHubSettings.EventProcessorHostFactory(consumerSettings);
var eventProcessorOptions = MessageBus.EventHubSettings.EventProcessorOptionsFactory(consumerSettings);
EventProcessorHost.RegisterEventProcessorFactoryAsync(this, eventProcessorOptions).Wait();
}
public static async Task <WarmStorageCoordinator> CreateAsync(string hostName, Configuration configuration)
{
WarmStorageEventSource.Log.InitializingEventHubListener(configuration.EventHubName, configuration.ConsumerGroupName);
Func <string, IElasticSearchWriter> elasticSearchWriterFactory = partitionId =>
new ElasticSearchWriter.ElasticSearchWriter(
configuration.ElasticSearchUrl,
configuration.ElasticSearchIndexPrefix,
configuration.ElasticSearchIndexType,
configuration.RetryCount
);
var ns = NamespaceManager.CreateFromConnectionString(configuration.EventHubConnectionString);
try
{
await ns.GetConsumerGroupAsync(configuration.EventHubName, configuration.ConsumerGroupName);
}
catch (Exception e)
{
WarmStorageEventSource.Log.InvalidEventHubConsumerGroupName(e, configuration.EventHubName, configuration.ConsumerGroupName);
throw;
}
WarmStorageEventSource.Log.ConsumerGroupFound(configuration.EventHubName, configuration.ConsumerGroupName);
var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, configuration.EventHubName);
var buildingLookupService = new BuildingLookupService();
await buildingLookupService.InitializeAsync();
var factory = new WarmStorageEventProcessorFactory(elasticSearchWriterFactory, eventHubId, buildingLookupService);
var options = new EventProcessorOptions()
{
MaxBatchSize = configuration.MaxBatchSize,
PrefetchCount = configuration.PreFetchCount,
ReceiveTimeOut = configuration.ReceiveTimeout
};
options.ExceptionReceived +=
(s, e) => WarmStorageEventSource.Log.ErrorProcessingMessage(e.Exception, e.Action);
var host = new EventProcessorHost(
hostName,
consumerGroupName: configuration.ConsumerGroupName,
eventHubPath: configuration.EventHubName,
eventHubConnectionString: configuration.EventHubConnectionString,
storageConnectionString: configuration.CheckpointStorageAccount);
await host.RegisterEventProcessorFactoryAsync(factory, options);
return(new WarmStorageCoordinator(host));
}
async Task InvokeAfterReceiveTimeoutFalse()
{
const int ReceiveTimeoutInSeconds = 15;
Log("Calling RegisterEventProcessorAsync with InvokeProcessorAfterReceiveTimeout=false");
var eventProcessorHost = new EventProcessorHost(
string.Empty,
PartitionReceiver.DefaultConsumerGroupName,
this.EventHubConnectionString,
this.StorageConnectionString,
this.LeaseContainerName);
var processorOptions = new EventProcessorOptions
{
ReceiveTimeout = TimeSpan.FromSeconds(ReceiveTimeoutInSeconds),
InvokeProcessorAfterReceiveTimeout = false,
MaxBatchSize = 100
};
var emptyBatchReceiveEvent = new AsyncAutoResetEvent(false);
var processorFactory = new TestEventProcessorFactory();
processorFactory.OnCreateProcessor += (f, createArgs) =>
{
var processor = createArgs.Item2;
string partitionId = createArgs.Item1.PartitionId;
processor.OnProcessEvents += (_, eventsArgs) =>
{
int eventCount = eventsArgs.Item2 != null?eventsArgs.Item2.events.Count() : 0;
Log($"Partition {partitionId} TestEventProcessor processing {eventCount} event(s)");
if (eventCount == 0)
{
emptyBatchReceiveEvent.Set();
}
};
};
await eventProcessorHost.RegisterEventProcessorFactoryAsync(processorFactory, processorOptions);
try
{
Log("Verifying no empty batches arrive...");
bool waitSucceeded = await emptyBatchReceiveEvent.WaitAsync(TimeSpan.FromSeconds(ReceiveTimeoutInSeconds * 2));
Assert.False(waitSucceeded, "No empty batch should have been received!");
}
finally
{
Log("Calling UnregisterEventProcessorAsync");
await eventProcessorHost.UnregisterEventProcessorAsync();
}
}
public async Task Start()
{
_eventProcessorHost = new EventProcessorHost(
EventHubConstants.EventHubName,
PartitionReceiver.DefaultConsumerGroupName,
EventHubConstants.EventHubConnectionString,
EventHubConstants.StorageConnectionString,
EventHubConstants.StorageContainerName);
await _eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventProcessorFactory(MessageReceivedHandler));
}
private async Task <bool> OnStart(StartListening arg)
{
_eventProcessorHost = new EventProcessorHost(
"akka-iot",
PartitionReceiver.DefaultConsumerGroupName,
Environment.GetEnvironmentVariable("akka-iot-hub-endpoint"),
Environment.GetEnvironmentVariable("akka-iot-checkpoint-storage"),
"akka-iot-checkpoints");
await _eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventProcessorFactory(_system, _tableStorage));
return(true);
}
protected GroupTopicConsumer(EventHubMessageBus messageBus, TopicGroup topicGroup, Func <PartitionConsumer> partitionConsumerFactory)
{
MessageBus = messageBus;
_partitionConsumerFactory = partitionConsumerFactory;
Log.InfoFormat(CultureInfo.InvariantCulture, "Creating EventProcessorHost for EventHub with Topic: {0}, Group: {1}", topicGroup.Topic, topicGroup.Group);
_processorHost = MessageBus.EventHubSettings.EventProcessorHostFactory(topicGroup);
var eventProcessorOptions = MessageBus.EventHubSettings.EventProcessorOptionsFactory(topicGroup);
_processorHost.RegisterEventProcessorFactoryAsync(this, eventProcessorOptions).Wait();
}
public static async Task<WarmStorageCoordinator> CreateAsync(string hostName, Configuration configuration)
{
WarmStorageEventSource.Log.InitializingEventHubListener(configuration.EventHubName, configuration.ConsumerGroupName);
Func<string, IElasticSearchWriter> elasticSearchWriterFactory = partitionId =>
new ElasticSearchWriter.ElasticSearchWriter(
configuration.ElasticSearchUrl,
configuration.ElasticSearchIndexPrefix,
configuration.ElasticSearchIndexType,
configuration.RetryCount
);
var ns = NamespaceManager.CreateFromConnectionString(configuration.EventHubConnectionString);
try
{
await ns.GetConsumerGroupAsync(configuration.EventHubName, configuration.ConsumerGroupName);
}
catch (Exception e)
{
WarmStorageEventSource.Log.InvalidEventHubConsumerGroupName(e, configuration.EventHubName, configuration.ConsumerGroupName);
throw;
}
WarmStorageEventSource.Log.ConsumerGroupFound(configuration.EventHubName, configuration.ConsumerGroupName);
var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, configuration.EventHubName);
var buildingLookupService = new BuildingLookupService();
await buildingLookupService.InitializeAsync();
var factory = new WarmStorageEventProcessorFactory(elasticSearchWriterFactory, eventHubId, buildingLookupService);
var options = new EventProcessorOptions()
{
MaxBatchSize = configuration.MaxBatchSize,
PrefetchCount = configuration.PreFetchCount,
ReceiveTimeOut = configuration.ReceiveTimeout
};
options.ExceptionReceived +=
(s, e) => WarmStorageEventSource.Log.ErrorProcessingMessage(e.Exception, e.Action);
var host = new EventProcessorHost(
hostName,
consumerGroupName: configuration.ConsumerGroupName,
eventHubPath: configuration.EventHubName,
eventHubConnectionString: configuration.EventHubConnectionString,
storageConnectionString: configuration.CheckpointStorageAccount);
await host.RegisterEventProcessorFactoryAsync(factory, options);
return new WarmStorageCoordinator(host);
}
public async Task ConsumeEventsFromHubAsync(string eventHubPath)
{
var connectionStringBuilder = new EventHubsConnectionStringBuilder(_eventHubConnectionString)
{
EntityPath = eventHubPath
};
var client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString());
_eventProcessorHost = new EventProcessorHost(eventHubPath, PartitionReceiver.DefaultConsumerGroupName, _eventHubConnectionString, _storageConnectionString, _containerName);
await _eventProcessorHost.RegisterEventProcessorFactoryAsync(_azureEventProcessorFactory);
}
public async Task StartAsync()
{
Console.WriteLine($"Registering EventHubProcessor, connecting to {_config.ConnectionString.Split(';')[0]}...");
var eventProcessorHost = new EventProcessorHost(_config.EntityPath, _config.ConsumerGroup, _config.ConnectionString, _config.StorageConnection, _config.StorageContainer);
await eventProcessorHost.RegisterEventProcessorFactoryAsync(new DeserializingEventProcessorFactory(_config.Deserializer));
Console.WriteLine("Receiving. Press ENTER to stop worker.");
Console.ReadLine();
await eventProcessorHost.UnregisterEventProcessorAsync();
}
private async Task StartProcessor(CancellationToken token)
{
try
{
string hostName = Environment.MachineName;
string eventHubPath = _configurationProvider.GetConfigurationSettingValue("RulesEventHub.Name");
string consumerGroup = EventHubConsumerGroup.DefaultGroupName;
string eventHubConnectionString = _configurationProvider.GetConfigurationSettingValue("RulesEventHub.ConnectionString");
string storageConnectionString = _configurationProvider.GetConfigurationSettingValue("device.StorageConnectionString");
_eventProcessorHost = new EventProcessorHost(
hostName,
eventHubPath,
consumerGroup,
eventHubConnectionString,
storageConnectionString);
_factory = new ActionProcessorFactory(
_actionLogic,
_actionMappingLogic,
_configurationProvider);
Trace.TraceInformation("ActionEventProcessor: Registering host...");
var options = new EventProcessorOptions();
options.ExceptionReceived += OptionsOnExceptionReceived;
await _eventProcessorHost.RegisterEventProcessorFactoryAsync(_factory);
// processing loop
while (!token.IsCancellationRequested)
{
Trace.TraceInformation("ActionEventProcessor: Processing...");
await Task.Delay(TimeSpan.FromMinutes(5), token);
}
// cleanup
await _eventProcessorHost.UnregisterEventProcessorAsync();
}
catch (Exception e)
{
Trace.TraceError("Error in ActionProcessor.StartProcessor, Exception: {0}", e.ToString());
}
_isRunning = false;
}
static void Main(string[] args)
{
Bifrost.Configuration.Configure.DiscoverAndConfigure();
var eventProcessorHostName = "";
var eventHubConnectionString = "";
var eventHubName = "";
var storageAccountName = "";
var storageAccountKey = "";
var storageConnectionString = $"DefaultEndpointsProtocol=https;AccountName={storageAccountName};AccountKey={storageAccountKey}";
var eventProcessorHost = new EventProcessorHost(
eventProcessorHostName,
eventHubName,
EventHubConsumerGroup.DefaultGroupName,
eventHubConnectionString,
storageConnectionString);
eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventProcessorFactory());
try
{
#if (DEBUG)
Console.WriteLine("Processing");
Console.ReadLine();
#else
var config = new JobHostConfiguration();
var host = new JobHost(config);
host.RunAndBlock();
#endif
}
finally
{
eventProcessorHost.UnregisterEventProcessorAsync().Wait();
}
}
async Task StartProcessor(CancellationToken token)
{
try
{
// Initialize
_eventProcessorHost = new EventProcessorHost(
Environment.MachineName,
_configurationProvider.GetConfigurationSettingValue("eventHub.HubName").ToLowerInvariant(),
EventHubConsumerGroup.DefaultGroupName,
_configurationProvider.GetConfigurationSettingValue("eventHub.ConnectionString"),
_configurationProvider.GetConfigurationSettingValue("eventHub.StorageConnectionString"));
_factory = new DeviceAdministrationProcessorFactory(_deviceLogic, _configurationProvider);
Trace.TraceInformation("DeviceEventProcessor: Registering host...");
var options = new EventProcessorOptions();
options.ExceptionReceived += OptionsOnExceptionReceived;
await _eventProcessorHost.RegisterEventProcessorFactoryAsync(_factory);
// processing loop
while (!token.IsCancellationRequested)
{
Trace.TraceInformation("DeviceEventProcessor: Processing...");
await Task.Delay(TimeSpan.FromMinutes(5), token);
// Any additional incremental processing can be done here (like checking states, etc).
}
// cleanup
await _eventProcessorHost.UnregisterEventProcessorAsync();
}
catch (Exception e)
{
Trace.TraceInformation("Error in DeviceEventProcessor.StartProcessor, Exception: {0}", e.Message);
}
_running = false;
}
// ReSharper disable once FunctionComplexityOverflow
private async void btnStart_Click(object sender, EventArgs e)
{
try
{
Cursor.Current = Cursors.WaitCursor;
if (string.Compare(btnStart.Text, Start, StringComparison.OrdinalIgnoreCase) == 0)
{
// Validate parameters
if (!ValidateParameters())
{
return;
}
btnStart.Enabled = false;
var eventHubClient = EventHubClient.CreateFromConnectionString(txtServiceBusConnectionString.Text,
cboEventHub.Text);
// Get the default Consumer Group
eventProcessorHost = new EventProcessorHost(Guid.NewGuid().ToString(),
eventHubClient.Path.ToLower(),
txtConsumerGroup.Text.ToLower(),
txtServiceBusConnectionString.Text,
txtStorageAccountConnectionString.Text)
{
PartitionManagerOptions = new PartitionManagerOptions
{
AcquireInterval = TimeSpan.FromSeconds(10), // Default is 10 seconds
RenewInterval = TimeSpan.FromSeconds(10), // Default is 10 seconds
LeaseInterval = TimeSpan.FromSeconds(30) // Default value is 30 seconds
}
};
WriteToLog(RegisteringEventProcessor);
var eventProcessorOptions = new EventProcessorOptions
{
InvokeProcessorAfterReceiveTimeout = true,
MaxBatchSize = 100,
PrefetchCount = 100,
ReceiveTimeOut = TimeSpan.FromSeconds(30),
};
eventProcessorOptions.ExceptionReceived += EventProcessorOptions_ExceptionReceived;
var eventProcessorFactoryConfiguration = new EventProcessorFactoryConfiguration
{
TrackEvent = a => Invoke(new Action<Alert>(i => alertBindingList.Add(i)), a),
WriteToLog = m => Invoke(new Action<string>(WriteToLog), m)
};
await
eventProcessorHost.RegisterEventProcessorFactoryAsync(
new EventProcessorFactory<EventProcessor>(eventProcessorFactoryConfiguration),
eventProcessorOptions);
WriteToLog(EventProcessorRegistered);
// Change button text
btnStart.Text = Stop;
}
else
{
// Stop Event Processor
if (eventProcessorHost != null)
{
await eventProcessorHost.UnregisterEventProcessorAsync();
}
// Change button text
btnStart.Text = Start;
}
}
catch (Exception ex)
{
HandleException(ex);
}
finally
{
Cursor.Current = Cursors.Default;
btnStart.Enabled = true;
}
}
/// <summary>
/// Starts the specified host control.
/// </summary>
/// <param name="hostControl">The host control.</param>
/// <returns></returns>
public override bool Start(HostControl hostControl)
{
var processorOptions = new EventProcessorOptions
{
MaxBatchSize = 5000, // default is 300
PrefetchCount = 1000
};
_processorHost = new EventProcessorHost(
Environment.MachineName,
eventHubPath: ConfigurationManager.AppSettings["eventHubPath"],
consumerGroupName: ConfigurationManager.AppSettings["consumerGroupName"],
storageConnectionString: ConfigurationManager.ConnectionStrings["StorageConnectionString"].ConnectionString,
eventHubConnectionString: ConfigurationManager.ConnectionStrings["EventHubConnectionString"].ConnectionString);
using (Logger.BeginTimedOperation("RegisterEventProcessorAsync"))
_processorHost.RegisterEventProcessorFactoryAsync(this, processorOptions).Wait();
return base.Start(hostControl);
}
private async Task StartEventProcessorAsync()
{
try
{
var eventHubClient = EventHubClient.CreateFromConnectionString(serviceBusConnectionString, eventHubName);
// Get the default Consumer Group
eventProcessorHost = new EventProcessorHost(Guid.NewGuid().ToString(),
eventHubClient.Path.ToLower(),
consumerGroupName.ToLower(),
serviceBusConnectionString,
storageAccountConnectionString)
{
PartitionManagerOptions = new PartitionManagerOptions
{
AcquireInterval = TimeSpan.FromSeconds(10), // Default is 10 seconds
RenewInterval = TimeSpan.FromSeconds(10), // Default is 10 seconds
LeaseInterval = TimeSpan.FromSeconds(30) // Default value is 30 seconds
}
};
ServiceEventSource.Current.Message(RegisteringEventProcessor);
var eventProcessorOptions = new EventProcessorOptions
{
InvokeProcessorAfterReceiveTimeout = true,
MaxBatchSize = 100,
PrefetchCount = 100,
ReceiveTimeOut = TimeSpan.FromSeconds(30),
};
eventProcessorOptions.ExceptionReceived += EventProcessorOptions_ExceptionReceived;
await eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventProcessorFactory<EventProcessor>(deviceActorServiceUri),
eventProcessorOptions);
ServiceEventSource.Current.Message(EventProcessorRegistered);
}
catch (Exception ex)
{
// Trace Error
ServiceEventSource.Current.Message(ex.Message);
throw;
}
}
protected async Task InitializeAsync(
string hostName,
string eventHubName,
string consumerGroupName,
string eventHubConnectionString,
string checkpointStorageAccount,
int maxBatchSize,
int prefetchCount,
TimeSpan receiveTimeout,
int maxConcurrencyPerProcessor,
IEnumerable<Type> typesToSearch,
Func<string, string, ICircuitBreaker> circuitBreakerFactory,
IDispatcherInstrumentationPublisher instrumentationPublisher)
{
Logger.Info("Initializing event hub listener for {0} ({1})", eventHubName, consumerGroupName);
// Get the consumer group via the Service Bus namespace (identifies the
// consumer)
var ns = NamespaceManager.CreateFromConnectionString(eventHubConnectionString);
try
{
await ns.GetConsumerGroupAsync(eventHubName, consumerGroupName);
Logger.Info("Found consumer group {1} for {0}", eventHubName, consumerGroupName);
}
catch (Exception e)
{
Logger.Error(e, "Could not establish connection to {0} in event hub {1}", consumerGroupName, eventHubName);
throw;
}
var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, eventHubName);
// Use a custom event processor factory to pass parameters to the
// event host processor
var factory = new EventProcessorFactory(
handlerResolver: new MessageHandlerResolver(typesToSearch, DependencyResolverFactory.GetResolver()),
maxConcurrency: maxConcurrencyPerProcessor,
circuitBreakerFactory: circuitBreakerFactory,
eventHubName: eventHubId,
instrumentationPublisher: instrumentationPublisher);
var options = new EventProcessorOptions
{
MaxBatchSize = maxBatchSize,
PrefetchCount = prefetchCount,
ReceiveTimeOut = receiveTimeout
};
options.ExceptionReceived += options_ExceptionReceived;
// Create the event processor host and register via the factory
_host = new EventProcessorHost(
hostName,
consumerGroupName: consumerGroupName,
eventHubPath: eventHubName,
eventHubConnectionString: eventHubConnectionString,
storageConnectionString: checkpointStorageAccount
);
await _host.RegisterEventProcessorFactoryAsync(factory, options);
Logger.Info("Event processor registered for {0} ({1})", eventHubName, consumerGroupName);
}
public static async Task<ColdStorageCoordinator> CreateAsync(
string hostName,
string eventHubName,
string consumerGroupName,
string eventHubConnectionString,
string checkpointStorageAccount,
int maxBatchSize,
int prefetchCount,
TimeSpan receiveTimeout,
IReadOnlyList<string> blobWriterStorageAccounts,
string containerName,
int rollSizeMb,
string blobPrefix,
int warningLevel,
int tripLevel,
TimeSpan stallInterval,
TimeSpan logCooldownInterval,
IColdStorageInstrumentationPublisher instrumentationPublisher)
{
Logger.Info("Initializing event hub listener for {0} ({1})", eventHubName, consumerGroupName);
var storageAccounts = blobWriterStorageAccounts
.Select(CloudStorageAccount.Parse)
.ToList();
Func<string, IBlobWriter> blobWriterFactory =
partitionId =>
new RollingBlobWriter(new PartitionAndDateNamingStrategy(partitionId, blobPrefix),
instrumentationPublisher,
storageAccounts[Int32.Parse(partitionId) % storageAccounts.Count],
containerName,
rollSizeMb);
var ns = NamespaceManager.CreateFromConnectionString(eventHubConnectionString);
try
{
await ns.GetConsumerGroupAsync(eventHubName, consumerGroupName);
}
catch (Exception e)
{
Logger.Error(e, "Invalid consumer group name {0} in event hub {1}", consumerGroupName, eventHubName);
throw;
}
Logger.Info("Found consumer group {1} for {0}", eventHubName, consumerGroupName);
var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, eventHubName);
var factory = new ColdStorageEventProcessorFactory(
blobWriterFactory,
instrumentationPublisher,
CancellationToken.None,
warningLevel,
tripLevel,
stallInterval,
logCooldownInterval,
eventHubId
);
var options = new EventProcessorOptions()
{
MaxBatchSize = maxBatchSize,
PrefetchCount = prefetchCount,
ReceiveTimeOut = receiveTimeout,
InvokeProcessorAfterReceiveTimeout = true
};
options.ExceptionReceived +=
(s, e) => Logger.Error(
e.Exception,
"Error on message processing, action {0}",
e.Action);
var host = new EventProcessorHost(
hostName,
consumerGroupName: consumerGroupName,
eventHubPath: eventHubName,
eventHubConnectionString: eventHubConnectionString,
storageConnectionString: checkpointStorageAccount);
await host.RegisterEventProcessorFactoryAsync(factory, options);
return new ColdStorageCoordinator(host);
}