static void Main(string[] args) { string eventHubConnectionString = Environment.GetEnvironmentVariable("APIMEVENTS-EVENTHUB-CONNECTIONSTRING", EnvironmentVariableTarget.Process); string eventHubName = Environment.GetEnvironmentVariable("APIMEVENTS-EVENTHUB-NAME", EnvironmentVariableTarget.Process); string storageAccountName = Environment.GetEnvironmentVariable("APIMEVENTS-STORAGEACCOUNT-NAME", EnvironmentVariableTarget.Process); string storageAccountKey = Environment.GetEnvironmentVariable("APIMEVENTS-STORAGEACCOUNT-KEY", EnvironmentVariableTarget.Process); string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", storageAccountName, storageAccountKey); string eventProcessorHostName = Guid.NewGuid().ToString(); var eventProcessorHost = new EventProcessorHost( eventProcessorHostName, eventHubName, EventHubConsumerGroup.DefaultGroupName, eventHubConnectionString, storageConnectionString); var logger = new ConsoleLogger(LogLevel.Debug); logger.LogDebug("Registering EventProcessor..."); var httpMessageProcessor = new RunscopeHttpMessageProcessor(new HttpClient(), logger); eventProcessorHost.RegisterEventProcessorFactoryAsync( new ApimHttpEventProcessorFactory(httpMessageProcessor, logger)); Console.WriteLine("Receiving. Press enter key to stop worker."); Console.ReadLine(); eventProcessorHost.UnregisterEventProcessorAsync().Wait(); }
public static async Task<ColdStorageCoordinator> CreateAsync(string hostName, Configuration configuration) { ColdStorageEventSource.Log.InitializingEventHubListener(configuration.EventHubName, configuration.ConsumerGroupName); var storageAccount = CloudStorageAccount.Parse(configuration.BlobWriterStorageAccount); Func<string, IBlobWriter> blobWriterFactory = partitionId => new RollingBlobWriter.RollingBlobWriter(new PartitionAndDateNamingStrategy(partitionId, configuration.BlobPrefix), storageAccount, configuration.ContainerName, configuration.RollSizeForBlobWriterMb); var ns = NamespaceManager.CreateFromConnectionString(configuration.EventHubConnectionString); try { await ns.GetConsumerGroupAsync(configuration.EventHubName, configuration.ConsumerGroupName); } catch (Exception e) { ColdStorageEventSource.Log.InvalidEventHubConsumerGroupName(e, configuration.EventHubName, configuration.ConsumerGroupName); throw; } ColdStorageEventSource.Log.ConsumerGroupFound(configuration.EventHubName, configuration.ConsumerGroupName); var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, configuration.EventHubName); var factory = new ColdStorageEventProcessorFactory( blobWriterFactory, configuration.CircuitBreakerWarningLevel, configuration.CircuitBreakerTripLevel, configuration.CircuitBreakerStallInterval, configuration.CircuitBreakerLogCooldownInterval, eventHubId ); var options = new EventProcessorOptions() { MaxBatchSize = configuration.MaxBatchSize, PrefetchCount = configuration.PreFetchCount, ReceiveTimeOut = configuration.ReceiveTimeout, InvokeProcessorAfterReceiveTimeout = true }; options.ExceptionReceived += (s, e) => ColdStorageEventSource.Log.ErrorProcessingMessage(e.Exception, e.Action); var host = new EventProcessorHost( hostName, consumerGroupName: configuration.ConsumerGroupName, eventHubPath: configuration.EventHubName, eventHubConnectionString: configuration.EventHubConnectionString, storageConnectionString: configuration.CheckpointStorageAccount); await host.RegisterEventProcessorFactoryAsync(factory, options); return new ColdStorageCoordinator(host); }
private static async Task MainAsync(string[] args) { //carica le impostazioni in appsettings var builder = new ConfigurationBuilder() .SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile( "appsettings.json", optional: true, reloadOnChange: true); var configuration = builder.Build(); Console.WriteLine("Registering EventProcessor..."); var eventProcessorHost = new EventProcessorHost( configuration["ConnectionStrings:EventHubPath"], PartitionReceiver.DefaultConsumerGroupName, configuration["ConnectionStrings:EventHubCS"], configuration["ConnectionStrings:StorageAccountCS"], configuration["ConnectionStrings:StorageContainer"]); // Registers the Event Processor Host and starts receiving messages await eventProcessorHost.RegisterEventProcessorFactoryAsync( new TeleEventProcessorFactory(configuration["ConnectionStrings:DefaultConnection"])); Console.WriteLine("Receiving. Press ENTER to stop worker."); Console.ReadLine(); // Disposes of the Event Processor Host await eventProcessorHost.UnregisterEventProcessorAsync(); }
private static async Task StartHost() { //Event Hub var eventHubConnectionString = GetEventHubConnectionString(); var storageConnectionString = GetStorageConnectionString(); string hostname = Guid.NewGuid().ToString(); host = new EventProcessorHost( hostname, Properties.Settings.Default.EventHubName, EventHubConsumerGroup.DefaultGroupName, eventHubConnectionString, storageConnectionString, Properties.Settings.Default.EventHubName.ToLowerInvariant()); factory = new HeartbeatEventProcessorFactory(hostname); try { Console.WriteLine("{0} > Registering host: {1}", DateTime.Now.ToString(), hostname); var options = new EventProcessorOptions(); options.ExceptionReceived += OptionsOnExceptionReceived; await host.RegisterEventProcessorFactoryAsync(factory); } catch (Exception exception) { Console.ForegroundColor = ConsoleColor.Red; Console.WriteLine("{0} > Exception: {1}", DateTime.Now.ToString(), exception.Message); Console.ResetColor(); } }
protected override async Task ExecuteAsync(CancellationToken cancellationToken) { string hubNamespace = _demoSettings.EventHubNamespace; var endpoint = new Uri($"sb://{hubNamespace}.servicebus.windows.net/"); string hubName = _demoSettings.EventHubName; var storageAccount = CloudStorageAccount.Parse(_demoSettings.EventHubStorageConnectionString); var host = new EventProcessorHost( endpoint, hubName, "$Default", new ManagedIdentityEventHubsTokenProvider(_demoSettings.ManagedIdentityTenantId), storageAccount, _demoSettings.EventHubStorageContainerName); try { await host.RegisterEventProcessorFactoryAsync(this); await Task.Delay(-1, cancellationToken); } finally { await host.UnregisterEventProcessorAsync(); } }
public async Task Start() { _eventProcessorHost = new EventProcessorHost( _settings.EntityPath, PartitionReceiver.DefaultConsumerGroupName, _settings.ConnectionString, string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", _settings.StorageAccountName, _settings.StorageAccountKey), _settings.StorageContainerName); EventProcessorFactory <T> eventProcessorFactory = new EventProcessorFactory <T>(); await _eventProcessorHost.RegisterEventProcessorFactoryAsync(eventProcessorFactory); eventProcessorFactory.Processor.OnEventClose += (context, reason) => { OnEventClose?.Invoke(context, reason); }; eventProcessorFactory.Processor.OnEventOpen += (context) => { OnEventOpen?.Invoke(context); }; eventProcessorFactory.Processor.OnEventError += (context, error) => { OnEventError?.Invoke(context, error); }; eventProcessorFactory.Processor.OnProcessEvents += (context, eventMessages) => { OnProcessEvents?.Invoke(context, eventMessages); }; }
public static async Task Start() { var eventHubConnectionString = GetEventHubConnectionString(); var storageConnectionString = ConfigManager.Config.TableConnectionString; var eventHubName = ConfigManager.Config.EventHubName; // here it's using eventhub as lease name. but it can be specified as any you want. // if the host is having same lease name, it will be shared between hosts. // by default it is using eventhub name as lease name. host = new EventProcessorHost( hostName, eventHubName, consumerGroupName, eventHubConnectionString, storageConnectionString, eventHubName.ToLowerInvariant()); factory = new EventProcessorFactory(hostName); try { Trace.WriteLine(string.Format("{0} > Registering host: {1}", DateTime.Now.ToString(), hostName), "Information"); var options = new EventProcessorOptions(); options.ExceptionReceived += OptionsOnExceptionReceived; await host.RegisterEventProcessorFactoryAsync(factory); } catch (Exception exception) { Trace.WriteLine(string.Format("{0} > Exception: {1}", DateTime.Now.ToString(), exception.Message), "Error"); Console.ResetColor(); } }
static void Main(string[] args) { var logger = new ConsoleLogger(LogLevel.Debug); logger.LogInfo("STARTING Moesif API Management Event Processor. Reading environment variables"); // Load configuration paramaters from Environment string eventHubConnectionString = ParamConfig.loadNonEmpty(AzureAppParamNames.EVENTHUB_CONN_STRING); string eventHubName = ParamConfig.loadNonEmpty(AzureAppParamNames.EVENTHUB_NAME); string storageAccountName = ParamConfig.loadNonEmpty(AzureAppParamNames.STORAGEACCOUNT_NAME); string storageAccountKey = ParamConfig.loadNonEmpty(AzureAppParamNames.STORAGEACCOUNT_KEY); // This App utilizes the "$Default" Eventhub consumer group // In future, we should make this configurable via environment variables string eventHubConsumerGroupName = EventHubConsumerGroup.DefaultGroupName; // Create connection string for azure storage account to store checkpoints string storageConnectionString = makeStorageAccountConnString(storageAccountName, storageAccountKey); string eventProcessorHostName = Guid.NewGuid().ToString(); var eventProcessorHost = new EventProcessorHost( eventProcessorHostName, eventHubName, eventHubConsumerGroupName, eventHubConnectionString, storageConnectionString); logger.LogDebug("Registering EventProcessor..."); var httpMessageProcessor = new MoesifHttpMessageProcessor(logger); eventProcessorHost.RegisterEventProcessorFactoryAsync( new ApimHttpEventProcessorFactory(httpMessageProcessor, logger)); logger.LogInfo("Process is running. Press enter key to end..."); Console.ReadLine(); logger.LogInfo("STOPPING Moesif API Management Event Processor"); eventProcessorHost.UnregisterEventProcessorAsync().Wait(); }
static async Task MainAsync(string[] args) { string eventHubPath = args[0]; // same as EntityPath string ehConnectionString = args[1]; string leaseContainerName = args[2]; string storageAccountName = args[3]; string storageAccountKey = args[4]; var eventProcessorHost = new EventProcessorHost( eventHubPath, PartitionReceiver.DefaultConsumerGroupName, ehConnectionString, string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", storageAccountName, storageAccountKey), leaseContainerName); // Registers the Event Processor Host and starts receiving messages await eventProcessorHost.RegisterEventProcessorFactoryAsync(new SimpleEventProcessorFactory("hello"), new EventProcessorOptions { InitialOffsetProvider = (partitionId) => DateTime.UtcNow }); Console.WriteLine("Receiving. Press ENTER to stop worker."); Console.ReadLine(); // Disposes of the Event Processor Host await eventProcessorHost.UnregisterEventProcessorAsync(); }
static void Main(string[] args) { string eventHubConnectionString = Environment.GetEnvironmentVariable("APIMEVENTS-EVENTHUB-CONNECTIONSTRING", EnvironmentVariableTarget.Process); string eventHubName = Environment.GetEnvironmentVariable("APIMEVENTS-EVENTHUB-NAME", EnvironmentVariableTarget.Process); string storageAccountName = Environment.GetEnvironmentVariable("APIMEVENTS-STORAGEACCOUNT-NAME", EnvironmentVariableTarget.Process); string storageAccountKey = Environment.GetEnvironmentVariable("APIMEVENTS-STORAGEACCOUNT-KEY", EnvironmentVariableTarget.Process); string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", storageAccountName, storageAccountKey); string eventProcessorHostName = Guid.NewGuid().ToString(); var eventProcessorHost = new EventProcessorHost( eventProcessorHostName, eventHubName, EventHubConsumerGroup.DefaultGroupName, eventHubConnectionString, storageConnectionString); var logger = new ConsoleLogger(LogLevel.Debug); logger.LogDebug("Registering EventProcessor..."); var httpMessageProcessor = new RunscopeHttpMessageProcessor(new HttpClient(), logger); eventProcessorHost.RegisterEventProcessorFactoryAsync( new ApimHttpEventProcessorFactory(httpMessageProcessor, logger)); Console.WriteLine("Receiving. Press enter key to stop worker."); Console.ReadLine(); eventProcessorHost.UnregisterEventProcessorAsync().Wait(); }
public static async Task Start(string hostName, string eventHubName, string consumerGroupName, string containerName, string blobPrefix, string serviceBusConnectionString, string storageConnectionString) { // Create Consumer Group if it doesn't exist if (consumerGroupName != null) { NamespaceManager manager = NamespaceManager.CreateFromConnectionString(serviceBusConnectionString); ConsumerGroupDescription description = new ConsumerGroupDescription(eventHubName, consumerGroupName); manager.CreateConsumerGroupIfNotExists(description); } // Create the Event Processor Host var host = new EventProcessorHost( hostName, eventHubName, consumerGroupName == null ? EventHubConsumerGroup.DefaultGroupName : consumerGroupName, serviceBusConnectionString, storageConnectionString, eventHubName); // Create the Factory var factory = new BlobEventProcessorHostFactory(storageConnectionString, containerName, blobPrefix); // Register the Factory await host.RegisterEventProcessorFactoryAsync(factory); }
public EventHubConsumer(ILogger logger, IConfigurationSection config, MessageHandler handler) { if (logger == null) { throw new ArgumentNullException(nameof(logger)); } _logger = logger; var options = new EventHubConsumerConfig(); config.Bind(options); _eventHubName = options.TopicName; logger.LogInformation("Starting consumer"); var connectionStringBuilder = new EventHubsConnectionStringBuilder($"Endpoint=sb://{options.Namespace}.servicebus.windows.net/;SharedAccessKeyName={options.AuthKeyName};SharedAccessKey={options.AuthKey}") { EntityPath = options.TopicName }; var eventProcessorHostName = Guid.NewGuid().ToString(); _eventProcessorHost = new EventProcessorHost( eventProcessorHostName, options.TopicName, options.Group, connectionStringBuilder.ToString(), $"DefaultEndpointsProtocol=https;AccountName={options.StorageAccountName};AccountKey={options.StorageAccountKey};EndpointSuffix=core.windows.net", options.StorageContainerName); _eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventHubEventProcessorFactory(logger, handler)).Wait(); }
static async Task Main(string[] args) { var builder = new ConfigurationBuilder().SetBasePath(Directory.GetCurrentDirectory()) .AddJsonFile("appSettings.json", optional: true, reloadOnChange: true); var configuration = builder.Build(); var host = new EventProcessorHost( configuration["eventHubPath"], configuration["eventHubConsumerGroup"], configuration["eventHubConnectionString"], configuration["StorageConnectionString"], configuration["leaseContainerName"] ); var factory = new IotHubProcessorFactory( configuration["StorageConnectionString"]); await host.RegisterEventProcessorFactoryAsync(factory); Console.WriteLine("IotHubEventProcessor is running..."); Console.WriteLine(""); Console.WriteLine(""); Console.WriteLine("ManulaConnectedCiotola"); Console.WriteLine(""); Console.WriteLine(""); Console.ReadLine(); }
private async static void Run(TypeContainer tc) { string eventHubConfigFile = _settings.EventHubConfigFile; var eventHubConfig = ConfigUtility.ReadConfig<Dictionary<string, string>>(eventHubConfigFile); string eventHubName = eventHubConfig["EventHubName"]; string eventHubConnectionString = eventHubConfig["EventHubConnectionString"]; string azureStorageConnectionString = eventHubConfig["AzureStorageConnectionString"]; string eventProcessorHostName = "IoTEventHubTSDBWriter"; try { EventProcessorHost host = new EventProcessorHost( eventProcessorHostName, eventHubName, EventHubConsumerGroup.DefaultGroupName, eventHubConnectionString, azureStorageConnectionString, eventHubName.ToLowerInvariant()); IEventProcessorFactory factory = new TsdbEventProcessorFactory(); IInitializer factoryInitializer = factory as IInitializer; factoryInitializer.Initialize(_settings); var options = new EventProcessorOptions(); options.ExceptionReceived += Options_ExceptionReceived; await host.RegisterEventProcessorFactoryAsync(factory, options); } catch (Exception ex) { Debug.WriteLine(ex.Message); } }
private static async Task MainAsync(string[] args) { Console.WriteLine("Registering EventProcessor...."); var eventProcessorHost = new EventProcessorHost( EventHubName, PartitionReceiver.DefaultConsumerGroupName, EventHubConnectionString, StorageConnectionString, StorageContainerName ); // Create an instance of IEventProcessorFactory with dependencies (connection string in this case) to inject into host IEventProcessorFactory processorFactory = new ProcessorFactory("some connection string"); // Register the Event Processor host and starts receiving messages await eventProcessorHost.RegisterEventProcessorFactoryAsync(processorFactory); Console.WriteLine("Receiving. Press Enter to stop worker"); Console.ReadLine(); //Dispose of the Event Processor Host await eventProcessorHost.UnregisterEventProcessorAsync(); }
private static async Task CheckpointLatest(EventHubRuntimeInformation runtimeInformation) { var leasedPartitionIds = new HashSet <string>(); var stopwatch = new Stopwatch(); EventProcessorHost processorHost = GetEventProcessorHost(); var factory = new CheckpointerFactory(() => new Checkpointer { OnOpen = partitionContext => leasedPartitionIds.Add(partitionContext.PartitionId), OnCheckpoint = eventData => stopwatch.Restart() }); await processorHost.RegisterEventProcessorFactoryAsync(factory); do { await Task.Delay(10); }while (leasedPartitionIds.Count < runtimeInformation.PartitionCount); stopwatch.Start(); do { await Task.Delay(10); }while (stopwatch.Elapsed.TotalSeconds < 1.0); stopwatch.Stop(); await processorHost.UnregisterEventProcessorAsync(); }
public AzureEventHubListener(SynchronizedInputWriter synchronizedInputWriter, ILogger logger, string eventHubConnectionString, string eventHubName, string consumerGroupName, string storageConnectionString, string storageContainerName) { _eventProcessorHost = new EventProcessorHost( eventHubName, consumerGroupName, eventHubConnectionString, storageConnectionString, storageContainerName) { PartitionManagerOptions = new PartitionManagerOptions() { RenewInterval = TimeSpan.FromSeconds(10), LeaseDuration = TimeSpan.FromSeconds(20), } }; var eventProcessorOptions = new EventProcessorOptions() { InvokeProcessorAfterReceiveTimeout = false, EnableReceiverRuntimeMetric = false, MaxBatchSize = 100, ReceiveTimeout = TimeSpan.FromSeconds(25), }; // Registers ClefEventProcessor in running EventProcessorHost instance var factory = new InputEventProcessorFactory <ClefEventProcessor>(synchronizedInputWriter, logger); _eventProcessorHost.RegisterEventProcessorFactoryAsync(factory, eventProcessorOptions); }
protected override async Task RunAsync(CancellationToken cancellationToken) { var configurationPackage = Context.CodePackageActivationContext.GetConfigurationPackageObject("Config"); var connectionString = configurationPackage.Settings.Sections["ConnectionString"].Parameters["SFTestDBConnection"].Value; var serviceBusConnection = configurationPackage.Settings.Sections["ConnectionString"] .Parameters["SFTestServiceBus"].Value; var eventProcessorHost = new EventProcessorHost( configurationPackage.Settings.Sections["ConnectionString"].Parameters["EntityPath"].Value, PartitionReceiver.DefaultConsumerGroupName, configurationPackage.Settings.Sections["ConnectionString"].Parameters["SFTestEventHub"].Value, configurationPackage.Settings.Sections["ConnectionString"].Parameters["SFTestDB"].Value, LeaseConnection); try { await eventProcessorHost.RegisterEventProcessorFactoryAsync(new A(connectionString, serviceBusConnection, Context), new EventProcessorOptions { MaxBatchSize = 50 }); cancellationToken.Register(async() => await eventProcessorHost.UnregisterEventProcessorAsync()); } catch (Exception e) { } finally { await base.RunAsync(cancellationToken); } }
private static void ProcessEventHub(TextWriter log) { string eventHubConnectionString = ConfigurationManager.AppSettings["eventHubConnectionString"]; string eventHubName = ConfigurationManager.AppSettings["eventHubName"]; string consumerGroup = ConfigurationManager.AppSettings["consumerGroup"]; string storageAccountName = ConfigurationManager.AppSettings["storageAccountName"]; string storageAccountKey = ConfigurationManager.AppSettings["storageAccountKey"]; bool newEventsOnly = ConfigurationManager.AppSettings["newEventsOnly"] == "true"; string storageConnectionString = string.Format("DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}", storageAccountName, storageAccountKey); ManualResetEvent oSignalProcessEnd = new ManualResetEvent(false); string eventProcessorHostName = Guid.NewGuid().ToString(); EventProcessorHost eventProcessorHost = newEventsOnly ? new EventProcessorHost(eventProcessorHostName, eventHubName, consumerGroup, eventHubConnectionString, storageConnectionString, eventProcessorHostName) : new EventProcessorHost(eventProcessorHostName, eventHubName, consumerGroup, eventHubConnectionString, storageConnectionString); log.WriteLine("Registering EventProcessor..."); var options = newEventsOnly ? new EventProcessorOptions { InitialOffsetProvider = (partitionId) => DateTime.UtcNow } : new EventProcessorOptions(); options.ExceptionReceived += (sender, e) => ProcessEventHubException(sender, e, log, oSignalProcessEnd); eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventProcessorFactory(log), options).Wait(); oSignalProcessEnd.WaitOne(); eventProcessorHost.UnregisterEventProcessorAsync().Wait(); }
public async void ReceiveOffers() { var eventProcessorFactory = new Fac(OnOfferAdded); await eventProcessorHost.RegisterEventProcessorFactoryAsync(eventProcessorFactory); //await eventProcessorHost.RegisterEventProcessorAsync<OfferEventProcessor>(); }
public async Task StartAsync(CancellationToken cancellationToken) { eventProcessorHost = new EventProcessorHost(eventProcessorHostConfig.HostName, eventProcessorHostConfig.EventHubPath, eventProcessorHostConfig.ConsumerGroupName, eventProcessorHostConfig.EventHubConnectionString, eventProcessorHostConfig.StorageConnectionString, eventProcessorHostConfig.LeaseContainerName); if (eventProcessorOptions == null) { await eventProcessorHost.RegisterEventProcessorFactoryAsync(iotHubEventProcessorFactory); } else { await eventProcessorHost.RegisterEventProcessorFactoryAsync(iotHubEventProcessorFactory, eventProcessorOptions); } }
public override async Task ConnectHandlerAsync(string queueName, Func <IEventMessage, CancellationToken, Task> messageReceived, CancellationToken cancellationToken) { _eventProcessors.Add(queueName, new AzureEventHubProcessorFactory(messageReceived, cancellationToken)); await _eventHost.RegisterEventProcessorFactoryAsync( new AzureEventHubProcessorFactory(messageReceived, cancellationToken), EventProcessorOptions.DefaultOptions); }
private async Task StartListenerAsync(EventProcessorHost eventProcessorHost, CancellationToken cancellationToken) { _logger.LogInformation("StartListenerAsync: MetricsIngestor/Starting"); // start the event listener await eventProcessorHost.RegisterEventProcessorFactoryAsync(new IngestorEventProcessorFactory(cancellationToken, _logger), GetEventProcessorOptions(60)); _serviceState = ServiceState.Running; _logger.LogInformation("StartListenerAsync: MetricsIngestor/Started"); }
protected override async Task ExecuteAsync(CancellationToken stoppingToken) { await _eventProcessor.RegisterEventProcessorFactoryAsync(new BaseballEventProcessorFactory(_hub, _config)); while (!stoppingToken.IsCancellationRequested) { await Task.Delay(500); } await _eventProcessor.UnregisterEventProcessorAsync(); }
public async Task ReceiveAsync(Action <T> action) { _eventProcessorHost = new EventProcessorHost( _eventHubName, PartitionReceiver.DefaultConsumerGroupName, _eventHubConnectionString, _storageConnectionString, _storageContainerName); await _eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventProcessorFactory <T>(action)); }
protected EventHubConsumer(EventHubMessageBus messageBus, ITopicGroupConsumerSettings consumerSettings, Func <EventHubConsumer, EventProcessor> eventProcessorFactory) { MessageBus = messageBus; EventProcessorFactory = eventProcessorFactory; Log.InfoFormat("Creating EventProcessorHost for Topic: {0}, Group: {1}", consumerSettings.Topic, consumerSettings.Group); EventProcessorHost = MessageBus.EventHubSettings.EventProcessorHostFactory(consumerSettings); var eventProcessorOptions = MessageBus.EventHubSettings.EventProcessorOptionsFactory(consumerSettings); EventProcessorHost.RegisterEventProcessorFactoryAsync(this, eventProcessorOptions).Wait(); }
public static async Task <WarmStorageCoordinator> CreateAsync(string hostName, Configuration configuration) { WarmStorageEventSource.Log.InitializingEventHubListener(configuration.EventHubName, configuration.ConsumerGroupName); Func <string, IElasticSearchWriter> elasticSearchWriterFactory = partitionId => new ElasticSearchWriter.ElasticSearchWriter( configuration.ElasticSearchUrl, configuration.ElasticSearchIndexPrefix, configuration.ElasticSearchIndexType, configuration.RetryCount ); var ns = NamespaceManager.CreateFromConnectionString(configuration.EventHubConnectionString); try { await ns.GetConsumerGroupAsync(configuration.EventHubName, configuration.ConsumerGroupName); } catch (Exception e) { WarmStorageEventSource.Log.InvalidEventHubConsumerGroupName(e, configuration.EventHubName, configuration.ConsumerGroupName); throw; } WarmStorageEventSource.Log.ConsumerGroupFound(configuration.EventHubName, configuration.ConsumerGroupName); var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, configuration.EventHubName); var buildingLookupService = new BuildingLookupService(); await buildingLookupService.InitializeAsync(); var factory = new WarmStorageEventProcessorFactory(elasticSearchWriterFactory, eventHubId, buildingLookupService); var options = new EventProcessorOptions() { MaxBatchSize = configuration.MaxBatchSize, PrefetchCount = configuration.PreFetchCount, ReceiveTimeOut = configuration.ReceiveTimeout }; options.ExceptionReceived += (s, e) => WarmStorageEventSource.Log.ErrorProcessingMessage(e.Exception, e.Action); var host = new EventProcessorHost( hostName, consumerGroupName: configuration.ConsumerGroupName, eventHubPath: configuration.EventHubName, eventHubConnectionString: configuration.EventHubConnectionString, storageConnectionString: configuration.CheckpointStorageAccount); await host.RegisterEventProcessorFactoryAsync(factory, options); return(new WarmStorageCoordinator(host)); }
async Task InvokeAfterReceiveTimeoutFalse() { const int ReceiveTimeoutInSeconds = 15; Log("Calling RegisterEventProcessorAsync with InvokeProcessorAfterReceiveTimeout=false"); var eventProcessorHost = new EventProcessorHost( string.Empty, PartitionReceiver.DefaultConsumerGroupName, this.EventHubConnectionString, this.StorageConnectionString, this.LeaseContainerName); var processorOptions = new EventProcessorOptions { ReceiveTimeout = TimeSpan.FromSeconds(ReceiveTimeoutInSeconds), InvokeProcessorAfterReceiveTimeout = false, MaxBatchSize = 100 }; var emptyBatchReceiveEvent = new AsyncAutoResetEvent(false); var processorFactory = new TestEventProcessorFactory(); processorFactory.OnCreateProcessor += (f, createArgs) => { var processor = createArgs.Item2; string partitionId = createArgs.Item1.PartitionId; processor.OnProcessEvents += (_, eventsArgs) => { int eventCount = eventsArgs.Item2 != null?eventsArgs.Item2.events.Count() : 0; Log($"Partition {partitionId} TestEventProcessor processing {eventCount} event(s)"); if (eventCount == 0) { emptyBatchReceiveEvent.Set(); } }; }; await eventProcessorHost.RegisterEventProcessorFactoryAsync(processorFactory, processorOptions); try { Log("Verifying no empty batches arrive..."); bool waitSucceeded = await emptyBatchReceiveEvent.WaitAsync(TimeSpan.FromSeconds(ReceiveTimeoutInSeconds * 2)); Assert.False(waitSucceeded, "No empty batch should have been received!"); } finally { Log("Calling UnregisterEventProcessorAsync"); await eventProcessorHost.UnregisterEventProcessorAsync(); } }
public async Task Start() { _eventProcessorHost = new EventProcessorHost( EventHubConstants.EventHubName, PartitionReceiver.DefaultConsumerGroupName, EventHubConstants.EventHubConnectionString, EventHubConstants.StorageConnectionString, EventHubConstants.StorageContainerName); await _eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventProcessorFactory(MessageReceivedHandler)); }
private async Task <bool> OnStart(StartListening arg) { _eventProcessorHost = new EventProcessorHost( "akka-iot", PartitionReceiver.DefaultConsumerGroupName, Environment.GetEnvironmentVariable("akka-iot-hub-endpoint"), Environment.GetEnvironmentVariable("akka-iot-checkpoint-storage"), "akka-iot-checkpoints"); await _eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventProcessorFactory(_system, _tableStorage)); return(true); }
protected GroupTopicConsumer(EventHubMessageBus messageBus, TopicGroup topicGroup, Func <PartitionConsumer> partitionConsumerFactory) { MessageBus = messageBus; _partitionConsumerFactory = partitionConsumerFactory; Log.InfoFormat(CultureInfo.InvariantCulture, "Creating EventProcessorHost for EventHub with Topic: {0}, Group: {1}", topicGroup.Topic, topicGroup.Group); _processorHost = MessageBus.EventHubSettings.EventProcessorHostFactory(topicGroup); var eventProcessorOptions = MessageBus.EventHubSettings.EventProcessorOptionsFactory(topicGroup); _processorHost.RegisterEventProcessorFactoryAsync(this, eventProcessorOptions).Wait(); }
public static async Task<WarmStorageCoordinator> CreateAsync(string hostName, Configuration configuration) { WarmStorageEventSource.Log.InitializingEventHubListener(configuration.EventHubName, configuration.ConsumerGroupName); Func<string, IElasticSearchWriter> elasticSearchWriterFactory = partitionId => new ElasticSearchWriter.ElasticSearchWriter( configuration.ElasticSearchUrl, configuration.ElasticSearchIndexPrefix, configuration.ElasticSearchIndexType, configuration.RetryCount ); var ns = NamespaceManager.CreateFromConnectionString(configuration.EventHubConnectionString); try { await ns.GetConsumerGroupAsync(configuration.EventHubName, configuration.ConsumerGroupName); } catch (Exception e) { WarmStorageEventSource.Log.InvalidEventHubConsumerGroupName(e, configuration.EventHubName, configuration.ConsumerGroupName); throw; } WarmStorageEventSource.Log.ConsumerGroupFound(configuration.EventHubName, configuration.ConsumerGroupName); var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, configuration.EventHubName); var buildingLookupService = new BuildingLookupService(); await buildingLookupService.InitializeAsync(); var factory = new WarmStorageEventProcessorFactory(elasticSearchWriterFactory, eventHubId, buildingLookupService); var options = new EventProcessorOptions() { MaxBatchSize = configuration.MaxBatchSize, PrefetchCount = configuration.PreFetchCount, ReceiveTimeOut = configuration.ReceiveTimeout }; options.ExceptionReceived += (s, e) => WarmStorageEventSource.Log.ErrorProcessingMessage(e.Exception, e.Action); var host = new EventProcessorHost( hostName, consumerGroupName: configuration.ConsumerGroupName, eventHubPath: configuration.EventHubName, eventHubConnectionString: configuration.EventHubConnectionString, storageConnectionString: configuration.CheckpointStorageAccount); await host.RegisterEventProcessorFactoryAsync(factory, options); return new WarmStorageCoordinator(host); }
public async Task ConsumeEventsFromHubAsync(string eventHubPath) { var connectionStringBuilder = new EventHubsConnectionStringBuilder(_eventHubConnectionString) { EntityPath = eventHubPath }; var client = EventHubClient.CreateFromConnectionString(connectionStringBuilder.ToString()); _eventProcessorHost = new EventProcessorHost(eventHubPath, PartitionReceiver.DefaultConsumerGroupName, _eventHubConnectionString, _storageConnectionString, _containerName); await _eventProcessorHost.RegisterEventProcessorFactoryAsync(_azureEventProcessorFactory); }
public async Task StartAsync() { Console.WriteLine($"Registering EventHubProcessor, connecting to {_config.ConnectionString.Split(';')[0]}..."); var eventProcessorHost = new EventProcessorHost(_config.EntityPath, _config.ConsumerGroup, _config.ConnectionString, _config.StorageConnection, _config.StorageContainer); await eventProcessorHost.RegisterEventProcessorFactoryAsync(new DeserializingEventProcessorFactory(_config.Deserializer)); Console.WriteLine("Receiving. Press ENTER to stop worker."); Console.ReadLine(); await eventProcessorHost.UnregisterEventProcessorAsync(); }
private async Task StartProcessor(CancellationToken token) { try { string hostName = Environment.MachineName; string eventHubPath = _configurationProvider.GetConfigurationSettingValue("RulesEventHub.Name"); string consumerGroup = EventHubConsumerGroup.DefaultGroupName; string eventHubConnectionString = _configurationProvider.GetConfigurationSettingValue("RulesEventHub.ConnectionString"); string storageConnectionString = _configurationProvider.GetConfigurationSettingValue("device.StorageConnectionString"); _eventProcessorHost = new EventProcessorHost( hostName, eventHubPath, consumerGroup, eventHubConnectionString, storageConnectionString); _factory = new ActionProcessorFactory( _actionLogic, _actionMappingLogic, _configurationProvider); Trace.TraceInformation("ActionEventProcessor: Registering host..."); var options = new EventProcessorOptions(); options.ExceptionReceived += OptionsOnExceptionReceived; await _eventProcessorHost.RegisterEventProcessorFactoryAsync(_factory); // processing loop while (!token.IsCancellationRequested) { Trace.TraceInformation("ActionEventProcessor: Processing..."); await Task.Delay(TimeSpan.FromMinutes(5), token); } // cleanup await _eventProcessorHost.UnregisterEventProcessorAsync(); } catch (Exception e) { Trace.TraceError("Error in ActionProcessor.StartProcessor, Exception: {0}", e.ToString()); } _isRunning = false; }
static void Main(string[] args) { Bifrost.Configuration.Configure.DiscoverAndConfigure(); var eventProcessorHostName = ""; var eventHubConnectionString = ""; var eventHubName = ""; var storageAccountName = ""; var storageAccountKey = ""; var storageConnectionString = $"DefaultEndpointsProtocol=https;AccountName={storageAccountName};AccountKey={storageAccountKey}"; var eventProcessorHost = new EventProcessorHost( eventProcessorHostName, eventHubName, EventHubConsumerGroup.DefaultGroupName, eventHubConnectionString, storageConnectionString); eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventProcessorFactory()); try { #if (DEBUG) Console.WriteLine("Processing"); Console.ReadLine(); #else var config = new JobHostConfiguration(); var host = new JobHost(config); host.RunAndBlock(); #endif } finally { eventProcessorHost.UnregisterEventProcessorAsync().Wait(); } }
async Task StartProcessor(CancellationToken token) { try { // Initialize _eventProcessorHost = new EventProcessorHost( Environment.MachineName, _configurationProvider.GetConfigurationSettingValue("eventHub.HubName").ToLowerInvariant(), EventHubConsumerGroup.DefaultGroupName, _configurationProvider.GetConfigurationSettingValue("eventHub.ConnectionString"), _configurationProvider.GetConfigurationSettingValue("eventHub.StorageConnectionString")); _factory = new DeviceAdministrationProcessorFactory(_deviceLogic, _configurationProvider); Trace.TraceInformation("DeviceEventProcessor: Registering host..."); var options = new EventProcessorOptions(); options.ExceptionReceived += OptionsOnExceptionReceived; await _eventProcessorHost.RegisterEventProcessorFactoryAsync(_factory); // processing loop while (!token.IsCancellationRequested) { Trace.TraceInformation("DeviceEventProcessor: Processing..."); await Task.Delay(TimeSpan.FromMinutes(5), token); // Any additional incremental processing can be done here (like checking states, etc). } // cleanup await _eventProcessorHost.UnregisterEventProcessorAsync(); } catch (Exception e) { Trace.TraceInformation("Error in DeviceEventProcessor.StartProcessor, Exception: {0}", e.Message); } _running = false; }
// ReSharper disable once FunctionComplexityOverflow private async void btnStart_Click(object sender, EventArgs e) { try { Cursor.Current = Cursors.WaitCursor; if (string.Compare(btnStart.Text, Start, StringComparison.OrdinalIgnoreCase) == 0) { // Validate parameters if (!ValidateParameters()) { return; } btnStart.Enabled = false; var eventHubClient = EventHubClient.CreateFromConnectionString(txtServiceBusConnectionString.Text, cboEventHub.Text); // Get the default Consumer Group eventProcessorHost = new EventProcessorHost(Guid.NewGuid().ToString(), eventHubClient.Path.ToLower(), txtConsumerGroup.Text.ToLower(), txtServiceBusConnectionString.Text, txtStorageAccountConnectionString.Text) { PartitionManagerOptions = new PartitionManagerOptions { AcquireInterval = TimeSpan.FromSeconds(10), // Default is 10 seconds RenewInterval = TimeSpan.FromSeconds(10), // Default is 10 seconds LeaseInterval = TimeSpan.FromSeconds(30) // Default value is 30 seconds } }; WriteToLog(RegisteringEventProcessor); var eventProcessorOptions = new EventProcessorOptions { InvokeProcessorAfterReceiveTimeout = true, MaxBatchSize = 100, PrefetchCount = 100, ReceiveTimeOut = TimeSpan.FromSeconds(30), }; eventProcessorOptions.ExceptionReceived += EventProcessorOptions_ExceptionReceived; var eventProcessorFactoryConfiguration = new EventProcessorFactoryConfiguration { TrackEvent = a => Invoke(new Action<Alert>(i => alertBindingList.Add(i)), a), WriteToLog = m => Invoke(new Action<string>(WriteToLog), m) }; await eventProcessorHost.RegisterEventProcessorFactoryAsync( new EventProcessorFactory<EventProcessor>(eventProcessorFactoryConfiguration), eventProcessorOptions); WriteToLog(EventProcessorRegistered); // Change button text btnStart.Text = Stop; } else { // Stop Event Processor if (eventProcessorHost != null) { await eventProcessorHost.UnregisterEventProcessorAsync(); } // Change button text btnStart.Text = Start; } } catch (Exception ex) { HandleException(ex); } finally { Cursor.Current = Cursors.Default; btnStart.Enabled = true; } }
/// <summary> /// Starts the specified host control. /// </summary> /// <param name="hostControl">The host control.</param> /// <returns></returns> public override bool Start(HostControl hostControl) { var processorOptions = new EventProcessorOptions { MaxBatchSize = 5000, // default is 300 PrefetchCount = 1000 }; _processorHost = new EventProcessorHost( Environment.MachineName, eventHubPath: ConfigurationManager.AppSettings["eventHubPath"], consumerGroupName: ConfigurationManager.AppSettings["consumerGroupName"], storageConnectionString: ConfigurationManager.ConnectionStrings["StorageConnectionString"].ConnectionString, eventHubConnectionString: ConfigurationManager.ConnectionStrings["EventHubConnectionString"].ConnectionString); using (Logger.BeginTimedOperation("RegisterEventProcessorAsync")) _processorHost.RegisterEventProcessorFactoryAsync(this, processorOptions).Wait(); return base.Start(hostControl); }
private async Task StartEventProcessorAsync() { try { var eventHubClient = EventHubClient.CreateFromConnectionString(serviceBusConnectionString, eventHubName); // Get the default Consumer Group eventProcessorHost = new EventProcessorHost(Guid.NewGuid().ToString(), eventHubClient.Path.ToLower(), consumerGroupName.ToLower(), serviceBusConnectionString, storageAccountConnectionString) { PartitionManagerOptions = new PartitionManagerOptions { AcquireInterval = TimeSpan.FromSeconds(10), // Default is 10 seconds RenewInterval = TimeSpan.FromSeconds(10), // Default is 10 seconds LeaseInterval = TimeSpan.FromSeconds(30) // Default value is 30 seconds } }; ServiceEventSource.Current.Message(RegisteringEventProcessor); var eventProcessorOptions = new EventProcessorOptions { InvokeProcessorAfterReceiveTimeout = true, MaxBatchSize = 100, PrefetchCount = 100, ReceiveTimeOut = TimeSpan.FromSeconds(30), }; eventProcessorOptions.ExceptionReceived += EventProcessorOptions_ExceptionReceived; await eventProcessorHost.RegisterEventProcessorFactoryAsync(new EventProcessorFactory<EventProcessor>(deviceActorServiceUri), eventProcessorOptions); ServiceEventSource.Current.Message(EventProcessorRegistered); } catch (Exception ex) { // Trace Error ServiceEventSource.Current.Message(ex.Message); throw; } }
protected async Task InitializeAsync( string hostName, string eventHubName, string consumerGroupName, string eventHubConnectionString, string checkpointStorageAccount, int maxBatchSize, int prefetchCount, TimeSpan receiveTimeout, int maxConcurrencyPerProcessor, IEnumerable<Type> typesToSearch, Func<string, string, ICircuitBreaker> circuitBreakerFactory, IDispatcherInstrumentationPublisher instrumentationPublisher) { Logger.Info("Initializing event hub listener for {0} ({1})", eventHubName, consumerGroupName); // Get the consumer group via the Service Bus namespace (identifies the // consumer) var ns = NamespaceManager.CreateFromConnectionString(eventHubConnectionString); try { await ns.GetConsumerGroupAsync(eventHubName, consumerGroupName); Logger.Info("Found consumer group {1} for {0}", eventHubName, consumerGroupName); } catch (Exception e) { Logger.Error(e, "Could not establish connection to {0} in event hub {1}", consumerGroupName, eventHubName); throw; } var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, eventHubName); // Use a custom event processor factory to pass parameters to the // event host processor var factory = new EventProcessorFactory( handlerResolver: new MessageHandlerResolver(typesToSearch, DependencyResolverFactory.GetResolver()), maxConcurrency: maxConcurrencyPerProcessor, circuitBreakerFactory: circuitBreakerFactory, eventHubName: eventHubId, instrumentationPublisher: instrumentationPublisher); var options = new EventProcessorOptions { MaxBatchSize = maxBatchSize, PrefetchCount = prefetchCount, ReceiveTimeOut = receiveTimeout }; options.ExceptionReceived += options_ExceptionReceived; // Create the event processor host and register via the factory _host = new EventProcessorHost( hostName, consumerGroupName: consumerGroupName, eventHubPath: eventHubName, eventHubConnectionString: eventHubConnectionString, storageConnectionString: checkpointStorageAccount ); await _host.RegisterEventProcessorFactoryAsync(factory, options); Logger.Info("Event processor registered for {0} ({1})", eventHubName, consumerGroupName); }
public static async Task<ColdStorageCoordinator> CreateAsync( string hostName, string eventHubName, string consumerGroupName, string eventHubConnectionString, string checkpointStorageAccount, int maxBatchSize, int prefetchCount, TimeSpan receiveTimeout, IReadOnlyList<string> blobWriterStorageAccounts, string containerName, int rollSizeMb, string blobPrefix, int warningLevel, int tripLevel, TimeSpan stallInterval, TimeSpan logCooldownInterval, IColdStorageInstrumentationPublisher instrumentationPublisher) { Logger.Info("Initializing event hub listener for {0} ({1})", eventHubName, consumerGroupName); var storageAccounts = blobWriterStorageAccounts .Select(CloudStorageAccount.Parse) .ToList(); Func<string, IBlobWriter> blobWriterFactory = partitionId => new RollingBlobWriter(new PartitionAndDateNamingStrategy(partitionId, blobPrefix), instrumentationPublisher, storageAccounts[Int32.Parse(partitionId) % storageAccounts.Count], containerName, rollSizeMb); var ns = NamespaceManager.CreateFromConnectionString(eventHubConnectionString); try { await ns.GetConsumerGroupAsync(eventHubName, consumerGroupName); } catch (Exception e) { Logger.Error(e, "Invalid consumer group name {0} in event hub {1}", consumerGroupName, eventHubName); throw; } Logger.Info("Found consumer group {1} for {0}", eventHubName, consumerGroupName); var eventHubId = ConfigurationHelper.GetEventHubName(ns.Address, eventHubName); var factory = new ColdStorageEventProcessorFactory( blobWriterFactory, instrumentationPublisher, CancellationToken.None, warningLevel, tripLevel, stallInterval, logCooldownInterval, eventHubId ); var options = new EventProcessorOptions() { MaxBatchSize = maxBatchSize, PrefetchCount = prefetchCount, ReceiveTimeOut = receiveTimeout, InvokeProcessorAfterReceiveTimeout = true }; options.ExceptionReceived += (s, e) => Logger.Error( e.Exception, "Error on message processing, action {0}", e.Action); var host = new EventProcessorHost( hostName, consumerGroupName: consumerGroupName, eventHubPath: eventHubName, eventHubConnectionString: eventHubConnectionString, storageConnectionString: checkpointStorageAccount); await host.RegisterEventProcessorFactoryAsync(factory, options); return new ColdStorageCoordinator(host); }