public async Task <string> OpenAsync(CancellationToken cancellationToken) { var listener = new EventHubsListener(_configuration); var parser = new EventHubListenerMessageJsonParser(); var builder = new MessageRouterBuilder(); var clusteringSerializer = new CsvOutputFormatter("id", "type", "version", "enqueueTimeUtc", "gameSessionId", "lat", "lon", "geoHash", "geoHashPrecision", "geoHashCenterLat", "geoHashCenterLon", "rnd"); var clusteringDlsOutputManager = new DataLakeStoreOutputManager( clusteringSerializer, new PipelineDateFilePathAlgorithm(newFileOption: NewFileNameOptions.Every5Minutes), _serviceClientCredentials, _subscription, _dataLakeName); var clusteringConsoleOutputManager = new ConsoleOutputManager(clusteringSerializer); builder.Pipeline("clustering") .HandlesMessageType("geo-location", "1.0.0") .HandlesMessageType("geo-location", "1.0.1") .AddHandler(new RandomIntMessageHandler()) .OutputTo(clusteringConsoleOutputManager, clusteringDlsOutputManager); // Build all pipelines var router = builder.Build(); // Attach the differeing parts of the message processor together _messageProcessor = new MessageProcessor <EventHubListenerMessage>(listener, parser, router); // The following method will never exit //await messageProcessor.ProcessAndBlockAsync(); return(_configuration.EventHubPath); }
public async Task RunAsync() { // Check that all configurations are set before continuing if (!Config.Check()) { // Exiting due to missing configuration Console.WriteLine("Press any key to continue"); Console.ReadKey(true); return; } // Setup Listener. This will be the same for all pipelines we are building. var listenerConfig = new EventProcessorHostOptions { EventHubConnectionString = Config.Root[Config.NAH_EHLISTENER_CONNECTIONSTRING], EventHubPath = Config.Root[Config.NAH_EHLISTENER_EVENTHUBPATH], ConsumerGroupName = Config.Root[Config.NAH_EHLISTENER_CONSUMERGROUP], StorageConnectionString = Config.Root[Config.NAH_EHLISTENER_STORAGECONNECTIONSTRING], LeaseContainerName = Config.Root[Config.NAH_EHLISTENER_LEASECONTAINERNAME] }; var listener = new EventHubsListener(listenerConfig, new EventProcessorOptions { MaxBatchSize = 1000, PrefetchCount = 30000 }); // Setup Message Parser. By default we are using Nether JSON Messages // Setting up parser that knows how to parse those messages. var parser = new EventHubListenerMessageJsonParser { AllowDbgEnqueuedTime = true, CorruptMessageAsyncFunc = OnCorruptMessageAsync, MessageTypePropertyName = "type", UseStaticMessageVersion = true, StaticMessageVersion = "1.0.0" }; // User a builder to create routing infrastructure for messages and the pipelines var builder = new MessageRouterBuilder(); Dictionary <string, Tuple <SqlDbType, int> > columntoDatatypeMapping = new Dictionary <string, Tuple <SqlDbType, int> >() { { "event_time", Tuple.Create <SqlDbType, int>(SqlDbType.DateTime, 0) }, { "install_time", Tuple.Create <SqlDbType, int>(SqlDbType.DateTime, 0) }, { "enqueuedTimeUtc", Tuple.Create <SqlDbType, int>(SqlDbType.DateTime, 0) }, { "offline", Tuple.Create <SqlDbType, int>(SqlDbType.Int, 0) }, { "resource_change", Tuple.Create <SqlDbType, int>(SqlDbType.Int, 0) }, { "amount", Tuple.Create <SqlDbType, int>(SqlDbType.Int, 0) } }; var sqlOutputManager = new SQLDatabaseOutputManager(Config.Root[Config.NAH_AZURE_SQLUTPUTMANAGER_CONNECTIONSTRING], columntoDatatypeMapping, true); builder.DefaultPipeline .AddHandler(new UnixTimeToDateTimeMessageHandler("install_time", "event_time")) .OutputTo(sqlOutputManager); // Build all pipelines var router = builder.Build(); // Attach the differeing parts of the message processor together var messageProcessor = new MessageProcessor <EventHubListenerMessage>(listener, parser, router, OnMessageProcessorInfoAsync); // The following method will never exit await messageProcessor.ProcessAndBlockAsync(); }
public async Task RunAsync() { // Check that all configurations are set before continuing if (!Config.Check()) { // Exiting due to missing configuration Console.WriteLine("Press any key to continue"); Console.ReadKey(true); return; } // Authenticate against Azure AD once and re-use for all needed purposes var serviceClientCretentials = await ApplicationTokenProvider.LoginSilentAsync(Config.Root[Config.NAH_AAD_Domain], new ClientCredential(Config.Root[Config.NAH_AAD_CLIENTID], Config.Root[Config.NAH_AAD_CLIENTSECRET])); // Setup Listener. This will be the same for all pipelines we are building. var listenerConfig = new EventProcessorHostOptions { EventHubConnectionString = Config.Root[Config.NAH_EHLISTENER_CONNECTIONSTRING], EventHubPath = Config.Root[Config.NAH_EHLISTENER_EVENTHUBPATH], ConsumerGroupName = Config.Root[Config.NAH_EHLISTENER_CONSUMERGROUP], StorageConnectionString = Config.Root[Config.NAH_EHLISTENER_STORAGECONNECTIONSTRING], LeaseContainerName = Config.Root[Config.NAH_EHLISTENER_LEASECONTAINERNAME] }; var listener = new EventHubsListener(listenerConfig, new EventProcessorOptions { MaxBatchSize = 1000, PrefetchCount = 30000 }); // Setup Message Parser. By default we are using Nether JSON Messages // Setting up parser that knows how to parse those messages. var parser = new EventHubListenerMessageJsonParser(new ConsoleCorruptMessageHandler()) { AllowDbgEnqueuedTime = true }; // User a builder to create routing infrastructure for messages and the pipelines var builder = new MessageRouterBuilder(); var filePathAlgorithm = new DateFolderStructure(newFileOption: NewFileNameOptions.Every3Hours); // Setting up "Geo Clustering Recipe" var clusteringSerializer = new CsvMessageFormatter("id", "type", "version", "enqueuedTimeUtc", "gameSession", "lat", "lon", "geoHash", "geoHashPrecision", "geoHashCenterLat", "geoHashCenterLon", "geoHashCenterDist", "rnd"); builder.Pipeline("geoclusters") .HandlesMessageType("geo-location", 1, 0) .AddHandler(new GeoHashMessageHandler { CalculateGeoHashCenterCoordinates = true }) .AddHandler(new RandomIntMessageHandler()) .OutputTo( new FileOutputManager(clusteringSerializer, filePathAlgorithm, Config.Root[Config.NAH_FILEOUTPUTMANAGER_LOCALDATAFOLDER]), new DataLakeStoreOutputManager( clusteringSerializer, filePathAlgorithm, serviceClientCretentials, Config.Root[Config.NAH_AZURE_SUBSCRIPTIONID], Config.Root[Config.NAH_AZURE_DLSOUTPUTMANAGER_ACCOUNTNAME]) ); // Setting up "Daily Active Users Recipe" var dauSerializer = new CsvMessageFormatter("id", "type", "version", "enqueuedTimeUtc", "gameSession", "gamerTag"); builder.Pipeline("dau") .HandlesMessageType("session-start", 1, 0) .OutputTo( new FileOutputManager(dauSerializer, filePathAlgorithm, Config.Root[Config.NAH_FILEOUTPUTMANAGER_LOCALDATAFOLDER]), new DataLakeStoreOutputManager( dauSerializer, filePathAlgorithm, serviceClientCretentials, Config.Root[Config.NAH_AZURE_SUBSCRIPTIONID], Config.Root[Config.NAH_AZURE_DLSOUTPUTMANAGER_ACCOUNTNAME]) ); var sessionSerializer = new CsvMessageFormatter("id", "type", "version", "enqueuedTimeUtc", "gameSession"); builder.Pipeline("sessions") .HandlesMessageType("heartbeat", 1, 0) .OutputTo( new FileOutputManager(sessionSerializer, filePathAlgorithm, Config.Root[Config.NAH_FILEOUTPUTMANAGER_LOCALDATAFOLDER]), new DataLakeStoreOutputManager( sessionSerializer, filePathAlgorithm, serviceClientCretentials, Config.Root[Config.NAH_AZURE_SUBSCRIPTIONID], Config.Root[Config.NAH_AZURE_DLSOUTPUTMANAGER_ACCOUNTNAME]) ); builder.DefaultPipeline .AddHandler(new RandomIntMessageHandler()) .OutputTo(new ConsoleOutputManager(new CsvMessageFormatter { IncludeHeaders = false })); // Build all pipelines var router = builder.Build(); // Attach the differeing parts of the message processor together var messageProcessor = new MessageProcessor <EventHubListenerMessage>(listener, parser, router, OnMessageProcessorInfoAsync); // The following method will never exit await messageProcessor.ProcessAndBlockAsync(); }
private static void Main(string[] args) { Greet(); SetupConfigurationProviders(); // Check that all configurations are set before continuing var configStatus = CheckConfigurationStatus( NAH_EHListener_ConnectionString, NAH_EHListener_EventHubPath, NAH_EHListener_ConsumerGroup, NAH_EHListener_StorageConnectionString, NAH_EHListener_LeaseContainerName, NAH_AAD_Domain, NAH_AAD_ClientId, NAH_AAD_ClientSecret, NAH_Azure_SubscriptionId, NAH_Azure_DLSOutputManager_AccountName); if (configStatus != ConfigurationStatus.Ok) { // Exiting due to missing configuration Console.WriteLine("Press any key to continue"); Console.ReadKey(true); return; } // Setup Listener. This will be the same for all pipelines we are building. var listenerConfig = new EventHubsListenerConfiguration { EventHubConnectionString = s_configuration[NAH_EHListener_ConnectionString], EventHubPath = s_configuration[NAH_EHListener_EventHubPath], ConsumerGroupName = s_configuration[NAH_EHListener_ConsumerGroup], StorageConnectionString = s_configuration[NAH_EHListener_StorageConnectionString], LeaseContainerName = s_configuration[NAH_EHListener_LeaseContainerName] }; var listener = new EventHubsListener(listenerConfig); // Setup Message Parser. By default we are using Nether JSON Messages // Setting up parser that knows how to parse those messages. var parser = new EventHubListenerMessageJsonParser(); // User a builder to create routing infrastructure for messages and the pipelines var builder = new MessageRouterBuilder(); // Setting up "Geo Clustering Recipe" var clusteringSerializer = new CsvOutputFormatter("id", "type", "version", "enqueueTimeUtc", "gameSessionId", "lat", "lon", "geoHash", "geoHashPrecision", "geoHashCenterLat", "geoHashCenterLon"); var clusteringDlsOutputManager = new DataLakeStoreOutputManager( clusteringSerializer, new PipelineDateFilePathAlgorithm(newFileOption: NewFileNameOptions.Every5Minutes), domain: s_configuration[NAH_AAD_Domain], clientId: s_configuration[NAH_AAD_ClientId], clientSecret: s_configuration[NAH_AAD_ClientSecret], subscriptionId: s_configuration[NAH_Azure_SubscriptionId], adlsAccountName: s_configuration[NAH_Azure_DLSOutputManager_AccountName]); var clusteringConsoleOutputManager = new ConsoleOutputManager(clusteringSerializer); builder.Pipeline("clustering") .HandlesMessageType("geo-location", "1.0.0") .HandlesMessageType("geo-location", "1.0.1") .AddHandler(new GeoHashMessageHandler { CalculateGeoHashCenterCoordinates = true }) .OutputTo(clusteringConsoleOutputManager, clusteringDlsOutputManager); // Build all pipelines var router = builder.Build(); // Attach the differeing parts of the message processor together var messageProcessor = new MessageProcessor <EventHubListenerMessage>(listener, parser, router); // Run in an async context since main method is not allowed to be marked as async Task.Run(async() => { await messageProcessor.ProcessAndBlockAsync(); }).GetAwaiter().GetResult(); }