private void GetResults(IMessageFormatter formatter, string pipeline, string messageType) { var start = GetDate("Start date"); var end = GetDate("End date"); if (start == null || end == null) { return; } var filePathAlgorithm = new DateFolderStructure(newFileOption: NewFileNameOptions.Every5Minutes); IFolderStructure _filePathAlgorithm = new DateFolderStructure(newFileOption: NewFileNameOptions.Every5Minutes); Console.Write("Root directory: "); var path = Console.ReadLine(); var f = new FileResultsReader(formatter, _filePathAlgorithm, path, pipeline, messageType); foreach (var msg in f.Get(start.Value, end.Value)) { Console.WriteLine(msg.ToString()); } }
public void GetLatestFromFileSystem(IMessageFormatter formatter, string pipeline, string messageType) { IFolderStructure _filePathAlgorithm = new DateFolderStructure(newFileOption: NewFileNameOptions.Every5Minutes); Console.Write("Root directory: "); var path = Console.ReadLine(); var f = new FileResultsReader(formatter, _filePathAlgorithm, path, pipeline, messageType); var messages = f.GetLatest(); foreach (var msg in messages) { Console.WriteLine(msg.ToString()); } }
public async Task RunAsync() { // Check that all configurations are set before continuing if (!Config.Check()) { // Exiting due to missing configuration Console.WriteLine("Press any key to continue"); Console.ReadKey(true); return; } // Authenticate against Azure AD once and re-use for all needed purposes var serviceClientCretentials = await ApplicationTokenProvider.LoginSilentAsync(Config.Root[Config.NAH_AAD_Domain], new ClientCredential(Config.Root[Config.NAH_AAD_CLIENTID], Config.Root[Config.NAH_AAD_CLIENTSECRET])); // Setup Listener. This will be the same for all pipelines we are building. var listenerConfig = new EventProcessorHostOptions { EventHubConnectionString = Config.Root[Config.NAH_EHLISTENER_CONNECTIONSTRING], EventHubPath = Config.Root[Config.NAH_EHLISTENER_EVENTHUBPATH], ConsumerGroupName = Config.Root[Config.NAH_EHLISTENER_CONSUMERGROUP], StorageConnectionString = Config.Root[Config.NAH_EHLISTENER_STORAGECONNECTIONSTRING], LeaseContainerName = Config.Root[Config.NAH_EHLISTENER_LEASECONTAINERNAME] }; var listener = new EventHubsListener(listenerConfig, new EventProcessorOptions { MaxBatchSize = 1000, PrefetchCount = 30000 }); // Setup Message Parser. By default we are using Nether JSON Messages // Setting up parser that knows how to parse those messages. var parser = new EventHubListenerMessageJsonParser(new ConsoleCorruptMessageHandler()) { AllowDbgEnqueuedTime = true }; // User a builder to create routing infrastructure for messages and the pipelines var builder = new MessageRouterBuilder(); var filePathAlgorithm = new DateFolderStructure(newFileOption: NewFileNameOptions.Every3Hours); // Setting up "Geo Clustering Recipe" var clusteringSerializer = new CsvMessageFormatter("id", "type", "version", "enqueuedTimeUtc", "gameSession", "lat", "lon", "geoHash", "geoHashPrecision", "geoHashCenterLat", "geoHashCenterLon", "geoHashCenterDist", "rnd"); builder.Pipeline("geoclusters") .HandlesMessageType("geo-location", 1, 0) .AddHandler(new GeoHashMessageHandler { CalculateGeoHashCenterCoordinates = true }) .AddHandler(new RandomIntMessageHandler()) .OutputTo( new FileOutputManager(clusteringSerializer, filePathAlgorithm, Config.Root[Config.NAH_FILEOUTPUTMANAGER_LOCALDATAFOLDER]), new DataLakeStoreOutputManager( clusteringSerializer, filePathAlgorithm, serviceClientCretentials, Config.Root[Config.NAH_AZURE_SUBSCRIPTIONID], Config.Root[Config.NAH_AZURE_DLSOUTPUTMANAGER_ACCOUNTNAME]) ); // Setting up "Daily Active Users Recipe" var dauSerializer = new CsvMessageFormatter("id", "type", "version", "enqueuedTimeUtc", "gameSession", "gamerTag"); builder.Pipeline("dau") .HandlesMessageType("session-start", 1, 0) .OutputTo( new FileOutputManager(dauSerializer, filePathAlgorithm, Config.Root[Config.NAH_FILEOUTPUTMANAGER_LOCALDATAFOLDER]), new DataLakeStoreOutputManager( dauSerializer, filePathAlgorithm, serviceClientCretentials, Config.Root[Config.NAH_AZURE_SUBSCRIPTIONID], Config.Root[Config.NAH_AZURE_DLSOUTPUTMANAGER_ACCOUNTNAME]) ); var sessionSerializer = new CsvMessageFormatter("id", "type", "version", "enqueuedTimeUtc", "gameSession"); builder.Pipeline("sessions") .HandlesMessageType("heartbeat", 1, 0) .OutputTo( new FileOutputManager(sessionSerializer, filePathAlgorithm, Config.Root[Config.NAH_FILEOUTPUTMANAGER_LOCALDATAFOLDER]), new DataLakeStoreOutputManager( sessionSerializer, filePathAlgorithm, serviceClientCretentials, Config.Root[Config.NAH_AZURE_SUBSCRIPTIONID], Config.Root[Config.NAH_AZURE_DLSOUTPUTMANAGER_ACCOUNTNAME]) ); builder.DefaultPipeline .AddHandler(new RandomIntMessageHandler()) .OutputTo(new ConsoleOutputManager(new CsvMessageFormatter { IncludeHeaders = false })); // Build all pipelines var router = builder.Build(); // Attach the differeing parts of the message processor together var messageProcessor = new MessageProcessor <EventHubListenerMessage>(listener, parser, router, OnMessageProcessorInfoAsync); // The following method will never exit await messageProcessor.ProcessAndBlockAsync(); }