private void GetResults(IMessageFormatter formatter, string pipeline, string messageType)
{
var start = GetDate("Start date");
var end = GetDate("End date");
if (start == null || end == null)
{
return;
}
var filePathAlgorithm = new DateFolderStructure(newFileOption: NewFileNameOptions.Every5Minutes);
IFolderStructure _filePathAlgorithm = new DateFolderStructure(newFileOption: NewFileNameOptions.Every5Minutes);
Console.Write("Root directory: ");
var path = Console.ReadLine();
var f = new FileResultsReader(formatter, _filePathAlgorithm, path, pipeline, messageType);
foreach (var msg in f.Get(start.Value, end.Value))
{
Console.WriteLine(msg.ToString());
}
}
public void GetLatestFromFileSystem(IMessageFormatter formatter, string pipeline, string messageType)
{
IFolderStructure _filePathAlgorithm = new DateFolderStructure(newFileOption: NewFileNameOptions.Every5Minutes);
Console.Write("Root directory: ");
var path = Console.ReadLine();
var f = new FileResultsReader(formatter, _filePathAlgorithm, path, pipeline, messageType);
var messages = f.GetLatest();
foreach (var msg in messages)
{
Console.WriteLine(msg.ToString());
}
}
public async Task RunAsync()
{
// Check that all configurations are set before continuing
if (!Config.Check())
{
// Exiting due to missing configuration
Console.WriteLine("Press any key to continue");
Console.ReadKey(true);
return;
}
// Authenticate against Azure AD once and re-use for all needed purposes
var serviceClientCretentials = await ApplicationTokenProvider.LoginSilentAsync(Config.Root[Config.NAH_AAD_Domain],
new ClientCredential(Config.Root[Config.NAH_AAD_CLIENTID], Config.Root[Config.NAH_AAD_CLIENTSECRET]));
// Setup Listener. This will be the same for all pipelines we are building.
var listenerConfig = new EventProcessorHostOptions
{
EventHubConnectionString = Config.Root[Config.NAH_EHLISTENER_CONNECTIONSTRING],
EventHubPath = Config.Root[Config.NAH_EHLISTENER_EVENTHUBPATH],
ConsumerGroupName = Config.Root[Config.NAH_EHLISTENER_CONSUMERGROUP],
StorageConnectionString = Config.Root[Config.NAH_EHLISTENER_STORAGECONNECTIONSTRING],
LeaseContainerName = Config.Root[Config.NAH_EHLISTENER_LEASECONTAINERNAME]
};
var listener = new EventHubsListener(listenerConfig, new EventProcessorOptions {
MaxBatchSize = 1000, PrefetchCount = 30000
});
// Setup Message Parser. By default we are using Nether JSON Messages
// Setting up parser that knows how to parse those messages.
var parser = new EventHubListenerMessageJsonParser(new ConsoleCorruptMessageHandler())
{
AllowDbgEnqueuedTime = true
};
// User a builder to create routing infrastructure for messages and the pipelines
var builder = new MessageRouterBuilder();
var filePathAlgorithm = new DateFolderStructure(newFileOption: NewFileNameOptions.Every3Hours);
// Setting up "Geo Clustering Recipe"
var clusteringSerializer = new CsvMessageFormatter("id", "type", "version", "enqueuedTimeUtc", "gameSession", "lat", "lon", "geoHash", "geoHashPrecision", "geoHashCenterLat", "geoHashCenterLon", "geoHashCenterDist", "rnd");
builder.Pipeline("geoclusters")
.HandlesMessageType("geo-location", 1, 0)
.AddHandler(new GeoHashMessageHandler {
CalculateGeoHashCenterCoordinates = true
})
.AddHandler(new RandomIntMessageHandler())
.OutputTo(
new FileOutputManager(clusteringSerializer, filePathAlgorithm, Config.Root[Config.NAH_FILEOUTPUTMANAGER_LOCALDATAFOLDER]),
new DataLakeStoreOutputManager(
clusteringSerializer,
filePathAlgorithm,
serviceClientCretentials,
Config.Root[Config.NAH_AZURE_SUBSCRIPTIONID],
Config.Root[Config.NAH_AZURE_DLSOUTPUTMANAGER_ACCOUNTNAME])
);
// Setting up "Daily Active Users Recipe"
var dauSerializer = new CsvMessageFormatter("id", "type", "version", "enqueuedTimeUtc", "gameSession", "gamerTag");
builder.Pipeline("dau")
.HandlesMessageType("session-start", 1, 0)
.OutputTo(
new FileOutputManager(dauSerializer, filePathAlgorithm, Config.Root[Config.NAH_FILEOUTPUTMANAGER_LOCALDATAFOLDER]),
new DataLakeStoreOutputManager(
dauSerializer,
filePathAlgorithm,
serviceClientCretentials,
Config.Root[Config.NAH_AZURE_SUBSCRIPTIONID],
Config.Root[Config.NAH_AZURE_DLSOUTPUTMANAGER_ACCOUNTNAME])
);
var sessionSerializer = new CsvMessageFormatter("id", "type", "version", "enqueuedTimeUtc", "gameSession");
builder.Pipeline("sessions")
.HandlesMessageType("heartbeat", 1, 0)
.OutputTo(
new FileOutputManager(sessionSerializer, filePathAlgorithm, Config.Root[Config.NAH_FILEOUTPUTMANAGER_LOCALDATAFOLDER]),
new DataLakeStoreOutputManager(
sessionSerializer,
filePathAlgorithm,
serviceClientCretentials,
Config.Root[Config.NAH_AZURE_SUBSCRIPTIONID],
Config.Root[Config.NAH_AZURE_DLSOUTPUTMANAGER_ACCOUNTNAME])
);
builder.DefaultPipeline
.AddHandler(new RandomIntMessageHandler())
.OutputTo(new ConsoleOutputManager(new CsvMessageFormatter {
IncludeHeaders = false
}));
// Build all pipelines
var router = builder.Build();
// Attach the differeing parts of the message processor together
var messageProcessor = new MessageProcessor <EventHubListenerMessage>(listener, parser, router, OnMessageProcessorInfoAsync);
// The following method will never exit
await messageProcessor.ProcessAndBlockAsync();
}
