public AnalyticsListener(EventHubsListenerConfiguration configuration,
ServiceClientCredentials servicePrinciple, string subscription, string dataLakeName)
{
_configuration = configuration;
_serviceClientCredentials = servicePrinciple;
_dataLakeName = dataLakeName;
_subscription = subscription;
}
public async Task RunAsync()
{
// Check that all configurations are set before continuing
if (!Config.Check())
{
// Exiting due to missing configuration
Console.WriteLine("Press any key to continue");
Console.ReadKey(true);
return;
}
// Authenticate against Azure AD once and re-use for all needed purposes
var serviceClientCretentials = await ApplicationTokenProvider.LoginSilentAsync(Config.Root[Config.NAH_AAD_Domain],
new ClientCredential(Config.Root[Config.NAH_AAD_CLIENTID], Config.Root[Config.NAH_AAD_CLIENTSECRET]));
// Setup Listener. This will be the same for all pipelines we are building.
var listenerConfig = new EventHubsListenerConfiguration
{
EventHubConnectionString = Config.Root[Config.NAH_EHLISTENER_CONNECTIONSTRING],
EventHubPath = Config.Root[Config.NAH_EHLISTENER_EVENTHUBPATH],
ConsumerGroupName = Config.Root[Config.NAH_EHLISTENER_CONSUMERGROUP],
StorageConnectionString = Config.Root[Config.NAH_EHLISTENER_STORAGECONNECTIONSTRING],
LeaseContainerName = Config.Root[Config.NAH_EHLISTENER_LEASECONTAINERNAME]
};
var listener = new EventHubsListener(listenerConfig);
// Setup Message Parser. By default we are using Nether JSON Messages
// Setting up parser that knows how to parse those messages.
var parser = new EventHubListenerMessageJsonParser(new ConsoleCorruptMessageHandler()) { AllowDbgEnqueuedTime = true };
// User a builder to create routing infrastructure for messages and the pipelines
var builder = new MessageRouterBuilder();
var filePathAlgorithm = new PipelineDateFilePathAlgorithm(newFileOption: NewFileNameOptions.Every3Hours);
// Setting up "Geo Clustering Recipe"
var clusteringSerializer = new CsvOutputFormatter("id", "type", "version", "enqueuedTimeUtc", "gameSession", "lat", "lon", "geoHash", "geoHashPrecision", "geoHashCenterLat", "geoHashCenterLon", "geoHashCenterDist", "rnd");
builder.Pipeline("clustering")
.HandlesMessageType("geo-location", "1.0.0")
.AddHandler(new GeoHashMessageHandler { CalculateGeoHashCenterCoordinates = true })
.AddHandler(new RandomIntMessageHandler())
.OutputTo(new ConsoleOutputManager(clusteringSerializer)
, new FileOutputManager(clusteringSerializer, filePathAlgorithm, Config.Root[Config.NAH_FILEOUTPUTMANAGER_LOCALDATAFOLDER])
, new DataLakeStoreOutputManager(
clusteringSerializer,
filePathAlgorithm,
serviceClientCretentials,
Config.Root[Config.NAH_AZURE_SUBSCRIPTIONID],
Config.Root[Config.NAH_AZURE_DLSOUTPUTMANAGER_ACCOUNTNAME])
);
// Setting up "Daily Active Users Recipe"
var dauSerializer = new CsvOutputFormatter("id", "type", "version", "enqueuedTimeUtc", "gameSession", "gamerTag");
builder.Pipeline("dau")
.HandlesMessageType("session-start", "1.0.0")
.OutputTo(new ConsoleOutputManager(dauSerializer)
, new FileOutputManager(dauSerializer, filePathAlgorithm, Config.Root[Config.NAH_FILEOUTPUTMANAGER_LOCALDATAFOLDER])
, new DataLakeStoreOutputManager(
dauSerializer,
filePathAlgorithm,
serviceClientCretentials,
Config.Root[Config.NAH_AZURE_SUBSCRIPTIONID],
Config.Root[Config.NAH_AZURE_DLSOUTPUTMANAGER_ACCOUNTNAME])
);
var sessionSerializer = new CsvOutputFormatter("id", "type", "version", "enqueuedTimeUtc", "gameSession");
builder.Pipeline("sessions")
.HandlesMessageType("heartbeat", "1.0.0")
.OutputTo(new ConsoleOutputManager(sessionSerializer)
, new FileOutputManager(sessionSerializer, filePathAlgorithm, Config.Root[Config.NAH_FILEOUTPUTMANAGER_LOCALDATAFOLDER])
, new DataLakeStoreOutputManager(
sessionSerializer,
filePathAlgorithm,
serviceClientCretentials,
Config.Root[Config.NAH_AZURE_SUBSCRIPTIONID],
Config.Root[Config.NAH_AZURE_DLSOUTPUTMANAGER_ACCOUNTNAME])
);
builder.DefaultPipeline
.AddHandler(new RandomIntMessageHandler())
.OutputTo(new ConsoleOutputManager(new CsvOutputFormatter { IncludeHeaders = false }));
// Build all pipelines
var router = builder.Build();
// Attach the differeing parts of the message processor together
var messageProcessor = new MessageProcessor<EventHubListenerMessage>(listener, parser, router);
// The following method will never exit
await messageProcessor.ProcessAndBlockAsync();
}
private static void Main(string[] args)
{
Greet();
SetupConfigurationProviders();
// Check that all configurations are set before continuing
var configStatus = CheckConfigurationStatus(
NAH_EHListener_ConnectionString,
NAH_EHListener_EventHubPath,
NAH_EHListener_ConsumerGroup,
NAH_EHListener_StorageConnectionString,
NAH_EHListener_LeaseContainerName,
NAH_AAD_Domain,
NAH_AAD_ClientId,
NAH_AAD_ClientSecret,
NAH_Azure_SubscriptionId,
NAH_Azure_DLSOutputManager_AccountName);
if (configStatus != ConfigurationStatus.Ok)
{
// Exiting due to missing configuration
Console.WriteLine("Press any key to continue");
Console.ReadKey(true);
return;
}
// Setup Listener. This will be the same for all pipelines we are building.
var listenerConfig = new EventHubsListenerConfiguration
{
EventHubConnectionString = s_configuration[NAH_EHListener_ConnectionString],
EventHubPath = s_configuration[NAH_EHListener_EventHubPath],
ConsumerGroupName = s_configuration[NAH_EHListener_ConsumerGroup],
StorageConnectionString = s_configuration[NAH_EHListener_StorageConnectionString],
LeaseContainerName = s_configuration[NAH_EHListener_LeaseContainerName]
};
var listener = new EventHubsListener(listenerConfig);
// Setup Message Parser. By default we are using Nether JSON Messages
// Setting up parser that knows how to parse those messages.
var parser = new EventHubListenerMessageJsonParser();
// User a builder to create routing infrastructure for messages and the pipelines
var builder = new MessageRouterBuilder();
// Setting up "Geo Clustering Recipe"
var clusteringSerializer = new CsvOutputFormatter("id", "type", "version", "enqueueTimeUtc", "gameSessionId", "lat", "lon", "geoHash", "geoHashPrecision", "geoHashCenterLat", "geoHashCenterLon");
var clusteringDlsOutputManager = new DataLakeStoreOutputManager(
clusteringSerializer,
new PipelineDateFilePathAlgorithm(newFileOption: NewFileNameOptions.Every5Minutes),
domain: s_configuration[NAH_AAD_Domain],
clientId: s_configuration[NAH_AAD_ClientId],
clientSecret: s_configuration[NAH_AAD_ClientSecret],
subscriptionId: s_configuration[NAH_Azure_SubscriptionId],
adlsAccountName: s_configuration[NAH_Azure_DLSOutputManager_AccountName]);
var clusteringConsoleOutputManager = new ConsoleOutputManager(clusteringSerializer);
builder.Pipeline("clustering")
.HandlesMessageType("geo-location", "1.0.0")
.HandlesMessageType("geo-location", "1.0.1")
.AddHandler(new GeoHashMessageHandler {
CalculateGeoHashCenterCoordinates = true
})
.OutputTo(clusteringConsoleOutputManager, clusteringDlsOutputManager);
// Build all pipelines
var router = builder.Build();
// Attach the differeing parts of the message processor together
var messageProcessor = new MessageProcessor <EventHubListenerMessage>(listener, parser, router);
// Run in an async context since main method is not allowed to be marked as async
Task.Run(async() =>
{
await messageProcessor.ProcessAndBlockAsync();
}).GetAwaiter().GetResult();
}
