static async Task Main(string[] args)
{
//Configure Twitter OAuth
var oauthToken = ConfigurationManager.AppSettings["oauth_token"];
var oauthTokenSecret = ConfigurationManager.AppSettings["oauth_token_secret"];
var oauthCustomerKey = ConfigurationManager.AppSettings["oauth_consumer_key"];
var oauthConsumerSecret = ConfigurationManager.AppSettings["oauth_consumer_secret"];
var keywords = ConfigurationManager.AppSettings["twitter_keywords"];
var producer = new EventHubProducerClient(
ConfigurationManager.AppSettings["EventHubConnectionString"],
ConfigurationManager.AppSettings["EventHubName"],
new EventHubProducerClientOptions()
{
}
);
Console.WriteLine($"Sending data eventhub : {producer.EventHubName} PartitionCount = {(await producer.GetPartitionIdsAsync()).Count()}");
IObservable <string> twitterStream = TwitterStream.StreamStatuses(
new TwitterConfig(
oauthToken,
oauthTokenSecret,
oauthCustomerKey,
oauthConsumerSecret,
keywords))
.ToObservable();
int maxMessageSizeInBytes = 250 * 1024;
int maxSecondsToBuffer = 20;
IObservable <EventData> eventDataObserver = Observable.Create <EventData>(
outputObserver => twitterStream.Subscribe(
new EventDataGenerator(outputObserver, maxMessageSizeInBytes, maxSecondsToBuffer)));
// keep upto 5 ongoing requests.
int maxRequestsInProgress = 5;
IObservable <Task> sendTasks = eventDataObserver
.Select(e =>
{
var batch = producer.CreateBatchAsync().Result;
if (!batch.TryAdd(e))
{
throw new ArgumentOutOfRangeException("Content too big to send in a single eventhub message");
}
return(producer.SendAsync(batch));
})
.Buffer(TimeSpan.FromMinutes(1), maxRequestsInProgress)
.Select(sendTaskList => Task.WhenAll(sendTaskList));
var subscription = sendTasks.Subscribe(
sendEventDatasTask => sendEventDatasTask.Wait(),
e => Console.WriteLine(e));
}
static void Main(string[] args)
{
//Configure Twitter OAuth
var oauthToken = ConfigurationManager.AppSettings["oauth_token"];
var oauthTokenSecret = ConfigurationManager.AppSettings["oauth_token_secret"];
var oauthCustomerKey = ConfigurationManager.AppSettings["oauth_consumer_key"];
var oauthConsumerSecret = ConfigurationManager.AppSettings["oauth_consumer_secret"];
var keywords = ConfigurationManager.AppSettings["twitter_keywords"];
//Configure EventHub
var ehConnectionBuilder = new EventHubsConnectionStringBuilder(ConfigurationManager.AppSettings["EventHubConnectionString"])
{
EntityPath = ConfigurationManager.AppSettings["EventHubName"]
};
EventHubClient client = EventHubClient.CreateFromConnectionString(ehConnectionBuilder.ToString());
Console.WriteLine($"Sending data eventhub : {client.EventHubName} PartitionCount = {client.GetRuntimeInformationAsync().Result.PartitionCount}");
IObservable <string> twitterStream = TwitterStream.StreamStatuses(
new TwitterConfig(
oauthToken,
oauthTokenSecret,
oauthCustomerKey,
oauthConsumerSecret,
keywords))
.ToObservable();
int maxMessageSizeInBytes = 250 * 1024;
int maxSecondsToBuffer = 20;
IObservable <EventData> eventDataObserver = Observable.Create <EventData>(
outputObserver => twitterStream.Subscribe(
new EventDataGenerator(outputObserver, maxMessageSizeInBytes, maxSecondsToBuffer)));
// keep upto 5 ongoing requests.
int maxRequestsInProgress = 5;
IObservable <Task> sendTasks = eventDataObserver
.Select(e => client.SendAsync(e))
.Buffer(TimeSpan.FromMinutes(1), maxRequestsInProgress)
.Select(sendTaskList => Task.WhenAll(sendTaskList));
var subscription = sendTasks.Subscribe(
sendEventDatasTask => sendEventDatasTask.Wait(),
e => Console.WriteLine(e));
}
static async Task Main(string[] args)
{
// Create Azure Key Vault Helper
AzureKeyVaultHelper azureKeyVaultHelper = new AzureKeyVaultHelper();
// Set variables
string keyVaultName = ConfigurationManager.AppSettings["KeyVaultName"];
string stAccountName = ConfigurationManager.AppSettings["StAccountName"];
string stAccountSecretName = ConfigurationManager.AppSettings["StAccountSecretName"];
string stAccountKey = azureKeyVaultHelper.GetSecret(stAccountSecretName, keyVaultName);
string stAccountContainer = ConfigurationManager.AppSettings["StAccountContainer"];
string stAccountDirectory = ConfigurationManager.AppSettings["StAccountDirectory"];
string keywordFilename = ConfigurationManager.AppSettings["KeywordFilename"];
string synonymsFilename = ConfigurationManager.AppSettings["TeamsSynonyms"];
string keywordFilePath = Path.GetTempFileName();
string synonymsFilePath = Path.GetTempFileName();
string ehConnectionStringSecretName = ConfigurationManager.AppSettings["EhConnectionString"];
string ehConnectionStringSecretValue = azureKeyVaultHelper.GetSecret(ehConnectionStringSecretName, keyVaultName);
// Create Data Lake Service Client
AzureDataLakeHelper azureDataLakeHelper = new AzureDataLakeHelper(stAccountDirectory, stAccountName, stAccountKey);
DataLakeServiceClient dataLakeServiceClient = azureDataLakeHelper.GetDataLakeServiceClient();
// Get file system
DataLakeFileSystemClient dataLakeFileSystemClient = dataLakeServiceClient.GetFileSystemClient(stAccountContainer);
// Download CSVs file
await azureDataLakeHelper.DownloadFile(dataLakeFileSystemClient, keywordFilename, keywordFilePath);
await azureDataLakeHelper.DownloadFile(dataLakeFileSystemClient, synonymsFilename, synonymsFilePath);
// Create adf helper
AdfOutputReaderHelper adfOutputReaderHelper = new AdfOutputReaderHelper(keywordFilePath, synonymsFilePath);
// Read Synonyms file
List <TeamSynonym> synonymsList = adfOutputReaderHelper.ReadSynonymsFile();
// Read Keyword's file (used by EventDataGenerator)
List <EventKeyword> keywordList = adfOutputReaderHelper.ReadAdfOutputFile(synonymsList);
// Create string with all the kewyords (used by TwitterStream)
string allKeywords = String.Empty;
foreach (EventKeyword keyword in keywordList)
{
allKeywords = allKeywords == String.Empty ? keyword.Values : allKeywords + "," + keyword.Values;
}
Console.WriteLine("Keywords readed: " + allKeywords);
//Configure Twitter OAuth
var oauthCustomerKeyName = ConfigurationManager.AppSettings["OauthConsumerKey"];
var oauthConsumerSecretName = ConfigurationManager.AppSettings["OauthConsumerSecret"];
var oauthTokenName = ConfigurationManager.AppSettings["OauthToken"];
var oauthTokenSecretName = ConfigurationManager.AppSettings["OauthTokenSecret"];
var oauthCustomerKeyValue = azureKeyVaultHelper.GetSecret(oauthCustomerKeyName, keyVaultName);
var oauthConsumerSecretValue = azureKeyVaultHelper.GetSecret(oauthConsumerSecretName, keyVaultName);
var oauthTokenValue = azureKeyVaultHelper.GetSecret(oauthTokenName, keyVaultName);
var oauthTokenSecretValue = azureKeyVaultHelper.GetSecret(oauthTokenSecretName, keyVaultName);
string keywords = allKeywords;
//Configure EventHub
var ehConnectionBuilder = new EventHubsConnectionStringBuilder(ehConnectionStringSecretValue)
{
EntityPath = ConfigurationManager.AppSettings["EventHubName"]
};
EventHubClient client = EventHubClient.CreateFromConnectionString(ehConnectionBuilder.ToString());
Console.WriteLine($"Sending data eventhub : {client.EventHubName} PartitionCount = {client.GetRuntimeInformationAsync().Result.PartitionCount}");
IObservable <string> twitterStream = TwitterStream.StreamStatuses(
new TwitterConfig(
oauthTokenValue,
oauthTokenSecretValue,
oauthCustomerKeyValue,
oauthConsumerSecretValue,
keywords))
.ToObservable();
//int maxMessageSizeInBytes = 250 * 1024;
//int maxSecondsToBuffer = 20;
int maxMessageSizeInBytes = 1 * 1024;
int maxSecondsToBuffer = 1;
IObservable <EventData> eventDataObserver = Observable.Create <EventData>(
outputObserver => twitterStream.Subscribe(
new EventDataGenerator(outputObserver, maxMessageSizeInBytes, maxSecondsToBuffer, keywordList)));
// keep upto 5 ongoing requests.
int maxRequestsInProgress = 5;
IObservable <Task> sendTasks = eventDataObserver
.Select(e => client.SendAsync(e))
.Buffer(TimeSpan.FromMinutes(1), maxRequestsInProgress)
.Select(sendTaskList => Task.WhenAll(sendTaskList));
var subscription = sendTasks.Subscribe(
sendEventDatasTask => sendEventDatasTask.Wait(),
e => Console.WriteLine(e));
}
