private void WriteToFile(string rawJsonLine, TwitterConfig config)
{
// need to deserialize and re-serialize to get rid of \u escaped text
dynamic jsonObject = JsonConvert.DeserializeObject(rawJsonLine);
var rawJson = JsonConvert.SerializeObject(jsonObject);
// Add a new line to the end of the raw JSON for easier separation in the file:
rawJson += Environment.NewLine;
if (config.CreateBigFile)
{
string path = Path.Combine(config.FolderName, config.BigFileName);
if (File.Exists(path))
{
var currentFileSize = new FileInfo(path).Length;
if (currentFileSize > config.FileSizeLimit)
{
// overwrite config and local variable
config.InitializeBigFileName();
path = Path.Combine(config.FolderName, config.BigFileName);
}
}
File.AppendAllText(path, rawJson, Encoding.UTF8);
}
else
{
string path = Path.Combine(config.FolderName, TwitterConfig.GetTweetFilename());
// Without Encoding it will be written as UTF-8 w/o BOM
File.WriteAllText(path, rawJson);
}
}
private static string CreateOAuthHeader(TwitterConfig config, string baseUrl)
{
var oauth_version = "1.0";
var oauth_signature_method = "HMAC-SHA1";
// unique request details
var oauth_nonce = Convert.ToBase64String(new ASCIIEncoding().GetBytes(DateTime.Now.Ticks.ToString()));
var oauth_timestamp = Convert.ToInt64(
(DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc))
.TotalSeconds).ToString();
// create oauth signature
var baseString = string.Format(
"oauth_consumer_key={0}&oauth_nonce={1}&oauth_signature_method={2}&" +
"oauth_timestamp={3}&oauth_token={4}&oauth_version={5}&track={6}",
config.OAuthConsumerKey,
oauth_nonce,
oauth_signature_method,
oauth_timestamp,
config.OAuthToken,
oauth_version,
Uri.EscapeDataString(config.Keywords));
baseString = string.Concat("POST&", Uri.EscapeDataString(baseUrl), "&", Uri.EscapeDataString(baseString));
var compositeKey = string.Concat(Uri.EscapeDataString(config.OAuthConsumerSecret),
"&", Uri.EscapeDataString(config.OAuthTokenSecret));
string oauth_signature;
using (var hasher = new HMACSHA1(Encoding.ASCII.GetBytes(compositeKey)))
{
oauth_signature = Convert.ToBase64String(
hasher.ComputeHash(Encoding.ASCII.GetBytes(baseString)));
}
// create the request header
return(string.Format(
"OAuth oauth_nonce=\"{0}\", oauth_signature_method=\"{1}\", " +
"oauth_timestamp=\"{2}\", oauth_consumer_key=\"{3}\", " +
"oauth_token=\"{4}\", oauth_signature=\"{5}\", " +
"oauth_version=\"{6}\"",
Uri.EscapeDataString(oauth_nonce),
Uri.EscapeDataString(oauth_signature_method),
Uri.EscapeDataString(oauth_timestamp),
Uri.EscapeDataString(config.OAuthConsumerKey),
Uri.EscapeDataString(config.OAuthToken),
Uri.EscapeDataString(oauth_signature),
Uri.EscapeDataString(oauth_version)
));
}
static TextReader ReadTweets(TwitterConfig config)
{
const string TwitterBaseUrlStreamingTweets = "https://stream.twitter.com/1.1/statuses/filter.json";
string authHeader = CreateOAuthHeader(config, TwitterBaseUrlStreamingTweets);
// make the request
ServicePointManager.Expect100Continue = false;
var postBody = "track=" + HttpUtility.UrlEncode(config.Keywords);
string resource_url = TwitterBaseUrlStreamingTweets + "?" + postBody;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(resource_url);
request.Headers.Add("Authorization", authHeader);
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.PreAuthenticate = true;
request.AllowWriteStreamBuffering = true;
request.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache);
// bail out and retry after 5 seconds
var responseTask = request.GetResponseAsync();
try
{
if (responseTask.Wait(5000))
{
return(new StreamReader(responseTask.Result.GetResponseStream(), Encoding.UTF8)); // TWE: added Encoding.UTF8
}
else
{
request.Abort();
return(StreamReader.Null);
}
}
catch (AggregateException ae)
{
WriteException("Exception while reading tweets: {0}", ae);
request.Abort();
return(StreamReader.Null);
}
}
public IEnumerable <Tweet> StreamStatuses(TwitterConfig config)
{
DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(Tweet));
var streamReader = ReadTweets(config);
while (keepRunning)
{
string line = null;
try { line = streamReader.ReadLine(); }
catch (Exception) { }
if (!string.IsNullOrWhiteSpace(line) && !line.StartsWith("{\"delete\"") && !line.StartsWith("{\"limit\""))
{
Debug.WriteLine(line);
// Sometimes the line is not correctly read which will end up in a serialization exception
dynamic result = null;
try
{
result = (Tweet)jsonSerializer.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(line)));
numberTweets++;
if (result.CreatedAt == null)
{
Console.Write("potential limit");
}
else
{
if ((config.IncludeRetweets == false) && IsRetweet(result))
{
var previousColor = Console.ForegroundColor;
Console.ForegroundColor = ConsoleColor.DarkYellow;
Console.WriteLine("{0} Retweet will not be processed ****", numberTweets);
Console.ForegroundColor = previousColor;
}
else
{
WriteToFile(line, config);
WriteToConsole(line);
}
}
}
catch (SerializationException ex1)
{
WriteException("Line could not be de-serialized: {0}" + line, ex1);
}
catch (JsonSerializationException ex2)
{
WriteException("Line could not be de-serialized: {0}" + line, ex2);
}
if (result != null)
{
yield return(result);
}
}
// Oops the Twitter has ended... or more likely some error have occurred.
// Reconnect to the twitter feed.
if (line == null)
{
Console.BackgroundColor = ConsoleColor.Red;
Console.WriteLine("Potential Limit reached... will reconnect to feed in an instant.");
Console.BackgroundColor = ConsoleColor.Black;
// As an exception we use Thread.Sleep here
Thread.Sleep(1000);
streamReader = ReadTweets(config);
}
}
}
static void Main(string[] args)
{
var appSettings = ConfigurationManager.AppSettings;
//Configure Twitter OAuth
var oauthToken = appSettings["oauth_token"];
var oauthTokenSecret = appSettings["oauth_token_secret"];
var oauthCustomerKey = appSettings["oauth_consumer_key"];
var oauthConsumerSecret = appSettings["oauth_consumer_secret"];
var searchGroups = appSettings["twitter_keywords"];
var removeAllUndefined = !string.IsNullOrWhiteSpace(appSettings["clear_all_with_undefined_sentiment"]) ?
Convert.ToBoolean(appSettings["clear_all_with_undefined_sentiment"])
: false;
var sendExtendedInformation = !string.IsNullOrWhiteSpace(appSettings["send_extended_information"]) ?
Convert.ToBoolean(appSettings["send_extended_information"])
: false;
var AzureOn = !string.IsNullOrWhiteSpace(appSettings["AzureOn"]) ?
Convert.ToBoolean(appSettings["AzureOn"])
: false;
var mode = appSettings["match_mode"];
var createBigFile = !string.IsNullOrWhiteSpace(appSettings["create_big_file"]) ?
Convert.ToBoolean(appSettings["create_big_file"]) : false;
long fileSizeLimit = !string.IsNullOrWhiteSpace(appSettings["filesizelimit"]) ?
Convert.ToInt64(appSettings["filesizelimit"])
: defaultFileSizeLimit;
if (fileSizeLimit < minFileSizeLimit)
{
fileSizeLimit = minFileSizeLimit;
Console.WriteLine("File size limit in config was too small and has been set to {0:N0}", fileSizeLimit);
}
var includeRetweets = !string.IsNullOrWhiteSpace(appSettings["IncludeRetweets"]) ?
Convert.ToBoolean(appSettings["IncludeRetweets"])
: false;
//Configure EventHub
var config = new EventHubConfig();
config.ConnectionString = appSettings["EventHubConnectionString"];
config.EventHubName = appSettings["EventHubName"];
//var myEventHubObserver = new EventHubObserver(config, AzureOn);
var keywords = searchGroups.Contains('|') ? string.Join(",", searchGroups.Split('|')) : searchGroups;
var tweet = new Tweet();
Console.WriteLine("Searching Tweets for keywords: {0}", keywords);
var folderName = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), @"tweets\");
EnsureDirectory(folderName);
var twitterConfig = new TwitterConfig(oauthToken, oauthTokenSecret, oauthCustomerKey, oauthConsumerSecret,
keywords, searchGroups, createBigFile, folderName, includeRetweets, fileSizeLimit);
// Write out the config especially the keywords... (can only happen AFTER TwitterConfig has been created!
string path = Path.Combine(folderName, Path.ChangeExtension(twitterConfig.BigFileName, ".config"));
File.WriteAllText(path, @"CAS BDA Search (Team Pharma) for Tweets was started with the following keywords:" + Environment.NewLine + Environment.NewLine + keywords, Encoding.UTF8);
// test
foreach (var sendingPayload in tweet.StreamStatuses(twitterConfig))
{
}
// end test
//**var sendingPayload = tweet.StreamStatuses(twitterConfig).Where(e => !string.IsNullOrWhiteSpace(e.Text)).Select(t => Sentiment.ComputeScore(t, searchGroups, mode)).Select(t => new Payload { CreatedAt = t.CreatedAt, Topic = t.Topic, SentimentScore = t.SentimentScore, Author = t.UserName, Text = t.Text, SendExtended = sendExtendedInformation, Language = t.Language});
//if (removeAllUndefined)
//{
// sendingPayload = sendingPayload.Where(e => e.SentimentScore > -1);
//}
//sendingPayload.Where(e => e.Topic != "No Match").ToObservable().Subscribe(myEventHubObserver);
}