public static IEnumerable <Tweet> StreamStatuses(TwitterConfig config)
{
DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(Tweet));
var streamReader = ReadTweets(config);
while (true)
{
string line = null;
try { line = streamReader.ReadLine(); }
catch (Exception) { }
if (!string.IsNullOrWhiteSpace(line) && !line.StartsWith("{\"delete\""))
{
var result = (Tweet)jsonSerializer.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(line)));
result.RawJson = line;
yield return(result);
}
// Oops the Twitter has ended... or more likely some error have occurred.
// Reconnect to the twitter feed.
if (line == null)
{
streamReader = ReadTweets(config);
}
}
}
public static IEnumerable<Tweet> StreamStatuses(TwitterConfig config)
{
DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(Tweet));
var streamReader = ReadTweets(config);
while (true)
{
string line = null;
try { line = streamReader.ReadLine(); }
catch (Exception) { }
if (!string.IsNullOrWhiteSpace(line) && !line.StartsWith("{\"delete\""))
{
var result = (Tweet)jsonSerializer.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(line)));
result.RawJson = line;
yield return result;
}
// Oops the Twitter has ended... or more likely some error have occurred.
// Reconnect to the twitter feed.
if (line == null)
{
streamReader = ReadTweets(config);
}
}
}
public static IEnumerable <string> StreamStatuses(TwitterConfig config)
{
TextReader streamReader = ReadTweets(config);
if (streamReader == StreamReader.Null)
{
throw new Exception("Could not connect to twitter with credentials provided");
}
while (true)
{
string line = null;
try
{
line = streamReader.ReadLine();
}
catch (Exception e)
{
Console.WriteLine($"Ignoring :{e}");
}
if (!string.IsNullOrWhiteSpace(line))
{
yield return(line);
}
// Reconnect to the twitter feed.
if (line == null)
{
streamReader = ReadTweets(config);
}
}
}
static TextReader ReadTweets(TwitterConfig config)
{
var oauth_version = "1.0";
var oauth_signature_method = "HMAC-SHA1";
// unique request details
var oauth_nonce = Convert.ToBase64String(new ASCIIEncoding().GetBytes(DateTime.Now.Ticks.ToString()));
var oauth_timestamp = Convert.ToInt64(
(DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc))
.TotalSeconds).ToString();
var resource_url = "https://stream.twitter.com/1.1/statuses/filter.json";
// create oauth signature
var baseString = string.Format(
"oauth_consumer_key={0}&oauth_nonce={1}&oauth_signature_method={2}&" +
"oauth_timestamp={3}&oauth_token={4}&oauth_version={5}&track={6}",
config.OAuthConsumerKey,
oauth_nonce,
oauth_signature_method,
oauth_timestamp,
config.OAuthToken,
oauth_version,
Uri.EscapeDataString(config.Keywords));
baseString = string.Concat("POST&", Uri.EscapeDataString(resource_url), "&", Uri.EscapeDataString(baseString));
var compositeKey = string.Concat(Uri.EscapeDataString(config.OAuthConsumerSecret),
"&", Uri.EscapeDataString(config.OAuthTokenSecret));
string oauth_signature;
using (var hasher = new HMACSHA1(ASCIIEncoding.ASCII.GetBytes(compositeKey)))
{
oauth_signature = Convert.ToBase64String(
hasher.ComputeHash(ASCIIEncoding.ASCII.GetBytes(baseString)));
}
// create the request header
var authHeader = string.Format(
"OAuth oauth_nonce=\"{0}\", oauth_signature_method=\"{1}\", " +
"oauth_timestamp=\"{2}\", oauth_consumer_key=\"{3}\", " +
"oauth_token=\"{4}\", oauth_signature=\"{5}\", " +
"oauth_version=\"{6}\"",
Uri.EscapeDataString(oauth_nonce),
Uri.EscapeDataString(oauth_signature_method),
Uri.EscapeDataString(oauth_timestamp),
Uri.EscapeDataString(config.OAuthConsumerKey),
Uri.EscapeDataString(config.OAuthToken),
Uri.EscapeDataString(oauth_signature),
Uri.EscapeDataString(oauth_version)
);
// make the request
ServicePointManager.Expect100Continue = false;
var postBody = "track=" + config.Keywords;
resource_url += "?" + postBody;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(resource_url);
request.Headers.Add("Authorization", authHeader);
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.PreAuthenticate = true;
request.AllowWriteStreamBuffering = true;
request.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache);
// bail out and retry after 5 seconds
var tresponse = request.GetResponseAsync();
if (tresponse.Wait(5000))
{
return(new StreamReader(tresponse.Result.GetResponseStream()));
}
else
{
request.Abort();
return(StreamReader.Null);
}
}
static void Main(string[] args)
{
var appSettings = ConfigurationManager.AppSettings;
//Configure Twitter OAuth
var oauthToken = appSettings["oauth_token"];
var oauthTokenSecret = appSettings["oauth_token_secret"];
var oauthCustomerKey = appSettings["oauth_consumer_key"];
var oauthConsumerSecret = appSettings["oauth_consumer_secret"];
var searchGroups = appSettings["twitter_keywords"];
var removeAllUndefined = !string.IsNullOrWhiteSpace(appSettings["clear_all_with_undefined_sentiment"]) ?
Convert.ToBoolean(appSettings["clear_all_with_undefined_sentiment"])
: false;
var sendExtendedInformation = !string.IsNullOrWhiteSpace(appSettings["send_extended_information"]) ?
Convert.ToBoolean(appSettings["send_extended_information"])
: false;
var AzureOn = !string.IsNullOrWhiteSpace(appSettings["AzureOn"]) ?
Convert.ToBoolean(appSettings["AzureOn"])
: false;
var mode = appSettings["match_mode"];
var createBigFile = !string.IsNullOrWhiteSpace(appSettings["create_big_file"]) ?
Convert.ToBoolean(appSettings["create_big_file"]) : false;
long fileSizeLimit = !string.IsNullOrWhiteSpace(appSettings["filesizelimit"]) ?
Convert.ToInt64(appSettings["filesizelimit"])
: defaultFileSizeLimit;
if (fileSizeLimit < minFileSizeLimit)
{
fileSizeLimit = minFileSizeLimit;
Console.WriteLine("File size limit in config was too small and has been set to {0:N0}", fileSizeLimit);
}
var includeRetweets = !string.IsNullOrWhiteSpace(appSettings["IncludeRetweets"]) ?
Convert.ToBoolean(appSettings["IncludeRetweets"])
: false;
//Configure EventHub
var config = new EventHubConfig();
config.ConnectionString = appSettings["EventHubConnectionString"];
config.EventHubName = appSettings["EventHubName"];
var myEventHubObserver = new EventHubObserver(config, AzureOn);
var keywords = searchGroups.Contains('|') ? string.Join(",", searchGroups.Split('|')) : searchGroups;
var tweet = new Tweet();
Console.WriteLine("Searching Tweets for keywords: {0}", keywords);
var folderName = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), @"tweets\");
EnsureDirectory(folderName);
var twitterConfig = new TwitterConfig(oauthToken, oauthTokenSecret, oauthCustomerKey, oauthConsumerSecret,
keywords, searchGroups, createBigFile, folderName, includeRetweets, fileSizeLimit);
// Write out the config especially the keywords... (can only happen AFTER TwitterConfig has been created!
string path = Path.Combine(folderName, Path.ChangeExtension(twitterConfig.BigFileName, ".config"));
File.WriteAllText(path, @"CAS BDA Search (Team Pharma) for Tweets was started with the following keywords:" + Environment.NewLine + Environment.NewLine + keywords, Encoding.UTF8);
// test
foreach (var sendingPayload in tweet.StreamStatuses(twitterConfig))
{
}
// end test
//**var sendingPayload = tweet.StreamStatuses(twitterConfig).Where(e => !string.IsNullOrWhiteSpace(e.Text)).Select(t => Sentiment.ComputeScore(t, searchGroups, mode)).Select(t => new Payload { CreatedAt = t.CreatedAt, Topic = t.Topic, SentimentScore = t.SentimentScore, Author = t.UserName, Text = t.Text, SendExtended = sendExtendedInformation, Language = t.Language});
//if (removeAllUndefined)
//{
// sendingPayload = sendingPayload.Where(e => e.SentimentScore > -1);
//}
//sendingPayload.Where(e => e.Topic != "No Match").ToObservable().Subscribe(myEventHubObserver);
}
static TextReader ReadTweets(TwitterConfig config)
{
var oauth_version = "1.0";
var oauth_signature_method = "HMAC-SHA1";
// unique request details
var oauth_nonce = Convert.ToBase64String(new ASCIIEncoding().GetBytes(DateTime.Now.Ticks.ToString()));
var oauth_timestamp = Convert.ToInt64(
(DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc))
.TotalSeconds).ToString();
var resource_url = "https://stream.twitter.com/1.1/statuses/filter.json";
// create oauth signature
var baseString = string.Format(
"oauth_consumer_key={0}&oauth_nonce={1}&oauth_signature_method={2}&" +
"oauth_timestamp={3}&oauth_token={4}&oauth_version={5}&track={6}",
config.OAuthConsumerKey,
oauth_nonce,
oauth_signature_method,
oauth_timestamp,
config.OAuthToken,
oauth_version,
Uri.EscapeDataString(config.Keywords));
baseString = string.Concat("POST&", Uri.EscapeDataString(resource_url), "&", Uri.EscapeDataString(baseString));
var compositeKey = string.Concat(Uri.EscapeDataString(config.OAuthConsumerSecret),
"&", Uri.EscapeDataString(config.OAuthTokenSecret));
string oauth_signature;
using (var hasher = new HMACSHA1(ASCIIEncoding.ASCII.GetBytes(compositeKey)))
{
oauth_signature = Convert.ToBase64String(
hasher.ComputeHash(ASCIIEncoding.ASCII.GetBytes(baseString)));
}
// create the request header
var authHeader = string.Format(
"OAuth oauth_nonce=\"{0}\", oauth_signature_method=\"{1}\", " +
"oauth_timestamp=\"{2}\", oauth_consumer_key=\"{3}\", " +
"oauth_token=\"{4}\", oauth_signature=\"{5}\", " +
"oauth_version=\"{6}\"",
Uri.EscapeDataString(oauth_nonce),
Uri.EscapeDataString(oauth_signature_method),
Uri.EscapeDataString(oauth_timestamp),
Uri.EscapeDataString(config.OAuthConsumerKey),
Uri.EscapeDataString(config.OAuthToken),
Uri.EscapeDataString(oauth_signature),
Uri.EscapeDataString(oauth_version)
);
// make the request
ServicePointManager.Expect100Continue = false;
var postBody = "track=" + config.Keywords;
resource_url += "?" + postBody;
HttpWebRequest request = (HttpWebRequest)WebRequest.Create(resource_url);
request.Headers.Add("Authorization", authHeader);
request.Method = "POST";
request.ContentType = "application/x-www-form-urlencoded";
request.PreAuthenticate = true;
request.AllowWriteStreamBuffering = true;
request.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache);
// bail out and retry after 5 seconds
var tresponse = request.GetResponseAsync();
if (tresponse.Wait(5000))
return new StreamReader(tresponse.Result.GetResponseStream());
else
{
request.Abort();
return StreamReader.Null;
}
}