public static IEnumerable <Tweet> StreamStatuses(TwitterConfig config) { DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(Tweet)); var streamReader = ReadTweets(config); while (true) { string line = null; try { line = streamReader.ReadLine(); } catch (Exception) { } if (!string.IsNullOrWhiteSpace(line) && !line.StartsWith("{\"delete\"")) { var result = (Tweet)jsonSerializer.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(line))); result.RawJson = line; yield return(result); } // Oops the Twitter has ended... or more likely some error have occurred. // Reconnect to the twitter feed. if (line == null) { streamReader = ReadTweets(config); } } }
public static IEnumerable<Tweet> StreamStatuses(TwitterConfig config) { DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(Tweet)); var streamReader = ReadTweets(config); while (true) { string line = null; try { line = streamReader.ReadLine(); } catch (Exception) { } if (!string.IsNullOrWhiteSpace(line) && !line.StartsWith("{\"delete\"")) { var result = (Tweet)jsonSerializer.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(line))); result.RawJson = line; yield return result; } // Oops the Twitter has ended... or more likely some error have occurred. // Reconnect to the twitter feed. if (line == null) { streamReader = ReadTweets(config); } } }
public static IEnumerable <string> StreamStatuses(TwitterConfig config) { TextReader streamReader = ReadTweets(config); if (streamReader == StreamReader.Null) { throw new Exception("Could not connect to twitter with credentials provided"); } while (true) { string line = null; try { line = streamReader.ReadLine(); } catch (Exception e) { Console.WriteLine($"Ignoring :{e}"); } if (!string.IsNullOrWhiteSpace(line)) { yield return(line); } // Reconnect to the twitter feed. if (line == null) { streamReader = ReadTweets(config); } } }
static TextReader ReadTweets(TwitterConfig config) { var oauth_version = "1.0"; var oauth_signature_method = "HMAC-SHA1"; // unique request details var oauth_nonce = Convert.ToBase64String(new ASCIIEncoding().GetBytes(DateTime.Now.Ticks.ToString())); var oauth_timestamp = Convert.ToInt64( (DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc)) .TotalSeconds).ToString(); var resource_url = "https://stream.twitter.com/1.1/statuses/filter.json"; // create oauth signature var baseString = string.Format( "oauth_consumer_key={0}&oauth_nonce={1}&oauth_signature_method={2}&" + "oauth_timestamp={3}&oauth_token={4}&oauth_version={5}&track={6}", config.OAuthConsumerKey, oauth_nonce, oauth_signature_method, oauth_timestamp, config.OAuthToken, oauth_version, Uri.EscapeDataString(config.Keywords)); baseString = string.Concat("POST&", Uri.EscapeDataString(resource_url), "&", Uri.EscapeDataString(baseString)); var compositeKey = string.Concat(Uri.EscapeDataString(config.OAuthConsumerSecret), "&", Uri.EscapeDataString(config.OAuthTokenSecret)); string oauth_signature; using (var hasher = new HMACSHA1(ASCIIEncoding.ASCII.GetBytes(compositeKey))) { oauth_signature = Convert.ToBase64String( hasher.ComputeHash(ASCIIEncoding.ASCII.GetBytes(baseString))); } // create the request header var authHeader = string.Format( "OAuth oauth_nonce=\"{0}\", oauth_signature_method=\"{1}\", " + "oauth_timestamp=\"{2}\", oauth_consumer_key=\"{3}\", " + "oauth_token=\"{4}\", oauth_signature=\"{5}\", " + "oauth_version=\"{6}\"", Uri.EscapeDataString(oauth_nonce), Uri.EscapeDataString(oauth_signature_method), Uri.EscapeDataString(oauth_timestamp), Uri.EscapeDataString(config.OAuthConsumerKey), Uri.EscapeDataString(config.OAuthToken), Uri.EscapeDataString(oauth_signature), Uri.EscapeDataString(oauth_version) ); // make the request ServicePointManager.Expect100Continue = false; var postBody = "track=" + config.Keywords; resource_url += "?" + postBody; HttpWebRequest request = (HttpWebRequest)WebRequest.Create(resource_url); request.Headers.Add("Authorization", authHeader); request.Method = "POST"; request.ContentType = "application/x-www-form-urlencoded"; request.PreAuthenticate = true; request.AllowWriteStreamBuffering = true; request.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache); // bail out and retry after 5 seconds var tresponse = request.GetResponseAsync(); if (tresponse.Wait(5000)) { return(new StreamReader(tresponse.Result.GetResponseStream())); } else { request.Abort(); return(StreamReader.Null); } }
static void Main(string[] args) { var appSettings = ConfigurationManager.AppSettings; //Configure Twitter OAuth var oauthToken = appSettings["oauth_token"]; var oauthTokenSecret = appSettings["oauth_token_secret"]; var oauthCustomerKey = appSettings["oauth_consumer_key"]; var oauthConsumerSecret = appSettings["oauth_consumer_secret"]; var searchGroups = appSettings["twitter_keywords"]; var removeAllUndefined = !string.IsNullOrWhiteSpace(appSettings["clear_all_with_undefined_sentiment"]) ? Convert.ToBoolean(appSettings["clear_all_with_undefined_sentiment"]) : false; var sendExtendedInformation = !string.IsNullOrWhiteSpace(appSettings["send_extended_information"]) ? Convert.ToBoolean(appSettings["send_extended_information"]) : false; var AzureOn = !string.IsNullOrWhiteSpace(appSettings["AzureOn"]) ? Convert.ToBoolean(appSettings["AzureOn"]) : false; var mode = appSettings["match_mode"]; var createBigFile = !string.IsNullOrWhiteSpace(appSettings["create_big_file"]) ? Convert.ToBoolean(appSettings["create_big_file"]) : false; long fileSizeLimit = !string.IsNullOrWhiteSpace(appSettings["filesizelimit"]) ? Convert.ToInt64(appSettings["filesizelimit"]) : defaultFileSizeLimit; if (fileSizeLimit < minFileSizeLimit) { fileSizeLimit = minFileSizeLimit; Console.WriteLine("File size limit in config was too small and has been set to {0:N0}", fileSizeLimit); } var includeRetweets = !string.IsNullOrWhiteSpace(appSettings["IncludeRetweets"]) ? Convert.ToBoolean(appSettings["IncludeRetweets"]) : false; //Configure EventHub var config = new EventHubConfig(); config.ConnectionString = appSettings["EventHubConnectionString"]; config.EventHubName = appSettings["EventHubName"]; var myEventHubObserver = new EventHubObserver(config, AzureOn); var keywords = searchGroups.Contains('|') ? string.Join(",", searchGroups.Split('|')) : searchGroups; var tweet = new Tweet(); Console.WriteLine("Searching Tweets for keywords: {0}", keywords); var folderName = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), @"tweets\"); EnsureDirectory(folderName); var twitterConfig = new TwitterConfig(oauthToken, oauthTokenSecret, oauthCustomerKey, oauthConsumerSecret, keywords, searchGroups, createBigFile, folderName, includeRetweets, fileSizeLimit); // Write out the config especially the keywords... (can only happen AFTER TwitterConfig has been created! string path = Path.Combine(folderName, Path.ChangeExtension(twitterConfig.BigFileName, ".config")); File.WriteAllText(path, @"CAS BDA Search (Team Pharma) for Tweets was started with the following keywords:" + Environment.NewLine + Environment.NewLine + keywords, Encoding.UTF8); // test foreach (var sendingPayload in tweet.StreamStatuses(twitterConfig)) { } // end test //**var sendingPayload = tweet.StreamStatuses(twitterConfig).Where(e => !string.IsNullOrWhiteSpace(e.Text)).Select(t => Sentiment.ComputeScore(t, searchGroups, mode)).Select(t => new Payload { CreatedAt = t.CreatedAt, Topic = t.Topic, SentimentScore = t.SentimentScore, Author = t.UserName, Text = t.Text, SendExtended = sendExtendedInformation, Language = t.Language}); //if (removeAllUndefined) //{ // sendingPayload = sendingPayload.Where(e => e.SentimentScore > -1); //} //sendingPayload.Where(e => e.Topic != "No Match").ToObservable().Subscribe(myEventHubObserver); }
static TextReader ReadTweets(TwitterConfig config) { var oauth_version = "1.0"; var oauth_signature_method = "HMAC-SHA1"; // unique request details var oauth_nonce = Convert.ToBase64String(new ASCIIEncoding().GetBytes(DateTime.Now.Ticks.ToString())); var oauth_timestamp = Convert.ToInt64( (DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc)) .TotalSeconds).ToString(); var resource_url = "https://stream.twitter.com/1.1/statuses/filter.json"; // create oauth signature var baseString = string.Format( "oauth_consumer_key={0}&oauth_nonce={1}&oauth_signature_method={2}&" + "oauth_timestamp={3}&oauth_token={4}&oauth_version={5}&track={6}", config.OAuthConsumerKey, oauth_nonce, oauth_signature_method, oauth_timestamp, config.OAuthToken, oauth_version, Uri.EscapeDataString(config.Keywords)); baseString = string.Concat("POST&", Uri.EscapeDataString(resource_url), "&", Uri.EscapeDataString(baseString)); var compositeKey = string.Concat(Uri.EscapeDataString(config.OAuthConsumerSecret), "&", Uri.EscapeDataString(config.OAuthTokenSecret)); string oauth_signature; using (var hasher = new HMACSHA1(ASCIIEncoding.ASCII.GetBytes(compositeKey))) { oauth_signature = Convert.ToBase64String( hasher.ComputeHash(ASCIIEncoding.ASCII.GetBytes(baseString))); } // create the request header var authHeader = string.Format( "OAuth oauth_nonce=\"{0}\", oauth_signature_method=\"{1}\", " + "oauth_timestamp=\"{2}\", oauth_consumer_key=\"{3}\", " + "oauth_token=\"{4}\", oauth_signature=\"{5}\", " + "oauth_version=\"{6}\"", Uri.EscapeDataString(oauth_nonce), Uri.EscapeDataString(oauth_signature_method), Uri.EscapeDataString(oauth_timestamp), Uri.EscapeDataString(config.OAuthConsumerKey), Uri.EscapeDataString(config.OAuthToken), Uri.EscapeDataString(oauth_signature), Uri.EscapeDataString(oauth_version) ); // make the request ServicePointManager.Expect100Continue = false; var postBody = "track=" + config.Keywords; resource_url += "?" + postBody; HttpWebRequest request = (HttpWebRequest)WebRequest.Create(resource_url); request.Headers.Add("Authorization", authHeader); request.Method = "POST"; request.ContentType = "application/x-www-form-urlencoded"; request.PreAuthenticate = true; request.AllowWriteStreamBuffering = true; request.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache); // bail out and retry after 5 seconds var tresponse = request.GetResponseAsync(); if (tresponse.Wait(5000)) return new StreamReader(tresponse.Result.GetResponseStream()); else { request.Abort(); return StreamReader.Null; } }