private void WriteToFile(string rawJsonLine, TwitterConfig config) { // need to deserialize and re-serialize to get rid of \u escaped text dynamic jsonObject = JsonConvert.DeserializeObject(rawJsonLine); var rawJson = JsonConvert.SerializeObject(jsonObject); // Add a new line to the end of the raw JSON for easier separation in the file: rawJson += Environment.NewLine; if (config.CreateBigFile) { string path = Path.Combine(config.FolderName, config.BigFileName); if (File.Exists(path)) { var currentFileSize = new FileInfo(path).Length; if (currentFileSize > config.FileSizeLimit) { // overwrite config and local variable config.InitializeBigFileName(); path = Path.Combine(config.FolderName, config.BigFileName); } } File.AppendAllText(path, rawJson, Encoding.UTF8); } else { string path = Path.Combine(config.FolderName, TwitterConfig.GetTweetFilename()); // Without Encoding it will be written as UTF-8 w/o BOM File.WriteAllText(path, rawJson); } }
private static string CreateOAuthHeader(TwitterConfig config, string baseUrl) { var oauth_version = "1.0"; var oauth_signature_method = "HMAC-SHA1"; // unique request details var oauth_nonce = Convert.ToBase64String(new ASCIIEncoding().GetBytes(DateTime.Now.Ticks.ToString())); var oauth_timestamp = Convert.ToInt64( (DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc)) .TotalSeconds).ToString(); // create oauth signature var baseString = string.Format( "oauth_consumer_key={0}&oauth_nonce={1}&oauth_signature_method={2}&" + "oauth_timestamp={3}&oauth_token={4}&oauth_version={5}&track={6}", config.OAuthConsumerKey, oauth_nonce, oauth_signature_method, oauth_timestamp, config.OAuthToken, oauth_version, Uri.EscapeDataString(config.Keywords)); baseString = string.Concat("POST&", Uri.EscapeDataString(baseUrl), "&", Uri.EscapeDataString(baseString)); var compositeKey = string.Concat(Uri.EscapeDataString(config.OAuthConsumerSecret), "&", Uri.EscapeDataString(config.OAuthTokenSecret)); string oauth_signature; using (var hasher = new HMACSHA1(Encoding.ASCII.GetBytes(compositeKey))) { oauth_signature = Convert.ToBase64String( hasher.ComputeHash(Encoding.ASCII.GetBytes(baseString))); } // create the request header return(string.Format( "OAuth oauth_nonce=\"{0}\", oauth_signature_method=\"{1}\", " + "oauth_timestamp=\"{2}\", oauth_consumer_key=\"{3}\", " + "oauth_token=\"{4}\", oauth_signature=\"{5}\", " + "oauth_version=\"{6}\"", Uri.EscapeDataString(oauth_nonce), Uri.EscapeDataString(oauth_signature_method), Uri.EscapeDataString(oauth_timestamp), Uri.EscapeDataString(config.OAuthConsumerKey), Uri.EscapeDataString(config.OAuthToken), Uri.EscapeDataString(oauth_signature), Uri.EscapeDataString(oauth_version) )); }
static TextReader ReadTweets(TwitterConfig config) { const string TwitterBaseUrlStreamingTweets = "https://stream.twitter.com/1.1/statuses/filter.json"; string authHeader = CreateOAuthHeader(config, TwitterBaseUrlStreamingTweets); // make the request ServicePointManager.Expect100Continue = false; var postBody = "track=" + HttpUtility.UrlEncode(config.Keywords); string resource_url = TwitterBaseUrlStreamingTweets + "?" + postBody; HttpWebRequest request = (HttpWebRequest)WebRequest.Create(resource_url); request.Headers.Add("Authorization", authHeader); request.Method = "POST"; request.ContentType = "application/x-www-form-urlencoded"; request.PreAuthenticate = true; request.AllowWriteStreamBuffering = true; request.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache); // bail out and retry after 5 seconds var responseTask = request.GetResponseAsync(); try { if (responseTask.Wait(5000)) { return(new StreamReader(responseTask.Result.GetResponseStream(), Encoding.UTF8)); // TWE: added Encoding.UTF8 } else { request.Abort(); return(StreamReader.Null); } } catch (AggregateException ae) { WriteException("Exception while reading tweets: {0}", ae); request.Abort(); return(StreamReader.Null); } }
public IEnumerable <Tweet> StreamStatuses(TwitterConfig config) { DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(Tweet)); var streamReader = ReadTweets(config); while (keepRunning) { string line = null; try { line = streamReader.ReadLine(); } catch (Exception) { } if (!string.IsNullOrWhiteSpace(line) && !line.StartsWith("{\"delete\"") && !line.StartsWith("{\"limit\"")) { Debug.WriteLine(line); // Sometimes the line is not correctly read which will end up in a serialization exception dynamic result = null; try { result = (Tweet)jsonSerializer.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(line))); numberTweets++; if (result.CreatedAt == null) { Console.Write("potential limit"); } else { if ((config.IncludeRetweets == false) && IsRetweet(result)) { var previousColor = Console.ForegroundColor; Console.ForegroundColor = ConsoleColor.DarkYellow; Console.WriteLine("{0} Retweet will not be processed ****", numberTweets); Console.ForegroundColor = previousColor; } else { WriteToFile(line, config); WriteToConsole(line); } } } catch (SerializationException ex1) { WriteException("Line could not be de-serialized: {0}" + line, ex1); } catch (JsonSerializationException ex2) { WriteException("Line could not be de-serialized: {0}" + line, ex2); } if (result != null) { yield return(result); } } // Oops the Twitter has ended... or more likely some error have occurred. // Reconnect to the twitter feed. if (line == null) { Console.BackgroundColor = ConsoleColor.Red; Console.WriteLine("Potential Limit reached... will reconnect to feed in an instant."); Console.BackgroundColor = ConsoleColor.Black; // As an exception we use Thread.Sleep here Thread.Sleep(1000); streamReader = ReadTweets(config); } } }
static void Main(string[] args) { var appSettings = ConfigurationManager.AppSettings; //Configure Twitter OAuth var oauthToken = appSettings["oauth_token"]; var oauthTokenSecret = appSettings["oauth_token_secret"]; var oauthCustomerKey = appSettings["oauth_consumer_key"]; var oauthConsumerSecret = appSettings["oauth_consumer_secret"]; var searchGroups = appSettings["twitter_keywords"]; var removeAllUndefined = !string.IsNullOrWhiteSpace(appSettings["clear_all_with_undefined_sentiment"]) ? Convert.ToBoolean(appSettings["clear_all_with_undefined_sentiment"]) : false; var sendExtendedInformation = !string.IsNullOrWhiteSpace(appSettings["send_extended_information"]) ? Convert.ToBoolean(appSettings["send_extended_information"]) : false; var AzureOn = !string.IsNullOrWhiteSpace(appSettings["AzureOn"]) ? Convert.ToBoolean(appSettings["AzureOn"]) : false; var mode = appSettings["match_mode"]; var createBigFile = !string.IsNullOrWhiteSpace(appSettings["create_big_file"]) ? Convert.ToBoolean(appSettings["create_big_file"]) : false; long fileSizeLimit = !string.IsNullOrWhiteSpace(appSettings["filesizelimit"]) ? Convert.ToInt64(appSettings["filesizelimit"]) : defaultFileSizeLimit; if (fileSizeLimit < minFileSizeLimit) { fileSizeLimit = minFileSizeLimit; Console.WriteLine("File size limit in config was too small and has been set to {0:N0}", fileSizeLimit); } var includeRetweets = !string.IsNullOrWhiteSpace(appSettings["IncludeRetweets"]) ? Convert.ToBoolean(appSettings["IncludeRetweets"]) : false; //Configure EventHub var config = new EventHubConfig(); config.ConnectionString = appSettings["EventHubConnectionString"]; config.EventHubName = appSettings["EventHubName"]; //var myEventHubObserver = new EventHubObserver(config, AzureOn); var keywords = searchGroups.Contains('|') ? string.Join(",", searchGroups.Split('|')) : searchGroups; var tweet = new Tweet(); Console.WriteLine("Searching Tweets for keywords: {0}", keywords); var folderName = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), @"tweets\"); EnsureDirectory(folderName); var twitterConfig = new TwitterConfig(oauthToken, oauthTokenSecret, oauthCustomerKey, oauthConsumerSecret, keywords, searchGroups, createBigFile, folderName, includeRetweets, fileSizeLimit); // Write out the config especially the keywords... (can only happen AFTER TwitterConfig has been created! string path = Path.Combine(folderName, Path.ChangeExtension(twitterConfig.BigFileName, ".config")); File.WriteAllText(path, @"CAS BDA Search (Team Pharma) for Tweets was started with the following keywords:" + Environment.NewLine + Environment.NewLine + keywords, Encoding.UTF8); // test foreach (var sendingPayload in tweet.StreamStatuses(twitterConfig)) { } // end test //**var sendingPayload = tweet.StreamStatuses(twitterConfig).Where(e => !string.IsNullOrWhiteSpace(e.Text)).Select(t => Sentiment.ComputeScore(t, searchGroups, mode)).Select(t => new Payload { CreatedAt = t.CreatedAt, Topic = t.Topic, SentimentScore = t.SentimentScore, Author = t.UserName, Text = t.Text, SendExtended = sendExtendedInformation, Language = t.Language}); //if (removeAllUndefined) //{ // sendingPayload = sendingPayload.Where(e => e.SentimentScore > -1); //} //sendingPayload.Where(e => e.Topic != "No Match").ToObservable().Subscribe(myEventHubObserver); }