public IEnumerable <Tweet> StreamStatuses(TwitterConfig config)
        {
            DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(Tweet));

            var streamReader = ReadTweets(config);

            while (keepRunning)
            {
                string line = null;
                try { line = streamReader.ReadLine(); }
                catch (Exception) { }

                if (!string.IsNullOrWhiteSpace(line) && !line.StartsWith("{\"delete\""))
                {
                    var result = (Tweet)jsonSerializer.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(line)));
                    result.RawJson = line;
                    yield return(result);
                }

                // Oops the Twitter has ended... or more likely some error have occurred.
                // Reconnect to the twitter feed.
                if (line == null)
                {
                    streamReader = ReadTweets(config);
                }
            }
        }
Exemple #2
0
        private void WriteToFile(string rawJsonLine, TwitterConfig config)
        {
            // need to deserialize and re-serialize to get rid of \u escaped text
            dynamic jsonObject = JsonConvert.DeserializeObject(rawJsonLine);
            var     rawJson    = JsonConvert.SerializeObject(jsonObject);

            // Add a new line to the end of the raw JSON for easier separation in the file:
            rawJson += Environment.NewLine;

            if (config.CreateBigFile)
            {
                string path = Path.Combine(config.FolderName, config.BigFileName);
                if (File.Exists(path))
                {
                    var currentFileSize = new FileInfo(path).Length;
                    if (currentFileSize > config.FileSizeLimit)
                    {
                        // overwrite config and local variable
                        config.InitializeBigFileName();
                        path = Path.Combine(config.FolderName, config.BigFileName);
                    }
                }
                File.AppendAllText(path, rawJson, Encoding.UTF8);
            }
            else
            {
                string path = Path.Combine(config.FolderName, TwitterConfig.GetTweetFilename());
                // Without Encoding it will be written as UTF-8 w/o BOM
                File.WriteAllText(path, rawJson);
            }
        }
Exemple #3
0
        private static string CreateOAuthHeader(TwitterConfig config, string baseUrl)
        {
            var oauth_version          = "1.0";
            var oauth_signature_method = "HMAC-SHA1";

            // unique request details
            var oauth_nonce     = Convert.ToBase64String(new ASCIIEncoding().GetBytes(DateTime.Now.Ticks.ToString()));
            var oauth_timestamp = Convert.ToInt64(
                (DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc))
                .TotalSeconds).ToString();

            // create oauth signature
            var baseString = string.Format(
                "oauth_consumer_key={0}&oauth_nonce={1}&oauth_signature_method={2}&" +
                "oauth_timestamp={3}&oauth_token={4}&oauth_version={5}&track={6}",
                config.OAuthConsumerKey,
                oauth_nonce,
                oauth_signature_method,
                oauth_timestamp,
                config.OAuthToken,
                oauth_version,
                Uri.EscapeDataString(config.Keywords));

            baseString = string.Concat("POST&", Uri.EscapeDataString(baseUrl), "&", Uri.EscapeDataString(baseString));

            var compositeKey = string.Concat(Uri.EscapeDataString(config.OAuthConsumerSecret),
                                             "&", Uri.EscapeDataString(config.OAuthTokenSecret));

            string oauth_signature;

            using (var hasher = new HMACSHA1(Encoding.ASCII.GetBytes(compositeKey)))
            {
                oauth_signature = Convert.ToBase64String(
                    hasher.ComputeHash(Encoding.ASCII.GetBytes(baseString)));
            }

            // create the request header
            return(string.Format(
                       "OAuth oauth_nonce=\"{0}\", oauth_signature_method=\"{1}\", " +
                       "oauth_timestamp=\"{2}\", oauth_consumer_key=\"{3}\", " +
                       "oauth_token=\"{4}\", oauth_signature=\"{5}\", " +
                       "oauth_version=\"{6}\"",
                       Uri.EscapeDataString(oauth_nonce),
                       Uri.EscapeDataString(oauth_signature_method),
                       Uri.EscapeDataString(oauth_timestamp),
                       Uri.EscapeDataString(config.OAuthConsumerKey),
                       Uri.EscapeDataString(config.OAuthToken),
                       Uri.EscapeDataString(oauth_signature),
                       Uri.EscapeDataString(oauth_version)
                       ));
        }
Exemple #4
0
        static TextReader ReadTweets(TwitterConfig config)
        {
            const string TwitterBaseUrlStreamingTweets = "https://stream.twitter.com/1.1/statuses/filter.json";

            string authHeader = CreateOAuthHeader(config, TwitterBaseUrlStreamingTweets);

            // make the request
            ServicePointManager.Expect100Continue = false;

            var            postBody     = "track=" + HttpUtility.UrlEncode(config.Keywords);
            string         resource_url = TwitterBaseUrlStreamingTweets + "?" + postBody;
            HttpWebRequest request      = (HttpWebRequest)WebRequest.Create(resource_url);

            request.Headers.Add("Authorization", authHeader);
            request.Method                    = "POST";
            request.ContentType               = "application/x-www-form-urlencoded";
            request.PreAuthenticate           = true;
            request.AllowWriteStreamBuffering = true;
            request.CachePolicy               = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache);

            // bail out and retry after 5 seconds
            var responseTask = request.GetResponseAsync();

            try
            {
                if (responseTask.Wait(5000))
                {
                    return(new StreamReader(responseTask.Result.GetResponseStream(), Encoding.UTF8)); // TWE: added Encoding.UTF8
                }
                else
                {
                    request.Abort();
                    return(StreamReader.Null);
                }
            }
            catch (AggregateException ae)
            {
                WriteException("Exception while reading tweets: ", ae);
                request.Abort();
                return(StreamReader.Null);
            }
        }
        static TextReader ReadTweets(TwitterConfig config)
        {
            var oauth_version          = "1.0";
            var oauth_signature_method = "HMAC-SHA1";

            // unique request details
            var oauth_nonce     = Convert.ToBase64String(new ASCIIEncoding().GetBytes(DateTime.Now.Ticks.ToString()));
            var oauth_timestamp = Convert.ToInt64(
                (DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc))
                .TotalSeconds).ToString();

            var resource_url = "https://stream.twitter.com/1.1/statuses/filter.json";

            // create oauth signature
            var baseString = string.Format(
                "oauth_consumer_key={0}&oauth_nonce={1}&oauth_signature_method={2}&" +
                "oauth_timestamp={3}&oauth_token={4}&oauth_version={5}&track={6}",
                config.OAuthConsumerKey,
                oauth_nonce,
                oauth_signature_method,
                oauth_timestamp,
                config.OAuthToken,
                oauth_version,
                Uri.EscapeDataString(config.Keywords));

            baseString = string.Concat("POST&", Uri.EscapeDataString(resource_url), "&", Uri.EscapeDataString(baseString));

            var compositeKey = string.Concat(Uri.EscapeDataString(config.OAuthConsumerSecret),
                                             "&", Uri.EscapeDataString(config.OAuthTokenSecret));

            string oauth_signature;

            using (var hasher = new HMACSHA1(ASCIIEncoding.ASCII.GetBytes(compositeKey)))
            {
                oauth_signature = Convert.ToBase64String(
                    hasher.ComputeHash(ASCIIEncoding.ASCII.GetBytes(baseString)));
            }

            // create the request header
            var authHeader = string.Format(
                "OAuth oauth_nonce=\"{0}\", oauth_signature_method=\"{1}\", " +
                "oauth_timestamp=\"{2}\", oauth_consumer_key=\"{3}\", " +
                "oauth_token=\"{4}\", oauth_signature=\"{5}\", " +
                "oauth_version=\"{6}\"",
                Uri.EscapeDataString(oauth_nonce),
                Uri.EscapeDataString(oauth_signature_method),
                Uri.EscapeDataString(oauth_timestamp),
                Uri.EscapeDataString(config.OAuthConsumerKey),
                Uri.EscapeDataString(config.OAuthToken),
                Uri.EscapeDataString(oauth_signature),
                Uri.EscapeDataString(oauth_version)
                );


            // make the request
            ServicePointManager.Expect100Continue = false;
            // updating TLS version "TLS  <  1.2 is unsupported by  Tweeter starting July 15, 2019"
            ServicePointManager.SecurityProtocol |= SecurityProtocolType.Tls12;
            var postBody = "track=" + HttpUtility.UrlEncode(config.Keywords);

            resource_url += "?" + postBody;
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(resource_url);

            request.Headers.Add("Authorization", authHeader);
            request.Method                    = "POST";
            request.ContentType               = "application/x-www-form-urlencoded";
            request.PreAuthenticate           = true;
            request.AllowWriteStreamBuffering = true;
            request.CachePolicy               = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache);

            // bail out and retry after 5 seconds
            var tresponse = request.GetResponseAsync();

            if (tresponse.Wait(5000))
            {
                return(new StreamReader(tresponse.Result.GetResponseStream()));
            }
            else
            {
                request.Abort();
                return(StreamReader.Null);
            }
        }
Exemple #6
0
        public IEnumerable <Tweet> StreamStatuses(TwitterConfig config)
        {
            DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(Tweet));

            var streamReader = ReadTweets(config);

            while (keepRunning)
            {
                string line = null;
                try { line = streamReader.ReadLine(); }
                catch (Exception) { }

                if (!string.IsNullOrWhiteSpace(line) && !line.StartsWith("{\"delete\"") && !line.StartsWith("{\"limit\""))
                {
                    Debug.WriteLine(line);
                    // Sometimes the line is not correctly read which will end up in a serialization exception
                    dynamic result = null;
                    try
                    {
                        result = (Tweet)jsonSerializer.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(line)));
                        numberTweets++;

                        if (result.CreatedAt == null)
                        {
                            Console.Write("potential limit");
                        }
                        else
                        {
                            if ((config.IncludeRetweets == false) && IsRetweet(result))
                            {
                                var previousColor = Console.ForegroundColor;
                                Console.ForegroundColor = ConsoleColor.DarkYellow;
                                Console.WriteLine("{0} Retweet will not be processed ****", numberTweets);
                                Console.ForegroundColor = previousColor;
                            }
                            else
                            {
                                WriteToFile(line, config);
                                WriteToConsole(line);
                            }
                        }
                    }
                    catch (SerializationException ex1)
                    {
                        WriteException("Line could not be de-serialized: " + line, ex1);
                    }
                    catch (JsonSerializationException ex2)
                    {
                        WriteException("Line could not be de-serialized: " + line, ex2);
                    }

                    if (result != null)
                    {
                        yield return(result);
                    }
                }

                // Oops the Twitter has ended... or more likely some error have occurred.
                // Reconnect to the twitter feed.
                if (line == null)
                {
                    Console.BackgroundColor = ConsoleColor.Red;
                    Console.WriteLine("Potential Limit reached... will reconnect to feed in an instant.");
                    Console.BackgroundColor = ConsoleColor.Black;

                    // As an exception we use Thread.Sleep here
                    Thread.Sleep(1000);

                    streamReader = ReadTweets(config);
                }
            }
        }