コード例 #1
0
        public static IEnumerable <Tweet> StreamStatuses(TwitterConfig config)
        {
            DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(Tweet));

            var streamReader = ReadTweets(config);

            while (true)
            {
                string line = null;
                try { line = streamReader.ReadLine(); }
                catch (Exception) { }

                if (!string.IsNullOrWhiteSpace(line) && !line.StartsWith("{\"delete\""))
                {
                    var result = (Tweet)jsonSerializer.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(line)));
                    result.RawJson = line;
                    yield return(result);
                }

                // Oops the Twitter has ended... or more likely some error have occurred.
                // Reconnect to the twitter feed.
                if (line == null)
                {
                    streamReader = ReadTweets(config);
                }
            }
        }
コード例 #2
0
        public static IEnumerable<Tweet> StreamStatuses(TwitterConfig config)
        {
            DataContractJsonSerializer jsonSerializer = new DataContractJsonSerializer(typeof(Tweet));

            var streamReader = ReadTweets(config);

            while (true)
            {
                string line = null;
                try { line = streamReader.ReadLine(); }
                catch (Exception) { }

                if (!string.IsNullOrWhiteSpace(line) && !line.StartsWith("{\"delete\""))
                {
                    var result = (Tweet)jsonSerializer.ReadObject(new MemoryStream(Encoding.UTF8.GetBytes(line)));
                    result.RawJson = line;
                    yield return result;
                }

                // Oops the Twitter has ended... or more likely some error have occurred.
                // Reconnect to the twitter feed.
                if (line == null)
                {
                    streamReader = ReadTweets(config);
                }
            }
        }
コード例 #3
0
        public static IEnumerable <string> StreamStatuses(TwitterConfig config)
        {
            TextReader streamReader = ReadTweets(config);

            if (streamReader == StreamReader.Null)
            {
                throw new Exception("Could not connect to twitter with credentials provided");
            }

            while (true)
            {
                string line = null;
                try
                {
                    line = streamReader.ReadLine();
                }
                catch (Exception e)
                {
                    Console.WriteLine($"Ignoring :{e}");
                }

                if (!string.IsNullOrWhiteSpace(line))
                {
                    yield return(line);
                }

                // Reconnect to the twitter feed.
                if (line == null)
                {
                    streamReader = ReadTweets(config);
                }
            }
        }
コード例 #4
0
        static TextReader ReadTweets(TwitterConfig config)
        {
            var oauth_version          = "1.0";
            var oauth_signature_method = "HMAC-SHA1";

            // unique request details
            var oauth_nonce     = Convert.ToBase64String(new ASCIIEncoding().GetBytes(DateTime.Now.Ticks.ToString()));
            var oauth_timestamp = Convert.ToInt64(
                (DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc))
                .TotalSeconds).ToString();

            var resource_url = "https://stream.twitter.com/1.1/statuses/filter.json";

            // create oauth signature
            var baseString = string.Format(
                "oauth_consumer_key={0}&oauth_nonce={1}&oauth_signature_method={2}&" +
                "oauth_timestamp={3}&oauth_token={4}&oauth_version={5}&track={6}",
                config.OAuthConsumerKey,
                oauth_nonce,
                oauth_signature_method,
                oauth_timestamp,
                config.OAuthToken,
                oauth_version,
                Uri.EscapeDataString(config.Keywords));

            baseString = string.Concat("POST&", Uri.EscapeDataString(resource_url), "&", Uri.EscapeDataString(baseString));

            var compositeKey = string.Concat(Uri.EscapeDataString(config.OAuthConsumerSecret),
                                             "&", Uri.EscapeDataString(config.OAuthTokenSecret));

            string oauth_signature;

            using (var hasher = new HMACSHA1(ASCIIEncoding.ASCII.GetBytes(compositeKey)))
            {
                oauth_signature = Convert.ToBase64String(
                    hasher.ComputeHash(ASCIIEncoding.ASCII.GetBytes(baseString)));
            }

            // create the request header
            var authHeader = string.Format(
                "OAuth oauth_nonce=\"{0}\", oauth_signature_method=\"{1}\", " +
                "oauth_timestamp=\"{2}\", oauth_consumer_key=\"{3}\", " +
                "oauth_token=\"{4}\", oauth_signature=\"{5}\", " +
                "oauth_version=\"{6}\"",
                Uri.EscapeDataString(oauth_nonce),
                Uri.EscapeDataString(oauth_signature_method),
                Uri.EscapeDataString(oauth_timestamp),
                Uri.EscapeDataString(config.OAuthConsumerKey),
                Uri.EscapeDataString(config.OAuthToken),
                Uri.EscapeDataString(oauth_signature),
                Uri.EscapeDataString(oauth_version)
                );


            // make the request
            ServicePointManager.Expect100Continue = false;

            var postBody = "track=" + config.Keywords;

            resource_url += "?" + postBody;
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(resource_url);

            request.Headers.Add("Authorization", authHeader);
            request.Method                    = "POST";
            request.ContentType               = "application/x-www-form-urlencoded";
            request.PreAuthenticate           = true;
            request.AllowWriteStreamBuffering = true;
            request.CachePolicy               = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache);

            // bail out and retry after 5 seconds
            var tresponse = request.GetResponseAsync();

            if (tresponse.Wait(5000))
            {
                return(new StreamReader(tresponse.Result.GetResponseStream()));
            }
            else
            {
                request.Abort();
                return(StreamReader.Null);
            }
        }
コード例 #5
0
ファイル: Program.cs プロジェクト: tweing/CasBda
        static void Main(string[] args)
        {
            var appSettings = ConfigurationManager.AppSettings;

            //Configure Twitter OAuth
            var oauthToken          = appSettings["oauth_token"];
            var oauthTokenSecret    = appSettings["oauth_token_secret"];
            var oauthCustomerKey    = appSettings["oauth_consumer_key"];
            var oauthConsumerSecret = appSettings["oauth_consumer_secret"];
            var searchGroups        = appSettings["twitter_keywords"];
            var removeAllUndefined  = !string.IsNullOrWhiteSpace(appSettings["clear_all_with_undefined_sentiment"]) ?
                                      Convert.ToBoolean(appSettings["clear_all_with_undefined_sentiment"])
                : false;
            var sendExtendedInformation = !string.IsNullOrWhiteSpace(appSettings["send_extended_information"]) ?
                                          Convert.ToBoolean(appSettings["send_extended_information"])
                : false;
            var AzureOn = !string.IsNullOrWhiteSpace(appSettings["AzureOn"]) ?
                          Convert.ToBoolean(appSettings["AzureOn"])
                : false;
            var mode          = appSettings["match_mode"];
            var createBigFile = !string.IsNullOrWhiteSpace(appSettings["create_big_file"]) ?
                                Convert.ToBoolean(appSettings["create_big_file"]) : false;

            long fileSizeLimit = !string.IsNullOrWhiteSpace(appSettings["filesizelimit"]) ?
                                 Convert.ToInt64(appSettings["filesizelimit"])
                : defaultFileSizeLimit;

            if (fileSizeLimit < minFileSizeLimit)
            {
                fileSizeLimit = minFileSizeLimit;
                Console.WriteLine("File size limit in config was too small and has been set to {0:N0}", fileSizeLimit);
            }

            var includeRetweets = !string.IsNullOrWhiteSpace(appSettings["IncludeRetweets"]) ?
                                  Convert.ToBoolean(appSettings["IncludeRetweets"])
                : false;

            //Configure EventHub
            var config = new EventHubConfig();

            config.ConnectionString = appSettings["EventHubConnectionString"];
            config.EventHubName     = appSettings["EventHubName"];

            var myEventHubObserver = new EventHubObserver(config, AzureOn);
            var keywords           = searchGroups.Contains('|') ? string.Join(",", searchGroups.Split('|')) : searchGroups;
            var tweet = new Tweet();

            Console.WriteLine("Searching Tweets for keywords: {0}", keywords);

            var folderName = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.MyDocuments), @"tweets\");

            EnsureDirectory(folderName);

            var twitterConfig = new TwitterConfig(oauthToken, oauthTokenSecret, oauthCustomerKey, oauthConsumerSecret,
                                                  keywords, searchGroups, createBigFile, folderName, includeRetweets, fileSizeLimit);

            // Write out the config especially the keywords... (can only happen AFTER TwitterConfig has been created!
            string path = Path.Combine(folderName, Path.ChangeExtension(twitterConfig.BigFileName, ".config"));

            File.WriteAllText(path, @"CAS BDA Search (Team Pharma) for Tweets was started with the following keywords:" + Environment.NewLine + Environment.NewLine + keywords, Encoding.UTF8);

            // test
            foreach (var sendingPayload in tweet.StreamStatuses(twitterConfig))
            {
            }

            // end test
            //**var sendingPayload = tweet.StreamStatuses(twitterConfig).Where(e => !string.IsNullOrWhiteSpace(e.Text)).Select(t => Sentiment.ComputeScore(t, searchGroups, mode)).Select(t => new Payload { CreatedAt = t.CreatedAt, Topic = t.Topic, SentimentScore = t.SentimentScore, Author = t.UserName, Text = t.Text, SendExtended = sendExtendedInformation, Language = t.Language});
            //if (removeAllUndefined)
            //{
            //	sendingPayload = sendingPayload.Where(e => e.SentimentScore > -1);
            //}
            //sendingPayload.Where(e => e.Topic != "No Match").ToObservable().Subscribe(myEventHubObserver);
        }
コード例 #6
0
        static TextReader ReadTweets(TwitterConfig config)
        {
            var oauth_version = "1.0";
            var oauth_signature_method = "HMAC-SHA1";

            // unique request details
            var oauth_nonce = Convert.ToBase64String(new ASCIIEncoding().GetBytes(DateTime.Now.Ticks.ToString()));
            var oauth_timestamp = Convert.ToInt64(
                (DateTime.UtcNow - new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc))
                    .TotalSeconds).ToString();

            var resource_url = "https://stream.twitter.com/1.1/statuses/filter.json";

            // create oauth signature
            var baseString = string.Format(
                "oauth_consumer_key={0}&oauth_nonce={1}&oauth_signature_method={2}&" +
                "oauth_timestamp={3}&oauth_token={4}&oauth_version={5}&track={6}",
                config.OAuthConsumerKey,
                oauth_nonce,
                oauth_signature_method,
                oauth_timestamp,
                config.OAuthToken,
                oauth_version,
                Uri.EscapeDataString(config.Keywords));

            baseString = string.Concat("POST&", Uri.EscapeDataString(resource_url), "&", Uri.EscapeDataString(baseString));

            var compositeKey = string.Concat(Uri.EscapeDataString(config.OAuthConsumerSecret),
            "&", Uri.EscapeDataString(config.OAuthTokenSecret));

            string oauth_signature;
            using (var hasher = new HMACSHA1(ASCIIEncoding.ASCII.GetBytes(compositeKey)))
            {
                oauth_signature = Convert.ToBase64String(
                hasher.ComputeHash(ASCIIEncoding.ASCII.GetBytes(baseString)));
            }

            // create the request header
            var authHeader = string.Format(
                "OAuth oauth_nonce=\"{0}\", oauth_signature_method=\"{1}\", " +
                "oauth_timestamp=\"{2}\", oauth_consumer_key=\"{3}\", " +
                "oauth_token=\"{4}\", oauth_signature=\"{5}\", " +
                "oauth_version=\"{6}\"",
                Uri.EscapeDataString(oauth_nonce),
                Uri.EscapeDataString(oauth_signature_method),
                Uri.EscapeDataString(oauth_timestamp),
                Uri.EscapeDataString(config.OAuthConsumerKey),
                Uri.EscapeDataString(config.OAuthToken),
                Uri.EscapeDataString(oauth_signature),
                Uri.EscapeDataString(oauth_version)
            );


            // make the request
            ServicePointManager.Expect100Continue = false;

            var postBody = "track=" + config.Keywords;
            resource_url += "?" + postBody;
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(resource_url);
            request.Headers.Add("Authorization", authHeader);
            request.Method = "POST";
            request.ContentType = "application/x-www-form-urlencoded";
            request.PreAuthenticate = true;
            request.AllowWriteStreamBuffering = true;
            request.CachePolicy = new System.Net.Cache.RequestCachePolicy(System.Net.Cache.RequestCacheLevel.BypassCache);

            // bail out and retry after 5 seconds
            var tresponse = request.GetResponseAsync();
            if (tresponse.Wait(5000))
                return new StreamReader(tresponse.Result.GetResponseStream());
            else
            {
                request.Abort(); 
                return StreamReader.Null;
            }
        }