private static async Task HandleTweetArrival(StreamContent streamContent) { if (streamContent == null) return; var status = streamContent.Entity as Status; if (status == null) return; var tweet = new Tweet { TwitterUserIdentifier = status.User.UserIDResponse, ScreenName = status.User.ScreenNameResponse, UserName = status.User.Name, HashTags = status.Entities.HashTagEntities.Select(h => h.Tag), Urls = status.Entities.UrlEntities.Select(x => x.Url), Text = status.Text, TimeStamp = status.CreatedAt, TweetId_num = status.StatusID, TweetId = status.StatusID.ToString(), Media = status.Entities.MediaEntities }; try { TweetParser.Main(tweet); } catch (System.Exception e) { Debug.Write(e.ToString()); throw; } await Task.Yield(); }
public void Start() { log.WriteLine("{0}: Started", DateTime.Now); log.WriteLine("{0}: Initializing IProcessingStep", DateTime.Now); GetIProcessingStep().Init(log); var screenname = UsersCollection.PrimaryUser().TwitterScreenName; log.WriteLine("{0}: Getting Friends for {1}", DateTime.Now, screenname); TwitterModel.Instance.UpdateFriendsForPrimaryUser(); log.WriteLine("{0}: Finished Getting Friends for {1}", DateTime.Now, screenname); log.WriteLine("{0}: Listening to Stream", DateTime.Now); var userStreamContext = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname); var trackerStreamContext = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname); streams = StartTwitterStream(userStreamContext, trackerStreamContext); userStream = streams.FirstOrDefault(); trackerStream = streams.LastOrDefault(); StartProcessingQueue(userStreamContext, trackerStreamContext); }
private void StartProcessingQueue(TwitterContext userStreamContext, TwitterContext trackerStreamContext) { var screenname = UsersCollection.PrimaryUser().TwitterScreenName; var queueTime = int.Parse(ConfigurationManager.AppSettings["QueueTime"] ?? "60000"); queueTimer = new Timer(queueTime); queueTimer.Elapsed += new ElapsedEventHandler((x, y) => { queueTimer.Enabled = false; bool restartTracker = false; try { log.WriteLine("{0}: Processing Queue", DateTime.Now); lock (queue_lock) { if (queue.Count == 0) { log.WriteLine("{0}: No Items to Process", DateTime.Now); return; } tweets = new Tweet[queue.Count]; queue.CopyTo(tweets); queue.Clear(); } tweets = tweets.Distinct().ToArray(); log.WriteLine("{0}: Processing {1} Items from Queue", DateTime.Now, tweets.Length); //Currently there is only one step but there could potentially be multiple user defined steps GetIProcessingStep().ProcessItems(tweets).Wait(); if (processingStep is IKeywordSuggestionStep) { try { restartTracker = (processingStep as IKeywordSuggestionStep).HasNewKeywordSuggestions(); } catch { } } tweets = null; } catch (Exception ex) { log.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString()); } finally { try { if (userStream != null && Math.Abs((lastCallBackTimeUserStream - DateTime.Now).TotalSeconds) > 90) //The User Stream Stalled or was Closed { if (hadUserStreamFailure) log.WriteLine("{0}: LinqToTwitter User Stream Was Closed Attempting to Reconnect", DateTime.Now); else log.WriteLine("{0}: LinqToTwitter User Stream Stalled Attempting to Restart It", DateTime.Now); userStreamContext = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname); var task = StartTwitterUserStream(userStreamContext); task.Wait(); userStream = task.Result; } } catch (Exception ex) { log.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString()); } try { if (restartTracker || //The tracker should be restarted because we have new potential keywords to track (trackerStream != null && Math.Abs((lastCallBackTimeTrackerStream - DateTime.Now).TotalSeconds) > 90)) //The Tracker Stream Stalled or was Closed { if (hadTrackerStreamFailure) log.WriteLine("{0}: LinqToTwitter Tracker Stream was Closed Attempting to Reconnect", DateTime.Now); else if (restartTracker) log.WriteLine("{0}: LinqToTwitter Tracker Stream will be Restarted to Track more Keywords", DateTime.Now); else log.WriteLine("{0}: LinqToTwitter Tracker Stream Stalled Attempting to Restart It", DateTime.Now); if (trackerStream != null) trackerStream.CloseStream(); trackerStreamContext = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname); var task = StartTwitterTrackerStream(trackerStreamContext); task.Wait(); trackerStream = task.Result; } } catch (Exception ex) { log.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString()); } queueTimer.Enabled = true; log.WriteLine("{0}: Completed Processing Queue", DateTime.Now); } }); log.WriteLine("{0}: Processing Queue every {1} milliseconds", DateTime.Now, queueTime); queueTimer.Start(); }
static void HandleStreamResponse(StreamContent strm) { switch (strm.EntityType) { case StreamEntityType.Control: var control = strm.Entity as Control; Console.WriteLine("Control URI: {0}", control.URL); break; case StreamEntityType.Delete: var delete = strm.Entity as Delete; Console.WriteLine("Delete - User ID: {0}, Status ID: {1}", delete.UserID, delete.StatusID); break; case StreamEntityType.DirectMessage: var dm = strm.Entity as DirectMessage; Console.WriteLine("Direct Message - Sender: {0}, Text: {1}", dm.Sender, dm.Text); break; case StreamEntityType.Disconnect: var disconnect = strm.Entity as Disconnect; Console.WriteLine("Disconnect - {0}", disconnect.Reason); break; case StreamEntityType.Event: var evt = strm.Entity as Event; Console.WriteLine("Event - Event Name: {0}", evt.EventName); break; case StreamEntityType.ForUser: var user = strm.Entity as ForUser; Console.WriteLine("For User - User ID: {0}, # Friends: {1}", user.UserID, user.Friends.Count); break; case StreamEntityType.FriendsList: var friends = strm.Entity as FriendsList; Console.WriteLine("Friends List - # Friends: {0}", friends.Friends.Count); break; case StreamEntityType.GeoScrub: var scrub = strm.Entity as GeoScrub; Console.WriteLine("GeoScrub - User ID: {0}, Up to Status ID: {1}", scrub.UserID, scrub.UpToStatusID); break; case StreamEntityType.Limit: var limit = strm.Entity as Limit; Console.WriteLine("Limit - Track: {0}", limit.Track); break; case StreamEntityType.Stall: var stall = strm.Entity as Stall; Console.WriteLine("Stall - Code: {0}, Message: {1}, % Full: {2}", stall.Code, stall.Message, stall.PercentFull); break; case StreamEntityType.Status: var status = strm.Entity as Status; Console.WriteLine("Status - @{0}: {1}", status.User.ScreenNameResponse, status.Text); break; case StreamEntityType.StatusWithheld: var statusWithheld = strm.Entity as StatusWithheld; Console.WriteLine("Status Withheld - Status ID: {0}, # Countries: {1}", statusWithheld.StatusID, statusWithheld.WithheldInCountries.Count); break; case StreamEntityType.TooManyFollows: var follows = strm.Entity as TooManyFollows; Console.WriteLine("Too Many Follows - Message: {0}", follows.Message); break; case StreamEntityType.UserWithheld: var userWithheld = strm.Entity as UserWithheld; Console.WriteLine("User Withheld - User ID: {0}, # Countries: {1}", userWithheld.UserID, userWithheld.WithheldInCountries.Count); break; case StreamEntityType.ParseError: var unparsedJson = strm.Entity as string; Console.WriteLine("Parse Error - {0}", unparsedJson); break; case StreamEntityType.Unknown: default: Console.WriteLine("Unknown - " + strm.Content + "\n"); break; } }
static void Main(string[] args) { if (!EnsureSingleLoad()) { Console.WriteLine("{0}: Another Instance Currently Runing", DateTime.Now); return; } Console.WriteLine("{0}: Started", DateTime.Now); var screenname = UsersCollection.PrimaryUser().TwitterScreenName; var secret = ConfigurationManager.AppSettings["TwitterCustomerSecret"]; HubConnection hubConnection = null; IHubProxy streamingHub = null; while (streamingHubConnectAttempts++ < 3) { if (streamingHubConnectAttempts > 1) System.Threading.Thread.Sleep(5000); Console.WriteLine("{0}: Attempting To Connect To PushURL '{1}' (Attempt: {2})", DateTime.Now, ConfigurationManager.AppSettings["PushURL"], streamingHubConnectAttempts); hubConnection = (!string.IsNullOrEmpty(ConfigurationManager.AppSettings["PushURL"])) ? new HubConnection(ConfigurationManager.AppSettings["PushURL"]) : null; if (hubConnection != null) { try { streamingHub = hubConnection.CreateHubProxy("streamingHub"); hubConnection.StateChanged += new Action<Microsoft.AspNet.SignalR.Client.StateChange>(sc => { if (sc.NewState == Microsoft.AspNet.SignalR.Client.ConnectionState.Connected) { Console.WriteLine("{0}: Push Connection Established", DateTime.Now); lock (queue_push_lock) { if (queue_push.Count > 0) { Console.WriteLine("{0}: Pushing {1} Tweets to Web Application", DateTime.Now, queue_push.Count()); streamingHub.Invoke("Send", new StreamItem() { Secret = secret, Data = queue_push }).Wait(); queue_push.Clear(); } } } else if (sc.NewState == Microsoft.AspNet.SignalR.Client.ConnectionState.Disconnected) Console.WriteLine("{0}: Push Connection Lost", DateTime.Now); else if (sc.NewState == Microsoft.AspNet.SignalR.Client.ConnectionState.Reconnecting) Console.WriteLine("{0}: Reestablishing Push Connection", DateTime.Now); else if (sc.NewState == Microsoft.AspNet.SignalR.Client.ConnectionState.Connecting) Console.WriteLine("{0}: Establishing Push Connection", DateTime.Now); }); var startHubTask = hubConnection.Start(); startHubTask.Wait(); if (!startHubTask.IsFaulted) break; } catch (Exception ex) { hubConnection = null; Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString()); } } } Console.WriteLine("{0}: Getting Friends for {1}", DateTime.Now, screenname); TwitterModel.Instance.UpdateFriendsForPrimaryUser(); Console.WriteLine("{0}: Finished Getting Friends for {1}", DateTime.Now, screenname); Console.WriteLine("{0}: Listening to Stream", DateTime.Now); var context = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname); stream = StartTwitterStream(context); queueTimer = new Timer(60000); queueTimer.Elapsed += new ElapsedEventHandler((x, y) => { queueTimer.Enabled = false; try { Console.WriteLine("{0}: Processing Queue", DateTime.Now); lock (queue_lock) { if (queue.Count == 0) { Console.WriteLine("{0}: No Items to Process", DateTime.Now); return; } tweets = new Tweet[queue.Count]; queue.CopyTo(tweets); queue.Clear(); } Console.WriteLine("{0}: Processing {1} Items from Queue", DateTime.Now, tweets.Length); var tp = new TweetProcessor(tweets, true); tp.Start(); tweets .GroupBy(t => t.User.Identifier.ScreenName) .ToList() .ForEach(g => { CachedRepository<Tweet>.Instance.Save(g.Key + TwitterModel.TWEETS, g.OrderBy(t => t.CreatedAt).Select(t => t).ToList()); Console.WriteLine("{0}: {1} Tweets Saved for {2}", DateTime.Now, g.Count(), g.Key); }); //CachedRepository<Tweet>.Instance.FlushChanges(); if (hubConnection != null && streamingHub != null) { int retweetThreshold = UsersCollection.PrimaryUser().RetweetThreshold; tweets = tweets.Where(t => t.RetweetCount >= retweetThreshold).ToArray(); if (hubConnection.State == Microsoft.AspNet.SignalR.Client.ConnectionState.Connected) { if (tweets.Length > 0) { Console.WriteLine("{0}: Pushing {1} Tweets to Web Application", DateTime.Now, tweets.Count()); streamingHub.Invoke("Send", new StreamItem() { Secret = secret, Data = tweets }).Wait(); } } else { lock (queue_push_lock) { queue_push.AddRange(tweets); } } } tweets = null; } catch (Exception ex) { Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString()); } finally { try { if (Math.Abs((lastCallBackTime - DateTime.Now).TotalSeconds) > 90) //The Stream Stalled or was Closed { if (hadStreamFailure) Console.WriteLine("{0}: LinqToTwitter UserStream Was Closed Attempting to Reconnect", DateTime.Now); else Console.WriteLine("{0}: LinqToTwitter UserStream Stalled Attempting to Restart It", DateTime.Now); context = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname); stream = StartTwitterStream(context); } } catch (Exception ex) { Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString()); } queueTimer.Enabled = true; Console.WriteLine("{0}: Completed Processing Queue", DateTime.Now); } }); queueTimer.Start(); /* * It appears like firing off the friends update while running the stream will * cause the stream to stop working. * var friendTimer = new Timer(3600000); friendTimer.Elapsed += new ElapsedEventHandler((x, y) => { friendTimer.Enabled = false; try { Console.WriteLine("{0}: Getting Friends for {1}", DateTime.Now, screenname); Friends.Update(); Console.WriteLine("{0}: Finished Getting Friends for {1}", DateTime.Now, screenname); } catch (Exception ex) { Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString()); } finally { friendTimer.Enabled = true; } }); friendTimer.Start(); */ while(Console.ReadLine() != "exit"); Console.WriteLine("{0}: Exiting", DateTime.Now); stream.CloseStream(); }
private string ConnectToStream(string message) { try { JsonSerializerSettings settings = new JsonSerializerSettings(); settings.MissingMemberHandling = Newtonsoft.Json.MissingMemberHandling.Ignore; twitterCtx = new TwitterContext((WinRtAuthorizer)_authenticationService.Authorizer); Debug.WriteLine("\nStreamed Content: \n"); int count = 0; (from strm in twitterCtx.UserStream where strm.Type == UserStreamType.User && strm.With == "followings" select strm) .StreamingCallback(strm => { _currentStream = strm; Debug.WriteLine(strm.Content + "\n" + strm.Error); if (strm.Error == null && !string.IsNullOrEmpty(strm.Content)) { Tweet currentTweet = JsonConvert.DeserializeObject<Tweet>(strm.Content, settings); count++; if (strm.Content.Contains(@"@" + _authenticationService.AuthorizedUserScreenName)) { SendToastNotification(currentTweet.text, ""); SendTileTextNotification(currentTweet.text); } SendBadgeNotification(count); } }) .SingleOrDefault(); } catch (TwitterQueryException ex) { message = ex.Message; } return message; }
/// <summary> /// Performs a query on the Twitter Stream. /// </summary> /// <param name="request">Request with url endpoint and all query parameters.</param> /// <returns> /// Caller expects an JSON formatted string response, but /// real response(s) with streams is fed to the callback. /// </returns> public async Task <string> QueryTwitterStreamAsync(Request request) { const int CarriageReturn = 0x0D; const int LineFeed = 0x0A; const int EndOfStream = 0xFF; WriteLog(request.FullUrl, "QueryTwitterStreamAsync"); IDictionary <string, string> reqParams = request.RequestParameters.ToDictionary(key => key.Name, val => val.Value); var baseFilter = new HttpBaseProtocolFilter { AutomaticDecompression = Authorizer.SupportsCompression, ProxyCredential = Authorizer.ProxyCredential, UseProxy = Authorizer.UseProxy }; var streamFilter = new GetMessageFilter(this, reqParams, request.FullUrl, baseFilter, CancellationToken); using (StreamingClient = new HttpClient(streamFilter)) { var httpRequest = new HttpRequestMessage(HttpMethod.Get, new Uri(request.FullUrl)); var response = await StreamingClient.SendRequestAsync( httpRequest, HttpCompletionOption.ResponseHeadersRead); await TwitterErrorHandler.ThrowIfErrorAsync(response).ConfigureAwait(false); var inputStream = await response.Content.ReadAsInputStreamAsync(); Stream stream = inputStream.AsStreamForRead(); var memStr = new MemoryStream(); byte[] readByte; while (stream.CanRead && !IsStreamClosed) { readByte = new byte[1]; await stream.ReadAsync(readByte, 0, 1, CancellationToken).ConfigureAwait(false); byte nextByte = readByte.SingleOrDefault(); CancellationToken.ThrowIfCancellationRequested(); if (IsStreamClosed) { break; } // TODO: review end-of-stream protocol if (nextByte == EndOfStream) { break; } if (nextByte != CarriageReturn && nextByte != LineFeed) { memStr.WriteByte(nextByte); } if (nextByte == LineFeed) { int byteCount = (int)memStr.Length; byte[] tweetBytes = new byte[byteCount]; memStr.Position = 0; await memStr.ReadAsync(tweetBytes, 0, byteCount, CancellationToken).ConfigureAwait(false); string tweet = Encoding.UTF8.GetString(tweetBytes, 0, byteCount); var strmContent = new StreamContent(this, tweet); await StreamingCallbackAsync(strmContent).ConfigureAwait(false); memStr.Dispose(); memStr = new MemoryStream(); } } } IsStreamClosed = false; return("{}"); }
/// <summary> /// Performs a query on the Twitter Stream. /// </summary> /// <param name="request">Request with url endpoint and all query parameters.</param> /// <returns> /// Caller expects an JSON formatted string response, but /// real response(s) with streams is fed to the callback. /// </returns> public async Task <string> QueryTwitterStreamAsync(Request request) { WriteLog(request.FullUrl, nameof(QueryTwitterStreamAsync)); var handler = new HttpClientHandler(); if (Authorizer.Proxy != null && handler.SupportsProxy) { handler.Proxy = Authorizer.Proxy; } using (StreamingClient = new HttpClient(handler)) { StreamingClient.Timeout = TimeSpan.FromMilliseconds(System.Threading.Timeout.Infinite); var httpRequest = ConfigureRequest(request); var response = await StreamingClient.SendAsync( httpRequest, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false); await TwitterErrorHandler.ThrowIfErrorAsync(response).ConfigureAwait(false); Stream stream = await CreateStream(response); const int CarriageReturn = 0x0D; const int LineFeed = 0x0A; var memStr = new MemoryStream(); byte[] readByte; while (stream.CanRead && !IsStreamClosed) { readByte = new byte[1]; await stream.ReadAsync(readByte, 0, 1, CancellationToken).ConfigureAwait(false); byte nextByte = readByte.SingleOrDefault(); CancellationToken.ThrowIfCancellationRequested(); if (IsStreamClosed) { break; } if (nextByte == 0xff) { break; } if (nextByte != CarriageReturn && nextByte != LineFeed) { memStr.WriteByte(nextByte); } if (nextByte == LineFeed) { int byteCount = (int)memStr.Length; byte[] tweetBytes = new byte[byteCount]; memStr.Position = 0; await memStr.ReadAsync(tweetBytes, 0, byteCount, CancellationToken).ConfigureAwait(false); string tweet = Encoding.UTF8.GetString(tweetBytes, 0, byteCount); var strmContent = new StreamContent(this, tweet); await StreamingCallbackAsync(strmContent).ConfigureAwait(false); memStr.Dispose(); memStr = new MemoryStream(); } } } IsStreamClosed = false; return("{}"); }
private void Start(string track = null) { TwitterContext context = new TwitterContext(new MvcAuthorizer() { CredentialStore = new LinqToTwitter.InMemoryCredentialStore() { OAuthTokenSecret = ConfigurationManager.AppSettings["OAuthTokenSecret"], ConsumerKey = ConfigurationManager.AppSettings["ConsumerKey"], ConsumerSecret = ConfigurationManager.AppSettings["ConsumerSecret"], OAuthToken = ConfigurationManager.AppSettings["OAuthToken"] } }); if (log != null) context.Log = log; track = track ?? ConfigurationManager.AppSettings["Track"]; var trackList = !string.IsNullOrEmpty(track) ? track.ToLower().Split(',').OrderByDescending(x => x.Length).ToList() : new List<string>(); try { streamTask = context.Streaming .Where(s => s.Type == LinqToTwitter.StreamingType.Filter && s.Track == string.Join(",", trackList.Distinct())) .Select(strm => strm) .StartAsync(async strm => { await Task.Run(() => { try { stream = strm; if (strm != null) { if (!string.IsNullOrEmpty(strm.Content)) { var status = new LinqToTwitter.Status(LitJson.JsonMapper.ToObject(strm.Content)); if (status != null && status.StatusID > 0) { string statusText = status.Text.ToLower(); if (trackList.Any(x => statusText.Contains(x))) { statusHandler(status.Text); if (log != null) log.WriteLine("{0}: Status Handled: @{1} said [{2}]", DateTime.Now, status.User.ScreenName, status.Text); } } else if (log != null) log.WriteLine("{0}: Unhandled Item in Stream: {1}", DateTime.Now, strm.Content); } else if (log != null) log.WriteLine("{0}: Twitter Keep Alive", DateTime.Now); } else throw new ArgumentNullException("strm", "This value should never be null!"); } catch (Exception ex) { if (log != null) log.WriteLine("{0}: Error (TrackerStream): {1}", DateTime.Now, ex.ToString()); } }); }); } catch (Exception ex) { if (log != null) log.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString()); } }
private void StartTwitterUserStream(TwitterContext context) { hadUserStreamFailure = false; context.Log = log; try { userStreamTask = context.Streaming .Where(s => s.Type == LinqToTwitter.StreamingType.User) .Select(strm => strm) .StartAsync(async strm => { await Task.Run(() => { try { lastCallBackTimeUserStream = DateTime.Now; if (userStream == null) log.WriteLine("{0}: Twitter Connection Established (UserStream)", DateTime.Now); userStream = strm; if (strm != null) { /* LinqToTwitter v3.0 no longer has *.Status if (strm.Status == TwitterErrorStatus.RequestProcessingException) { var wex = strm.Error as WebException; if (wex != null && wex.Status == WebExceptionStatus.ConnectFailure) { log.WriteLine("{0}: LinqToTwitter UserStream Connection Failure (UserStream)", DateTime.Now); hadUserStreamFailure = true; //Will Be Restarted By Processing Queue } } else */ if (!string.IsNullOrEmpty(strm.Content)) { var status = new LinqToTwitter.Status(LitJson.JsonMapper.ToObject(strm.Content)); if (status != null && status.StatusID > 0) { var tweet = new Tweet(status.RetweetedStatus.StatusID == 0 ? status : status.RetweetedStatus); lock (queue_lock) { queue.Add(tweet); } log.WriteLine("{0}: Added Item to Queue (UserStream): {1}", DateTime.Now, tweet.TweetText); } else { //If you can handle friends we will look for them if (processingStep is ITweepProcessingStep) { var jsonDataFriends = LitJson.JsonMapper.ToObject(strm.Content) .FirstOrDefault(x => x.Key == "friends"); //If this is a friends collection update we will notify you if (!jsonDataFriends.Equals(default(KeyValuePair<string, LitJson.JsonData>)) && jsonDataFriends.Value != null && jsonDataFriends.Value.IsArray) { var friends = new List<LazyLoader<Tweep>>(); for (int i = 0; i < jsonDataFriends.Value.Count; i++) { friends.Add(TwitterModel.Instance(PrimaryUser.TwitterScreenName).GetLazyLoadedTweep(ulong.Parse(jsonDataFriends.Value[i].ToString()), Tweep.TweepType.Follower)); } (processingStep as ITweepProcessingStep).ProcessTweeps(friends); } else log.WriteLine("{0}: Unhandled Item in Stream (UserStream): {1}", DateTime.Now, strm.Content); } else log.WriteLine("{0}: Unhandled Item in Stream (UserStream): {1}", DateTime.Now, strm.Content); } } else log.WriteLine("{0}: Twitter Keep Alive (UserStream)", DateTime.Now); } else throw new ArgumentNullException("strm", "This value should never be null!"); } catch (Exception ex) { log.WriteLine("{0}: Error (UserStream): {1}", DateTime.Now, ex.ToString()); } }); }); } catch (Exception ex) { log.WriteLine("{0}: Error (UserStream): {1}", DateTime.Now, ex.ToString()); } }
private void StartTwitterTrackerStream(TwitterContext context) { hadTrackerStreamFailure = false; List<string> trackList = null; context.Log = log; string track = ConfigurationManager.AppSettings["Track"] ?? (PrimaryUser.Track ?? ""); string[] ignore = (ConfigurationManager.AppSettings["Ignore"] ?? "").ToLower().Split(','); int minFollowers = int.Parse(ConfigurationManager.AppSettings["MinFollowerCount"] ?? "0"); try { if (string.IsNullOrEmpty(track) && !(processingStep is IKeywordSuggestionStep)) { log.WriteLine("{0}: To track keywords one of the following must be true: \n\t1) AppSetting Property 'Track' Cannot be Null or Empty.\n\t2) UserCollection Property 'Track' Cannot be Null or Empty.\n\t3) ProcessingStep must Implement IKeywordSuggestionStep.", DateTime.Now); return; } else { var processIgnoreWords = !string.IsNullOrEmpty(track) ? track.ToLower().Split(',').ToList() : new List<string>(); trackList = !string.IsNullOrEmpty(track) ? track.ToLower().Split(',').ToList() : new List<string>(); if (processingStep is IKeywordSuggestionStep) { var keywordSuggestionStep = processingStep as IKeywordSuggestionStep; if (keywordSuggestionStep != null) { /* I have chosen to wrap these calls in seperate try catch statements incase one fails * the other can still run. This way if the get fails we may still have hope of a reset. */ try { keywordSuggestionStep.SetIgnoreKeywords(processIgnoreWords); } catch { } try { trackList.AddRange(keywordSuggestionStep.GetKeywordSuggestions().Select(x => x.ToLower()).ToList()); } catch { } try { keywordSuggestionStep.ResetHasNewKeywordSuggestions(); } catch { } } } if (trackList.Count == 0) { log.WriteLine("{0}: No Keywords to Track at this time.", DateTime.Now); return; } else { if (trackList.Count > MAX_TRACK) { trackList = trackList.OrderByDescending(x => x.Length).Take(400).ToList(); log.WriteLine("{0}: Tracking List Exceeds Max {1} Reducing List", DateTime.Now, MAX_TRACK); } log.WriteLine("{0}: Attempting to Track: {1}", DateTime.Now, string.Join(",", trackList)); log.WriteLine("{0}: Ignoring : {1}", DateTime.Now, string.Join(",", ignore)); } trackerStreamTask = context.Streaming .Where(s => s.Type == LinqToTwitter.StreamingType.Filter && s.Track == string.Join(",", trackList.Distinct())) .Select(strm => strm) .StartAsync(async strm => { await Task.Run(() => { try { lastCallBackTimeTrackerStream = DateTime.Now; if (trackerStream == null) log.WriteLine("{0}: Twitter Connection Established (TrackerStream)", DateTime.Now); trackerStream = strm; if (strm != null) { /* LinqToTwitter v3.0 no longer has *.Status if (strm.Status == TwitterErrorStatus.RequestProcessingException) { var wex = strm.Error as WebException; if (wex != null && wex.Status == WebExceptionStatus.ConnectFailure) { log.WriteLine("{0}: LinqToTwitter Stream Connection Failure (TrackerStream)", DateTime.Now); hadTrackerStreamFailure = true; //Will Be Restarted By Processing Queue } } else */ if (!string.IsNullOrEmpty(strm.Content)) { var status = new LinqToTwitter.Status(LitJson.JsonMapper.ToObject(strm.Content)); if (status != null && status.StatusID > 0) { string statusText = status.Text.ToLower(); if ( trackList.Any(x => statusText.Contains(x)) && //Looking for exact matches status.User.FollowersCount >= minFollowers && //Meets the follower cutoff !ignore.Any(x => x != "" && statusText.Contains(x)) //Ignore these ) { var tweet = new Tweet(status.RetweetedStatus.StatusID == 0 ? status : status.RetweetedStatus); lock (queue_lock) { queue.Add(tweet); } log.WriteLine("{0}: Added Item to Queue (TrackerStream): @{1} said [{2}]", DateTime.Now, tweet.User.ScreenName, tweet.TweetText); } } else log.WriteLine("{0}: Unhandled Item in Stream (TrackerStream): {1}", DateTime.Now, strm.Content); } else log.WriteLine("{0}: Twitter Keep Alive (TrackerStream)", DateTime.Now); } else throw new ArgumentNullException("strm", "This value should never be null!"); } catch (Exception ex) { log.WriteLine("{0}: Error (TrackerStream): {1}", DateTime.Now, ex.ToString()); } }); }); } } catch (Exception ex) { log.WriteLine("{0}: Error (TrackerStream): {1}", DateTime.Now, ex.ToString()); } }
private void HandleTweet(StreamContent content) { if (content.Status != TwitterErrorStatus.Success) { Debug.WriteLine(content.Error.ToString()); return; } var statusJson = ContentToJson(content); if(statusJson == null) { return; } var status = new Status(statusJson); if(status.Text == null) { return; } if (this._cultures != null) { foreach (var culture in this._cultures) { if(status.Lang.Equals(culture)) { this._listener(status); break; } } } else { this._listener(status); } }
private static JsonData ContentToJson(StreamContent content) { JsonData statusJson; try { statusJson = JsonMapper.ToObject(content.Content); } catch (JsonException ex) { Console.WriteLine("Could not convert stream content to json data, content: {0}, message: {1}", content.Content, ex.Message); return null; } return statusJson; }
private bool IsKeepAliveMessageFromTwitterApi(StreamContent strm) { return string.IsNullOrWhiteSpace(strm.Content); }