private static async Task HandleTweetArrival(StreamContent streamContent)
{
if (streamContent == null)
return;
var status = streamContent.Entity as Status;
if (status == null)
return;
var tweet = new Tweet
{
TwitterUserIdentifier = status.User.UserIDResponse,
ScreenName = status.User.ScreenNameResponse,
UserName = status.User.Name,
HashTags = status.Entities.HashTagEntities.Select(h => h.Tag),
Urls = status.Entities.UrlEntities.Select(x => x.Url),
Text = status.Text,
TimeStamp = status.CreatedAt,
TweetId_num = status.StatusID,
TweetId = status.StatusID.ToString(),
Media = status.Entities.MediaEntities
};
try
{
TweetParser.Main(tweet);
}
catch (System.Exception e)
{
Debug.Write(e.ToString());
throw;
}
await Task.Yield();
}
public void Start()
{
log.WriteLine("{0}: Started", DateTime.Now);
log.WriteLine("{0}: Initializing IProcessingStep", DateTime.Now);
GetIProcessingStep().Init(log);
var screenname = UsersCollection.PrimaryUser().TwitterScreenName;
log.WriteLine("{0}: Getting Friends for {1}", DateTime.Now, screenname);
TwitterModel.Instance.UpdateFriendsForPrimaryUser();
log.WriteLine("{0}: Finished Getting Friends for {1}", DateTime.Now, screenname);
log.WriteLine("{0}: Listening to Stream", DateTime.Now);
var userStreamContext = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname);
var trackerStreamContext = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname);
streams = StartTwitterStream(userStreamContext, trackerStreamContext);
userStream = streams.FirstOrDefault();
trackerStream = streams.LastOrDefault();
StartProcessingQueue(userStreamContext, trackerStreamContext);
}
private void StartProcessingQueue(TwitterContext userStreamContext, TwitterContext trackerStreamContext)
{
var screenname = UsersCollection.PrimaryUser().TwitterScreenName;
var queueTime = int.Parse(ConfigurationManager.AppSettings["QueueTime"] ?? "60000");
queueTimer = new Timer(queueTime);
queueTimer.Elapsed += new ElapsedEventHandler((x, y) =>
{
queueTimer.Enabled = false;
bool restartTracker = false;
try
{
log.WriteLine("{0}: Processing Queue", DateTime.Now);
lock (queue_lock)
{
if (queue.Count == 0)
{
log.WriteLine("{0}: No Items to Process", DateTime.Now);
return;
}
tweets = new Tweet[queue.Count];
queue.CopyTo(tweets);
queue.Clear();
}
tweets = tweets.Distinct().ToArray();
log.WriteLine("{0}: Processing {1} Items from Queue", DateTime.Now, tweets.Length);
//Currently there is only one step but there could potentially be multiple user defined steps
GetIProcessingStep().ProcessItems(tweets).Wait();
if (processingStep is IKeywordSuggestionStep)
{
try { restartTracker = (processingStep as IKeywordSuggestionStep).HasNewKeywordSuggestions(); } catch { }
}
tweets = null;
}
catch (Exception ex)
{
log.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
finally
{
try
{
if (userStream != null && Math.Abs((lastCallBackTimeUserStream - DateTime.Now).TotalSeconds) > 90) //The User Stream Stalled or was Closed
{
if (hadUserStreamFailure)
log.WriteLine("{0}: LinqToTwitter User Stream Was Closed Attempting to Reconnect", DateTime.Now);
else
log.WriteLine("{0}: LinqToTwitter User Stream Stalled Attempting to Restart It", DateTime.Now);
userStreamContext = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname);
var task = StartTwitterUserStream(userStreamContext);
task.Wait();
userStream = task.Result;
}
}
catch (Exception ex)
{
log.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
try
{
if (restartTracker || //The tracker should be restarted because we have new potential keywords to track
(trackerStream != null && Math.Abs((lastCallBackTimeTrackerStream - DateTime.Now).TotalSeconds) > 90)) //The Tracker Stream Stalled or was Closed
{
if (hadTrackerStreamFailure)
log.WriteLine("{0}: LinqToTwitter Tracker Stream was Closed Attempting to Reconnect", DateTime.Now);
else if (restartTracker)
log.WriteLine("{0}: LinqToTwitter Tracker Stream will be Restarted to Track more Keywords", DateTime.Now);
else
log.WriteLine("{0}: LinqToTwitter Tracker Stream Stalled Attempting to Restart It", DateTime.Now);
if (trackerStream != null)
trackerStream.CloseStream();
trackerStreamContext = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname);
var task = StartTwitterTrackerStream(trackerStreamContext);
task.Wait();
trackerStream = task.Result;
}
}
catch (Exception ex)
{
log.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
queueTimer.Enabled = true;
log.WriteLine("{0}: Completed Processing Queue", DateTime.Now);
}
});
log.WriteLine("{0}: Processing Queue every {1} milliseconds", DateTime.Now, queueTime);
queueTimer.Start();
}
static void HandleStreamResponse(StreamContent strm)
{
switch (strm.EntityType)
{
case StreamEntityType.Control:
var control = strm.Entity as Control;
Console.WriteLine("Control URI: {0}", control.URL);
break;
case StreamEntityType.Delete:
var delete = strm.Entity as Delete;
Console.WriteLine("Delete - User ID: {0}, Status ID: {1}", delete.UserID, delete.StatusID);
break;
case StreamEntityType.DirectMessage:
var dm = strm.Entity as DirectMessage;
Console.WriteLine("Direct Message - Sender: {0}, Text: {1}", dm.Sender, dm.Text);
break;
case StreamEntityType.Disconnect:
var disconnect = strm.Entity as Disconnect;
Console.WriteLine("Disconnect - {0}", disconnect.Reason);
break;
case StreamEntityType.Event:
var evt = strm.Entity as Event;
Console.WriteLine("Event - Event Name: {0}", evt.EventName);
break;
case StreamEntityType.ForUser:
var user = strm.Entity as ForUser;
Console.WriteLine("For User - User ID: {0}, # Friends: {1}", user.UserID, user.Friends.Count);
break;
case StreamEntityType.FriendsList:
var friends = strm.Entity as FriendsList;
Console.WriteLine("Friends List - # Friends: {0}", friends.Friends.Count);
break;
case StreamEntityType.GeoScrub:
var scrub = strm.Entity as GeoScrub;
Console.WriteLine("GeoScrub - User ID: {0}, Up to Status ID: {1}", scrub.UserID, scrub.UpToStatusID);
break;
case StreamEntityType.Limit:
var limit = strm.Entity as Limit;
Console.WriteLine("Limit - Track: {0}", limit.Track);
break;
case StreamEntityType.Stall:
var stall = strm.Entity as Stall;
Console.WriteLine("Stall - Code: {0}, Message: {1}, % Full: {2}", stall.Code, stall.Message, stall.PercentFull);
break;
case StreamEntityType.Status:
var status = strm.Entity as Status;
Console.WriteLine("Status - @{0}: {1}", status.User.ScreenNameResponse, status.Text);
break;
case StreamEntityType.StatusWithheld:
var statusWithheld = strm.Entity as StatusWithheld;
Console.WriteLine("Status Withheld - Status ID: {0}, # Countries: {1}", statusWithheld.StatusID, statusWithheld.WithheldInCountries.Count);
break;
case StreamEntityType.TooManyFollows:
var follows = strm.Entity as TooManyFollows;
Console.WriteLine("Too Many Follows - Message: {0}", follows.Message);
break;
case StreamEntityType.UserWithheld:
var userWithheld = strm.Entity as UserWithheld;
Console.WriteLine("User Withheld - User ID: {0}, # Countries: {1}", userWithheld.UserID, userWithheld.WithheldInCountries.Count);
break;
case StreamEntityType.ParseError:
var unparsedJson = strm.Entity as string;
Console.WriteLine("Parse Error - {0}", unparsedJson);
break;
case StreamEntityType.Unknown:
default:
Console.WriteLine("Unknown - " + strm.Content + "\n");
break;
}
}
static void Main(string[] args)
{
if (!EnsureSingleLoad())
{
Console.WriteLine("{0}: Another Instance Currently Runing", DateTime.Now);
return;
}
Console.WriteLine("{0}: Started", DateTime.Now);
var screenname = UsersCollection.PrimaryUser().TwitterScreenName;
var secret = ConfigurationManager.AppSettings["TwitterCustomerSecret"];
HubConnection hubConnection = null;
IHubProxy streamingHub = null;
while (streamingHubConnectAttempts++ < 3)
{
if (streamingHubConnectAttempts > 1) System.Threading.Thread.Sleep(5000);
Console.WriteLine("{0}: Attempting To Connect To PushURL '{1}' (Attempt: {2})", DateTime.Now, ConfigurationManager.AppSettings["PushURL"], streamingHubConnectAttempts);
hubConnection = (!string.IsNullOrEmpty(ConfigurationManager.AppSettings["PushURL"])) ? new HubConnection(ConfigurationManager.AppSettings["PushURL"]) : null;
if (hubConnection != null)
{
try
{
streamingHub = hubConnection.CreateHubProxy("streamingHub");
hubConnection.StateChanged += new Action<Microsoft.AspNet.SignalR.Client.StateChange>(sc =>
{
if (sc.NewState == Microsoft.AspNet.SignalR.Client.ConnectionState.Connected)
{
Console.WriteLine("{0}: Push Connection Established", DateTime.Now);
lock (queue_push_lock)
{
if (queue_push.Count > 0)
{
Console.WriteLine("{0}: Pushing {1} Tweets to Web Application", DateTime.Now, queue_push.Count());
streamingHub.Invoke("Send", new StreamItem() { Secret = secret, Data = queue_push }).Wait();
queue_push.Clear();
}
}
}
else if (sc.NewState == Microsoft.AspNet.SignalR.Client.ConnectionState.Disconnected)
Console.WriteLine("{0}: Push Connection Lost", DateTime.Now);
else if (sc.NewState == Microsoft.AspNet.SignalR.Client.ConnectionState.Reconnecting)
Console.WriteLine("{0}: Reestablishing Push Connection", DateTime.Now);
else if (sc.NewState == Microsoft.AspNet.SignalR.Client.ConnectionState.Connecting)
Console.WriteLine("{0}: Establishing Push Connection", DateTime.Now);
});
var startHubTask = hubConnection.Start();
startHubTask.Wait();
if (!startHubTask.IsFaulted) break;
}
catch (Exception ex)
{
hubConnection = null;
Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
}
}
Console.WriteLine("{0}: Getting Friends for {1}", DateTime.Now, screenname);
TwitterModel.Instance.UpdateFriendsForPrimaryUser();
Console.WriteLine("{0}: Finished Getting Friends for {1}", DateTime.Now, screenname);
Console.WriteLine("{0}: Listening to Stream", DateTime.Now);
var context = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname);
stream = StartTwitterStream(context);
queueTimer = new Timer(60000);
queueTimer.Elapsed += new ElapsedEventHandler((x, y) =>
{
queueTimer.Enabled = false;
try
{
Console.WriteLine("{0}: Processing Queue", DateTime.Now);
lock (queue_lock)
{
if (queue.Count == 0)
{
Console.WriteLine("{0}: No Items to Process", DateTime.Now);
return;
}
tweets = new Tweet[queue.Count];
queue.CopyTo(tweets);
queue.Clear();
}
Console.WriteLine("{0}: Processing {1} Items from Queue", DateTime.Now, tweets.Length);
var tp = new TweetProcessor(tweets, true);
tp.Start();
tweets
.GroupBy(t => t.User.Identifier.ScreenName)
.ToList()
.ForEach(g =>
{
CachedRepository<Tweet>.Instance.Save(g.Key + TwitterModel.TWEETS, g.OrderBy(t => t.CreatedAt).Select(t => t).ToList());
Console.WriteLine("{0}: {1} Tweets Saved for {2}", DateTime.Now, g.Count(), g.Key);
});
//CachedRepository<Tweet>.Instance.FlushChanges();
if (hubConnection != null && streamingHub != null)
{
int retweetThreshold = UsersCollection.PrimaryUser().RetweetThreshold;
tweets = tweets.Where(t => t.RetweetCount >= retweetThreshold).ToArray();
if (hubConnection.State == Microsoft.AspNet.SignalR.Client.ConnectionState.Connected)
{
if (tweets.Length > 0)
{
Console.WriteLine("{0}: Pushing {1} Tweets to Web Application", DateTime.Now, tweets.Count());
streamingHub.Invoke("Send", new StreamItem() { Secret = secret, Data = tweets }).Wait();
}
}
else
{
lock (queue_push_lock)
{
queue_push.AddRange(tweets);
}
}
}
tweets = null;
}
catch (Exception ex)
{
Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
finally
{
try
{
if (Math.Abs((lastCallBackTime - DateTime.Now).TotalSeconds) > 90) //The Stream Stalled or was Closed
{
if (hadStreamFailure)
Console.WriteLine("{0}: LinqToTwitter UserStream Was Closed Attempting to Reconnect", DateTime.Now);
else
Console.WriteLine("{0}: LinqToTwitter UserStream Stalled Attempting to Restart It", DateTime.Now);
context = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname);
stream = StartTwitterStream(context);
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
queueTimer.Enabled = true;
Console.WriteLine("{0}: Completed Processing Queue", DateTime.Now);
}
});
queueTimer.Start();
/*
* It appears like firing off the friends update while running the stream will
* cause the stream to stop working.
*
var friendTimer = new Timer(3600000);
friendTimer.Elapsed += new ElapsedEventHandler((x, y) =>
{
friendTimer.Enabled = false;
try
{
Console.WriteLine("{0}: Getting Friends for {1}", DateTime.Now, screenname);
Friends.Update();
Console.WriteLine("{0}: Finished Getting Friends for {1}", DateTime.Now, screenname);
}
catch (Exception ex)
{
Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
finally
{
friendTimer.Enabled = true;
}
});
friendTimer.Start();
*/
while(Console.ReadLine() != "exit");
Console.WriteLine("{0}: Exiting", DateTime.Now);
stream.CloseStream();
}
private string ConnectToStream(string message)
{
try
{
JsonSerializerSettings settings = new JsonSerializerSettings();
settings.MissingMemberHandling = Newtonsoft.Json.MissingMemberHandling.Ignore;
twitterCtx = new TwitterContext((WinRtAuthorizer)_authenticationService.Authorizer);
Debug.WriteLine("\nStreamed Content: \n");
int count = 0;
(from strm in twitterCtx.UserStream
where strm.Type == UserStreamType.User &&
strm.With == "followings"
select strm)
.StreamingCallback(strm =>
{
_currentStream = strm;
Debug.WriteLine(strm.Content + "\n" + strm.Error);
if (strm.Error == null && !string.IsNullOrEmpty(strm.Content))
{
Tweet currentTweet = JsonConvert.DeserializeObject<Tweet>(strm.Content, settings);
count++;
if (strm.Content.Contains(@"@" + _authenticationService.AuthorizedUserScreenName))
{
SendToastNotification(currentTweet.text, "");
SendTileTextNotification(currentTweet.text);
}
SendBadgeNotification(count);
}
})
.SingleOrDefault();
}
catch (TwitterQueryException ex)
{
message = ex.Message;
}
return message;
}
/// <summary>
/// Performs a query on the Twitter Stream.
/// </summary>
/// <param name="request">Request with url endpoint and all query parameters.</param>
/// <returns>
/// Caller expects an JSON formatted string response, but
/// real response(s) with streams is fed to the callback.
/// </returns>
public async Task <string> QueryTwitterStreamAsync(Request request)
{
const int CarriageReturn = 0x0D;
const int LineFeed = 0x0A;
const int EndOfStream = 0xFF;
WriteLog(request.FullUrl, "QueryTwitterStreamAsync");
IDictionary <string, string> reqParams =
request.RequestParameters.ToDictionary(key => key.Name, val => val.Value);
var baseFilter = new HttpBaseProtocolFilter
{
AutomaticDecompression = Authorizer.SupportsCompression,
ProxyCredential = Authorizer.ProxyCredential,
UseProxy = Authorizer.UseProxy
};
var streamFilter = new GetMessageFilter(this, reqParams, request.FullUrl, baseFilter, CancellationToken);
using (StreamingClient = new HttpClient(streamFilter))
{
var httpRequest = new HttpRequestMessage(HttpMethod.Get, new Uri(request.FullUrl));
var response = await StreamingClient.SendRequestAsync(
httpRequest, HttpCompletionOption.ResponseHeadersRead);
await TwitterErrorHandler.ThrowIfErrorAsync(response).ConfigureAwait(false);
var inputStream = await response.Content.ReadAsInputStreamAsync();
Stream stream = inputStream.AsStreamForRead();
var memStr = new MemoryStream();
byte[] readByte;
while (stream.CanRead && !IsStreamClosed)
{
readByte = new byte[1];
await stream.ReadAsync(readByte, 0, 1, CancellationToken).ConfigureAwait(false);
byte nextByte = readByte.SingleOrDefault();
CancellationToken.ThrowIfCancellationRequested();
if (IsStreamClosed)
{
break;
}
// TODO: review end-of-stream protocol
if (nextByte == EndOfStream)
{
break;
}
if (nextByte != CarriageReturn && nextByte != LineFeed)
{
memStr.WriteByte(nextByte);
}
if (nextByte == LineFeed)
{
int byteCount = (int)memStr.Length;
byte[] tweetBytes = new byte[byteCount];
memStr.Position = 0;
await memStr.ReadAsync(tweetBytes, 0, byteCount, CancellationToken).ConfigureAwait(false);
string tweet = Encoding.UTF8.GetString(tweetBytes, 0, byteCount);
var strmContent = new StreamContent(this, tweet);
await StreamingCallbackAsync(strmContent).ConfigureAwait(false);
memStr.Dispose();
memStr = new MemoryStream();
}
}
}
IsStreamClosed = false;
return("{}");
}
/// <summary>
/// Performs a query on the Twitter Stream.
/// </summary>
/// <param name="request">Request with url endpoint and all query parameters.</param>
/// <returns>
/// Caller expects an JSON formatted string response, but
/// real response(s) with streams is fed to the callback.
/// </returns>
public async Task <string> QueryTwitterStreamAsync(Request request)
{
WriteLog(request.FullUrl, nameof(QueryTwitterStreamAsync));
var handler = new HttpClientHandler();
if (Authorizer.Proxy != null && handler.SupportsProxy)
{
handler.Proxy = Authorizer.Proxy;
}
using (StreamingClient = new HttpClient(handler))
{
StreamingClient.Timeout = TimeSpan.FromMilliseconds(System.Threading.Timeout.Infinite);
var httpRequest = ConfigureRequest(request);
var response = await StreamingClient.SendAsync(
httpRequest, HttpCompletionOption.ResponseHeadersRead).ConfigureAwait(false);
await TwitterErrorHandler.ThrowIfErrorAsync(response).ConfigureAwait(false);
Stream stream = await CreateStream(response);
const int CarriageReturn = 0x0D;
const int LineFeed = 0x0A;
var memStr = new MemoryStream();
byte[] readByte;
while (stream.CanRead && !IsStreamClosed)
{
readByte = new byte[1];
await stream.ReadAsync(readByte, 0, 1, CancellationToken).ConfigureAwait(false);
byte nextByte = readByte.SingleOrDefault();
CancellationToken.ThrowIfCancellationRequested();
if (IsStreamClosed)
{
break;
}
if (nextByte == 0xff)
{
break;
}
if (nextByte != CarriageReturn && nextByte != LineFeed)
{
memStr.WriteByte(nextByte);
}
if (nextByte == LineFeed)
{
int byteCount = (int)memStr.Length;
byte[] tweetBytes = new byte[byteCount];
memStr.Position = 0;
await memStr.ReadAsync(tweetBytes, 0, byteCount, CancellationToken).ConfigureAwait(false);
string tweet = Encoding.UTF8.GetString(tweetBytes, 0, byteCount);
var strmContent = new StreamContent(this, tweet);
await StreamingCallbackAsync(strmContent).ConfigureAwait(false);
memStr.Dispose();
memStr = new MemoryStream();
}
}
}
IsStreamClosed = false;
return("{}");
}
private void Start(string track = null)
{
TwitterContext context = new TwitterContext(new MvcAuthorizer()
{
CredentialStore = new LinqToTwitter.InMemoryCredentialStore()
{
OAuthTokenSecret = ConfigurationManager.AppSettings["OAuthTokenSecret"],
ConsumerKey = ConfigurationManager.AppSettings["ConsumerKey"],
ConsumerSecret = ConfigurationManager.AppSettings["ConsumerSecret"],
OAuthToken = ConfigurationManager.AppSettings["OAuthToken"]
}
});
if (log != null)
context.Log = log;
track = track ?? ConfigurationManager.AppSettings["Track"];
var trackList = !string.IsNullOrEmpty(track) ? track.ToLower().Split(',').OrderByDescending(x => x.Length).ToList() : new List<string>();
try
{
streamTask = context.Streaming
.Where(s => s.Type == LinqToTwitter.StreamingType.Filter && s.Track == string.Join(",", trackList.Distinct()))
.Select(strm => strm)
.StartAsync(async strm =>
{
await Task.Run(() =>
{
try
{
stream = strm;
if (strm != null)
{
if (!string.IsNullOrEmpty(strm.Content))
{
var status = new LinqToTwitter.Status(LitJson.JsonMapper.ToObject(strm.Content));
if (status != null && status.StatusID > 0)
{
string statusText = status.Text.ToLower();
if (trackList.Any(x => statusText.Contains(x)))
{
statusHandler(status.Text);
if (log != null)
log.WriteLine("{0}: Status Handled: @{1} said [{2}]", DateTime.Now, status.User.ScreenName, status.Text);
}
}
else if (log != null)
log.WriteLine("{0}: Unhandled Item in Stream: {1}", DateTime.Now, strm.Content);
}
else if (log != null)
log.WriteLine("{0}: Twitter Keep Alive", DateTime.Now);
}
else
throw new ArgumentNullException("strm", "This value should never be null!");
}
catch (Exception ex)
{
if (log != null)
log.WriteLine("{0}: Error (TrackerStream): {1}", DateTime.Now, ex.ToString());
}
});
});
}
catch (Exception ex)
{
if (log != null)
log.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
}
private void StartTwitterUserStream(TwitterContext context)
{
hadUserStreamFailure = false;
context.Log = log;
try
{
userStreamTask = context.Streaming
.Where(s => s.Type == LinqToTwitter.StreamingType.User)
.Select(strm => strm)
.StartAsync(async strm =>
{
await Task.Run(() =>
{
try
{
lastCallBackTimeUserStream = DateTime.Now;
if (userStream == null)
log.WriteLine("{0}: Twitter Connection Established (UserStream)", DateTime.Now);
userStream = strm;
if (strm != null)
{
/* LinqToTwitter v3.0 no longer has *.Status
if (strm.Status == TwitterErrorStatus.RequestProcessingException)
{
var wex = strm.Error as WebException;
if (wex != null && wex.Status == WebExceptionStatus.ConnectFailure)
{
log.WriteLine("{0}: LinqToTwitter UserStream Connection Failure (UserStream)", DateTime.Now);
hadUserStreamFailure = true;
//Will Be Restarted By Processing Queue
}
}
else
*/
if (!string.IsNullOrEmpty(strm.Content))
{
var status = new LinqToTwitter.Status(LitJson.JsonMapper.ToObject(strm.Content));
if (status != null && status.StatusID > 0)
{
var tweet = new Tweet(status.RetweetedStatus.StatusID == 0 ? status : status.RetweetedStatus);
lock (queue_lock)
{
queue.Add(tweet);
}
log.WriteLine("{0}: Added Item to Queue (UserStream): {1}", DateTime.Now, tweet.TweetText);
}
else
{
//If you can handle friends we will look for them
if (processingStep is ITweepProcessingStep)
{
var jsonDataFriends = LitJson.JsonMapper.ToObject(strm.Content)
.FirstOrDefault(x => x.Key == "friends");
//If this is a friends collection update we will notify you
if (!jsonDataFriends.Equals(default(KeyValuePair<string, LitJson.JsonData>)) &&
jsonDataFriends.Value != null &&
jsonDataFriends.Value.IsArray)
{
var friends = new List<LazyLoader<Tweep>>();
for (int i = 0; i < jsonDataFriends.Value.Count; i++)
{
friends.Add(TwitterModel.Instance(PrimaryUser.TwitterScreenName).GetLazyLoadedTweep(ulong.Parse(jsonDataFriends.Value[i].ToString()), Tweep.TweepType.Follower));
}
(processingStep as ITweepProcessingStep).ProcessTweeps(friends);
}
else
log.WriteLine("{0}: Unhandled Item in Stream (UserStream): {1}", DateTime.Now, strm.Content);
}
else
log.WriteLine("{0}: Unhandled Item in Stream (UserStream): {1}", DateTime.Now, strm.Content);
}
}
else
log.WriteLine("{0}: Twitter Keep Alive (UserStream)", DateTime.Now);
}
else
throw new ArgumentNullException("strm", "This value should never be null!");
}
catch (Exception ex)
{
log.WriteLine("{0}: Error (UserStream): {1}", DateTime.Now, ex.ToString());
}
});
});
}
catch (Exception ex)
{
log.WriteLine("{0}: Error (UserStream): {1}", DateTime.Now, ex.ToString());
}
}
private void StartTwitterTrackerStream(TwitterContext context)
{
hadTrackerStreamFailure = false;
List<string> trackList = null;
context.Log = log;
string track = ConfigurationManager.AppSettings["Track"] ?? (PrimaryUser.Track ?? "");
string[] ignore = (ConfigurationManager.AppSettings["Ignore"] ?? "").ToLower().Split(',');
int minFollowers = int.Parse(ConfigurationManager.AppSettings["MinFollowerCount"] ?? "0");
try
{
if (string.IsNullOrEmpty(track) && !(processingStep is IKeywordSuggestionStep))
{
log.WriteLine("{0}: To track keywords one of the following must be true: \n\t1) AppSetting Property 'Track' Cannot be Null or Empty.\n\t2) UserCollection Property 'Track' Cannot be Null or Empty.\n\t3) ProcessingStep must Implement IKeywordSuggestionStep.", DateTime.Now);
return;
}
else
{
var processIgnoreWords = !string.IsNullOrEmpty(track) ? track.ToLower().Split(',').ToList() : new List<string>();
trackList = !string.IsNullOrEmpty(track) ? track.ToLower().Split(',').ToList() : new List<string>();
if (processingStep is IKeywordSuggestionStep)
{
var keywordSuggestionStep = processingStep as IKeywordSuggestionStep;
if (keywordSuggestionStep != null)
{
/* I have chosen to wrap these calls in seperate try catch statements incase one fails
* the other can still run. This way if the get fails we may still have hope of a reset.
*/
try { keywordSuggestionStep.SetIgnoreKeywords(processIgnoreWords); }
catch { }
try { trackList.AddRange(keywordSuggestionStep.GetKeywordSuggestions().Select(x => x.ToLower()).ToList()); }
catch { }
try { keywordSuggestionStep.ResetHasNewKeywordSuggestions(); }
catch { }
}
}
if (trackList.Count == 0)
{
log.WriteLine("{0}: No Keywords to Track at this time.", DateTime.Now);
return;
}
else
{
if (trackList.Count > MAX_TRACK)
{
trackList = trackList.OrderByDescending(x => x.Length).Take(400).ToList();
log.WriteLine("{0}: Tracking List Exceeds Max {1} Reducing List", DateTime.Now, MAX_TRACK);
}
log.WriteLine("{0}: Attempting to Track: {1}", DateTime.Now, string.Join(",", trackList));
log.WriteLine("{0}: Ignoring : {1}", DateTime.Now, string.Join(",", ignore));
}
trackerStreamTask = context.Streaming
.Where(s => s.Type == LinqToTwitter.StreamingType.Filter && s.Track == string.Join(",", trackList.Distinct()))
.Select(strm => strm)
.StartAsync(async strm =>
{
await Task.Run(() =>
{
try
{
lastCallBackTimeTrackerStream = DateTime.Now;
if (trackerStream == null)
log.WriteLine("{0}: Twitter Connection Established (TrackerStream)", DateTime.Now);
trackerStream = strm;
if (strm != null)
{
/* LinqToTwitter v3.0 no longer has *.Status
if (strm.Status == TwitterErrorStatus.RequestProcessingException)
{
var wex = strm.Error as WebException;
if (wex != null && wex.Status == WebExceptionStatus.ConnectFailure)
{
log.WriteLine("{0}: LinqToTwitter Stream Connection Failure (TrackerStream)", DateTime.Now);
hadTrackerStreamFailure = true;
//Will Be Restarted By Processing Queue
}
}
else
*/
if (!string.IsNullOrEmpty(strm.Content))
{
var status = new LinqToTwitter.Status(LitJson.JsonMapper.ToObject(strm.Content));
if (status != null && status.StatusID > 0)
{
string statusText = status.Text.ToLower();
if (
trackList.Any(x => statusText.Contains(x)) && //Looking for exact matches
status.User.FollowersCount >= minFollowers && //Meets the follower cutoff
!ignore.Any(x => x != "" && statusText.Contains(x)) //Ignore these
)
{
var tweet = new Tweet(status.RetweetedStatus.StatusID == 0 ? status : status.RetweetedStatus);
lock (queue_lock)
{
queue.Add(tweet);
}
log.WriteLine("{0}: Added Item to Queue (TrackerStream): @{1} said [{2}]", DateTime.Now, tweet.User.ScreenName, tweet.TweetText);
}
}
else
log.WriteLine("{0}: Unhandled Item in Stream (TrackerStream): {1}", DateTime.Now, strm.Content);
}
else
log.WriteLine("{0}: Twitter Keep Alive (TrackerStream)", DateTime.Now);
}
else
throw new ArgumentNullException("strm", "This value should never be null!");
}
catch (Exception ex)
{
log.WriteLine("{0}: Error (TrackerStream): {1}", DateTime.Now, ex.ToString());
}
});
});
}
}
catch (Exception ex)
{
log.WriteLine("{0}: Error (TrackerStream): {1}", DateTime.Now, ex.ToString());
}
}
private void HandleTweet(StreamContent content)
{
if (content.Status != TwitterErrorStatus.Success)
{
Debug.WriteLine(content.Error.ToString());
return;
}
var statusJson = ContentToJson(content);
if(statusJson == null)
{
return;
}
var status = new Status(statusJson);
if(status.Text == null)
{
return;
}
if (this._cultures != null)
{
foreach (var culture in this._cultures)
{
if(status.Lang.Equals(culture))
{
this._listener(status);
break;
}
}
}
else
{
this._listener(status);
}
}
private static JsonData ContentToJson(StreamContent content)
{
JsonData statusJson;
try
{
statusJson = JsonMapper.ToObject(content.Content);
}
catch (JsonException ex)
{
Console.WriteLine("Could not convert stream content to json data, content: {0}, message: {1}", content.Content, ex.Message);
return null;
}
return statusJson;
}
private bool IsKeepAliveMessageFromTwitterApi(StreamContent strm)
{
return string.IsNullOrWhiteSpace(strm.Content);
}
