public void PopulateExtendedData()
{
var tp = new TweetProcessor(new List <Tweet> {
this
}, true);
tp.Start();
}
public void PopulateExtendedData()
{
if (Links == null || Links.Count == 0)
{
var tp = new TweetProcessor(new List <Tweet> {
this
}, 0, true);
tp.Start();
}
}
public virtual Task<IEnumerable<Tweet>> ProcessItems(IEnumerable<Tweet> tweets)
{
return Task<IEnumerable<Tweet>>.Factory.StartNew(new Func<IEnumerable<Tweet>>(() =>
{
var tp = new TweetProcessor(tweets, 0, true);
tp.Start();
StoreInRepository(tweets);
return tweets;
}));
}
static void Main(string[] args)
{
if (!EnsureSingleLoad())
{
Console.WriteLine("{0}: Another Instance Currently Runing", DateTime.Now);
return;
}
Console.WriteLine("{0}: Started", DateTime.Now);
var screenname = UsersCollection.PrimaryUser().TwitterScreenName;
var secret = ConfigurationManager.AppSettings["TwitterCustomerSecret"];
HubConnection hubConnection = null;
IHubProxy streamingHub = null;
while (streamingHubConnectAttempts++ < 3)
{
if (streamingHubConnectAttempts > 1) System.Threading.Thread.Sleep(5000);
Console.WriteLine("{0}: Attempting To Connect To PushURL '{1}' (Attempt: {2})", DateTime.Now, ConfigurationManager.AppSettings["PushURL"], streamingHubConnectAttempts);
hubConnection = (!string.IsNullOrEmpty(ConfigurationManager.AppSettings["PushURL"])) ? new HubConnection(ConfigurationManager.AppSettings["PushURL"]) : null;
if (hubConnection != null)
{
try
{
streamingHub = hubConnection.CreateHubProxy("streamingHub");
hubConnection.StateChanged += new Action<Microsoft.AspNet.SignalR.Client.StateChange>(sc =>
{
if (sc.NewState == Microsoft.AspNet.SignalR.Client.ConnectionState.Connected)
{
Console.WriteLine("{0}: Push Connection Established", DateTime.Now);
lock (queue_push_lock)
{
if (queue_push.Count > 0)
{
Console.WriteLine("{0}: Pushing {1} Tweets to Web Application", DateTime.Now, queue_push.Count());
streamingHub.Invoke("Send", new StreamItem() { Secret = secret, Data = queue_push }).Wait();
queue_push.Clear();
}
}
}
else if (sc.NewState == Microsoft.AspNet.SignalR.Client.ConnectionState.Disconnected)
Console.WriteLine("{0}: Push Connection Lost", DateTime.Now);
else if (sc.NewState == Microsoft.AspNet.SignalR.Client.ConnectionState.Reconnecting)
Console.WriteLine("{0}: Reestablishing Push Connection", DateTime.Now);
else if (sc.NewState == Microsoft.AspNet.SignalR.Client.ConnectionState.Connecting)
Console.WriteLine("{0}: Establishing Push Connection", DateTime.Now);
});
var startHubTask = hubConnection.Start();
startHubTask.Wait();
if (!startHubTask.IsFaulted) break;
}
catch (Exception ex)
{
hubConnection = null;
Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
}
}
Console.WriteLine("{0}: Getting Friends for {1}", DateTime.Now, screenname);
TwitterModel.Instance.UpdateFriendsForPrimaryUser();
Console.WriteLine("{0}: Finished Getting Friends for {1}", DateTime.Now, screenname);
Console.WriteLine("{0}: Listening to Stream", DateTime.Now);
var context = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname);
stream = StartTwitterStream(context);
queueTimer = new Timer(60000);
queueTimer.Elapsed += new ElapsedEventHandler((x, y) =>
{
queueTimer.Enabled = false;
try
{
Console.WriteLine("{0}: Processing Queue", DateTime.Now);
lock (queue_lock)
{
if (queue.Count == 0)
{
Console.WriteLine("{0}: No Items to Process", DateTime.Now);
return;
}
tweets = new Tweet[queue.Count];
queue.CopyTo(tweets);
queue.Clear();
}
Console.WriteLine("{0}: Processing {1} Items from Queue", DateTime.Now, tweets.Length);
var tp = new TweetProcessor(tweets, true);
tp.Start();
tweets
.GroupBy(t => t.User.Identifier.ScreenName)
.ToList()
.ForEach(g =>
{
CachedRepository<Tweet>.Instance.Save(g.Key + TwitterModel.TWEETS, g.OrderBy(t => t.CreatedAt).Select(t => t).ToList());
Console.WriteLine("{0}: {1} Tweets Saved for {2}", DateTime.Now, g.Count(), g.Key);
});
//CachedRepository<Tweet>.Instance.FlushChanges();
if (hubConnection != null && streamingHub != null)
{
int retweetThreshold = UsersCollection.PrimaryUser().RetweetThreshold;
tweets = tweets.Where(t => t.RetweetCount >= retweetThreshold).ToArray();
if (hubConnection.State == Microsoft.AspNet.SignalR.Client.ConnectionState.Connected)
{
if (tweets.Length > 0)
{
Console.WriteLine("{0}: Pushing {1} Tweets to Web Application", DateTime.Now, tweets.Count());
streamingHub.Invoke("Send", new StreamItem() { Secret = secret, Data = tweets }).Wait();
}
}
else
{
lock (queue_push_lock)
{
queue_push.AddRange(tweets);
}
}
}
tweets = null;
}
catch (Exception ex)
{
Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
finally
{
try
{
if (Math.Abs((lastCallBackTime - DateTime.Now).TotalSeconds) > 90) //The Stream Stalled or was Closed
{
if (hadStreamFailure)
Console.WriteLine("{0}: LinqToTwitter UserStream Was Closed Attempting to Reconnect", DateTime.Now);
else
Console.WriteLine("{0}: LinqToTwitter UserStream Stalled Attempting to Restart It", DateTime.Now);
context = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname);
stream = StartTwitterStream(context);
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
queueTimer.Enabled = true;
Console.WriteLine("{0}: Completed Processing Queue", DateTime.Now);
}
});
queueTimer.Start();
/*
* It appears like firing off the friends update while running the stream will
* cause the stream to stop working.
*
var friendTimer = new Timer(3600000);
friendTimer.Elapsed += new ElapsedEventHandler((x, y) =>
{
friendTimer.Enabled = false;
try
{
Console.WriteLine("{0}: Getting Friends for {1}", DateTime.Now, screenname);
Friends.Update();
Console.WriteLine("{0}: Finished Getting Friends for {1}", DateTime.Now, screenname);
}
catch (Exception ex)
{
Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
finally
{
friendTimer.Enabled = true;
}
});
friendTimer.Start();
*/
while(Console.ReadLine() != "exit");
Console.WriteLine("{0}: Exiting", DateTime.Now);
stream.CloseStream();
}
public virtual Task<IEnumerable<Tweet>> ProcessItems(IEnumerable<Tweet> tweets)
{
return Task<IEnumerable<Tweet>>.Factory.StartNew(new Func<IEnumerable<Tweet>>(() =>
{
var tp = new TweetProcessor(tweets, true);
tp.Start();
StoreInRepository(tweets);
if (hubConnection != null && streamingHub != null)
{
int retweetThreshold = UsersCollection.PrimaryUser().RetweetThreshold;
tweets = tweets.Where(t => t.RetweetCount >= retweetThreshold).ToArray();
if (hubConnection.State == SignalR.Client.ConnectionState.Connected)
{
if (tweets.Count() > 0)
{
log.WriteLine("{0}: Pushing {1} Tweets to Web Application", DateTime.Now, tweets.Count());
streamingHub.Invoke("Send", new StreamItem() { Secret = secret, Data = tweets }).Wait();
}
}
else
{
lock (queue_push_lock)
{
queue_push.AddRange(tweets);
}
}
}
return tweets;
}));
}
static void Main(string[] args)
{
if (!EnsureSingleLoad())
{
Console.WriteLine("{0}: Another Instance Currently Runing", DateTime.Now);
return;
}
TweetProcessor tp;
List<Tweet> tweets;
var start = DateTime.Now;
Console.WriteLine("{0}: Started", start);
var primaryUser = UsersCollection.PrimaryUser();
Console.WriteLine("{0}: Getting Friends for {1}", DateTime.Now, primaryUser.TwitterScreenName);
Postworthy.Models.Twitter.TwitterModel.Instance.UpdateFriendsForPrimaryUser();
Console.WriteLine("{0}: Getting Tweets for {1}", DateTime.Now, primaryUser.TwitterScreenName);
tweets = StatusTimeline.Get();
if (tweets != null)
{
Console.WriteLine("{0}: Processing {1} Tweets", DateTime.Now, tweets.Count);
tp = new TweetProcessor(tweets);
tp.Start();
Console.WriteLine("{0}: Saving Tweets", DateTime.Now);
tweets
.GroupBy(t => t.User.Identifier.ScreenName)
.ToList()
.ForEach(g =>
{
Repository<Tweet>.Instance.Save(g.Key + TwitterModel.TWEETS, g.Select(x => x).ToList());
Console.WriteLine("{0}: {1} Tweets Saved for {2}", DateTime.Now, g.Count(), g.Key);
});
Repository<Tweet>.Instance.FlushChanges();
}
else
tweets = new List<Tweet>();
List<string> screenNames = new List<string>();
screenNames.AddRange(TwitterModel.Instance.GetRelevantScreenNames(primaryUser.TwitterScreenName));
Console.WriteLine("{0}: Update Retweet Counts", DateTime.Now);
List<Tweet> updateTweets = new List<Tweet>();
foreach (var screenName in screenNames)
{
var tweetsToUpdate = Repository<Tweet>.Instance.Query(screenName + TwitterModel.TWEETS, where: t => t.CreatedAt > DateTime.Now.AddHours(-48));
if (tweetsToUpdate != null && tweetsToUpdate.Count > 1)
{
tweetsToUpdate = tweetsToUpdate.Except(tweets).OrderByDescending(t => t.Status.CreatedAt).ToList();
if (tweetsToUpdate != null && tweetsToUpdate.Count > 1)
{
Console.WriteLine("{0}: Updating Retweet Counts for {1}", DateTime.Now, screenName);
var updatedStatuses = StatusTimeline.Get(screenName, tweetsToUpdate.First().StatusID);
if (updatedStatuses != null && updatedStatuses.Count > 0)
{
int tweetsAdded = 0;
foreach (var s in updatedStatuses)
{
var t = tweetsToUpdate.SingleOrDefault(x => x.StatusID == s.StatusID);
if (t != null && t.RetweetCount != s.RetweetCount)
{
t.Status.RetweetCount = s.RetweetCount;
updateTweets.Add(t);
tweetsAdded++;
}
}
if (tweetsAdded > 0) Console.WriteLine("{0}: {1} Retweet Counts Updated for {2}", DateTime.Now, tweetsAdded, screenName);
}
}
}
}
if (updateTweets.Count > 0)
{
Console.WriteLine("{0}: Processing {1} Tweets with New Retweet Counts", DateTime.Now, updateTweets.Count);
tp = new TweetProcessor(updateTweets);
tp.Start();
Repository<Tweet>.Instance.FlushChanges();
}
var end = DateTime.Now;
Console.WriteLine("{0}: Finished in {1} minutes", end, (end - start).TotalMinutes);
}
