//
// GET: /Reports/BasicWordCountData/5
public JsonResult BasicWordCountData(int id, DateTime?startRange, DateTime?endRange)
{
//var start = !String.IsNullOrWhiteSpace(startRange) ? DateTime.Parse(startRange) : DateTime.Now.AddHours(-4);
if (startRange == null)
{
startRange = DateTime.UtcNow.AddHours(-4);
}
else
{
startRange = startRange.Value.ToUniversalTime();
}
if (endRange == null)
{
endRange = DateTime.UtcNow;
}
else
{
endRange = endRange.Value.ToUniversalTime();
}
var processor = new TweetProcessor();
var tweetsText = _tweetsLogic.GetTweetTextInDateRange(id, startRange.Value, endRange.Value);
var words = processor.WordCountStats(tweetsText).OrderByDescending(w => w.Value).Take(20).ToList();
var dataModel = new { words = words.Select(w => w.Key), counts = words.Select(w => w.Value) };
return(new JsonResult
{
Data = dataModel,
JsonRequestBehavior = JsonRequestBehavior.AllowGet
});
}
public TweetProcessorTests()
{
_mockTwitterDataCapturer = new Mock <ITwitterDataCapturer>();
_mockUrlHandler = new Mock <IUrlHandler>();
_tweetProcessor = new TweetProcessor(_mockTwitterDataCapturer.Object, _mockUrlHandler.Object, new TelemetryClient());
_mockTwitterDataCapturer.Setup(mtdc => mtdc.IncrementTweetAtTime(It.IsAny <DateTime>()));
_mockTwitterDataCapturer.Setup(mtdc => mtdc.IncrementEmoji(It.IsAny <string>()));
_mockTwitterDataCapturer.Setup(mtdc => mtdc.IncrementHashtag(It.IsAny <string>()));
_mockTwitterDataCapturer.Setup(mtdc => mtdc.IncrementDomain(It.IsAny <string>()));
}
public virtual Task <IEnumerable <Tweet> > ProcessItems(IEnumerable <Tweet> tweets)
{
return(Task <IEnumerable <Tweet> > .Factory.StartNew(new Func <IEnumerable <Tweet> >(() =>
{
var tp = new TweetProcessor(tweets, 0, true);
tp.Start();
StoreInRepository(tweets);
return tweets;
})));
}
public static void Main(string[] args)
{
// TweetProcesser.JsonToText ("../../data/TexasDataProc.data.uncompressed", "../../data/TextTexasData.txt");
/*
* string[] disasters = new string[] {
* "flood",
* "erupt",
* "tsunami",
* "blizzard",
* "drought",
* "storm",
* "tornado",
* "epidem",
* "hurrican",
* "cyclon",
* "disast",
* "catastr",
* "thunderstorm",
* "earthquak"
* };
*
* List<string> data = TweetProcessor.GetFilteredDataFromText ("../../data/TextTexasData.txt", disasters);
* TweetProcessor.WriteToFile (data, "../../data/DisasterTweets.txt");
*/
var data = TweetProcessor.GetDataFromText("../../data/DisasterTweets.txt");
int topicCount = 20;
int topTermsCount = 15;
Console.WriteLine("There are {0} tweets.", data.Count);
var l = new LDA(data, topicCount);
Console.WriteLine("Starting inference...");
l.Inference(500, 100, 10);
for (int topic = 0; topic < topicCount; topic++)
{
Console.WriteLine("{0}:", topic);
string[] ret = l.GetTopTermsForTopic(topic, topTermsCount);
foreach (var s in ret)
{
Console.Write("{0} ", s);
}
Console.WriteLine();
}
Console.WriteLine("Done!");
}
public JsonResult AdvancedWordCountData(int id, int interval, DateTime?startRange, DateTime?endRange)
{
if (startRange == null)
{
startRange = DateTime.UtcNow.AddHours(-4);
}
else
{
startRange = startRange.Value.ToUniversalTime();
}
if (endRange == null)
{
endRange = DateTime.UtcNow;
}
else
{
endRange = endRange.Value.ToUniversalTime();
}
var processor = new TweetProcessor();
var tweets = _tweetsLogic.GetTweetsInDateRange(id, startRange.Value, endRange.Value);
var timeInterval = (int)Math.Floor((endRange - startRange).Value.TotalMilliseconds / interval);
var counts = new List <List <int> >();
var inset = 0;
var texts = tweets.Where(t => t.CreatedDate > endRange.Value.AddMilliseconds(-timeInterval) && t.CreatedDate < endRange).Select(t => t.Text).ToList();
var result = processor.WordCountStats(texts).OrderByDescending(w => w.Value).Take(10).ToList();
var categories = result.Select(w => w.Key).ToList();
foreach (var category in categories)
{
var innerCounts = new List <int>();
for (var i = 0; i < interval; i++)
{
var intervalStartRange = startRange.Value.AddMilliseconds(inset);
var intervalEndRange = startRange.Value.AddMilliseconds(inset + timeInterval);
texts = tweets.Where(t => t.CreatedDate > intervalStartRange && t.CreatedDate < intervalEndRange).Select(t => t.Text).ToList();
var wordsDict = processor.WordCountStats(texts);
innerCounts.Add(wordsDict.ContainsKey(category) ? wordsDict[category] : 0);
inset += timeInterval;
}
counts.Add(innerCounts);
inset = 0;
}
var dataModel = new { words = categories, counts, timeInterval, startRange };
return(new JsonResult
{
Data = dataModel,
JsonRequestBehavior = JsonRequestBehavior.AllowGet
});
}
public void ProcessTweetAsyncTest()
{
// Arrange
const string keyword = "I am happy!";
var tweetsRepository = new Mock <ITweetsRepository>();
var configuration = new Mock <ITextAnalyticsConfiguration>();
var handlerMock = new Mock <HttpMessageHandler>(MockBehavior.Strict);
handlerMock
.Protected()
// Setup the PROTECTED method to mock
.Setup <Task <HttpResponseMessage> >(
"SendAsync",
ItExpr.IsAny <HttpRequestMessage>(),
ItExpr.IsAny <CancellationToken>()
)
// prepare the expected response of the mocked http call
.ReturnsAsync(new HttpResponseMessage
{
StatusCode = HttpStatusCode.OK,
Content = new StringContent(
"{\r\n\"documents\":[\r\n{\r\n\"score\":0.9999237060546875,\r\n\"id\":\"1\"\r\n}\r\n],\r\n\"errors\":[]\r\n}")
})
.Verifiable();
// use real http client with mocked handler here
var httpClient = new HttpClient(handlerMock.Object);
var tweetProcessor = new TweetProcessor(tweetsRepository.Object, configuration.Object, httpClient);
var tweet = new Mock <ITweet>();
tweet.SetupGet(x => x.FullText).Returns(keyword);
var matchedTweetReceivedEventArgs = new MatchedTweetReceivedEventArgs(tweet.Object, "");
// Act
tweetProcessor.ProcessTweetAsync(keyword, matchedTweetReceivedEventArgs).Wait();
// Assert
tweetsRepository.Verify(x => x.SaveTweet(matchedTweetReceivedEventArgs));
tweetsRepository.Verify(x => x.SaveSentiment(It.Is <TweetSentiment>(t =>
t.FullText == "I am happy!" && Math.Abs(t.Score - 0.9999237060546875) < 0.01)));
}
public virtual Task <IEnumerable <Tweet> > ProcessItems(IEnumerable <Tweet> tweets)
{
return(Task <IEnumerable <Tweet> > .Factory.StartNew(new Func <IEnumerable <Tweet> >(() =>
{
var tp = new TweetProcessor(tweets, true);
tp.Start();
StoreInRepository(tweets);
if (hubConnection != null && streamingHub != null)
{
int retweetThreshold = UsersCollection.PrimaryUser().RetweetThreshold;
tweets = tweets.Where(t => t.RetweetCount >= retweetThreshold).ToArray();
if (hubConnection.State == SignalR.Client.ConnectionState.Connected)
{
if (tweets.Count() > 0)
{
log.WriteLine("{0}: Pushing {1} Tweets to Web Application", DateTime.Now, tweets.Count());
streamingHub.Invoke("Send", new StreamItem()
{
Secret = secret, Data = tweets
}).Wait();
}
}
else
{
lock (queue_push_lock)
{
queue_push.AddRange(tweets);
}
}
}
return tweets;
})));
}
static void Main(string[] args)
{
if (!EnsureSingleLoad())
{
Console.WriteLine("{0}: Another Instance Currently Runing", DateTime.Now);
return;
}
Console.WriteLine("{0}: Started", DateTime.Now);
var screenname = UsersCollection.PrimaryUser().TwitterScreenName;
var secret = ConfigurationManager.AppSettings["TwitterCustomerSecret"];
HubConnection hubConnection = null;
IHubProxy streamingHub = null;
while (streamingHubConnectAttempts++ < 3)
{
if (streamingHubConnectAttempts > 1)
{
System.Threading.Thread.Sleep(5000);
}
Console.WriteLine("{0}: Attempting To Connect To PushURL '{1}' (Attempt: {2})", DateTime.Now, ConfigurationManager.AppSettings["PushURL"], streamingHubConnectAttempts);
hubConnection = (!string.IsNullOrEmpty(ConfigurationManager.AppSettings["PushURL"])) ? new HubConnection(ConfigurationManager.AppSettings["PushURL"]) : null;
if (hubConnection != null)
{
try
{
streamingHub = hubConnection.CreateProxy("streamingHub");
hubConnection.StateChanged += new Action <SignalR.Client.StateChange>(sc =>
{
if (sc.NewState == SignalR.Client.ConnectionState.Connected)
{
Console.WriteLine("{0}: Push Connection Established", DateTime.Now);
lock (queue_push_lock)
{
if (queue_push.Count > 0)
{
Console.WriteLine("{0}: Pushing {1} Tweets to Web Application", DateTime.Now, queue_push.Count());
streamingHub.Invoke("Send", new StreamItem()
{
Secret = secret, Data = queue_push
}).Wait();
queue_push.Clear();
}
}
}
else if (sc.NewState == SignalR.Client.ConnectionState.Disconnected)
{
Console.WriteLine("{0}: Push Connection Lost", DateTime.Now);
}
else if (sc.NewState == SignalR.Client.ConnectionState.Reconnecting)
{
Console.WriteLine("{0}: Reestablishing Push Connection", DateTime.Now);
}
else if (sc.NewState == SignalR.Client.ConnectionState.Connecting)
{
Console.WriteLine("{0}: Establishing Push Connection", DateTime.Now);
}
});
var startHubTask = hubConnection.Start();
startHubTask.Wait();
if (!startHubTask.IsFaulted)
{
break;
}
}
catch (Exception ex)
{
hubConnection = null;
Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
}
}
Console.WriteLine("{0}: Getting Friends for {1}", DateTime.Now, screenname);
TwitterModel.Instance.UpdateFriendsForPrimaryUser();
Console.WriteLine("{0}: Finished Getting Friends for {1}", DateTime.Now, screenname);
Console.WriteLine("{0}: Listening to Stream", DateTime.Now);
var context = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname);
stream = StartTwitterStream(context);
queueTimer = new Timer(60000);
queueTimer.Elapsed += new ElapsedEventHandler((x, y) =>
{
queueTimer.Enabled = false;
try
{
Console.WriteLine("{0}: Processing Queue", DateTime.Now);
lock (queue_lock)
{
if (queue.Count == 0)
{
Console.WriteLine("{0}: No Items to Process", DateTime.Now);
return;
}
tweets = new Tweet[queue.Count];
queue.CopyTo(tweets);
queue.Clear();
}
Console.WriteLine("{0}: Processing {1} Items from Queue", DateTime.Now, tweets.Length);
var tp = new TweetProcessor(tweets, true);
tp.Start();
tweets
.GroupBy(t => t.User.Identifier.ScreenName)
.ToList()
.ForEach(g =>
{
Repository <Tweet> .Instance.Save(g.Key + TwitterModel.TWEETS, g.OrderBy(t => t.CreatedAt).Select(t => t).ToList());
Console.WriteLine("{0}: {1} Tweets Saved for {2}", DateTime.Now, g.Count(), g.Key);
});
Repository <Tweet> .Instance.FlushChanges();
if (hubConnection != null && streamingHub != null)
{
int retweetThreshold = UsersCollection.PrimaryUser().RetweetThreshold;
tweets = tweets.Where(t => t.RetweetCount >= retweetThreshold).ToArray();
if (hubConnection.State == SignalR.Client.ConnectionState.Connected)
{
if (tweets.Length > 0)
{
Console.WriteLine("{0}: Pushing {1} Tweets to Web Application", DateTime.Now, tweets.Count());
streamingHub.Invoke("Send", new StreamItem()
{
Secret = secret, Data = tweets
}).Wait();
}
}
else
{
lock (queue_push_lock)
{
queue_push.AddRange(tweets);
}
}
}
tweets = null;
}
catch (Exception ex)
{
Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
finally
{
try
{
if (Math.Abs((lastCallBackTime - DateTime.Now).TotalSeconds) > 90) //The Stream Stalled or was Closed
{
if (hadStreamFailure)
{
Console.WriteLine("{0}: LinqToTwitter UserStream Was Closed Attempting to Reconnect", DateTime.Now);
}
else
{
Console.WriteLine("{0}: LinqToTwitter UserStream Stalled Attempting to Restart It", DateTime.Now);
}
context = TwitterModel.Instance.GetAuthorizedTwitterContext(screenname);
stream = StartTwitterStream(context);
}
}
catch (Exception ex)
{
Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
}
queueTimer.Enabled = true;
Console.WriteLine("{0}: Completed Processing Queue", DateTime.Now);
}
});
queueTimer.Start();
/*
* It appears like firing off the friends update while running the stream will
* cause the stream to stop working.
*
* var friendTimer = new Timer(3600000);
* friendTimer.Elapsed += new ElapsedEventHandler((x, y) =>
* {
* friendTimer.Enabled = false;
* try
* {
* Console.WriteLine("{0}: Getting Friends for {1}", DateTime.Now, screenname);
* Friends.Update();
* Console.WriteLine("{0}: Finished Getting Friends for {1}", DateTime.Now, screenname);
* }
* catch (Exception ex)
* {
* Console.WriteLine("{0}: Error: {1}", DateTime.Now, ex.ToString());
* }
* finally
* {
* friendTimer.Enabled = true;
* }
* });
* friendTimer.Start();
*/
while (Console.ReadLine() != "exit")
{
;
}
Console.WriteLine("{0}: Exiting", DateTime.Now);
stream.CloseStream();
}
static void Main(string[] args)
{
if (!EnsureSingleLoad())
{
Console.WriteLine("{0}: Another Instance Currently Runing", DateTime.Now);
return;
}
TweetProcessor tp;
List <Tweet> tweets;
var start = DateTime.Now;
Console.WriteLine("{0}: Started", start);
var primaryUser = UsersCollection.PrimaryUser();
Console.WriteLine("{0}: Getting Friends for {1}", DateTime.Now, primaryUser.TwitterScreenName);
Postworthy.Models.Twitter.TwitterModel.Instance.UpdateFriendsForPrimaryUser();
Console.WriteLine("{0}: Getting Tweets for {1}", DateTime.Now, primaryUser.TwitterScreenName);
tweets = StatusTimeline.Get();
if (tweets != null)
{
Console.WriteLine("{0}: Processing {1} Tweets", DateTime.Now, tweets.Count);
tp = new TweetProcessor(tweets);
tp.Start();
Console.WriteLine("{0}: Saving Tweets", DateTime.Now);
tweets
.GroupBy(t => t.User.Identifier.ScreenName)
.ToList()
.ForEach(g =>
{
Repository <Tweet> .Instance.Save(g.Key + TwitterModel.TWEETS, g.Select(x => x).ToList());
Console.WriteLine("{0}: {1} Tweets Saved for {2}", DateTime.Now, g.Count(), g.Key);
});
Repository <Tweet> .Instance.FlushChanges();
}
else
{
tweets = new List <Tweet>();
}
List <string> screenNames = new List <string>();
screenNames.AddRange(TwitterModel.Instance.GetRelevantScreenNames(primaryUser.TwitterScreenName));
Console.WriteLine("{0}: Update Retweet Counts", DateTime.Now);
List <Tweet> updateTweets = new List <Tweet>();
foreach (var screenName in screenNames)
{
var tweetsToUpdate = Repository <Tweet> .Instance.Query(screenName + TwitterModel.TWEETS, where : t => t.CreatedAt > DateTime.Now.AddHours(-48));
if (tweetsToUpdate != null && tweetsToUpdate.Count > 1)
{
tweetsToUpdate = tweetsToUpdate.Except(tweets).OrderByDescending(t => t.Status.CreatedAt).ToList();
if (tweetsToUpdate != null && tweetsToUpdate.Count > 1)
{
Console.WriteLine("{0}: Updating Retweet Counts for {1}", DateTime.Now, screenName);
var updatedStatuses = StatusTimeline.Get(screenName, tweetsToUpdate.First().StatusID);
if (updatedStatuses != null && updatedStatuses.Count > 0)
{
int tweetsAdded = 0;
foreach (var s in updatedStatuses)
{
var t = tweetsToUpdate.SingleOrDefault(x => x.StatusID == s.StatusID);
if (t != null && t.RetweetCount != s.RetweetCount)
{
t.Status.RetweetCount = s.RetweetCount;
updateTweets.Add(t);
tweetsAdded++;
}
}
if (tweetsAdded > 0)
{
Console.WriteLine("{0}: {1} Retweet Counts Updated for {2}", DateTime.Now, tweetsAdded, screenName);
}
}
}
}
}
if (updateTweets.Count > 0)
{
Console.WriteLine("{0}: Processing {1} Tweets with New Retweet Counts", DateTime.Now, updateTweets.Count);
tp = new TweetProcessor(updateTweets);
tp.Start();
Repository <Tweet> .Instance.FlushChanges();
}
var end = DateTime.Now;
Console.WriteLine("{0}: Finished in {1} minutes", end, (end - start).TotalMinutes);
}
public ProcessEngine(TweetProcessor tweetProcessor, TweetStreamer tweetStreamer)
{
_tweetProcessor = tweetProcessor;
_tweetStreamer = tweetStreamer;
}
