/// <summary>
/// Updates the classifier guess on the training page.
/// </summary>
/// <param name="model">The model.</param>
/// <returns></returns>
private TrainerModel UpdateClassifier(TrainerModel model)
{
// ask the naive bayes classifier what it thinks
var classifier = new NaiveBayes();
model.UnigramSentiment = classifier.GetTextSentiment(model.CurrentTweet.text, 1, 1, false, null, OclumenContext.BasicNgrams, OclumenContext);
model.BigramSentiment = classifier.GetTextSentiment(model.CurrentTweet.text, 2, 1, false, null, OclumenContext.BasicNgrams, OclumenContext);
model.UnigramSentimentStemmed = classifier.GetTextSentiment(model.CurrentTweet.text, 1, 1, true, Dictionary, OclumenContext.StemmedNgrams, OclumenContext);
model.BigramSentimentStemmed = classifier.GetTextSentiment(model.CurrentTweet.text, 2, 1, true, Dictionary, OclumenContext.StemmedNgrams, OclumenContext);
return(model);
}
public static void TweetClassifier()
{
var context = new OclumenContext();
var classifier = new NaiveBayes();
var smoothingFactor = 1;
var unigrams = new Dictionary <string, List <KeyValuePair <Sentiment, decimal> > >();
var unigramsSt = new Dictionary <string, List <KeyValuePair <Sentiment, decimal> > >();
var bigrams = new Dictionary <string, List <KeyValuePair <Sentiment, decimal> > >();
var bigramsSt = new Dictionary <string, List <KeyValuePair <Sentiment, decimal> > >();
foreach (var ngram in context.BasicNgrams.Where(x => x.Cardinality == 1))
{
var ngramProbabilityList = new List <KeyValuePair <Sentiment, decimal> >(3);
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Positive, ngram.PositiveCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Neutral, ngram.NeutralCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Negative, ngram.NegativeCount + smoothingFactor));
unigrams.Add(ngram.Text.ToLower(), ngramProbabilityList);
ngramProbabilityList = new List <KeyValuePair <Sentiment, decimal> >(3);
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Positive, ngram.RtPositiveCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Neutral, ngram.RtNeutralCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Negative, ngram.RtNegativeCount + smoothingFactor));
unigrams.Add(ngram.Text.ToLower() + "_rt", ngramProbabilityList);
}
foreach (var ngram in context.StemmedNgrams.Where(x => x.Cardinality == 1))
{
var ngramProbabilityList = new List <KeyValuePair <Sentiment, decimal> >(3);
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Positive, ngram.PositiveCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Neutral, ngram.NeutralCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Negative, ngram.NegativeCount + smoothingFactor));
unigramsSt.Add(ngram.Text.ToLower(), ngramProbabilityList);
ngramProbabilityList = new List <KeyValuePair <Sentiment, decimal> >(3);
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Positive, ngram.RtPositiveCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Neutral, ngram.RtNeutralCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Negative, ngram.RtNegativeCount + smoothingFactor));
unigramsSt.Add(ngram.Text.ToLower() + "_rt", ngramProbabilityList);
}
foreach (var ngram in context.BasicNgrams.Where(x => x.Cardinality == 2))
{
var ngramProbabilityList = new List <KeyValuePair <Sentiment, decimal> >(3);
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Positive, ngram.PositiveCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Neutral, ngram.NeutralCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Negative, ngram.NegativeCount + smoothingFactor));
bigrams.Add(ngram.Text.ToLower(), ngramProbabilityList);
ngramProbabilityList = new List <KeyValuePair <Sentiment, decimal> >(3);
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Positive, ngram.RtPositiveCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Neutral, ngram.RtNeutralCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Negative, ngram.RtNegativeCount + smoothingFactor));
bigrams.Add(ngram.Text.ToLower() + "_rt", ngramProbabilityList);
}
foreach (var ngram in context.StemmedNgrams.Where(x => x.Cardinality == 2))
{
var ngramProbabilityList = new List <KeyValuePair <Sentiment, decimal> >(3);
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Positive, ngram.PositiveCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Neutral, ngram.NeutralCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Negative, ngram.NegativeCount + smoothingFactor));
bigramsSt.Add(ngram.Text.ToLower(), ngramProbabilityList);
ngramProbabilityList = new List <KeyValuePair <Sentiment, decimal> >(3);
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Positive, ngram.RtPositiveCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Neutral, ngram.RtNeutralCount + smoothingFactor));
ngramProbabilityList.Add(new KeyValuePair <Sentiment, decimal>(Sentiment.Negative, ngram.RtNegativeCount + smoothingFactor));
bigramsSt.Add(ngram.Text.ToLower() + "_rt", ngramProbabilityList);
}
int sleepTime = AppConfigSettings.ProcessTweetThreadSleepTime;
while (CrawlerStatus.KeepRunning)
{
var tweetToProcess = context.RawTweets.First(x => x.AutoSentimentTimestamp == DateTime.MinValue);
if (tweetToProcess == null)
{
Thread.Sleep(sleepTime);
continue;
}
// add the hashtags to process queue
String originalTweet;
var hashtags = TwitterTextUtility.GetHashtags(tweetToProcess.text);
var retweets = TwitterTextUtility.GetRetweets(tweetToProcess.text, out originalTweet);
lock (_syncRoot)
{
foreach (var hashtag in hashtags)
{
UnprocessedHashtags.Enqueue(hashtag);
}
foreach (var retweet in retweets)
{
UnprocessedRetweets.Enqueue(retweet);
}
}
// auto classify the tweets
tweetToProcess.AutoUnigram = (int)classifier.GetTextSentiment(tweetToProcess.text, 1, 1, false, Dictionary, context.BasicNgrams, context, unigrams);
tweetToProcess.AutoUnigramStemmed = (int)classifier.GetTextSentiment(tweetToProcess.text, 1, 1, true, Dictionary, context.StemmedNgrams, context, unigramsSt);
tweetToProcess.AutoBigram = (int)classifier.GetTextSentiment(tweetToProcess.text, 2, 1, false, Dictionary, context.BasicNgrams, context, bigrams);
tweetToProcess.AutoBigramStemmed = (int)classifier.GetTextSentiment(tweetToProcess.text, 2, 1, true, Dictionary, context.StemmedNgrams, context, bigramsSt);
tweetToProcess.AutoSentimentTimestamp = DateTime.UtcNow;
context.SaveChanges();
}
}
