private void refreshData()
{
sentiResults.InnerHtml = "<div class='card mb-4'>" +
"<div class='card-block'>" +
"<h3 class='card-title'>Results</h3>" +
"<div class='dropdown card-title-btn-container'>" +
"<div class='dropdown-menu dropdown-menu-right' aria-labelledby='dropdownMenuButton'> " +
"<a class='dropdown-item' href='#'><em class='fa fa-search mr-1'></em>More info</a>" +
"<a class='dropdown-item' href='#'><em class='fa fa-thumb-tack mr-1'></em>Pin Window</a>" +
"<a class='dropdown-item' href='#'><em class='fa fa-remove mr-1'></em>Close Window</a>" +
"</div>" +
"</div>" +
"<div class='table-responsive'>" +
"<table class='table table-striped'>" +
"<thead>" +
"<tr>" +
"<th>Sentiment ID</th>" +
"<th>Sentiment Total</th>" +
"<th>Primary Category</th>" +
"<th>Key Phrases</th>" +
"<th>Tweet ID</th>" +
"</tr>" +
"</thead>" +
"<tbody>";
List <DB_Service.Sentiments> sentiments = new List <DB_Service.Sentiments>();
DB_Service.ServiceClient service = new DB_Service.ServiceClient();
sentiments = service.getSentiments();
if (sentiments != null)
{
foreach (DB_Service.Sentiments senti in sentiments)
{
sentiResults.InnerHtml += "<tr>" +
"<td>" + senti.sentiment_id + "</td>" +
"<td>" + senti.sentiment_total + "</td>" +
"<td>" + senti.category_primary + "</td>" +
"<td>" + senti.key_phrases + "</td>" +
"<td>" + senti.tweet_id + "</td>" +
"</tr>";
}
}
sentiResults.InnerHtml += "</tbody>" +
"</table>" +
"</div>" +
"</div>" +
"</div>" +
"</div>";
}
protected void UpdatePage_Click(object sender, EventArgs e)
{
string category = txtCategory.Value;
string twitterHandle = txtTwitterHandle.Value;
string keyword = txtKeyword.Value;
if (category.Equals(""))
{
category = "Crime";
}
if (twitterHandle.Equals(""))
{
twitterHandle = "S.A Crime Watch";
}
if (keyword.Equals(""))
{
keyword = "Crime";
}
List <DB_Service.CrimeTweets> tweetQuery = Query.Search_SearchTweet(keyword);
DB_Service.ServiceClient service = new DB_Service.ServiceClient();
queryFeed.InnerHtml = "";
foreach (DB_Service.CrimeTweets tweet in tweetQuery)
{
queryFeed.InnerHtml += /*"<div class='col-lg-4 mb-4 bg-default'>" +*/
"<div class='card'>" +
"<div class='card-header'>Received At: " + DateTime.Now + "</div>" +
"<div class='card-block'>" +
"<p>" + tweet.message + "</p>" +
"</div>" +
//"</div>" +
"</div>";
}
queryFeed.InnerHtml += "<div class='alert bg-success' role='alert'> Success! Twitter has been queried and tweets have been added to the database.<a href= '#' class='float-right'></a></div>";
List <DB_Service.CrimeTweets> newTweetList = service.getCrimeTweetsToAnalyse();
List <SentimentResults> newResults = TextAnalyticsV2_1.fullAnalysis(newTweetList);
queryFeed.InnerHtml += "<div class='alert bg-success' role='alert'> Success! New tweets language, phrases, entities and sentiment have been analysed and added to the database.<a href= '#' class='float-right'></a></div>";
// Printing language results.
//foreach (SentimentResults document in newResults)
//{
// queryFeed.InnerHtml += "Document ID: " + document.getTweet_id() + ", Score: " + document.getSenti_score() + "\n";
//}
}
public static List <DB_Service.Sentiments> processTweetSentiments(List <DB_Service.CrimeTweets> crimeTweets)
{
List <DB_Service.Sentiments> mySentiments = new List <DB_Service.Sentiments>();
foreach (DB_Service.CrimeTweets ct in crimeTweets)
{
DB_Service.Sentiments senti = new DB_Service.Sentiments();
senti.sentiment_total = getCompoundTweetAnalysis(ct.message);
senti.category_primary = "To be Analysed";
senti.key_phrases = "To be Analysed";
senti.tweet_id = ct.tweet_id;
mySentiments.Add(senti);
}
if (mySentiments != null)
{
DB_Service.ServiceClient service = new DB_Service.ServiceClient();
service.addSentiments(mySentiments);
}
return(mySentiments);
}
//public List<SentimentResults> extractingKeyPhrases(List<GuestGeek_DBService.CrimeTweets> crimeTweets)
//{
// // Create a client.---------------------------------------------------------------------------------------------------------------------------------
// ITextAnalyticsClient client = new TextAnalyticsClient(new ApiKeyServiceClientCredentials())
// {
// Endpoint = "https://westeurope.api.cognitive.microsoft.com/text/analytics/v2.0/"
// };
// System.Console.OutputEncoding = System.Text.Encoding.UTF8;
// // Getting key phrases.---------------------------------------------------------------------------------------------------------------------------------
// Console.WriteLine("\n\n===== KEY-PHRASE EXTRACTION ======");
// List<MultiLanguageInput> keyPhrases = new List<MultiLanguageInput>();
// foreach (GuestGeek_DBService.CrimeTweets ct in crimeTweets)
// {
// MultiLanguageInput inp = new MultiLanguageInput(;
// myInp.Add(inp);
// }
// KeyPhraseBatchResult result2 = client.KeyPhrasesAsync(new MultiLanguageBatchInput(
// new List<MultiLanguageInput>()
// {
// new MultiLanguageInput("ja", "1", "猫は幸せ"),
// new MultiLanguageInput("de", "2", "Fahrt nach Stuttgart und dann zum Hotel zu Fu."),
// new MultiLanguageInput("en", "3", "My cat is stiff as a rock."),
// new MultiLanguageInput("es", "4", "A mi me encanta el fútbol!")
// })).Result;
// // Printing key phrases.
// foreach (var document in result2.Documents)
// {
// Console.WriteLine("Document ID: {0} ", document.Id);
// Console.WriteLine("\t Key phrases:");
// foreach (string keyphrase in document.KeyPhrases)
// {
// Console.WriteLine("\t\t" + keyphrase);
// }
// }
//}
//static async void MakeRequest()
//{
// var client = new HttpClient();
// var queryString = HttpUtility.ParseQueryString(string.Empty);
// // Request headers
// client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", TwitterKeys.getTextAnalyticsKey());
// var uri = "https://westus.api.cognitive.microsoft.com/text/analytics/v2.0/languages?" + queryString;
// HttpResponseMessage response;
// // Request body
// byte[] byteData = Encoding.UTF8.GetBytes("{body}");
// using (var content = new ByteArrayContent(byteData))
// {
// content.Headers.ContentType = new MediaTypeHeaderValue("application/json");
// response = await client.PostAsync(uri, content);
// }
//}
public List <SentimentResults> fullAnalysis(List <DB_Service.CrimeTweets> crimeTweets)
{
// Create a client.---------------------------------------------------------------------------------------------------------------------------------
ITextAnalyticsClient client = new TextAnalyticsClient(new ApiKeyServiceClientCredentials())
{
Endpoint = "https://westeurope.api.cognitive.microsoft.com/text/analytics/v2.0/"
};
System.Console.OutputEncoding = System.Text.Encoding.UTF8;
// Extracting language.---------------------------------------------------------------------------------------------------------------------------------
Console.WriteLine("===== LANGUAGE EXTRACTION ======");
List <Input> myInp = new List <Input>();//Languages
foreach (DB_Service.CrimeTweets ct in crimeTweets)
{
//Sentiment.Sentiment.sanitize(ct.message);
Input inp = new Input(ct.tweet_id.ToString(), ct.message);
myInp.Add(inp);
}
LanguageBatchResult result = null;
try
{
result = client.DetectLanguageAsync(new BatchInput(myInp)).Result;
//Task.Run(() => client.DetectLanguageAsync(new BatchInput(myInp)).Result).Wait();
}
catch (AggregateException aex)
{
string messages = "";
foreach (Exception ex in aex.InnerExceptions)
{
messages += ex.Message + "\r\n";
}
Debug.WriteLine(messages);
}
// Printing language results.
List <SentimentResults> tweetLangs = new List <SentimentResults>();//Language
foreach (var document in result.Documents)
{
SentimentResults tr = new SentimentResults();
tr.language_short = document.DetectedLanguages[0].Iso6391Name;
tr.tweet_id = Int32.Parse(document.Id);
tr.language = document.DetectedLanguages[0].Name;
tweetLangs.Add(tr);
}
// Getting key phrases.---------------------------------------------------------------------------------------------------------------------------------
Console.WriteLine("\n\n===== KEY-PHRASE EXTRACTION ======");
List <MultiLanguageInput> keyPhrases = new List <MultiLanguageInput>();//Key phrases
int count = 0;
foreach (DB_Service.CrimeTweets ct in crimeTweets)
{
string tempLang = tweetLangs.ElementAt <SentimentResults>(count).language_short;
MultiLanguageInput inp = new MultiLanguageInput(tempLang, ct.tweet_id.ToString(), ct.message);
keyPhrases.Add(inp);
count++;
}
KeyPhraseBatchResult result2 = client.KeyPhrasesAsync(new MultiLanguageBatchInput(keyPhrases)).Result;
// Printing key phrases.
List <string> phrases = new List <string>();
List <SentimentResults> tweetKeyPhrases = new List <SentimentResults>();
count = 0;
foreach (var document in result2.Documents)
{
//Console.WriteLine("Document ID: {0} ", document.Id);
//Console.WriteLine("\t Key phrases:");
foreach (string keyphrase in document.KeyPhrases)
{
//Console.WriteLine("\t\t" + keyphrase);
phrases.Add(keyphrase);
}
SentimentResults sr = new SentimentResults();
sr = tweetLangs.ElementAt <SentimentResults>(count);
sr.keyPhrases = phrases;
tweetKeyPhrases.Add(sr);
count++;
}
// Analyzing sentiment.---------------------------------------------------------------------------------------------------------------------------------
Console.WriteLine("\n\n===== SENTIMENT ANALYSIS ======");
List <MultiLanguageInput> sentiAni = new List <MultiLanguageInput>();//Sentiment Analysis
count = 0;
foreach (DB_Service.CrimeTweets ct in crimeTweets)
{
string tempLang = tweetKeyPhrases.ElementAt <SentimentResults>(count).language_short;
MultiLanguageInput inp = new MultiLanguageInput(tempLang, ct.tweet_id.ToString(), ct.message);
sentiAni.Add(inp);
count++;
}
SentimentBatchResult result3 = client.SentimentAsync(new MultiLanguageBatchInput(sentiAni)).Result;
// Printing sentiment results.
List <SentimentResults> tweetSentiments = new List <SentimentResults>();
count = 0;
foreach (var document in result3.Documents)
{
//Console.WriteLine("Document ID: {0} , Sentiment Score: {1:0.00}", document.Id, document.Score);
SentimentResults sr = new SentimentResults();
sr = tweetKeyPhrases.ElementAt <SentimentResults>(count);
sr.senti_score = (double)document.Score;
tweetSentiments.Add(sr);
count++;
}
List <DB_Service.Sentiments> completeSentiments = new List <DB_Service.Sentiments>();
foreach (SentimentResults finalResults in tweetSentiments)
{
DB_Service.Sentiments newSenti = new DB_Service.Sentiments();
newSenti.tweet_id = finalResults.tweet_id;
newSenti.sentiment_total = finalResults.senti_score;
newSenti.category_primary = finalResults.language + ", " + finalResults.language_short;
StringBuilder wholePhrase = new StringBuilder("");
count = 0;
foreach (String word in finalResults.keyPhrases)
{
count++;
if (finalResults.keyPhrases.Count > count)
{
wholePhrase.Append(word + ",");
}
else
{
wholePhrase.Append(word);
}
}
completeSentiments.Add(newSenti);
}
DB_Service.ServiceClient service = new DB_Service.ServiceClient();
service.addSentiments(completeSentiments);
return(tweetSentiments);
// Linking entities---------------------------------------------------------------------------------------------------------------------------------
//Console.WriteLine("\n\n===== ENTITY LINKING ======");
//EntitiesBatchResult result4 = client.EntitiesAsync(
// new MultiLanguageBatchInput(
// new List<MultiLanguageInput>()
// {
// new MultiLanguageInput("en", "0", "I really enjoy the new XBox One S. It has a clean look, it has 4K/HDR resolution and it is affordable."),
// new MultiLanguageInput("en", "1", "The Seattle Seahawks won the Super Bowl in 2014."),
// })).Result;
//// Printing entity results.
//foreach (var document in result4.Documents)
//{
// Console.WriteLine("Document ID: {0} , Entities: {1}", document.Id, String.Join(", ", document.Entities.Select(entity => entity.Name)));
//}
}
public static List <SentimentResults> fullAnalysis(List <DB_Service.CrimeTweets> crimeTweets)
{//-------------------------------------------------------------------------------------------------------------------------------------
// Create a client.
ITextAnalyticsClient client = new TextAnalyticsClient(new ApiKeyServiceClientCredentials())
{
Endpoint = "https://westeurope.api.cognitive.microsoft.com"
//https://westeurope.api.cognitive.microsoft.com/text/analytics/v2.0
};
//-------------------------------------------------------------------------------------------------------------------------------------
// Extracting language
List <Input> myInp = new List <Input>();
foreach (DB_Service.CrimeTweets ct in crimeTweets)
{
Input inp = new Input(ct.tweet_id.ToString(), ct.message);
myInp.Add(inp);
}
var result = client.DetectLanguageAsync(new BatchInput(myInp)).Result;
List <SentimentResults> tweetLangs = new List <SentimentResults>();
// Printing language results.
foreach (var document in result.Documents)
{
SentimentResults sr = new SentimentResults();
sr.setTweet_id(Int32.Parse(document.Id));
sr.setLanguage_short(document.DetectedLanguages[0].Iso6391Name);
sr.setLanguage(document.DetectedLanguages[0].Name);
tweetLangs.Add(sr);
}
//-------------------------------------------------------------------------------------------------------------------------------------
// Getting key-phrases
List <MultiLanguageInput> keyPhrases = new List <MultiLanguageInput>();//Key phrases
int count = 0;
foreach (DB_Service.CrimeTweets ct in crimeTweets)
{
string tempLang = tweetLangs.ElementAt <SentimentResults>(count).getLanguage_short();
MultiLanguageInput inp = new MultiLanguageInput(tempLang, ct.tweet_id.ToString(), ct.message);
keyPhrases.Add(inp);
count++;
}
KeyPhraseBatchResult result2 = client.KeyPhrasesAsync(new MultiLanguageBatchInput(keyPhrases)).Result;
// Printing keyphrases
List <string> phrases = new List <string>();
List <SentimentResults> tweetKeyPhrases = new List <SentimentResults>();
count = 0;
foreach (var document in result2.Documents)
{
foreach (string keyphrase in document.KeyPhrases)
{
phrases.Add(keyphrase);
}
SentimentResults sr = new SentimentResults();
sr = tweetLangs.ElementAt <SentimentResults>(count);
sr.setKeyPhrases(phrases);
tweetKeyPhrases.Add(sr);
count++;
}
//-------------------------------------------------------------------------------------------------------------------------------------
// Getting Sentiment Analysis
List <MultiLanguageInput> sentiAni = new List <MultiLanguageInput>();//Sentiment Analysis
count = 0;
foreach (DB_Service.CrimeTweets ct in crimeTweets)
{
string tempLang = tweetKeyPhrases.ElementAt <SentimentResults>(count).getLanguage_short();
MultiLanguageInput inp = new MultiLanguageInput(tempLang, ct.tweet_id.ToString(), ct.message);
sentiAni.Add(inp);
count++;
}
SentimentBatchResult result3 = client.SentimentAsync(new MultiLanguageBatchInput(sentiAni)).Result;
// Printing sentiment results
List <SentimentResults> tweetSentiments = new List <SentimentResults>();
count = 0;
foreach (var document in result3.Documents)
{
SentimentResults sr = new SentimentResults();
sr = tweetKeyPhrases.ElementAt <SentimentResults>(count);
sr.setSenti_score((double)document.Score);
tweetSentiments.Add(sr);
count++;
}
//-------------------------------------------------------------------------------------------------------------------------------------
// Getting Entities
//Continue using the same list so languages wont change
EntitiesBatchResult result4 = client.EntitiesAsync(new MultiLanguageBatchInput(sentiAni)).Result;
// Printing entities results
List <string> entitiySet = new List <string>();
List <SentimentResults> tweetEntities = new List <SentimentResults>();
count = 0;
foreach (var document in result4.Documents)
{
foreach (EntityRecord entitiy in document.Entities)
{
entitiySet.Add(entitiy.Name);
}
SentimentResults sr = new SentimentResults();
sr = tweetSentiments.ElementAt <SentimentResults>(count);
sr.setEntities(entitiySet);
tweetEntities.Add(sr);
count++;
}
//-------------------------------------------------------------------------------------------------------------------------------------
//Add Data to Database Service
List <DB_Service.Sentiments> completeSentiments = new List <DB_Service.Sentiments>();
List <DB_Service.Entities> completeEntities = new List <DB_Service.Entities>();
foreach (SentimentResults finalResults in tweetEntities)
{
//Start building Sentiment class
DB_Service.Sentiments newSenti = new DB_Service.Sentiments();
newSenti.tweet_id = finalResults.getTweet_id();
newSenti.sentiment_total = finalResults.getSenti_score();
newSenti.category_primary = finalResults.getLanguage() + ", " + finalResults.getLanguage_short();
List <string> entList = finalResults.getEntities();
List <string> phraseList = finalResults.getKeyPhrases();
StringBuilder wholePhrase = new StringBuilder("");
count = 0;
//Start building Entity Class
if (entList != null && entList.Count > 0)
{
foreach (string entity in entList)
{
wholePhrase.Append(entity + ",");
//DB_Service.Entities newEntity = new DB_Service.Entities();
//newEntity.name = entity;
//newEntity.sentiment_id = -1//this is a programming design problem
}
}
if (phraseList != null && phraseList.Count > 0)
{
foreach (string word in phraseList)
{
count++;
if (phraseList.Count > count)
{
wholePhrase.Append(word + ",");
}
else
{
wholePhrase.Append(word);
}
}
}
newSenti.key_phrases = wholePhrase.ToString();
//List<string> EntList = finalResults.getEntities();
//if(EntList != null && EntList.Count > 0)
//{
// //newSenti.category_primary = EntList.ElementAt<string>(0);
//}
//else
//{
// newSenti.category_primary = "";
//}
//Finish building Sentiment Class
completeSentiments.Add(newSenti);
}
//Add to service now
DB_Service.ServiceClient service = new DB_Service.ServiceClient();
service.addSentiments(completeSentiments);
return(tweetEntities);
}
public static List <DB_Service.CrimeTweets> Search_SearchTweet(String keyword)
{
checkHasSetUserCred();
var searchParameter = Search.CreateTweetSearchParameter(keyword);
//searchParameter.SetGeoCode(new Coordinates(-26.195246, 28.034088), 100, DistanceMeasure.Kilometers);
//searchParameter.Lang = LanguageFilter.English;
//searchParameter.SearchType = SearchResultType.Popular;
//searchParameter.MaximumNumberOfResults = 100;
//searchParameter.Since = new DateTime(2013, 12, 1);
//searchParameter.Until = new DateTime(2013, 12, 11);
//searchParameter.SinceId = 399616835892781056;
//searchParameter.MaxId = 405001488843284480;
searchParameter.TweetSearchType = TweetSearchType.OriginalTweetsOnly;
//searchParameter.Filters = TweetSearchFilters.Videos;
var tweets = Search.SearchTweets(searchParameter);
List <String> tweetText = new List <String>();
List <DB_Service.CrimeTweets> crimeTweets = new List <DB_Service.CrimeTweets>();
if (tweets == null)
{
tweetText.Add(ExceptionHandler.GetLastException().TwitterDescription);
}
if (tweets != null)
{
foreach (var tweet in tweets)
{
tweetText.Add(tweet.Text);
DB_Service.CrimeTweets myTweet = new DB_Service.CrimeTweets();
myTweet.message = tweet.Text;
ICoordinates newCood = tweet.Coordinates;
if (newCood == null)
{
myTweet.latitude = -1;
myTweet.longitude = -1;
}
else
{
myTweet.latitude = newCood.Latitude;
myTweet.longitude = newCood.Longitude;
}
IPlace newPlace = tweet.Place;
if (newPlace == null)
{
myTweet.location = "Not provided";
}
else
{
myTweet.location = tweet.Place.Name;
}
myTweet.post_datetime = tweet.CreatedAt;
myTweet.recieved_datetime = DateTime.Now;
myTweet.twitter_handle = tweet.Prefix;
myTweet.weather = "Needs to be Analysed";
List <IUserMentionEntity> myMention = tweet.UserMentions;
myTweet.mentions = "";
foreach (IUserMentionEntity ent in myMention)
{
if (myTweet.mentions.Length != 0)
{
myTweet.mentions += " ";
}
myTweet.mentions += ent.ScreenName;
}
myTweet.tags = "Needs to be analysed";
crimeTweets.Add(myTweet);
}
}
DB_Service.ServiceClient service = new DB_Service.ServiceClient();
service.addCrimeTweets(crimeTweets);
return(crimeTweets);
}
