private void refreshData() { sentiResults.InnerHtml = "<div class='card mb-4'>" + "<div class='card-block'>" + "<h3 class='card-title'>Results</h3>" + "<div class='dropdown card-title-btn-container'>" + "<div class='dropdown-menu dropdown-menu-right' aria-labelledby='dropdownMenuButton'> " + "<a class='dropdown-item' href='#'><em class='fa fa-search mr-1'></em>More info</a>" + "<a class='dropdown-item' href='#'><em class='fa fa-thumb-tack mr-1'></em>Pin Window</a>" + "<a class='dropdown-item' href='#'><em class='fa fa-remove mr-1'></em>Close Window</a>" + "</div>" + "</div>" + "<div class='table-responsive'>" + "<table class='table table-striped'>" + "<thead>" + "<tr>" + "<th>Sentiment ID</th>" + "<th>Sentiment Total</th>" + "<th>Primary Category</th>" + "<th>Key Phrases</th>" + "<th>Tweet ID</th>" + "</tr>" + "</thead>" + "<tbody>"; List <DB_Service.Sentiments> sentiments = new List <DB_Service.Sentiments>(); DB_Service.ServiceClient service = new DB_Service.ServiceClient(); sentiments = service.getSentiments(); if (sentiments != null) { foreach (DB_Service.Sentiments senti in sentiments) { sentiResults.InnerHtml += "<tr>" + "<td>" + senti.sentiment_id + "</td>" + "<td>" + senti.sentiment_total + "</td>" + "<td>" + senti.category_primary + "</td>" + "<td>" + senti.key_phrases + "</td>" + "<td>" + senti.tweet_id + "</td>" + "</tr>"; } } sentiResults.InnerHtml += "</tbody>" + "</table>" + "</div>" + "</div>" + "</div>" + "</div>"; }
protected void UpdatePage_Click(object sender, EventArgs e) { string category = txtCategory.Value; string twitterHandle = txtTwitterHandle.Value; string keyword = txtKeyword.Value; if (category.Equals("")) { category = "Crime"; } if (twitterHandle.Equals("")) { twitterHandle = "S.A Crime Watch"; } if (keyword.Equals("")) { keyword = "Crime"; } List <DB_Service.CrimeTweets> tweetQuery = Query.Search_SearchTweet(keyword); DB_Service.ServiceClient service = new DB_Service.ServiceClient(); queryFeed.InnerHtml = ""; foreach (DB_Service.CrimeTweets tweet in tweetQuery) { queryFeed.InnerHtml += /*"<div class='col-lg-4 mb-4 bg-default'>" +*/ "<div class='card'>" + "<div class='card-header'>Received At: " + DateTime.Now + "</div>" + "<div class='card-block'>" + "<p>" + tweet.message + "</p>" + "</div>" + //"</div>" + "</div>"; } queryFeed.InnerHtml += "<div class='alert bg-success' role='alert'> Success! Twitter has been queried and tweets have been added to the database.<a href= '#' class='float-right'></a></div>"; List <DB_Service.CrimeTweets> newTweetList = service.getCrimeTweetsToAnalyse(); List <SentimentResults> newResults = TextAnalyticsV2_1.fullAnalysis(newTweetList); queryFeed.InnerHtml += "<div class='alert bg-success' role='alert'> Success! New tweets language, phrases, entities and sentiment have been analysed and added to the database.<a href= '#' class='float-right'></a></div>"; // Printing language results. //foreach (SentimentResults document in newResults) //{ // queryFeed.InnerHtml += "Document ID: " + document.getTweet_id() + ", Score: " + document.getSenti_score() + "\n"; //} }
public static List <DB_Service.Sentiments> processTweetSentiments(List <DB_Service.CrimeTweets> crimeTweets) { List <DB_Service.Sentiments> mySentiments = new List <DB_Service.Sentiments>(); foreach (DB_Service.CrimeTweets ct in crimeTweets) { DB_Service.Sentiments senti = new DB_Service.Sentiments(); senti.sentiment_total = getCompoundTweetAnalysis(ct.message); senti.category_primary = "To be Analysed"; senti.key_phrases = "To be Analysed"; senti.tweet_id = ct.tweet_id; mySentiments.Add(senti); } if (mySentiments != null) { DB_Service.ServiceClient service = new DB_Service.ServiceClient(); service.addSentiments(mySentiments); } return(mySentiments); }
//public List<SentimentResults> extractingKeyPhrases(List<GuestGeek_DBService.CrimeTweets> crimeTweets) //{ // // Create a client.--------------------------------------------------------------------------------------------------------------------------------- // ITextAnalyticsClient client = new TextAnalyticsClient(new ApiKeyServiceClientCredentials()) // { // Endpoint = "https://westeurope.api.cognitive.microsoft.com/text/analytics/v2.0/" // }; // System.Console.OutputEncoding = System.Text.Encoding.UTF8; // // Getting key phrases.--------------------------------------------------------------------------------------------------------------------------------- // Console.WriteLine("\n\n===== KEY-PHRASE EXTRACTION ======"); // List<MultiLanguageInput> keyPhrases = new List<MultiLanguageInput>(); // foreach (GuestGeek_DBService.CrimeTweets ct in crimeTweets) // { // MultiLanguageInput inp = new MultiLanguageInput(; // myInp.Add(inp); // } // KeyPhraseBatchResult result2 = client.KeyPhrasesAsync(new MultiLanguageBatchInput( // new List<MultiLanguageInput>() // { // new MultiLanguageInput("ja", "1", "猫は幸せ"), // new MultiLanguageInput("de", "2", "Fahrt nach Stuttgart und dann zum Hotel zu Fu."), // new MultiLanguageInput("en", "3", "My cat is stiff as a rock."), // new MultiLanguageInput("es", "4", "A mi me encanta el fútbol!") // })).Result; // // Printing key phrases. // foreach (var document in result2.Documents) // { // Console.WriteLine("Document ID: {0} ", document.Id); // Console.WriteLine("\t Key phrases:"); // foreach (string keyphrase in document.KeyPhrases) // { // Console.WriteLine("\t\t" + keyphrase); // } // } //} //static async void MakeRequest() //{ // var client = new HttpClient(); // var queryString = HttpUtility.ParseQueryString(string.Empty); // // Request headers // client.DefaultRequestHeaders.Add("Ocp-Apim-Subscription-Key", TwitterKeys.getTextAnalyticsKey()); // var uri = "https://westus.api.cognitive.microsoft.com/text/analytics/v2.0/languages?" + queryString; // HttpResponseMessage response; // // Request body // byte[] byteData = Encoding.UTF8.GetBytes("{body}"); // using (var content = new ByteArrayContent(byteData)) // { // content.Headers.ContentType = new MediaTypeHeaderValue("application/json"); // response = await client.PostAsync(uri, content); // } //} public List <SentimentResults> fullAnalysis(List <DB_Service.CrimeTweets> crimeTweets) { // Create a client.--------------------------------------------------------------------------------------------------------------------------------- ITextAnalyticsClient client = new TextAnalyticsClient(new ApiKeyServiceClientCredentials()) { Endpoint = "https://westeurope.api.cognitive.microsoft.com/text/analytics/v2.0/" }; System.Console.OutputEncoding = System.Text.Encoding.UTF8; // Extracting language.--------------------------------------------------------------------------------------------------------------------------------- Console.WriteLine("===== LANGUAGE EXTRACTION ======"); List <Input> myInp = new List <Input>();//Languages foreach (DB_Service.CrimeTweets ct in crimeTweets) { //Sentiment.Sentiment.sanitize(ct.message); Input inp = new Input(ct.tweet_id.ToString(), ct.message); myInp.Add(inp); } LanguageBatchResult result = null; try { result = client.DetectLanguageAsync(new BatchInput(myInp)).Result; //Task.Run(() => client.DetectLanguageAsync(new BatchInput(myInp)).Result).Wait(); } catch (AggregateException aex) { string messages = ""; foreach (Exception ex in aex.InnerExceptions) { messages += ex.Message + "\r\n"; } Debug.WriteLine(messages); } // Printing language results. List <SentimentResults> tweetLangs = new List <SentimentResults>();//Language foreach (var document in result.Documents) { SentimentResults tr = new SentimentResults(); tr.language_short = document.DetectedLanguages[0].Iso6391Name; tr.tweet_id = Int32.Parse(document.Id); tr.language = document.DetectedLanguages[0].Name; tweetLangs.Add(tr); } // Getting key phrases.--------------------------------------------------------------------------------------------------------------------------------- Console.WriteLine("\n\n===== KEY-PHRASE EXTRACTION ======"); List <MultiLanguageInput> keyPhrases = new List <MultiLanguageInput>();//Key phrases int count = 0; foreach (DB_Service.CrimeTweets ct in crimeTweets) { string tempLang = tweetLangs.ElementAt <SentimentResults>(count).language_short; MultiLanguageInput inp = new MultiLanguageInput(tempLang, ct.tweet_id.ToString(), ct.message); keyPhrases.Add(inp); count++; } KeyPhraseBatchResult result2 = client.KeyPhrasesAsync(new MultiLanguageBatchInput(keyPhrases)).Result; // Printing key phrases. List <string> phrases = new List <string>(); List <SentimentResults> tweetKeyPhrases = new List <SentimentResults>(); count = 0; foreach (var document in result2.Documents) { //Console.WriteLine("Document ID: {0} ", document.Id); //Console.WriteLine("\t Key phrases:"); foreach (string keyphrase in document.KeyPhrases) { //Console.WriteLine("\t\t" + keyphrase); phrases.Add(keyphrase); } SentimentResults sr = new SentimentResults(); sr = tweetLangs.ElementAt <SentimentResults>(count); sr.keyPhrases = phrases; tweetKeyPhrases.Add(sr); count++; } // Analyzing sentiment.--------------------------------------------------------------------------------------------------------------------------------- Console.WriteLine("\n\n===== SENTIMENT ANALYSIS ======"); List <MultiLanguageInput> sentiAni = new List <MultiLanguageInput>();//Sentiment Analysis count = 0; foreach (DB_Service.CrimeTweets ct in crimeTweets) { string tempLang = tweetKeyPhrases.ElementAt <SentimentResults>(count).language_short; MultiLanguageInput inp = new MultiLanguageInput(tempLang, ct.tweet_id.ToString(), ct.message); sentiAni.Add(inp); count++; } SentimentBatchResult result3 = client.SentimentAsync(new MultiLanguageBatchInput(sentiAni)).Result; // Printing sentiment results. List <SentimentResults> tweetSentiments = new List <SentimentResults>(); count = 0; foreach (var document in result3.Documents) { //Console.WriteLine("Document ID: {0} , Sentiment Score: {1:0.00}", document.Id, document.Score); SentimentResults sr = new SentimentResults(); sr = tweetKeyPhrases.ElementAt <SentimentResults>(count); sr.senti_score = (double)document.Score; tweetSentiments.Add(sr); count++; } List <DB_Service.Sentiments> completeSentiments = new List <DB_Service.Sentiments>(); foreach (SentimentResults finalResults in tweetSentiments) { DB_Service.Sentiments newSenti = new DB_Service.Sentiments(); newSenti.tweet_id = finalResults.tweet_id; newSenti.sentiment_total = finalResults.senti_score; newSenti.category_primary = finalResults.language + ", " + finalResults.language_short; StringBuilder wholePhrase = new StringBuilder(""); count = 0; foreach (String word in finalResults.keyPhrases) { count++; if (finalResults.keyPhrases.Count > count) { wholePhrase.Append(word + ","); } else { wholePhrase.Append(word); } } completeSentiments.Add(newSenti); } DB_Service.ServiceClient service = new DB_Service.ServiceClient(); service.addSentiments(completeSentiments); return(tweetSentiments); // Linking entities--------------------------------------------------------------------------------------------------------------------------------- //Console.WriteLine("\n\n===== ENTITY LINKING ======"); //EntitiesBatchResult result4 = client.EntitiesAsync( // new MultiLanguageBatchInput( // new List<MultiLanguageInput>() // { // new MultiLanguageInput("en", "0", "I really enjoy the new XBox One S. It has a clean look, it has 4K/HDR resolution and it is affordable."), // new MultiLanguageInput("en", "1", "The Seattle Seahawks won the Super Bowl in 2014."), // })).Result; //// Printing entity results. //foreach (var document in result4.Documents) //{ // Console.WriteLine("Document ID: {0} , Entities: {1}", document.Id, String.Join(", ", document.Entities.Select(entity => entity.Name))); //} }
public static List <SentimentResults> fullAnalysis(List <DB_Service.CrimeTweets> crimeTweets) {//------------------------------------------------------------------------------------------------------------------------------------- // Create a client. ITextAnalyticsClient client = new TextAnalyticsClient(new ApiKeyServiceClientCredentials()) { Endpoint = "https://westeurope.api.cognitive.microsoft.com" //https://westeurope.api.cognitive.microsoft.com/text/analytics/v2.0 }; //------------------------------------------------------------------------------------------------------------------------------------- // Extracting language List <Input> myInp = new List <Input>(); foreach (DB_Service.CrimeTweets ct in crimeTweets) { Input inp = new Input(ct.tweet_id.ToString(), ct.message); myInp.Add(inp); } var result = client.DetectLanguageAsync(new BatchInput(myInp)).Result; List <SentimentResults> tweetLangs = new List <SentimentResults>(); // Printing language results. foreach (var document in result.Documents) { SentimentResults sr = new SentimentResults(); sr.setTweet_id(Int32.Parse(document.Id)); sr.setLanguage_short(document.DetectedLanguages[0].Iso6391Name); sr.setLanguage(document.DetectedLanguages[0].Name); tweetLangs.Add(sr); } //------------------------------------------------------------------------------------------------------------------------------------- // Getting key-phrases List <MultiLanguageInput> keyPhrases = new List <MultiLanguageInput>();//Key phrases int count = 0; foreach (DB_Service.CrimeTweets ct in crimeTweets) { string tempLang = tweetLangs.ElementAt <SentimentResults>(count).getLanguage_short(); MultiLanguageInput inp = new MultiLanguageInput(tempLang, ct.tweet_id.ToString(), ct.message); keyPhrases.Add(inp); count++; } KeyPhraseBatchResult result2 = client.KeyPhrasesAsync(new MultiLanguageBatchInput(keyPhrases)).Result; // Printing keyphrases List <string> phrases = new List <string>(); List <SentimentResults> tweetKeyPhrases = new List <SentimentResults>(); count = 0; foreach (var document in result2.Documents) { foreach (string keyphrase in document.KeyPhrases) { phrases.Add(keyphrase); } SentimentResults sr = new SentimentResults(); sr = tweetLangs.ElementAt <SentimentResults>(count); sr.setKeyPhrases(phrases); tweetKeyPhrases.Add(sr); count++; } //------------------------------------------------------------------------------------------------------------------------------------- // Getting Sentiment Analysis List <MultiLanguageInput> sentiAni = new List <MultiLanguageInput>();//Sentiment Analysis count = 0; foreach (DB_Service.CrimeTweets ct in crimeTweets) { string tempLang = tweetKeyPhrases.ElementAt <SentimentResults>(count).getLanguage_short(); MultiLanguageInput inp = new MultiLanguageInput(tempLang, ct.tweet_id.ToString(), ct.message); sentiAni.Add(inp); count++; } SentimentBatchResult result3 = client.SentimentAsync(new MultiLanguageBatchInput(sentiAni)).Result; // Printing sentiment results List <SentimentResults> tweetSentiments = new List <SentimentResults>(); count = 0; foreach (var document in result3.Documents) { SentimentResults sr = new SentimentResults(); sr = tweetKeyPhrases.ElementAt <SentimentResults>(count); sr.setSenti_score((double)document.Score); tweetSentiments.Add(sr); count++; } //------------------------------------------------------------------------------------------------------------------------------------- // Getting Entities //Continue using the same list so languages wont change EntitiesBatchResult result4 = client.EntitiesAsync(new MultiLanguageBatchInput(sentiAni)).Result; // Printing entities results List <string> entitiySet = new List <string>(); List <SentimentResults> tweetEntities = new List <SentimentResults>(); count = 0; foreach (var document in result4.Documents) { foreach (EntityRecord entitiy in document.Entities) { entitiySet.Add(entitiy.Name); } SentimentResults sr = new SentimentResults(); sr = tweetSentiments.ElementAt <SentimentResults>(count); sr.setEntities(entitiySet); tweetEntities.Add(sr); count++; } //------------------------------------------------------------------------------------------------------------------------------------- //Add Data to Database Service List <DB_Service.Sentiments> completeSentiments = new List <DB_Service.Sentiments>(); List <DB_Service.Entities> completeEntities = new List <DB_Service.Entities>(); foreach (SentimentResults finalResults in tweetEntities) { //Start building Sentiment class DB_Service.Sentiments newSenti = new DB_Service.Sentiments(); newSenti.tweet_id = finalResults.getTweet_id(); newSenti.sentiment_total = finalResults.getSenti_score(); newSenti.category_primary = finalResults.getLanguage() + ", " + finalResults.getLanguage_short(); List <string> entList = finalResults.getEntities(); List <string> phraseList = finalResults.getKeyPhrases(); StringBuilder wholePhrase = new StringBuilder(""); count = 0; //Start building Entity Class if (entList != null && entList.Count > 0) { foreach (string entity in entList) { wholePhrase.Append(entity + ","); //DB_Service.Entities newEntity = new DB_Service.Entities(); //newEntity.name = entity; //newEntity.sentiment_id = -1//this is a programming design problem } } if (phraseList != null && phraseList.Count > 0) { foreach (string word in phraseList) { count++; if (phraseList.Count > count) { wholePhrase.Append(word + ","); } else { wholePhrase.Append(word); } } } newSenti.key_phrases = wholePhrase.ToString(); //List<string> EntList = finalResults.getEntities(); //if(EntList != null && EntList.Count > 0) //{ // //newSenti.category_primary = EntList.ElementAt<string>(0); //} //else //{ // newSenti.category_primary = ""; //} //Finish building Sentiment Class completeSentiments.Add(newSenti); } //Add to service now DB_Service.ServiceClient service = new DB_Service.ServiceClient(); service.addSentiments(completeSentiments); return(tweetEntities); }
public static List <DB_Service.CrimeTweets> Search_SearchTweet(String keyword) { checkHasSetUserCred(); var searchParameter = Search.CreateTweetSearchParameter(keyword); //searchParameter.SetGeoCode(new Coordinates(-26.195246, 28.034088), 100, DistanceMeasure.Kilometers); //searchParameter.Lang = LanguageFilter.English; //searchParameter.SearchType = SearchResultType.Popular; //searchParameter.MaximumNumberOfResults = 100; //searchParameter.Since = new DateTime(2013, 12, 1); //searchParameter.Until = new DateTime(2013, 12, 11); //searchParameter.SinceId = 399616835892781056; //searchParameter.MaxId = 405001488843284480; searchParameter.TweetSearchType = TweetSearchType.OriginalTweetsOnly; //searchParameter.Filters = TweetSearchFilters.Videos; var tweets = Search.SearchTweets(searchParameter); List <String> tweetText = new List <String>(); List <DB_Service.CrimeTweets> crimeTweets = new List <DB_Service.CrimeTweets>(); if (tweets == null) { tweetText.Add(ExceptionHandler.GetLastException().TwitterDescription); } if (tweets != null) { foreach (var tweet in tweets) { tweetText.Add(tweet.Text); DB_Service.CrimeTweets myTweet = new DB_Service.CrimeTweets(); myTweet.message = tweet.Text; ICoordinates newCood = tweet.Coordinates; if (newCood == null) { myTweet.latitude = -1; myTweet.longitude = -1; } else { myTweet.latitude = newCood.Latitude; myTweet.longitude = newCood.Longitude; } IPlace newPlace = tweet.Place; if (newPlace == null) { myTweet.location = "Not provided"; } else { myTweet.location = tweet.Place.Name; } myTweet.post_datetime = tweet.CreatedAt; myTweet.recieved_datetime = DateTime.Now; myTweet.twitter_handle = tweet.Prefix; myTweet.weather = "Needs to be Analysed"; List <IUserMentionEntity> myMention = tweet.UserMentions; myTweet.mentions = ""; foreach (IUserMentionEntity ent in myMention) { if (myTweet.mentions.Length != 0) { myTweet.mentions += " "; } myTweet.mentions += ent.ScreenName; } myTweet.tags = "Needs to be analysed"; crimeTweets.Add(myTweet); } } DB_Service.ServiceClient service = new DB_Service.ServiceClient(); service.addCrimeTweets(crimeTweets); return(crimeTweets); }