private async Task AnalyzeTextAsync()
{
try
{
if (!string.IsNullOrEmpty(this.speechRecognitionTextBox.Text))
{
SentimentResult textAnalysisResult = await TextAnalyticsHelper.GetTextSentimentAsync(new string[] { this.speechRecognitionTextBox.Text });
double score = textAnalysisResult.Scores.ElementAt(0);
this.sentimentControl.Sentiment = score;
}
else
{
this.sentimentControl.Sentiment = 0.5;
}
this.OnSpeechRecognitionAndSentimentProcessed(new SpeechRecognitionAndSentimentResult {
SpeechRecognitionText = this.speechRecognitionTextBox.Text, TextAnalysisSentiment = this.sentimentControl.Sentiment
});
}
catch (Exception ex)
{
await Util.GenericApiCallExceptionHandler(ex, "Error during Text Analytics call.");
}
}
public static async Task Run(
[BlobTrigger("searchabledocuments/{name}.{ext}", Connection = "AzureWebJobsStorage")] Stream myBlob,
string name, string ext, TraceWriter log)
{
// Because suffix filters don't work yet - this should take non-pdfs off the todo list
if (ext.ToLower() != "pdf")
{
return;
}
log.Info($"Text Processing beginning for {name} ({myBlob.Length} Bytes)");
log.Info($"Extracting text from the PDF (including OCR");
var pages = iTextPDFHelper.GetPDFPages(myBlob, log, ocrImages: true);
log.Info($"Calling Text Analytics to determine key phrases");
Dictionary <string, int> keyPhrases = await TextAnalyticsHelper.GetKeyPhrases(pages, log);
log.Info($"Uploading document to Azure Search");
foreach (var page in pages)
{
string pageId = HttpServerUtility.UrlTokenEncode(Encoding.UTF8.GetBytes(name + "." + ext + page.Number));
await AzureSearchHelper.UploadToAzureSearch(pageId, name + "." + ext, page.Number, page.KeyPhrases, page.Text, log);
}
}
public static async Task Run(
[BlobTrigger("summariseddocuments/{name}.{ext}")] Stream myBlob,
[Blob("summariseddocuments/{name}.{ext}.summary.txt", FileAccess.Write)] CloudBlobStream summaryBlob,
string name,
string ext,
TraceWriter log
)
{
// Because suffix filters don't work yet - this should take non-pdfs off the todo list
if (ext.ToLower() != "pdf")
{
return;
}
log.Info($"Text Processing beginning for {name} ({myBlob.Length} Bytes)");
log.Info($"Extracting text from the PDF");
var pages = iTextPDFHelper.GetPDFPages(myBlob, log, ocrImages: true);
log.Info($"Calling Text Analytics to determine key phrases");
Dictionary <string, int> keyPhrases = await TextAnalyticsHelper.GetKeyPhrases(pages, log);
var topPhrases = keyPhrases.OrderByDescending(pair => pair.Value).Take(20).ToList();
log.Info($"Building summary");
string summary = TextAnalyticsHelper.BuildSummary(pages, topPhrases);
log.Info($"Saving summary to new blob");
using (var stream = new MemoryStream(Encoding.UTF8.GetBytes(summary), false))
{
stream.CopyTo(summaryBlob);
}
}
private async Task SearchAsync(string query)
{
try
{
this.TopKeyPhrases.Clear();
this.latestSearchResult.Clear();
this.FilteredNewsResults.Clear();
this.sentimentDistributionControl.UpdateData(Enumerable.Empty <double>());
this.progressRing.IsActive = true;
string userLanguage = this.languageComboBox.SelectedValue.ToString();
var news = await BingSearchHelper.GetNewsSearchResults(query, count : 50, offset : 0, market : GetBingSearchMarketFromLanguage(userLanguage));
Task <SentimentResult> sentimentTask = TextAnalyticsHelper.GetTextSentimentAsync(news.Select(n => n.Title).ToArray(), language: GetTextAnalyticsLanguageCodeFromLanguage(userLanguage));
Task <KeyPhrasesResult> keyPhrasesTask;
if (IsLanguageSupportedByKeyPhraseAPI(userLanguage))
{
keyPhrasesTask = TextAnalyticsHelper.GetKeyPhrasesAsync(news.Select(n => n.Title).ToArray(), language: GetTextAnalyticsLanguageCodeFromLanguage(userLanguage));
}
else
{
keyPhrasesTask = Task.FromResult(new KeyPhrasesResult {
KeyPhrases = new string[][] { new string [] { "Not available in this language" } }
});
}
await Task.WhenAll(sentimentTask, keyPhrasesTask);
var sentiment = sentimentTask.Result;
for (int i = 0; i < news.Count(); i++)
{
NewsArticle article = news.ElementAt(i);
this.latestSearchResult.Add(new NewsAndSentimentScore {
Article = article, TitleSentiment = Math.Round(sentiment.Scores.ElementAt(i), 2)
});
}
UpdateFilteredResults();
this.sentimentDistributionControl.UpdateData(this.latestSearchResult.Select(n => n.TitleSentiment));
var wordGroups = keyPhrasesTask.Result.KeyPhrases.SelectMany(k => k).GroupBy(phrase => phrase, StringComparer.OrdinalIgnoreCase).OrderByDescending(g => g.Count()).Take(10).OrderBy(g => g.Key);
this.TopKeyPhrases.AddRange(wordGroups.Select(g => new KeyPhraseCount {
KeyPhrase = g.Key, Count = g.Count()
}));
}
catch (HttpRequestException ex)
{
await Util.GenericApiCallExceptionHandler(ex, "Processing error");
}
finally
{
this.progressRing.IsActive = false;
}
}
public Form1()
{
InitializeComponent();
speechConfig = SpeechConfig.FromSubscription(subscriptionKey, speechRegion);
recognizer = new SpeechRecognizer(speechConfig);
stopRecognition = new TaskCompletionSource <int>();
textAnalyticsHelper = new TextAnalyticsHelper(textAnalyticsEndpoint, textAnalyticsKey);
}
private async Task GetKeyPhrasesAsync()
{
try
{
KeyPhrasesResult keyPhrasesResult = await TextAnalyticsHelper.GetKeyPhrasesAsync(this.CallerTextBox.Text);
this.KeyPhrases.AddRange(keyPhrasesResult.KeyPhrases.OrderBy(i => i));
}
catch (Exception ex)
{
await Util.GenericApiCallExceptionHandler(ex, "Error during Text Analytics 'Key Phrases' call.");
}
}
public Startup(IWebHostEnvironment env, ILoggerFactory loggerFactory)
{
var builder = new ConfigurationBuilder()
.SetBasePath(env.ContentRootPath)
.AddJsonFile("appsettings.json", optional: true, reloadOnChange: true)
.AddJsonFile($"appsettings.{env.EnvironmentName}.json", optional: true)
.AddJsonFile("cognitivemodels.json", optional: true)
.AddJsonFile($"cognitivemodels.{env.EnvironmentName}.json", optional: true)
.AddEnvironmentVariables();
Configuration = builder.Build();
DBHelper.Configuration = Configuration;
TextAnalyticsHelper.Configure(Configuration);
JournalHelper.Configure(Configuration);
TokensController.Configuration = Configuration;
GraphHelper.Configure(Configuration);
this.HostingEnvironment = env;
}
private async Task DetectLanguageAsync()
{
try
{
if (!string.IsNullOrEmpty(this.speechRecognitionTextBox.Text))
{
DetectLanguageResult textDetectResult = await TextAnalyticsHelper.GetDetectedLanguageAsync(this.speechRecognitionTextBox.Text);
this.detectedLanguage = new string[] { textDetectResult.Language["iso6391Name"], textDetectResult.Language["name"] };
}
else
{
this.detectedLanguage = new string[] { "en", string.Empty };
}
}
catch (Exception ex)
{
await Util.GenericApiCallExceptionHandler(ex, "Error during Text Analytics 'Detect' call.");
}
}
private async Task AnalyzeTextAsync()
{
try
{
if (!string.IsNullOrEmpty(this.speechRecognitionTextBox.Text))
{
SentimentResult textAnalysisResult = await TextAnalyticsHelper.GetTextSentimentAsync(this.translatedText, this.detectedLanguage[0]);
this.sentimentControl.Sentiment = textAnalysisResult.Score;
}
else
{
this.sentimentControl.Sentiment = 0.5;
}
this.OnSpeechRecognitionAndSentimentProcessed(new SpeechRecognitionAndSentimentResult {
SpeechRecognitionText = this.speechRecognitionTextBox.Text, TextAnalysisSentiment = this.sentimentControl.Sentiment, DetectedLanguage = this.detectedLanguage[1], TranslatedText = this.translatedText
});
}
catch (Exception ex)
{
await Util.GenericApiCallExceptionHandler(ex, "Error during Text Analytics 'Sentiment' call.");
}
}
//DelegateCommand _ShowBusyCommand;
//public DelegateCommand ShowBusyCommand
// => _ShowBusyCommand ?? (_ShowBusyCommand = new DelegateCommand(async () =>
// {
// Views.Busy.SetBusy(true, _BusyText);
// await Task.Delay(5000);
// Views.Busy.SetBusy(false);
// }, () => !string.IsNullOrEmpty(BusyText)));
public async Task GotoDetailsPage()
{
Views.Busy.SetBusy(true, "Getting your score...");
UnconsciousBiasResult result = null;
var graphClient = await AuthenticationHelper.GetAuthenticatedClientAsync();
if (graphClient != null)
{
var emails = await graphClient.Me
.Messages
.Request()
.Search($"\"to:{Value}\"")
.Select("UniqueBody")
.GetAsync();
StringBuilder sb = new StringBuilder();
sb.Append("{\"documents\":[");
int i = 1;
foreach (var email in emails)
{
Debug.WriteLine(email.UniqueBody);
// This is super-hacky processing out the HTML tags from the email.
// TODO: replace this with a proper library that will do this better.
string body = email.UniqueBody.Content;
string bodyWithoutHTMLtags = WebUtility.HtmlDecode(Regex.Replace(body, "<[^>]*(>|$)", string.Empty));
string step2 = Regex.Replace(bodyWithoutHTMLtags, @"[\s\r\n]+", " ");
sb.Append("{\"id\":\"" + i + "\",\"text\":\"" + step2 + "\"},");
i++;
}
// Remove the trailing comma to get well-formatted JSON
string tempString = sb.ToString();
if (tempString.LastIndexOf(",") == tempString.Length - 1)
{
sb.Remove(tempString.Length - 1, 1);
}
// Close JSON message
sb.Append("]}");
List <double> sentimentScores = await TextAnalyticsHelper.GetSentiment(sb.ToString());
// Calculate average sentiment score
double scoreSum = 0.0;
int scoreCount = 0;
foreach (double sentimentScore in sentimentScores)
{
scoreSum += sentimentScore;
scoreCount++;
}
double averageSentimentScore = scoreSum / scoreCount;
int sentimentPercentage = Convert.ToInt32(averageSentimentScore * 100);
result = new UnconsciousBiasResult()
{
Positivity = sentimentPercentage,
KeyWords = "TODO",
Topics = "TODO",
PositivityGraph = sentimentScores,
Person = Value
};
}
// can pass value to other screen and do fancy display
NavigationService.Navigate(typeof(Views.DetailPage), result);
await Task.CompletedTask;
}
private async Task <AnalyzeTextResult> AnalyzeNewsAsync(IEnumerable <NewsArticle> news, string userLanguage)
{
var scores = new List <double>();
var topKeyPhrases = new List <KeyPhraseCount>();
// prepare news titles to text string
string[] newsTitleList = news.Select(n => n.Title.Replace(".", ";")).ToArray();
string newsTitles = string.Join(". ", newsTitleList);
var strInfo = new StringInfo(newsTitles);
int lenInTextElements = strInfo.LengthInTextElements;
// check Text Analytics data limits
string languageCode = GetTextAnalyticsLanguageCodeFromLanguage(userLanguage);
bool isLangSupportedForKeyPhrases = IsLanguageSupportedByKeyPhraseAPI(userLanguage);
if (lenInTextElements < TextAnalyticsHelper.MaximumLengthOfTextDocument)
{
Task <DocumentSentiment> sentimentTask = TextAnalyticsHelper.AnalyzeSentimentAsync(newsTitles, languageCode);
Task <KeyPhraseCollection> keyPhrasesTask = isLangSupportedForKeyPhrases ? TextAnalyticsHelper.ExtractKeyPhrasesAsync(newsTitles, languageCode) : Task.FromResult <KeyPhraseCollection>(null);
await Task.WhenAll(sentimentTask, keyPhrasesTask);
var sentimentResult = sentimentTask.Result;
var keyPhrasesResult = keyPhrasesTask.Result;
scores = sentimentResult.Sentences.Select(s => GetSentimentScore(s)).ToList();
var wordGroups = keyPhrasesResult?.GroupBy(phrase => phrase, StringComparer.OrdinalIgnoreCase).OrderByDescending(g => g.Count()).Take(10).OrderBy(g => g.Key).ToList();
topKeyPhrases = wordGroups != null && wordGroups.Any()
? wordGroups.Select(w => new KeyPhraseCount {
KeyPhrase = w.Key, Count = w.Count()
}).ToList()
: new List <KeyPhraseCount>()
{
new KeyPhraseCount {
KeyPhrase = "Not available in this language", Count = 1
}
};
}
else
{
// if the input data is larger than max limit then split the input data into several different requests
var sentimentTaskList = new List <Task <AnalyzeSentimentResultCollection> >();
var keyPhrasesTaskList = new List <Task <ExtractKeyPhrasesResultCollection> >();
int maxDocsPerRequest = Math.Min(TextAnalyticsHelper.MaxDocumentsPerRequestForSentimentAnalysis, TextAnalyticsHelper.MaxDocumentsPerRequestForKeyPhraseExtraction);
int batchSize = Math.Min((int)Math.Ceiling((decimal)TotalNewsCount / maxDocsPerRequest), maxDocsPerRequest);
for (int i = 0; i < TextAnalyticsHelper.MaxRequestsPerSecond; i++)
{
int skip = i * batchSize;
string[] newsTitlesBatch = newsTitleList.Skip(skip).Take(batchSize).ToArray();
if (!newsTitlesBatch.Any())
{
break;
}
sentimentTaskList.Add(TextAnalyticsHelper.AnalyzeSentimentAsync(newsTitlesBatch, languageCode));
if (isLangSupportedForKeyPhrases)
{
keyPhrasesTaskList.Add(TextAnalyticsHelper.ExtractKeyPhrasesAsync(newsTitlesBatch, languageCode));
}
}
var taskList = new List <Task>();
taskList.AddRange(sentimentTaskList);
taskList.AddRange(keyPhrasesTaskList);
await Task.WhenAll(taskList);
foreach (var sentimentTask in sentimentTaskList)
{
AnalyzeSentimentResultCollection sentimentResult = sentimentTask.Result;
scores.AddRange(sentimentResult.SelectMany(d => d.DocumentSentiment.Sentences).Select(s => GetSentimentScore(s)).ToList());
}
var keyPhrasesList = new List <string>();
foreach (var keyPhrasesTask in keyPhrasesTaskList)
{
ExtractKeyPhrasesResultCollection keyPhrasesResult = keyPhrasesTask.Result;
keyPhrasesList.AddRange(keyPhrasesResult.SelectMany(k => k.KeyPhrases));
}
var wordGroups = keyPhrasesList.GroupBy(phrase => phrase, StringComparer.OrdinalIgnoreCase).OrderByDescending(g => g.Count()).Take(10).OrderBy(g => g.Key).ToList();
topKeyPhrases = wordGroups.Any()
? wordGroups.Select(w => new KeyPhraseCount {
KeyPhrase = w.Key, Count = w.Count()
}).ToList()
: new List <KeyPhraseCount>()
{
new KeyPhraseCount {
KeyPhrase = "Not available in this language", Count = 1
}
};
}
return(new AnalyzeTextResult
{
Scores = scores,
TopKeyPhrases = topKeyPhrases
});
}
/// <summary>
/// Logs an incoming message to the Bot. Core channel information is stored along with the original message and sentiment.
/// Logs an event called "BotMessageReceived".
/// Logs the following properties: conversation id, from user id, conversation name, channel id, client info.
/// Other info logged based on configuration: from user name, original message, sentiment.
/// </summary>
/// <param name="activity">The incoming Activity from the Bot Framework.</param>
/// <param name="additionalProperties">Additional properties to log.</param>
public void LogIncomingMessage(ITurnContext context, Dictionary <string, string> additionalProperties = null)
{
// Simple guard statements
EnsureNotNull(context, "Context activity cannot be null");
// Track the event with an exception wrapper
TrackEventWithExceptionWrapper((TelemetryClient tc) =>
{
var activity = context.Activity.AsMessageActivity();
Dictionary <string, string> telemetryProperties = new Dictionary <string, string>();
// Set the Telemetry context as we don't have a BF Telemetry Initializer at this time...
if (!string.IsNullOrEmpty(activity.Conversation.Id))
{
tc.Context.Session.Id = activity.Conversation.Id;
}
if (!string.IsNullOrEmpty(activity.From.Id))
{
tc.Context.User.Id = activity.From.Id;
}
// General message metadata, defensive to cover for discrepancies across channels
if (!string.IsNullOrEmpty(activity.ChannelId))
{
telemetryProperties.Add(TelemetryHelper.ChannelProperty, activity.ChannelId);
}
if (!string.IsNullOrEmpty(activity.From.Id))
{
telemetryProperties.Add(TelemetryHelper.FromIdProperty, activity.From.Id);
}
if (!string.IsNullOrEmpty(activity.Conversation.Id))
{
telemetryProperties.Add(TelemetryHelper.ConversationIdProperty, activity.Conversation.Id);
}
if (!string.IsNullOrEmpty(activity.Conversation.Name))
{
telemetryProperties.Add(TelemetryHelper.ConversationNameProperty, activity.Conversation.Name);
}
if (additionalProperties != null)
{
foreach (KeyValuePair <string, string> valuePair in additionalProperties)
{
if (telemetryProperties.ContainsKey(valuePair.Key))
{
telemetryProperties[valuePair.Key] = valuePair.Value;
}
else
{
telemetryProperties.Add(valuePair.Key, valuePair.Value);
}
}
}
// Now check for specific entities such as client info...
if (activity.Entities != null)
{
// Do we have any client info? e.g. language, locale, device type
var clientEntity = activity.Entities.FirstOrDefault(e => e.Type == TelemetryHelper.ClientInfoIdentifier);
if (clientEntity != null && clientEntity.Properties != null)
{
string prop = clientEntity.Properties.ToString();
telemetryProperties.Add(TelemetryHelper.ClientInfoProperty, prop);
}
}
// For some customers, logging user name within Application Insights might be an issue so have provided a config setting to disable this feature
if (logUserName && !string.IsNullOrEmpty(activity.From.Name))
{
telemetryProperties.Add(TelemetryHelper.FromNameProperty, activity.From.Name);
}
// For some customers, logging the utterances within Application Insights might be an so have provided a config setting to disable this feature
if (logOriginalMessages && !string.IsNullOrEmpty(activity.Text))
{
telemetryProperties.Add(TelemetryHelper.TextProperty, activity.Text);
}
// Only perform Text Analytics operations if we have a valid key and log sentiment is set to true
if (!string.IsNullOrEmpty(textAnalyticsKey) && logSentimentAndKeyPhrases && !string.IsNullOrEmpty(activity.Text))
{
try
{
// Crude but gets a good sense of the utterance length.
// NOTE: before splitting, replace multiple instances of spaces with a single space
// and trim either end, so that we do not skew the amount of words in the trimmed list.
string modifiedText = Regex.Replace(activity.Text, @"\s+", " ").Trim();
string[] words = activity.Text.Split(' ');
// Sentiment and Key Phrase extraction is not effective on short utterances so we skip if less than the provided threshold
if (words.Length >= sentimentWordThreshold)
{
// For each utterance we identify the language in order to then extract key-phrases and sentiment which is addded to the Telemetry
// LUIS now performs Key Phrase Extraction and Sentiment for you but not exposed as part of the v4SDK, once fixed this can be removed.
var(identifiedLanguage, keyPhrases, sentiment) = TextAnalyticsHelper.EvaluateUtterance(textAnalyticsClient, activity.Text);
if (!string.IsNullOrEmpty(identifiedLanguage))
{
telemetryProperties.Add(TelemetryHelper.LanguageProperty, identifiedLanguage);
}
if (!string.IsNullOrEmpty(keyPhrases))
{
telemetryProperties.Add(TelemetryHelper.KeyPhrasesProperty, keyPhrases);
}
if (sentiment != int.MinValue)
{
string sentimentScore = sentiment.ToString("N2");
telemetryProperties.Add(TelemetryHelper.SentimentProperty, sentimentScore);
}
}
else
{
tc.TrackTrace($"TelemetryHelper::LogIncomingMessage::No sentiment calculated for a utterance with {words.Length} word(s).");
}
}
catch (Exception e)
{
tc.TrackException(e);
tc.TrackTrace($"TelemetryHelper::Exception ocurred whilst calculating sentiment - skipping but still logging without it. {e.Message}");
}
}
// Log the event
tc.TrackEvent(TelemetryHelper.BotMessageReceivedEvent, telemetryProperties);
});
}
private async Task AnalyzeTextAsync()
{
try
{
this.progressControl.IsActive = true;
DisplayProcessingUI();
string input = this.inputText.Text;
var detectedLanguageTask = TextAnalyticsHelper.DetectLanguageAsync(input);
var detectedKeyPhrasesTask = TextAnalyticsHelper.ExtractKeyPhrasesAsync(input);
var documentSentimentTask = TextAnalyticsHelper.AnalyzeSentimentAsync(input, sentimentAnalyses: AdditionalSentimentAnalyses.OpinionMining);
var namedEntitiesResponseTask = TextAnalyticsHelper.RecognizeEntitiesAsync(input);
var linkedEntitiesResponseTask = TextAnalyticsHelper.RecognizeLinkedEntitiesAsync(input);
await Task.WhenAll(detectedLanguageTask, detectedKeyPhrasesTask, documentSentimentTask, namedEntitiesResponseTask, linkedEntitiesResponseTask);
var detectedLanguage = detectedLanguageTask.Result;
var detectedKeyPhrases = detectedKeyPhrasesTask.Result;
var documentSentiment = documentSentimentTask.Result;
var namedEntitiesResponse = namedEntitiesResponseTask.Result;
var linkedEntitiesResponse = linkedEntitiesResponseTask.Result;
// detected language and key phrases
this.detectedLangTextBlock.Text = !string.IsNullOrEmpty(detectedLanguage.Name) ? $"{detectedLanguage.Name} (confidence: {(int)(detectedLanguage.ConfidenceScore * 100)}%)" : "Not found";
this.detectedKeyPhrasesTextBlock.Text = detectedKeyPhrases.Any() ? string.Join(", ", detectedKeyPhrases) : "Not found";
// document sentiment
CreateSentimentChart(documentSentiment);
// mined opinions
OpinionMiningCollection.Clear();
var minedOpinions = documentSentiment?.Sentences.SelectMany(s => s.MinedOpinions);
if (minedOpinions != null && minedOpinions.Any())
{
var minedOpinionList = minedOpinions.Select(om => new MinedOpinion()
{
Aspect = om.Aspect.Text,
Opinions = string.Join(", ", om.Opinions.Select(o => $"{o.Text} ({o.Sentiment.ToString("G")})"))
});
OpinionMiningCollection.AddRange(minedOpinionList);
}
// entities
if (namedEntitiesResponse.Any())
{
this.namesEntitiesGridView.ItemsSource = namedEntitiesResponse.Select(x => new { x.Text, Category = $"[{x.Category}]" });
}
else
{
this.namesEntitiesGridView.ItemsSource = new[] { new { Text = "No entities" } };
}
// linked entities
if (linkedEntitiesResponse.Any())
{
this.linkedEntitiesGridView.ItemsSource = linkedEntitiesResponse.Select(x => new { Name = $"{x.Name} ({x.DataSource})", x.Url });
}
else
{
this.linkedEntitiesGridView.ItemsSource = new[] { new { Name = "No linked entities" } };
}
// prepare json result
var jsonResult = new
{
LanguageDetection = detectedLanguage,
KeyPhrases = detectedKeyPhrases,
Sentiment = documentSentiment,
Entities = namedEntitiesResponse,
EntityLinking = linkedEntitiesResponse
};
this.jsonResultTextBlock.Text = JsonConvert.SerializeObject(jsonResult, Formatting.Indented);
}
catch (Exception ex)
{
await Util.GenericApiCallExceptionHandler(ex, "Failure analyzing text");
}
finally
{
this.progressControl.IsActive = false;
}
}
private async Task AnalyzeTextAsync()
{
try
{
this.progressControl.IsActive = true;
DisplayProcessingUI();
// detect language
string input = this.inputText.Text;
DetectedLanguage detectedLanguage = await TextAnalyticsHelper.DetectLanguageAsync(input);
string languageCode = TextAnalyticsHelper.GetLanguageCode(detectedLanguage);
// check supported languages
bool isOpinionMiningSupported = TextAnalyticsHelper.OpinionMiningSupportedLanguages.Any(l => string.Equals(l, languageCode, StringComparison.OrdinalIgnoreCase));
bool isSentimentSupported = TextAnalyticsHelper.SentimentAnalysisSupportedLanguages.Any(l => string.Equals(l, languageCode, StringComparison.OrdinalIgnoreCase));
bool isKeyPhraseSupported = TextAnalyticsHelper.KeyPhraseExtractionSupportedLanguages.Any(l => string.Equals(l, languageCode, StringComparison.OrdinalIgnoreCase));
bool isNamedEntitySupported = TextAnalyticsHelper.NamedEntitySupportedLanguages.Any(l => string.Equals(l, languageCode, StringComparison.OrdinalIgnoreCase));
bool isEntityLinkingSupported = TextAnalyticsHelper.EntityLinkingSupportedLanguages.Any(l => string.Equals(l, languageCode, StringComparison.OrdinalIgnoreCase));
// sentiment analysis, key phrase extraction, named entity recognition and entity linking
Task <DocumentSentiment> documentSentimentTask = isSentimentSupported ? TextAnalyticsHelper.AnalyzeSentimentAsync(input, languageCode, isOpinionMiningSupported) : Task.FromResult <DocumentSentiment>(null);
Task <KeyPhraseCollection> detectedKeyPhrasesTask = isKeyPhraseSupported ? TextAnalyticsHelper.ExtractKeyPhrasesAsync(input, languageCode) : Task.FromResult <KeyPhraseCollection>(null);
Task <CategorizedEntityCollection> namedEntitiesResponseTask = isNamedEntitySupported ? TextAnalyticsHelper.RecognizeEntitiesAsync(input, languageCode) : Task.FromResult <CategorizedEntityCollection>(null);
Task <LinkedEntityCollection> linkedEntitiesResponseTask = isEntityLinkingSupported ? TextAnalyticsHelper.RecognizeLinkedEntitiesAsync(input, languageCode) : Task.FromResult <LinkedEntityCollection>(null);
await Task.WhenAll(documentSentimentTask, detectedKeyPhrasesTask, namedEntitiesResponseTask, linkedEntitiesResponseTask);
DocumentSentiment documentSentiment = documentSentimentTask.Result;
KeyPhraseCollection detectedKeyPhrases = detectedKeyPhrasesTask.Result;
CategorizedEntityCollection namedEntitiesResponse = namedEntitiesResponseTask.Result;
LinkedEntityCollection linkedEntitiesResponse = linkedEntitiesResponseTask.Result;
// display results
this.detectedLangTextBlock.Text = !string.IsNullOrEmpty(detectedLanguage.Name) ? $"{detectedLanguage.Name} (confidence: {(int)(detectedLanguage.ConfidenceScore * 100)}%)" : NotFound;
this.detectedKeyPhrasesTextBlock.Text = detectedKeyPhrases != null && detectedKeyPhrases.Any()
? string.Join(", ", detectedKeyPhrases)
: isKeyPhraseSupported?NotFound : LanguageNotSupported;
this.namesEntitiesGridView.ItemsSource = namedEntitiesResponse != null && namedEntitiesResponse.Any()
? namedEntitiesResponse.Select(x => new { x.Text, Category = $"[{x.Category}]" })
: new[] { new { Text = isNamedEntitySupported ? "No entities" : LanguageNotSupported, Category = "" } };
this.linkedEntitiesGridView.ItemsSource = linkedEntitiesResponse != null && linkedEntitiesResponse.Any()
? linkedEntitiesResponse.Select(x => new { Name = $"{x.Name} ({x.DataSource})", x.Url })
: new[] {
isEntityLinkingSupported
? new { Name = "No linked entities", Url = new Uri("about:blank") }
: new { Name = LanguageNotSupported, Url = TextAnalyticsHelper.LanguageSupportUri }
};
if (isSentimentSupported)
{
CreateSentimentChart(documentSentiment);
// mined opinions
OpinionMiningCollection.Clear();
var minedOpinions = documentSentiment?.Sentences.SelectMany(s => s.MinedOpinions);
if (minedOpinions != null && minedOpinions.Any())
{
var minedOpinionList = minedOpinions.Select(om => new MinedOpinion()
{
Aspect = om.Aspect.Text,
Opinions = string.Join(", ", om.Opinions.Select(o => $"{o.Text} ({o.Sentiment.ToString("G")})"))
});
OpinionMiningCollection.AddRange(minedOpinionList);
}
}
else
{
this.sentimentTextBlock.Text = LanguageNotSupported;
this.sentimentChart.Visibility = Visibility.Collapsed;
}
// prepare json result
var jsonResult = new
{
LanguageDetection = detectedLanguage,
KeyPhrases = detectedKeyPhrases,
Sentiment = documentSentiment,
Entities = namedEntitiesResponse,
EntityLinking = linkedEntitiesResponse
};
this.jsonResultTextBlock.Text = JsonConvert.SerializeObject(jsonResult, Formatting.Indented);
}
catch (Exception ex)
{
await Util.GenericApiCallExceptionHandler(ex, "Failure analyzing text");
}
finally
{
this.progressControl.IsActive = false;
}
}
private async Task <DialogTurnResult> CheckTodaysEntry(WaterfallStepContext stepContext, CancellationToken cancellationToken)
{
await PopulateStateObjects(stepContext);
_graphHelper.Authenticate();
var section = stepContext.Values["Section"] as OnenoteSection;
var contextId = _discussionState.SignedInUserId;
var currentPages = await _graphHelper.GetNotebookPagesInSection(section);
DateTime entryDate = DateTime.Today;
List <JournalPageMetadata> journalPages = new List <JournalPageMetadata>();
int currentDay = entryDate.Day;
int daysInMonth = DateTime.DaysInMonth(entryDate.Year, entryDate.Month);
int year = entryDate.Year;
int month = entryDate.Month;
List <JournalEntry> userEntries = DBHelper.GetJournalEntries(contextId,
new DateTime(year, month, 1),
new DateTime(year, month, daysInMonth));
var invalidEntries = new KeyValuePair <string, List <string> >(
contextId, new List <string>());
// Remove pages that do not exist in the current pages collection
for (int i = userEntries.Count - 1; i >= 0; i--)
{
var expectedPage = userEntries[i];
if (!currentPages.Exists(ce => ce.Id == expectedPage.Id))
{
invalidEntries.Value.Add(expectedPage.Id);
userEntries.Remove(expectedPage);
}
}
if (invalidEntries.Value.Count > 0)
{
DBHelper.RemoveInvalidJournalEntries(invalidEntries);
}
var sentimentNeeded = userEntries.Where(j => {
var cp = currentPages.First(p => p.Id == j.Id);
return(
(j.Sentiment == null) ||
((j.Sentiment != null) &&
(cp.LastModifiedDateTime > j.LastModified)));
}).ToList();
if (sentimentNeeded.Count > 0)
{
var sentimentDocuments = new Dictionary <String, TextDocumentInput[]>();
foreach (var item in sentimentNeeded)
{
var prompt = "Checking out your" +
(item.Sentiment != null ? " updated" : "") +
$" entry for {item.EntryDate.ToLongDateString()}";
await stepContext.Context.SendActivityAsync(MessageFactory.Text(prompt));
string content = await _graphHelper.GetNotebookPageContent(item.Id);
HtmlDocument doc = new HtmlDocument();
doc.LoadHtml(content);
var body = doc.DocumentNode.SelectSingleNode("//body");
var divHeader = doc.DocumentNode.SelectSingleNode("//div[@data-id='_default']");
var paragraphs = new List <String>();
var paragraphNodes = divHeader.SelectNodes(".//p");
if (paragraphNodes?.Count > 0)
{
paragraphs.AddRange(paragraphNodes.Select(p => p.InnerText).ToArray());
}
var remainingHtml = body.InnerHtml.Replace(divHeader.OuterHtml, "");
var remainingDoc = new HtmlDocument();
remainingDoc.LoadHtml(remainingHtml);
paragraphNodes = remainingDoc.DocumentNode.SelectNodes("//p");
if (paragraphNodes?.Count > 0)
{
paragraphs.AddRange(paragraphNodes.Select(p => p.InnerText).ToArray());
}
if (paragraphs.Count > 0)
{
var combinedText = System.Web.HttpUtility.HtmlDecode(String.Join("\n", paragraphs));
AddInputDocument(sentimentDocuments, item.Id, combinedText);
prompt = "That's new.. I'll have to consider what you wrote.";
}
else
{
prompt = "Nothing new for me to check out here!";
}
await stepContext.Context.SendActivityAsync(MessageFactory.Text(prompt));
}
if (sentimentDocuments.Count > 0)
{
await stepContext.Context.SendActivityAsync(MessageFactory.Text(
"Assessing what you wrote..."));
}
var sentimentResults = TextAnalyticsHelper.GetSentiment(sentimentDocuments);
foreach (var entryKey in sentimentResults.Keys)
{
var assessedPage = currentPages.First(e => e.Id == entryKey);
var lastModified = assessedPage.LastModifiedDateTime;
var ds = sentimentResults[entryKey];
DBHelper.SaveJournalEntryAssessment(new JournalEntryAssessment
{
Id = entryKey,
Sentiment = ds,
LastModified = lastModified?.UtcDateTime
});
var prompt = $"Your entry on {assessedPage.Title} was {ds.Sentiment}";
await stepContext.Context.SendActivityAsync(MessageFactory.Text(prompt));
}
}
var monthMode = false;
var createdPage = false;
OnenotePage currentPage = null;
var pageId = String.Empty;
//Month mode
if (monthMode)
{
for (int i = currentDay; i <= daysInMonth; i++)
{
DateTime current = new DateTime(year, month, i);
if (!userEntries.Exists(e => e.EntryDate == current))
{
journalPages.Add(new JournalPageMetadata(current));
}
}
}
else
{
await stepContext.Context.SendActivityAsync(MessageFactory.Text(
"Looking for today's journal entry"));
var existingPage = userEntries.FirstOrDefault(e => e.EntryDate == entryDate);
if (existingPage == null)
{
await stepContext.Context.SendActivityAsync(MessageFactory.Text(
"Hmmm... didn't find today's entry. Don't worry I'll create one for you."));
journalPages.Add(new JournalPageMetadata(entryDate));
}
else
{
pageId = existingPage.Id;
await stepContext.Context.SendActivityAsync(MessageFactory.Text(
"Found it!"));
}
}
if (String.IsNullOrEmpty(pageId))
{
var weeks = journalPages.GroupBy(p => p.Week).ToList();
var pageTemplate = ResourceHelper
.ReadManifestData <JournalingDialog>("JournalPageTemplate.htm");
var headerPageTemplate = ResourceHelper
.ReadManifestData <JournalingDialog>("JournalHeaderTemplate.htm");
foreach (var grp in weeks)
{
var headerPageTitle = $"Week {grp.Key}";
var headerEntryDate = entryDate.ToString("yyyy-MM-ddTHH:mm:ss.0000000");
Console.Write($" {headerPageTitle} ");
var headerPage = currentPages.FirstOrDefault(p => p.Title == headerPageTitle);
if (headerPage == null)
{
var headerHtml = String.Format(headerPageTemplate, headerPageTitle, headerEntryDate);
var headerPageId = await _graphHelper.CreateNotebookPage(section.Id, headerHtml);
Console.WriteLine(headerPageId);
}
else
{
Console.WriteLine(headerPage.Id);
}
foreach (var item in grp)
{
var jp = DBHelper.GetJournalPrompt();
var pageTitle = $"{item.DayOfWeek} the {item.Day.AsOrdinal()}";
Console.Write($"\t{pageTitle} ");
var newEntryDate = item.EntryDate.ToString("yyyy-MM-ddTHH:mm:ss.0000000");
var prompt = jp.Prompt;
var details = jp.Details;
var promptSource = jp.Source;
var html = String.Format(pageTemplate, pageTitle, newEntryDate,
prompt, details, promptSource,
entryDate.Ticks, jp.SourceIndex, jp.PromptIndex);
currentPage = await _graphHelper.CreateNotebookPage(section.Id, html);
pageId = currentPage.Id;
createdPage = true;
var je = new JournalEntry
{
UserContextId = contextId,
Id = pageId,
PromptSourceId = jp.SourceIndex,
PromptIndexId = jp.PromptIndex,
EntryDate = item.EntryDate
};
DBHelper.CreateJournalEntry(je);
await _graphHelper.UpdateNotebookPageLevel(pageId, 1);
await stepContext.Context.SendActivityAsync(MessageFactory.Text(
"All done setting up today's entry for you!. Ping me when you want me to take a look."));
Console.WriteLine($"{pageId}");
}
}
}
if ((currentPage == null) && !String.IsNullOrEmpty(pageId))
{
currentPage = await _graphHelper.GetNotebookPage(pageId);
}
AddUpdateStepContextValue(stepContext, "CurrentPage", currentPage);
return(await stepContext.NextAsync(null, cancellationToken));
}
