private string textAnalysis(string text) { MLContext mlContext = new MLContext(); TrainTestData splitDataView = LoadData(mlContext); ITransformer model = BuildAndTrainModel(mlContext, splitDataView.TrainSet); Evaluate(mlContext, model, splitDataView.TestSet); PredictionEngine <SentimentData, SentimentPrediction> predictionFunction = mlContext.Model.CreatePredictionEngine <SentimentData, SentimentPrediction>(model); SentimentData sampleStatement = new SentimentData { SentimentText = text }; //Console.WriteLine(); //Console.WriteLine($"Sentiment: {resultPrediction.SentimentText} | Prediction: {(Convert.ToBoolean(resultPrediction.Prediction) ? "Positive" : "Negative")} | Probability: {resultPrediction.Probability} "); var resultPrediction = predictionFunction.Predict(sampleStatement); var curseCount = helperMethods.profanityCheck(text); var punctuation = text.Where(Char.IsPunctuation).Distinct().ToArray(); var words = text.Split().Select(x => x.Trim(punctuation)); float wordslength = words.Count(); foreach (string word in words) { if (word.Length == 0) { wordslength = wordslength - 1; } } float curseRatio = curseCount / wordslength; var result = new TextAnalysisResult { Context = (Convert.ToBoolean(resultPrediction.Prediction) ? "Positive" : "Negative"), CurseCount = curseCount, CurseRatio = curseRatio }; var returnJson = JsonConvert.SerializeObject(result); return(returnJson); }
static TextAnalysisResult AnalyzeText(string inputText, string accountkey) { TextAnalysisResult result = new TextAnalysisResult(); using (var httpClient = new HttpClient()) { // Set http client parameters to call data market and encode text string inputTextEncoded = HttpUtility.UrlEncode(inputText); httpClient.BaseAddress = new Uri("https://api.datamarket.azure.com/"); string creds = "AccountKey:" + accountkey; string authorizationHeader = "Basic " + Convert.ToBase64String(Encoding.ASCII.GetBytes(creds)); httpClient.DefaultRequestHeaders.Add("Authorization", authorizationHeader); httpClient.DefaultRequestHeaders.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); // Get key phrases string keyPhrasesRequest = "data.ashx/amla/text-analytics/v1/GetKeyPhrases?Text=" + inputTextEncoded; HttpResponseMessage response = httpClient.GetAsync(keyPhrasesRequest).Result; string content = response.Content.ReadAsStringAsync().Result; if (!response.IsSuccessStatusCode) { throw new Exception("Call failed. HTTP status code: " + response.StatusCode + " and contents: " + content); } result = JsonConvert.DeserializeObject<TextAnalysisResult>(content); // Get sentiment string sentimentRequest = "data.ashx/amla/text-analytics/v1/GetSentiment?Text=" + inputTextEncoded; response = httpClient.GetAsync(sentimentRequest).Result; content = response.Content.ReadAsStringAsync().Result; if (!response.IsSuccessStatusCode) { throw new Exception("Call failed. HTTP status code: " + response.StatusCode + " and contents: " + content); } dynamic sentimentResult = Newtonsoft.Json.Linq.JObject.Parse(content); result.SentimentIndex = sentimentResult.Score; } return result; }
public async Task <TextAnalysisResult> AnalyzeTextAsync(string text) { // See: https://docs.microsoft.com/en-us/azure/cognitive-services/text-analytics/quickstarts/csharp var analysisResult = new TextAnalysisResult(); if (string.IsNullOrEmpty(text)) { Console.WriteLine("\t\t\tContentAnalyzer.AnalyzeTextAsync(): no text to analyze"); return(analysisResult); } string textToAnalyze = text; if (text.Length > 5000) { Console.WriteLine("\t\t\tContentAnalyzer.AnalyzeTextAsync(): text longer than supported length. Trimming it..."); textToAnalyze = text.Substring(0, 5000); } Console.WriteLine("\t\t\tContentAnalyzer.AnalyzeTextAsync(): initializing TextAnalyticsAPI"); ITextAnalyticsAPI m_TextAnalyticsClient = new TextAnalyticsAPI { AzureRegion = AzureRegions.Westeurope, SubscriptionKey = m_TextAnalyticsAPISubscriptionKey }; Console.WriteLine("\t\t\tContentAnalyzer.AnalyzeTextAsync(): detecting content language"); var batchLanguageResult = await m_TextAnalyticsClient.DetectLanguageAsync(new BatchInput(new List <Input>() { new Input("1", textToAnalyze) })).ConfigureAwait(false); if (batchLanguageResult.Errors.Count > 0) { Console.WriteLine("\t\t\tContentAnalyzer.AnalyzeTextAsync(): error while detecting language"); foreach (var errors in batchLanguageResult.Errors) { Console.WriteLine($"\t{errors.Message}"); } return(analysisResult); } analysisResult.DetectedLanguage = batchLanguageResult.Documents[0].DetectedLanguages[0].Name; analysisResult.DetectedLanguageScore = batchLanguageResult.Documents[0].DetectedLanguages[0].Score.GetValueOrDefault(); Console.WriteLine($"\t\t\tContentAnalyzer.AnalyzeTextAsync(): detected language is '{analysisResult.DetectedLanguage}' ({(analysisResult.DetectedLanguageScore * 100):0.00}%)"); Console.WriteLine("\t\t\tContentAnalyzer.AnalyzeTextAsync(): performing key-phrase extraction"); var multiLanguageInput = new MultiLanguageBatchInput(new List <MultiLanguageInput>() { new MultiLanguageInput(batchLanguageResult.Documents[0].DetectedLanguages[0].Iso6391Name, "1", textToAnalyze) }); var batchKeyphraseResult = await m_TextAnalyticsClient.KeyPhrasesAsync(multiLanguageInput).ConfigureAwait(false); if (batchKeyphraseResult.Errors.Count > 0) { Console.WriteLine("\t\t\tContentAnalyzer.AnalyzeTextAsync(): error while extracting key-phrases"); foreach (var errors in batchKeyphraseResult.Errors) { Console.WriteLine($"\t\t\t\t{errors.Message}"); } return(analysisResult); } Console.WriteLine($"\t\t\tContentAnalyzer.AnalyzeTextAsync(): retrieved {batchKeyphraseResult.Documents[0].KeyPhrases.Count} key-phrases:"); foreach (var keyphrase in batchKeyphraseResult.Documents[0].KeyPhrases) { analysisResult.KeyPhrases.Add(keyphrase); Console.WriteLine($"\t\t\t\t{keyphrase}"); } Console.WriteLine("\t\t\tContentAnalyzer.AnalyzeTextAsync(): performing sentiment analysis"); var batchSentimentResult = await m_TextAnalyticsClient.SentimentAsync(multiLanguageInput).ConfigureAwait(false); if (batchSentimentResult.Errors.Count > 0) { Console.WriteLine("\t\t\tContentAnalyzer.AnalyzeTextAsync(): error while detecting sentiment"); foreach (var errors in batchSentimentResult.Errors) { Console.WriteLine($"\t\t\t\t{errors.Message}"); } return(analysisResult); } analysisResult.SentimentScore = batchSentimentResult.Documents[0].Score.GetValueOrDefault(); analysisResult.Sentiment = GetSentiment(analysisResult.SentimentScore); Console.WriteLine($"\t\t\tContentAnalyzer.AnalyzeTextAsync(): sentiment is '{analysisResult.Sentiment}' ({(analysisResult.SentimentScore * 100):0.00}%)"); // Extend analysis by estimating reading time for content analysisResult.WordCount = TextTokenizer.GetWordCount(text); analysisResult.ReadingTimeInMinutes = ReadingTimeEstimator.GetEstimatedReadingTime(analysisResult.WordCount, analysisResult.DetectedLanguage); return(analysisResult); }
public async ValueTask <UserTypingStatistics> GenerateUserStatisticsAsync(string userId, string language, TextGenerationType textGenerationType) { var existingStatistics = await _userTypingStatisticsStore.GetUserTypingStatisticsAsync(userId, language, textGenerationType) .ConfigureAwait(false); var userSessions = Enumerable.Empty <UserSession>(); userSessions = existingStatistics != null ? await _userSessionRepository.FindAllForUserFromTypingResultsAsync(userId, existingStatistics.LastHandledResultUtc) .ConfigureAwait(false) : await _userSessionRepository.FindAllForUserAsync(userId) .ConfigureAwait(false); var results = new List <TextAnalysisResult>(); var textsTypedCount = 0; DateTime lastHandledResultUtc = default; foreach (var userSession in userSessions) { var typingSession = await _typingSessionRepository.FindAsync(userSession.TypingSessionId) .ConfigureAwait(false); if (typingSession == null) { throw new InvalidOperationException("Typing session is not found."); } foreach (var textTypingResult in userSession.GetTextTypingResults()) { textsTypedCount++; if (textTypingResult.SubmittedResultsUtc > lastHandledResultUtc) { lastHandledResultUtc = textTypingResult.SubmittedResultsUtc; } var text = typingSession.GetTypingSessionTextAtIndexOrDefault(textTypingResult.TypingSessionTextIndex); if (text == null) { throw new InvalidOperationException("Text is not found in typing session."); } var textEntity = await _textRepository.FindAsync(text.TextId) .ConfigureAwait(false); if (textEntity == null) { throw new InvalidOperationException("Text is not found."); } if (textEntity.Language != language) { continue; } if (textEntity.TextGenerationType != textGenerationType) { continue; // Generate statistics only for requested text generation type. } var textAnalysisResult = await _textTypingResultValidator.ValidateAsync(text.Value, textTypingResult) .ConfigureAwait(false); results.Add(textAnalysisResult); } } if (results.Count == 0) { if (existingStatistics != null) { // No new data yet. return(existingStatistics); } return(new UserTypingStatistics(0, 0, Enumerable.Empty <KeyPairAggregatedData>(), DateTime.UtcNow)); } var aggregatedResult = new TextAnalysisResult( results.Sum(x => x.SpeedCpm) / results.Count, results.SelectMany(x => x.KeyPairs)); var specificKeys = aggregatedResult.KeyPairs.GroupBy(x => new { x.FromKey, x.ShouldBeKey }); var aggregatedData = specificKeys.Select(x => new KeyPairAggregatedData( x.Key.FromKey, x.Key.ShouldBeKey, x.Where(y => y.Type == KeyPairType.Correct).Any() ? x.Where(y => y.Type == KeyPairType.Correct).Average(y => y.Delay) : 0, x.Where(y => y.Type == KeyPairType.Correct).Any() ? x.Where(y => y.Type == KeyPairType.Correct).Min(y => y.Delay) : 0, x.Where(y => y.Type == KeyPairType.Correct).Any() ? x.Where(y => y.Type == KeyPairType.Correct).Max(y => y.Delay) : 0, x.Count(y => y.Type == KeyPairType.Correct), x.Count(y => y.Type == KeyPairType.Mistake))); var result = MergeAndReturnNew(existingStatistics, new TypingReport(aggregatedResult, aggregatedData), textsTypedCount, lastHandledResultUtc); await _userTypingStatisticsStore.SaveAsync(userId, result, language, textGenerationType) .ConfigureAwait(false); return(result); }
public async ValueTask <TypingReport> GenerateReportForUserSessionAsync(string userSessionId, TextGenerationType textGenerationType) { var results = new List <TextAnalysisResult>(); var userSession = await _userSessionRepository.FindAsync(userSessionId) .ConfigureAwait(false); if (userSession == null) { throw new InvalidOperationException("Could not find user session."); } var typingSession = await _typingSessionRepository.FindAsync(userSession.TypingSessionId) .ConfigureAwait(false); if (typingSession == null) { throw new InvalidOperationException("Typing session is not found."); } foreach (var textTypingResult in userSession.GetTextTypingResults()) { var text = typingSession.GetTypingSessionTextAtIndexOrDefault(textTypingResult.TypingSessionTextIndex); if (text == null) { throw new InvalidOperationException("Text is not found in typing session."); } var textEntity = await _textRepository.FindAsync(text.TextId) .ConfigureAwait(false); if (textEntity == null) { throw new InvalidOperationException("Text is not found."); } if (textGenerationType != textEntity.TextGenerationType) { continue; } var textAnalysisResult = await _textTypingResultValidator.ValidateAsync(text.Value, textTypingResult) .ConfigureAwait(false); results.Add(textAnalysisResult); } if (results.Count == 0) { // No data yet. return(new TypingReport(new TextAnalysisResult(0, Enumerable.Empty <KeyPair>()), Enumerable.Empty <KeyPairAggregatedData>())); } var aggregatedResult = new TextAnalysisResult( results.Sum(x => x.SpeedCpm) / results.Count, results.SelectMany(x => x.KeyPairs)); var specificKeys = aggregatedResult.KeyPairs.GroupBy(x => new { x.FromKey, x.ShouldBeKey }); var aggregatedData = specificKeys.Select(x => new KeyPairAggregatedData( x.Key.FromKey, x.Key.ShouldBeKey, x.Where(y => y.Type == KeyPairType.Correct).Any() ? x.Where(y => y.Type == KeyPairType.Correct).Average(y => y.Delay) : 0, x.Where(y => y.Type == KeyPairType.Correct).Any() ? x.Where(y => y.Type == KeyPairType.Correct).Min(y => y.Delay) : 0, x.Where(y => y.Type == KeyPairType.Correct).Any() ? x.Where(y => y.Type == KeyPairType.Correct).Max(y => y.Delay) : 0, x.Count(y => y.Type == KeyPairType.Correct), x.Count(y => y.Type == KeyPairType.Mistake))); return(new TypingReport(aggregatedResult, aggregatedData)); }