private async Task <IWord> ProcessSingleWord(string cleanToken) { string msg = "'" + cleanToken + "' "; IWord w; if (this.Language == SupportedLanguage.English) { IList <IWord> offlineWords = await OfflineWord.Search(cleanToken); if (offlineWords.Count > 0) { w = WordSearchUtilities.GetMostProbable(offlineWords); } else { // TODO: find a better way to add a word that is not found in the database. msg += "SearchCount is zero, so I'm creating a new word with an unclassified part of speech and an unknown language level"; w = new OfflineWord() { Term = cleanToken, JsonLevel = LanguageLevelClassification.Unknown, PartOfSpeech = PartOfSpeech.Unclassified, Language = (string)new SupportedLanguageToLcidConverter().Convert(SupportedLanguage.English, typeof(string), null, CultureInfo.InvariantCulture) }; } Tools.Logger.Log("ProcessSingleWord", msg); return(w); } else if (this.Language == SupportedLanguage.English) { Tools.Logger.Log("ProcessSingleWord", "Call remote SpanishTagger"); SpanishPOSTagger es_tagger = new SpanishPOSTagger(App.OAuth2Account, cleanToken); IList <SpanishWord> results = await es_tagger.CallEndpointAsObjectAsync(); Tools.Logger.Log("ProcessSingleWord", "Got the results!"); try { return(results.First()); } catch (Exception) { return(new OfflineWord() { Term = cleanToken, JsonLevel = LanguageLevelClassification.Unknown, PartOfSpeech = PartOfSpeech.Unclassified, Language = (string)new SupportedLanguageToLcidConverter().Convert(SupportedLanguage.Spanish, typeof(string), null, CultureInfo.InvariantCulture) }); } } return(new OfflineWord() { Term = cleanToken, JsonLevel = LanguageLevelClassification.Unknown, PartOfSpeech = PartOfSpeech.Unclassified, Language = (string)new SupportedLanguageToLcidConverter().Convert(SupportedLanguage.English, typeof(string), null, CultureInfo.InvariantCulture) }); }
/// <summary> /// Initializes or reinitializes all properties which can be extracted from an offline analysis of the text. /// </summary> private void InitializeOfflineAnalysisProperties() { // TODO: catch errors in the UI! this.OfflineAnalysisResult = new AsyncLazy <List <IWord> >(async() => { List <IWord> analysisCache = new List <IWord>(); if (this.Language == SupportedLanguage.English) { IList <Task <IWord> > searchTokenTasks = new List <Task <IWord> >(); // Dirty tokens foreach (string token in Regex.Matches(this._essayContents, "[\\w']+").Cast <Match>().Select(m => m.Value)) { foreach (string cleanToken in Regex.Matches(await this.PreprocessSingleWord(token), "[\\w']+").Cast <Match>().Select(m => m.Value)) { searchTokenTasks.Add(this.ProcessSingleWord(cleanToken)); } } IEnumerable <IWord> results = await Task.WhenAll(searchTokenTasks); Tools.Logger.Log("EssayExercise", "I've waited all of them!"); foreach (IWord w in results) { if (this.ExcludeFunctionalWords && ( w.PartOfSpeech == PartOfSpeech.ClauseOpener || w.PartOfSpeech == PartOfSpeech.Conjunction || w.PartOfSpeech == PartOfSpeech.Determiner || w.PartOfSpeech == PartOfSpeech.DeterminerPronoun || w.PartOfSpeech == PartOfSpeech.ExistentialParticle || w.PartOfSpeech == PartOfSpeech.Genitive || w.PartOfSpeech == PartOfSpeech.InfinitiveMarker || w.PartOfSpeech == PartOfSpeech.InterjectionOrDiscourseMarker || w.PartOfSpeech == PartOfSpeech.NegativeMarker || w.PartOfSpeech == PartOfSpeech.CardinalNumber || w.PartOfSpeech == PartOfSpeech.Ordinal || w.PartOfSpeech == PartOfSpeech.Pronoun || w.PartOfSpeech == PartOfSpeech.ModalVerb)) { continue; } analysisCache.Add(w); } } else if (this.Language == SupportedLanguage.Spanish) { Tools.Logger.Log("EssayExerciseES", "Call remote SpanishTagger"); SpanishPOSTagger es_tagger = new SpanishPOSTagger(App.OAuth2Account, this._essayContents); IList <SpanishWord> results = await es_tagger.CallEndpointAsObjectAsync(); Tools.Logger.Log("EssayExerciseES", "Got the results!"); analysisCache.AddRange(results); } else if (this.Language == SupportedLanguage.German) { Tools.Logger.Log("EssayExerciseDE", "Call remote GermanTagger"); GermanPOSTagger de_tagger = new GermanPOSTagger(App.OAuth2Account, this._essayContents); IList <GermanWord> results = await de_tagger.CallEndpointAsObjectAsync(); Tools.Logger.Log("EssayExerciseDE", "Got the results!"); analysisCache.AddRange(results); } return(analysisCache); }); this.LevelClassification = new AsyncLazy <Dictionary <LanguageLevelClassification, List <IWord> > >(async() => { if (this.Language == SupportedLanguage.English) { List <IWord> offlineAnalysis = await this.OfflineAnalysisResult; Dictionary <LanguageLevelClassification, List <IWord> > result = new Dictionary <LanguageLevelClassification, List <IWord> >(); foreach (LanguageLevelClassification level in Enum.GetValues(typeof(LanguageLevelClassification))) { result.Add(level, new List <IWord>()); } foreach (IWord word in offlineAnalysis) { LanguageLevelClassification level = await word.Level; result[level].Add(word); } return(result); } return(new Dictionary <LanguageLevelClassification, List <IWord> >()); }); this.LevelClassificationDistribution = new AsyncLazy <Dictionary <LanguageLevelClassification, float> >(async() => { if (this.Language == SupportedLanguage.English) { // TODO: check for any possible loss of precision List <IWord> offlineAnalysis = await this.OfflineAnalysisResult; Dictionary <LanguageLevelClassification, float> result = new Dictionary <LanguageLevelClassification, float>(); foreach (LanguageLevelClassification level in Enum.GetValues(typeof(LanguageLevelClassification))) { result.Add(level, 0); } foreach (IWord word in offlineAnalysis) { LanguageLevelClassification level = await word.Level; result[level] = result[level] + 1; } foreach (LanguageLevelClassification level in Enum.GetValues(typeof(LanguageLevelClassification))) { result[level] = result[level] / (float)offlineAnalysis.Count; } return(result); } return(new Dictionary <LanguageLevelClassification, float>()); }); this.NumWords = new AsyncLazy <int>(async() => { List <IWord> offlineAnalysis = await this.OfflineAnalysisResult; return(offlineAnalysis.Count); }); this.Adjectives = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.Adjective)); }); this.Adverbs = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.Adverb)); }); this.ClauseOpeners = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.ClauseOpener)); }); this.Conjunctions = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.Conjunction)); }); this.Determiners = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.Determiner)); }); this.DeterminerPronouns = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.DeterminerPronoun)); }); this.ExistentialParticles = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.ExistentialParticle)); }); this.ForeignWords = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.ForeignWord)); }); this.Genitives = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.Genitive)); }); this.InfinitiveMarkers = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.InfinitiveMarker)); }); this.Interjections = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.InterjectionOrDiscourseMarker)); }); this.LettersOfAlphabet = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.LetterAsWord)); }); this.NegativeMarkers = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.NegativeMarker)); }); this.CommonNouns = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.CommonNoun)); }); this.ProperNouns = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.ProperNoun)); }); this.PartsOfProperNouns = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.PartOfProperNoun)); }); this.CardinalNumbers = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.CardinalNumber)); }); this.OrdinalNumbers = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.Ordinal)); }); this.Prepositions = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.Preposition)); }); this.Pronouns = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.Pronoun)); }); this.UnclassifiedWords = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.Unclassified)); }); this.Verbs = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.Verb)); }); this.ModalVerbs = new AsyncLazy <IDictionary <IWord, int> >(async() => { return(await this.GetWordsByPartOfSpeech(PartOfSpeech.ModalVerb)); }); }