private SuggestedWordListsCreator(WordVocabularySource source, StringTokens tokens, ScoredTokenPredictionMaker maker, Func <int, bool> isTokenVisible, bool startOnFirstWord, int lowerBound, int upperBound, int maxListCount, int maxListItemCount) { _source = source; _tokens = tokens; _maker = maker; _isTokenVisible = isTokenVisible; _startOnFirstWord = startOnFirstWord; _lowerBound = lowerBound; _upperBound = upperBound; _maxListCount = maxListCount; _maxListItemCount = maxListItemCount; var settings = source.Model.Environment.Settings; _allowDuplicateStems = settings.AllowDuplicateStems; _findFollowOnPredictions = settings.FindFollowOnPredictions; _combineCorePredictions = settings.CombineCorePredictions; _findCorePredictionPrefixes = settings.FindCorePredictionPrefixes; _findCorePredictionSuffixes = settings.FindCorePredictionSuffixes; _nascents = new List <NascentWordPredictionList>(_maxListCount); _capitalizer = source.Model.Environment.TryCapitalizeFirstWord; }
internal ScoredTokenPredictionMaker CreateNextPredictionMaker(int token, Func <int, bool> tokenFilter) { var databases = GetNextContextDatabases(_contextDatabases, token); var maker = new ScoredTokenPredictionMaker(_source, tokenFilter, databases); return(maker); }
internal NascentWordPredictionList(WordPrediction prediction, ScoredTokenPredictionMaker followOnMaker, WordPrediction followOn) { _first = prediction; _list = new List <WordPrediction>(1) { prediction }; _last = prediction; _followOnMaker = followOnMaker; _followOn = followOn; }
internal void MergeWithNext(NascentWordPredictionList next) { var targetPredictions = _list; var sourcePredictions = next._list; Debug.Assert(targetPredictions[0].Text.Length < sourcePredictions[0].Text.Length); Debug.Assert(sourcePredictions[0].Text.StartsWith(targetPredictions[0].Text)); var sourcePosition = 0; var targetPosition = 0; while (sourcePosition < sourcePredictions.Count && targetPosition < targetPredictions.Count) { if (sourcePredictions[sourcePosition].Index < targetPredictions[targetPosition].Index) { Debug.Assert(targetPredictions[targetPosition].Text.StartsWith(sourcePredictions[sourcePosition].Text)); targetPredictions.Insert(targetPosition, sourcePredictions[sourcePosition]); sourcePosition++; } else { Debug.Assert(targetPredictions[targetPosition].Index < sourcePredictions[sourcePosition].Index); Debug.Assert(sourcePredictions[sourcePosition].Text.StartsWith(targetPredictions[targetPosition].Text)); } targetPosition++; } if (sourcePosition < sourcePredictions.Count) { while (sourcePosition < sourcePredictions.Count) { targetPredictions.Add(sourcePredictions[sourcePosition]); sourcePosition++; } _followOnMaker = next._followOnMaker; _followOn = next._followOn; } else { Debug.WriteLine("TODO: Debug.Assert(ReferenceEquals(_followOn, next._followOn));"); } Debug.Assert(ReferenceEquals(_first, _list[0])); }
private WordPrediction GetTopPrediction(ScoredTokenPredictionMaker maker, bool isFirstWord) { WordPrediction value; if (_findFollowOnPredictions) { var topScores = maker.GetTopScores(0, _source.Count, true, false); using (var enumerator = topScores.GetEnumerator()) { value = GetNextCorePrediction(enumerator, isFirstWord); } } else { value = null; } Debug.Assert(value == null || value.Score.IsInContextScore, "Only true following predictions expected"); return(value); }
internal static void PrototypeTreePredictions(ScoredTokenPredictionMaker maker, StringTokens tokens, int maxCount) { string ToText(TokenPredictorInfo info) { var text = tokens[info.Token]; var nullIndex = text.IndexOf('\0'); if (nullIndex != -1) { text = text.Substring(0, nullIndex); } return(text); } var roots = maker.GetContextRoots(); Debug.WriteLine(string.Empty); Debug.WriteLine("Prototype tree prediction:"); var enumerable = TreePredictableItem <TokenPredictorInfo> .FindOrderedItems(roots.SortedEnumerable); using (var enumerator = enumerable.GetEnumerator()) { for (var count = 0; count < maxCount && enumerator.MoveNext(); count++) { var text = string.Empty; foreach (var info in enumerator.Current) { text += $"{ToText(info)} "; } Debug.WriteLine(text); } } Debug.WriteLine(string.Empty); }
internal ScoredTokenPredictionMaker CreatePredictionMaker(PredictiveVocabularySource source, Func <int, bool> tokenFilter, int[] context) { var maker = new ScoredTokenPredictionMaker(source, _database, tokenFilter, context); return(maker); }