private SuggestedWordListsCreator(WordVocabularySource source,
StringTokens tokens,
ScoredTokenPredictionMaker maker,
Func <int, bool> isTokenVisible,
bool startOnFirstWord,
int lowerBound,
int upperBound,
int maxListCount,
int maxListItemCount)
{
_source = source;
_tokens = tokens;
_maker = maker;
_isTokenVisible = isTokenVisible;
_startOnFirstWord = startOnFirstWord;
_lowerBound = lowerBound;
_upperBound = upperBound;
_maxListCount = maxListCount;
_maxListItemCount = maxListItemCount;
var settings = source.Model.Environment.Settings;
_allowDuplicateStems = settings.AllowDuplicateStems;
_findFollowOnPredictions = settings.FindFollowOnPredictions;
_combineCorePredictions = settings.CombineCorePredictions;
_findCorePredictionPrefixes = settings.FindCorePredictionPrefixes;
_findCorePredictionSuffixes = settings.FindCorePredictionSuffixes;
_nascents = new List <NascentWordPredictionList>(_maxListCount);
_capitalizer = source.Model.Environment.TryCapitalizeFirstWord;
}
internal ScoredTokenPredictionMaker CreateNextPredictionMaker(int token, Func <int, bool> tokenFilter)
{
var databases = GetNextContextDatabases(_contextDatabases, token);
var maker = new ScoredTokenPredictionMaker(_source, tokenFilter, databases);
return(maker);
}
internal NascentWordPredictionList(WordPrediction prediction, ScoredTokenPredictionMaker followOnMaker, WordPrediction followOn)
{
_first = prediction;
_list = new List <WordPrediction>(1)
{
prediction
};
_last = prediction;
_followOnMaker = followOnMaker;
_followOn = followOn;
}
internal void MergeWithNext(NascentWordPredictionList next)
{
var targetPredictions = _list;
var sourcePredictions = next._list;
Debug.Assert(targetPredictions[0].Text.Length < sourcePredictions[0].Text.Length);
Debug.Assert(sourcePredictions[0].Text.StartsWith(targetPredictions[0].Text));
var sourcePosition = 0;
var targetPosition = 0;
while (sourcePosition < sourcePredictions.Count && targetPosition < targetPredictions.Count)
{
if (sourcePredictions[sourcePosition].Index < targetPredictions[targetPosition].Index)
{
Debug.Assert(targetPredictions[targetPosition].Text.StartsWith(sourcePredictions[sourcePosition].Text));
targetPredictions.Insert(targetPosition, sourcePredictions[sourcePosition]);
sourcePosition++;
}
else
{
Debug.Assert(targetPredictions[targetPosition].Index < sourcePredictions[sourcePosition].Index);
Debug.Assert(sourcePredictions[sourcePosition].Text.StartsWith(targetPredictions[targetPosition].Text));
}
targetPosition++;
}
if (sourcePosition < sourcePredictions.Count)
{
while (sourcePosition < sourcePredictions.Count)
{
targetPredictions.Add(sourcePredictions[sourcePosition]);
sourcePosition++;
}
_followOnMaker = next._followOnMaker;
_followOn = next._followOn;
}
else
{
Debug.WriteLine("TODO: Debug.Assert(ReferenceEquals(_followOn, next._followOn));");
}
Debug.Assert(ReferenceEquals(_first, _list[0]));
}
private WordPrediction GetTopPrediction(ScoredTokenPredictionMaker maker, bool isFirstWord)
{
WordPrediction value;
if (_findFollowOnPredictions)
{
var topScores = maker.GetTopScores(0, _source.Count, true, false);
using (var enumerator = topScores.GetEnumerator())
{
value = GetNextCorePrediction(enumerator, isFirstWord);
}
}
else
{
value = null;
}
Debug.Assert(value == null || value.Score.IsInContextScore, "Only true following predictions expected");
return(value);
}
internal static void PrototypeTreePredictions(ScoredTokenPredictionMaker maker,
StringTokens tokens,
int maxCount)
{
string ToText(TokenPredictorInfo info)
{
var text = tokens[info.Token];
var nullIndex = text.IndexOf('\0');
if (nullIndex != -1)
{
text = text.Substring(0, nullIndex);
}
return(text);
}
var roots = maker.GetContextRoots();
Debug.WriteLine(string.Empty);
Debug.WriteLine("Prototype tree prediction:");
var enumerable = TreePredictableItem <TokenPredictorInfo> .FindOrderedItems(roots.SortedEnumerable);
using (var enumerator = enumerable.GetEnumerator())
{
for (var count = 0; count < maxCount && enumerator.MoveNext(); count++)
{
var text = string.Empty;
foreach (var info in enumerator.Current)
{
text += $"{ToText(info)} ";
}
Debug.WriteLine(text);
}
}
Debug.WriteLine(string.Empty);
}
internal ScoredTokenPredictionMaker CreatePredictionMaker(PredictiveVocabularySource source, Func <int, bool> tokenFilter, int[] context)
{
var maker = new ScoredTokenPredictionMaker(source, _database, tokenFilter, context);
return(maker);
}