internal ScoredTokenPredictionMaker(PredictiveVocabularySource source, TokenPredictorDatabase database, Func <int, bool> tokenFilter, int[] context)
: this(source, tokenFilter, GetContextDatabases(database, context))
{
}
internal IEnumerable <int> GetTopIndices <T>(PredictiveVocabularySource <T> source, ITokenTileFilter filter, int[] context, int minIndex, int limIndex, int count)
where T : ISuggestionItem
{
var toFindCount = count;
var foundTokens = new HashSet <int>();
var contextLimit = context.Length;
var contextStart = Math.Max(0, contextLimit - _width + 1);
var scanIndex = contextStart;
while (toFindCount != 0 && scanIndex <= contextLimit)
{
var processed = true;
var database = _database.GetChild(context, scanIndex, contextLimit - scanIndex);
if (database != null)
{
var candidates = new List <CandidatePair>();
var acceptedMin = int.MinValue;
foreach (var pair in database)
{
var token = pair.Key;
if (!foundTokens.Contains(token))
{
var tokenCount = pair.Value.Count;
var index = source.GetTokenIndex(token);
if (minIndex <= index && index < limIndex && acceptedMin <= tokenCount)
{
var candidateLimit = candidates.Count;
while (0 < candidateLimit && candidates[candidateLimit - 1].Count < tokenCount)
{
candidateLimit--;
}
candidates.Insert(candidateLimit, new CandidatePair(token, tokenCount));
if (toFindCount == candidates.Count)
{
acceptedMin = candidates[candidates.Count - 1].Count;
}
if (toFindCount < candidates.Count &&
candidates[candidates.Count - 1].Count < candidates[toFindCount - 1].Count)
{
Debug.Assert(candidates[toFindCount].Count < candidates[toFindCount - 1].Count);
candidates.RemoveRange(toFindCount, candidates.Count - toFindCount);
acceptedMin = candidates[candidates.Count - 1].Count;
}
}
}
}
var sortableCandidates = new List <int[]>();
foreach (var candidate in candidates)
{
var counts = new int[contextLimit - scanIndex + 2];
counts[0] = candidate.Count;
counts[contextLimit - scanIndex + 1] = candidate.Token;
sortableCandidates.Add(counts);
}
for (var subIndex = scanIndex + 1; subIndex <= contextLimit; subIndex++)
{
var dictionary = _database.GetChild(context, subIndex, contextLimit - subIndex);
if (dictionary != null)
{
foreach (var counts in sortableCandidates)
{
Debug.Assert(counts[subIndex - scanIndex] == 0);
if (dictionary.TryGetValue(counts[counts.Length - 1], out var subCount))
{
counts[subIndex - scanIndex] = subCount.Count;
}
}
}
}
sortableCandidates.Sort(RankSort);
var sliceCount = Math.Min(sortableCandidates.Count, toFindCount);
for (var i = 0; i < sliceCount; i++)
{
var counts = sortableCandidates[i];
var token = counts[counts.Length - 1];
var index = source.GetTokenIndex(token);
foundTokens.Add(token);
if (filter.IsTokenVisible(token))
{
yield return(index);
toFindCount--;
}
else
{
processed = false;
}
}
}
if (processed)
{
scanIndex++;
}
}
if (0 < toFindCount)
{
var tokens = source.GetTokens();
using (var enumerator = tokens.GetEnumerator())
{
while (0 < toFindCount && enumerator.MoveNext())
{
var token = enumerator.Current;
if (!foundTokens.Contains(token))
{
var index = source.GetTokenIndex(token);
if (minIndex <= index && index < limIndex)
{
foundTokens.Add(token);
if (filter.IsTokenVisible(token))
{
yield return(index);
toFindCount--;
}
}
}
}
}
}
int RankSort(int[] l, int[] r)
{
Debug.Assert(l.Length == r.Length);
var lim = l.Length;
var i = 0;
while (i < lim && l[i] == r[i])
{
i++;
}
var comparison = i == lim ? 0 : r[i].CompareTo(l[i]);
return(comparison);
}
}
private ScoredTokenPredictionMaker(PredictiveVocabularySource source, Func <int, bool> tokenFilter, TokenPredictorDatabase[] contextDatabases)
{
_source = source;
_contextDatabases = contextDatabases;
_tokenFilter = tokenFilter;
}
internal ScoredTokenPredictionMaker CreatePredictionMaker(PredictiveVocabularySource source, Func <int, bool> tokenFilter, int[] context)
{
var maker = new ScoredTokenPredictionMaker(source, _database, tokenFilter, context);
return(maker);
}
