private static TokenPredictorDatabase[] GetContextDatabases(TokenPredictorDatabase rootDatabase, int[] context)
{
var contextLength = context.Length;
var databases = new List <TokenPredictorDatabase>(contextLength + 1)
{
rootDatabase
};
var done = false;
for (var index = 1; index <= contextLength && !done; index++)
{
var database = GetContextDatabase(rootDatabase, context, contextLength - index);
if (database != null)
{
databases.Add(database);
}
else
{
done = true;
}
}
return(databases.ToArray());
}
private static void SubtractTokens(TokenPredictorDatabase accumulator, TokenPredictorDatabase subtractive)
{
var expired = default(List <int>);
foreach (var pair in subtractive)
{
var info = accumulator.GetValue(pair.Key);
if (0 < accumulator.DecrementCount(info, pair.Value.Count))
{
var subtractiveChlidren = pair.Value.TryGetChildren();
if (subtractiveChlidren != null)
{
SubtractTokens(info.GetChildren(), subtractiveChlidren);
}
}
else
{
Debug.Assert(info.Count == 0);
if (expired == null)
{
expired = new List <int>();
}
expired.Add(pair.Key);
}
}
if (expired != null)
{
foreach (var token in expired)
{
accumulator.Remove(token);
}
}
}
internal TokenPredictorDatabase GetChildren()
{
if (_children == null)
{
_children = new TokenPredictorDatabase();
}
return(_children);
}
private static void AddTokens(TokenPredictorDatabase accumulator, TokenPredictorDatabase additive)
{
foreach (var pair in additive)
{
var info = accumulator.GetValue(pair.Key);
accumulator.IncrementCount(info, pair.Value.Count);
var additiveChlidren = pair.Value.TryGetChildren();
if (additiveChlidren != null)
{
AddTokens(info.GetChildren(), additiveChlidren);
}
}
}
private static TokenPredictorDatabase GetContextDatabase(TokenPredictorDatabase rootDatabase, int[] context, int contextStart)
{
var contextLength = context.Length;
var database = rootDatabase;
for (var index = contextStart; database != null && index < contextLength; index++)
{
if (database.TryGetValue(context[index], out var info))
{
database = info.TryGetChildren();
}
else
{
database = null;
}
}
return(database);
}
internal int GetTopToken(int[] context)
{
var result = -1;
var contextLimit = context.Length;
var contextStart = Math.Max(0, contextLimit - _width + 1);
var scanStart = contextStart;
TokenPredictorDatabase leafDatabase = null;
for (; leafDatabase == null && scanStart < contextLimit; scanStart++)
{
leafDatabase = _database.GetChild(context, scanStart, contextLimit - scanStart);
}
if (leafDatabase != null)
{
var tokens = leafDatabase.GetTopRanked();
for (; 1 < tokens.Count && scanStart < contextLimit; scanStart++)
{
leafDatabase = _database.GetChild(context, scanStart, contextLimit - scanStart);
Debug.Assert(leafDatabase != null);
tokens = leafDatabase.GetTopRanked(tokens);
}
Debug.Assert(result == -1);
foreach (var token in tokens)
{
if (result < token)
{
result = token;
}
}
}
return(result);
}
internal ScoredTokenPredictionMaker(PredictiveVocabularySource source, TokenPredictorDatabase database, Func <int, bool> tokenFilter, int[] context)
: this(source, tokenFilter, GetContextDatabases(database, context))
{
}
private static IDictionary <string, object> ToJsonDictionary(StringTokens tokens, TokenPredictorDatabase database)
{
var json = new SortedDictionary <string, object>();
foreach (var pair in database)
{
var key = tokens.GetString(pair.Key);
var value = ToJsonDictionary(tokens, pair.Value);
json.Add(key, value);
}
return(json);
}