private static TokenPredictorDatabase[] GetContextDatabases(TokenPredictorDatabase rootDatabase, int[] context) { var contextLength = context.Length; var databases = new List <TokenPredictorDatabase>(contextLength + 1) { rootDatabase }; var done = false; for (var index = 1; index <= contextLength && !done; index++) { var database = GetContextDatabase(rootDatabase, context, contextLength - index); if (database != null) { databases.Add(database); } else { done = true; } } return(databases.ToArray()); }
private static void SubtractTokens(TokenPredictorDatabase accumulator, TokenPredictorDatabase subtractive) { var expired = default(List <int>); foreach (var pair in subtractive) { var info = accumulator.GetValue(pair.Key); if (0 < accumulator.DecrementCount(info, pair.Value.Count)) { var subtractiveChlidren = pair.Value.TryGetChildren(); if (subtractiveChlidren != null) { SubtractTokens(info.GetChildren(), subtractiveChlidren); } } else { Debug.Assert(info.Count == 0); if (expired == null) { expired = new List <int>(); } expired.Add(pair.Key); } } if (expired != null) { foreach (var token in expired) { accumulator.Remove(token); } } }
internal TokenPredictorDatabase GetChildren() { if (_children == null) { _children = new TokenPredictorDatabase(); } return(_children); }
private static void AddTokens(TokenPredictorDatabase accumulator, TokenPredictorDatabase additive) { foreach (var pair in additive) { var info = accumulator.GetValue(pair.Key); accumulator.IncrementCount(info, pair.Value.Count); var additiveChlidren = pair.Value.TryGetChildren(); if (additiveChlidren != null) { AddTokens(info.GetChildren(), additiveChlidren); } } }
private static TokenPredictorDatabase GetContextDatabase(TokenPredictorDatabase rootDatabase, int[] context, int contextStart) { var contextLength = context.Length; var database = rootDatabase; for (var index = contextStart; database != null && index < contextLength; index++) { if (database.TryGetValue(context[index], out var info)) { database = info.TryGetChildren(); } else { database = null; } } return(database); }
internal int GetTopToken(int[] context) { var result = -1; var contextLimit = context.Length; var contextStart = Math.Max(0, contextLimit - _width + 1); var scanStart = contextStart; TokenPredictorDatabase leafDatabase = null; for (; leafDatabase == null && scanStart < contextLimit; scanStart++) { leafDatabase = _database.GetChild(context, scanStart, contextLimit - scanStart); } if (leafDatabase != null) { var tokens = leafDatabase.GetTopRanked(); for (; 1 < tokens.Count && scanStart < contextLimit; scanStart++) { leafDatabase = _database.GetChild(context, scanStart, contextLimit - scanStart); Debug.Assert(leafDatabase != null); tokens = leafDatabase.GetTopRanked(tokens); } Debug.Assert(result == -1); foreach (var token in tokens) { if (result < token) { result = token; } } } return(result); }
internal ScoredTokenPredictionMaker(PredictiveVocabularySource source, TokenPredictorDatabase database, Func <int, bool> tokenFilter, int[] context) : this(source, tokenFilter, GetContextDatabases(database, context)) { }
private static IDictionary <string, object> ToJsonDictionary(StringTokens tokens, TokenPredictorDatabase database) { var json = new SortedDictionary <string, object>(); foreach (var pair in database) { var key = tokens.GetString(pair.Key); var value = ToJsonDictionary(tokens, pair.Value); json.Add(key, value); } return(json); }