internal string GetIndexWord(int index) { var token = _vocabularyList[index]; var word = _tokens.GetString(token); return(word); }
public bool IsTokenVisible(int token) { if (!_tokenToAcceptance.TryGetValue(token, out var value)) { var word = _tokens.GetString(token); // If there is content before a null character... var nullPosition = word.IndexOf('\0'); switch (nullPosition) { case 0: if (word == StringTokens.StopString) { value = true; } else { var command = (TileCommand)Enum.Parse(typeof(TileCommand), word.Substring(1)); value = _source.IsCommandVisible(command); } break; default: // ...ignore the content beyond the null. word = word.Substring(0, nullPosition); goto case -1; case -1: value = true; using (var enumerator = _set.GetEnumerator()) { while (value && enumerator.MoveNext()) { if (_compare.Compare(word, enumerator.Current, CompareOptions.IgnoreCase) == 0) { value = false; } } } if (value) { _set.Add(word); } _tokenToAcceptance.Add(token, value); break; } } return(value); }
private static IDictionary <string, object> ToJsonDictionary(StringTokens tokens, TokenPredictorDatabase database) { var json = new SortedDictionary <string, object>(); foreach (var pair in database) { var key = tokens.GetString(pair.Key); var value = ToJsonDictionary(tokens, pair.Value); json.Add(key, value); } return(json); }