internal string GetIndexWord(int index)
{
var token = _vocabularyList[index];
var word = _tokens.GetString(token);
return(word);
}
public bool IsTokenVisible(int token)
{
if (!_tokenToAcceptance.TryGetValue(token, out var value))
{
var word = _tokens.GetString(token);
// If there is content before a null character...
var nullPosition = word.IndexOf('\0');
switch (nullPosition)
{
case 0:
if (word == StringTokens.StopString)
{
value = true;
}
else
{
var command = (TileCommand)Enum.Parse(typeof(TileCommand), word.Substring(1));
value = _source.IsCommandVisible(command);
}
break;
default:
// ...ignore the content beyond the null.
word = word.Substring(0, nullPosition);
goto case -1;
case -1:
value = true;
using (var enumerator = _set.GetEnumerator())
{
while (value && enumerator.MoveNext())
{
if (_compare.Compare(word, enumerator.Current, CompareOptions.IgnoreCase) == 0)
{
value = false;
}
}
}
if (value)
{
_set.Add(word);
}
_tokenToAcceptance.Add(token, value);
break;
}
}
return(value);
}
private static IDictionary <string, object> ToJsonDictionary(StringTokens tokens, TokenPredictorDatabase database)
{
var json = new SortedDictionary <string, object>();
foreach (var pair in database)
{
var key = tokens.GetString(pair.Key);
var value = ToJsonDictionary(tokens, pair.Value);
json.Add(key, value);
}
return(json);
}