public virtual void ImportTokensFromTokensFile()
{
string vocab = GetOptionString("tokenVocab");
if (vocab != null)
{
TokenVocabParser vparser = new TokenVocabParser(this);
IDictionary<string, int> tokens = vparser.Load();
tool.Log("grammar", "tokens=" + tokens);
foreach (string t in tokens.Keys)
{
if (t[0] == '\'')
DefineStringLiteral(t, tokens[t]);
else
DefineTokenName(t, tokens[t]);
}
}
}
