/// <summary>
/// Creates a new tokenizer. Useful if for some reason
/// you don't want a full LALR parser, but are just interested in a tokenizer.
/// </summary>
/// <returns></returns>
public StringTokenizer CreateNewTokenizer()
{
DFA.State startState = dfaStates[content.InitialStates.DFA];
DFA.DFA dfa = new DFA.DFA(dfaStates, startState);
return(new StringTokenizer(dfa));
}
/// <summary>
/// Creates a new tokenizer. Useful if for some reason
/// you don't want a full LALR parser, but are just interested in a tokenizer.
/// </summary>
/// <returns></returns>
public StringTokenizer CreateNewTokenizer()
{
DFA.State startState = dfaStates[content.InitialStates.DFA];
DFA.DFA dfa = new DFA.DFA(dfaStates,startState);
return new StringTokenizer(dfa);
}