public Tokenizer(params Terminal[] rules)
{
terminals = rules;
var x = new DFA.TokenizerState(rules.Select(r => r.automaton).ToArray());
initialState = AddState(x);
// Detect if any rules are unmatched.
var found = new bool[rules.Length];
foreach (var state in states.Values)
{
if (state.acceptToken >= 0)
{
found[state.acceptToken] = true;
}
}
//
for (int state = 0; state < found.Length; ++state)
{
if (!found[state])
{
throw new UnmatchableTokenException(state);
}
}
}
private State AddState(DFA.TokenizerState ts)
{
if (!states.TryGetValue(ts, out State state))
{
state = new State();
states.Add(ts, state);
// Populate this state
state.transitions = new State[16];
for (int i = 0; i < 16; ++i)
{
state.transitions[i] = AddState(ts.Next(i));
}
state.acceptToken = ts.AcceptToken;
}
return(state);
}
