public IEnumerable <Token> Lex(string content)
{
StateContainer state = new StateContainer();
Scanner scanner = new Scanner(content);
Token token;
// Start off in the default state.
state.Push(State("default"));
while ((token = ReadToken(scanner, state)) != null)
{
yield return(token);
}
}
private void ProcessState(Word word, string match, StateContainer state)
{
if (word.doesPopState)
{
state.Pop();
}
if (word.pushState != null)
{
state.Push(word.pushState);
}
if (word.isNewLine)
{
state.NewLine();
}
else
{
state.IncrementChar(match.Length);
}
}
private Token ReadToken(Scanner scanner, StateContainer state)
{
string word = scanner.Read(state.GetDelims() as DelimiterCollection <Word>);
if (word == null)
{
return(null);
}
var(lexData, regexMatch) = state.words.WordMatch(word);
string invalid = word.Substring(regexMatch?.Length ?? 0);
if (!String.IsNullOrEmpty(invalid))
{
throw new SyntaxException(
$"\"{invalid}\" is not a valid token in the \"{state.name}\" state."
);
}
Token ret = new Token(lexData.token, regexMatch.Value, state.LineNum, state.CharNum);
ProcessState(lexData, regexMatch.Value, state);
return(ret);
}
