public IEnumerable <Token> Lex(string content) { StateContainer state = new StateContainer(); Scanner scanner = new Scanner(content); Token token; // Start off in the default state. state.Push(State("default")); while ((token = ReadToken(scanner, state)) != null) { yield return(token); } }
private void ProcessState(Word word, string match, StateContainer state) { if (word.doesPopState) { state.Pop(); } if (word.pushState != null) { state.Push(word.pushState); } if (word.isNewLine) { state.NewLine(); } else { state.IncrementChar(match.Length); } }
private Token ReadToken(Scanner scanner, StateContainer state) { string word = scanner.Read(state.GetDelims() as DelimiterCollection <Word>); if (word == null) { return(null); } var(lexData, regexMatch) = state.words.WordMatch(word); string invalid = word.Substring(regexMatch?.Length ?? 0); if (!String.IsNullOrEmpty(invalid)) { throw new SyntaxException( $"\"{invalid}\" is not a valid token in the \"{state.name}\" state." ); } Token ret = new Token(lexData.token, regexMatch.Value, state.LineNum, state.CharNum); ProcessState(lexData, regexMatch.Value, state); return(ret); }