public IEnumerable <SimpleLexer.Token> getTokenized()
{
SimpleLexer.Lexer lexer = new SimpleLexer.Lexer();
addDefinitionToLexer(lexer, "(operator)", @"\*|\/|\+|\-");
addDefinitionToLexer(lexer, "(literal)", @"\d+");
addDefinitionToLexer(lexer, "(white-space)", @"\s+");
IEnumerable <SimpleLexer.Token> tokens = lexer.Tokenize("1 * 2 / 3 + 4 - 5");
return(tokens);
}
private void addDefinitionToLexer(SimpleLexer.Lexer Lexer, string type, string pattern, bool isIgnored = false)
{
Lexer.AddDefinition(new SimpleLexer.TokenDefinition(type, new System.Text.RegularExpressions.Regex(pattern), isIgnored));
}
