public static Tokenizer <SimpleToken> Create()
{
var builder = new TokenizerBuilder <SimpleToken>()
.Match(Span.Regex(@"""((?:""""|[^""])*)"""), SimpleToken.Text)
.Match(ExtraParsers.SpanBetween('<', '>'), SimpleToken.Echo)
.Ignore(Span.WhiteSpace)
.BooleanOperators()
.Match(Character.EqualTo(':'), SimpleToken.Colon)
.Match(Character.EqualTo('!'), SimpleToken.Exclamation)
.Match(Character.EqualTo(','), SimpleToken.Comma)
.Match(Character.EqualTo('='), SimpleToken.Equal)
.Match(Character.EqualTo('('), SimpleToken.OpenParen)
.Match(Character.EqualTo(')'), SimpleToken.CloseParen)
.Match(Character.EqualTo('['), SimpleToken.OpenBracket)
.Match(Character.EqualTo(']'), SimpleToken.CloseBracket)
.Match(Character.EqualTo('{'), SimpleToken.OpenBrace)
.Match(Character.EqualTo('}'), SimpleToken.CloseBrace)
.Match(Character.EqualTo(';'), SimpleToken.Semicolon)
.Match(Span.EqualTo("if"), SimpleToken.If, true)
.Match(Span.EqualTo("else"), SimpleToken.Else, true)
.Match(Numerics.Integer, SimpleToken.Integer)
.Match(Numerics.DecimalDouble, SimpleToken.Double)
.Match(Span.Regex(IdentifierRegex), SimpleToken.Identifier)
.Build();
return(builder);
}
public static Tokenizer <MicroToken> Create()
{
return(new TokenizerBuilder <MicroToken>()
.Ignore(Character.WhiteSpace)
.Match(ExtraParsers.SpanBetween('\''), MicroToken.Text)
.Match(Character.EqualTo(','), MicroToken.Comma)
.Match(Character.EqualTo('='), MicroToken.Equal)
.Match(Numerics.Integer, MicroToken.Number)
.Match(Span.Regex(@"\w+[\d\w_]*"), MicroToken.Identifier)
.Build());
}