private static Tokenizer <Tokens> GetTokenizer(PegFacade <Tokens, RuleNames, double?> peg)
{
return(peg.Tokenizer(new[] {
peg.Token(Tokens.Plus, new Regex(@"\+")),
peg.Token(Tokens.Number, new Regex(@"[-+]?[0-9]+\.?[0-9]*")),
peg.Token(Tokens.Whitespace, new Regex(@"\s+"))
}));
}
private void CreateTokenizer()
{
// NB: This list of token definitions is prioritized
_tokenizer = _pegFacade.Tokenizer(new[] {
_pegFacade.Token(TokenNames.Comment, new Regex(@"//.*\n")),
_pegFacade.Token(TokenNames.String, new Regex(@"""(?:(?:\\\"")|[^\""])*?\""", RegexOptions.Compiled)), // \" is used to escape quote characters
_pegFacade.Token(TokenNames.Variable, new Regex(@"\:[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*", RegexOptions.Compiled)),
_pegFacade.Token(TokenNames.Identifier, new Regex(@"[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*", RegexOptions.Compiled)),
_pegFacade.Token(TokenNames.Whitespace, new Regex(@"\s+", RegexOptions.Compiled)),
_pegFacade.Token(TokenNames.SemiColon, new Regex(@";", RegexOptions.Compiled)),
_pegFacade.Token(TokenNames.Number, new Regex(@"[+-]?\d+\.?\d*", RegexOptions.Compiled)),
_pegFacade.Token(TokenNames.Comma, new Regex(@",", RegexOptions.Compiled)),
_pegFacade.Token(TokenNames.Colon, new Regex(@":", RegexOptions.Compiled)),
_pegFacade.Token(TokenNames.OpenRoundBracket, new Regex(@"\(", RegexOptions.Compiled)),
_pegFacade.Token(TokenNames.CloseRoundBracket, new Regex(@"\)", RegexOptions.Compiled)),
_pegFacade.Token(TokenNames.Arrow, new Regex(@"\=\>", RegexOptions.Compiled)),
_pegFacade.Token(TokenNames.OpenAngleBracket, new Regex(@"\<", RegexOptions.Compiled)),
_pegFacade.Token(TokenNames.CloseAngleBracket, new Regex(@"\>", RegexOptions.Compiled))
});
}
private void BuildTokenizer()
{
_tokenizer = _inputPeg.Tokenizer(new[] {
_inputPeg.Token(TokenNames.Identifier, new Regex(@"[a-zA-Z_][a-zA-Z\d_]*")),
_inputPeg.Token(TokenNames.OpenSquare, new Regex(@"\[")),
_inputPeg.Token(TokenNames.CloseSquare, new Regex(@"\]")),
_inputPeg.Token(TokenNames.OpenRound, new Regex(@"\(")),
_inputPeg.Token(TokenNames.CloseRound, new Regex(@"\)")),
_inputPeg.Token(TokenNames.WhiteSpace, new Regex(@"\s+")),
_inputPeg.Token(TokenNames.And, new Regex(@"\&")),
_inputPeg.Token(TokenNames.Not, new Regex(@"\!")),
_inputPeg.Token(TokenNames.OneOrMore, new Regex(@"\+")),
_inputPeg.Token(TokenNames.ZeroOrMore, new Regex(@"\*")),
_inputPeg.Token(TokenNames.Optional, new Regex(@"\?")),
_inputPeg.Token(TokenNames.Choice, new Regex(@"\|")),
_inputPeg.Token(TokenNames.Arrow, new Regex(@"\=\>")),
_inputPeg.Token(TokenNames.SemiColon, new Regex(@";"))
});
}
