private static Tokenizer <Tokens> GetTokenizer(PegFacade <Tokens, RuleNames, double?> peg) { return(peg.Tokenizer(new[] { peg.Token(Tokens.Plus, new Regex(@"\+")), peg.Token(Tokens.Number, new Regex(@"[-+]?[0-9]+\.?[0-9]*")), peg.Token(Tokens.Whitespace, new Regex(@"\s+")) })); }
private void CreateTokenizer() { // NB: This list of token definitions is prioritized _tokenizer = _pegFacade.Tokenizer(new[] { _pegFacade.Token(TokenNames.Comment, new Regex(@"//.*\n")), _pegFacade.Token(TokenNames.String, new Regex(@"""(?:(?:\\\"")|[^\""])*?\""", RegexOptions.Compiled)), // \" is used to escape quote characters _pegFacade.Token(TokenNames.Variable, new Regex(@"\:[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*", RegexOptions.Compiled)), _pegFacade.Token(TokenNames.Identifier, new Regex(@"[_a-zA-Z]\w*(?:\.[_a-zA-Z]\w*)*", RegexOptions.Compiled)), _pegFacade.Token(TokenNames.Whitespace, new Regex(@"\s+", RegexOptions.Compiled)), _pegFacade.Token(TokenNames.SemiColon, new Regex(@";", RegexOptions.Compiled)), _pegFacade.Token(TokenNames.Number, new Regex(@"[+-]?\d+\.?\d*", RegexOptions.Compiled)), _pegFacade.Token(TokenNames.Comma, new Regex(@",", RegexOptions.Compiled)), _pegFacade.Token(TokenNames.Colon, new Regex(@":", RegexOptions.Compiled)), _pegFacade.Token(TokenNames.OpenRoundBracket, new Regex(@"\(", RegexOptions.Compiled)), _pegFacade.Token(TokenNames.CloseRoundBracket, new Regex(@"\)", RegexOptions.Compiled)), _pegFacade.Token(TokenNames.Arrow, new Regex(@"\=\>", RegexOptions.Compiled)), _pegFacade.Token(TokenNames.OpenAngleBracket, new Regex(@"\<", RegexOptions.Compiled)), _pegFacade.Token(TokenNames.CloseAngleBracket, new Regex(@"\>", RegexOptions.Compiled)) }); }
private void BuildTokenizer() { _tokenizer = _inputPeg.Tokenizer(new[] { _inputPeg.Token(TokenNames.Identifier, new Regex(@"[a-zA-Z_][a-zA-Z\d_]*")), _inputPeg.Token(TokenNames.OpenSquare, new Regex(@"\[")), _inputPeg.Token(TokenNames.CloseSquare, new Regex(@"\]")), _inputPeg.Token(TokenNames.OpenRound, new Regex(@"\(")), _inputPeg.Token(TokenNames.CloseRound, new Regex(@"\)")), _inputPeg.Token(TokenNames.WhiteSpace, new Regex(@"\s+")), _inputPeg.Token(TokenNames.And, new Regex(@"\&")), _inputPeg.Token(TokenNames.Not, new Regex(@"\!")), _inputPeg.Token(TokenNames.OneOrMore, new Regex(@"\+")), _inputPeg.Token(TokenNames.ZeroOrMore, new Regex(@"\*")), _inputPeg.Token(TokenNames.Optional, new Regex(@"\?")), _inputPeg.Token(TokenNames.Choice, new Regex(@"\|")), _inputPeg.Token(TokenNames.Arrow, new Regex(@"\=\>")), _inputPeg.Token(TokenNames.SemiColon, new Regex(@";")) }); }