static MappingTokenizer() { var nameChars = CharMatcher.AnyOf( CharMatcher.Range('a', 'z'), CharMatcher.Range('A', 'Z'), CharMatcher.Range('0', '9'), CharMatcher.AnyOf('_', '-')); var newLineChars = CharMatcher.AnyOf('\r', '\n'); var whitespaceChars = CharMatcher.AnyOf(' ', '\t'); Tokeniser = new Tokenizer <TokenType>(TokenType.String) .Token( TokenType.Comment, o => o .Match(CharMatcher.Single('/')).CanStart() .Then(CharMatcher.Single('/')) .Then(CharMatcher.NoneOf(newLineChars)).CanRepeat().CanEnd()) .Token( TokenType.Comment, o => { var commentContent = o .Match(CharMatcher.Single('/')).CanStart() .Then(CharMatcher.Single('*')) .Then(CharMatcher.NoneOf('*')).CanRepeat(); var closingStar = commentContent .Then(CharMatcher.Single('*')).CanRepeat(); closingStar .Then(CharMatcher.Single('/')).CanEnd(); closingStar.CanFollowWith(commentContent); }) .Token( TokenType.QuotedItem, o => { var start = o.Match(CharMatcher.Single('"')).CanStart(); var end = start .Then(CharMatcher.NoneOf('"')).CanRepeat() .Then(CharMatcher.Single('"')).CanEnd(); end.CanFollowWith(start); start.CanFollowWith(end); }) .Token( TokenType.Partial, o => o .Match(CharMatcher.Single('<')).CanStart() .Then(CharMatcher.Single('<')) .Then(nameChars).CanRepeat() .Then(CharMatcher.Single('>')) .Then(CharMatcher.Single('>')).CanEnd()) .Token( TokenType.Variable, o => o .Match(CharMatcher.Single('$')).CanStart() .Then(CharMatcher.Single('(')) .Then(nameChars).CanRepeat() .Then(CharMatcher.Single(')')).CanEnd()) .Token( TokenType.LineContinuation, o => { var start = o.Match(newLineChars).CanRepeat().CanStart(); var end = o.Match(whitespaceChars).CanRepeat().CanEnd(); start.CanFollowWith(end); end.CanFollowWith(start); }) .Token( TokenType.NewLine, o => o.Match(newLineChars).CanStart().CanRepeat().CanEnd()) .Token( TokenType.Whitespace, o => o.Match(whitespaceChars).CanStart().CanRepeat().CanEnd()) .Token(TokenType.PartialAssignment, "::") .Token(TokenType.And, "&&") .Token(TokenType.Or, "||") .Token(TokenType.Equals, '=') .Token(TokenType.Dot, '.') .Token(TokenType.Array, "[]") .Token(TokenType.OpenSquareBracket, '[') .Token(TokenType.CloseSquareBracket, ']') .Token(TokenType.OpenCurlyBracket, '{') .Token(TokenType.CloseCurlyBracket, '}') .Token(TokenType.Comma, ',') .Token(TokenType.QuestionMark, '?') .Token(TokenType.ExclaimationMark, '!') .Token(TokenType.OpenRoundBracket, '(') .Token(TokenType.CloseRoundBracket, ')') .Token(TokenType.Colon, ":") .Token( TokenType.Item, o => o.Match(nameChars).CanStart().CanRepeat().CanEnd()) .Token(TokenType.WildCard, '*'); }