protected override Token IsMatchImpl(Tokenizer tokenizer) { var leftOperand = GetIntegers(tokenizer); if (leftOperand != null) { bool isFloat = false; string numberValue = leftOperand; if (tokenizer.Current == ".") { tokenizer.Consume(); var rightOperand = GetIntegers(tokenizer); if (rightOperand != null) { numberValue += "." + rightOperand; isFloat = true; } } if (tokenizer.Current == "e") { string rightOperand = ""; tokenizer.Consume(); if (tokenizer.Current == "-") { rightOperand = "-"; tokenizer.Consume(); } rightOperand += GetIntegers(tokenizer); if (rightOperand != null) { numberValue += "e" + rightOperand; isFloat = true; } } if (!tokenizer.End()) { if (Char.IsLetter(tokenizer.Current, 0)) { return(null); } } if (isFloat) { return(new Token(TokenType.FloatValue, numberValue)); } return(new Token(TokenType.IntValue, leftOperand)); } return(null); }
protected override Token IsMatchImpl(Tokenizer tokenizer) { if (new MatchKeyword(TokenType.Unknown, CommentDelim.Item1).IsMatch(tokenizer) == null) { return(null); } else { tokenizer.Consume(); while (!tokenizer.End() && new MatchKeyword(TokenType.Unknown, CommentDelim.Item2).IsMatch(tokenizer) == null) { tokenizer.Consume(); } if (new MatchKeyword(TokenType.Unknown, CommentDelim.Item2).IsMatch(tokenizer) == null) { tokenizer.Consume(); } TokenType tokenType = TokenType.OpenBlockComment; if (CommentDelim == LINE_COMMENT) { tokenType = TokenType.OpenLineComment; } return(new Token(tokenType, CommentDelim.Item1)); } }
protected override Token IsMatchImpl(Tokenizer tokenizer) { foreach (var character in Match) { if (tokenizer.Current == character.ToString(CultureInfo.InvariantCulture)) { tokenizer.Consume(); } else { return(null); } } bool found; if (!AllowAsSubString) { var next = tokenizer.Current; found = String.IsNullOrWhiteSpace(next) || SpecialCharacters.Any(character => character.Match == next); } else { found = true; } return(found ? new Token(TokenType, Match) : null); }
protected override Token IsMatchImpl(Tokenizer tokenizer) { var str = new StringBuilder(); if (tokenizer.Current == StringDelim) { tokenizer.Consume(); while (!tokenizer.End() && tokenizer.Current != StringDelim) { str.Append(tokenizer.Current); tokenizer.Consume(); } if (tokenizer.End()) { return(null); } if (tokenizer.Current == StringDelim) { tokenizer.Consume(); } } TokenType tokenType; if (StringDelim == TIC) { tokenType = TokenType.CharValue; if (str.Length > 1) { return(null); } } else { tokenType = TokenType.StringValue; } return(str.Length > 0 ? new Token(tokenType, StringDelim + str + StringDelim) : null); }
protected override Token IsMatchImpl(Tokenizer tokenizer) { bool foundWhiteSpace = false; while (!tokenizer.End() && String.IsNullOrWhiteSpace(tokenizer.Current)) { foundWhiteSpace = true; tokenizer.Consume(); } return(foundWhiteSpace ? new Token(TokenType.WhiteSpace) : null); }
private String GetIntegers(Tokenizer tokenizer) { var regex = new Regex("[0-9]"); String num = null; while (tokenizer.Current != null && regex.IsMatch(tokenizer.Current)) { num += tokenizer.Current; tokenizer.Consume(); } return(num != null ? num : null); }
protected override Token IsMatchImpl(Tokenizer tokenizer) { String current = null; while (!tokenizer.End() && !String.IsNullOrWhiteSpace(tokenizer.Current) && SpecialCharacters.All(m => m.Match != tokenizer.Current)) { current += tokenizer.Current; tokenizer.Consume(); } TokenType tokenType = TokenType.Unknown; if (current == null) { return(null); } if (Char.IsLetter(current, 0)) { tokenType = TokenType.Identifier; } return(new Token(tokenType, current)); }