public void ShouldReturnTrueIfNumericToken() { Assert.True(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.DecimalLiteral)); Assert.True(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.IntegerLiteral)); Assert.True(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.DoubleLiteral)); Assert.True(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Int64Literal)); Assert.True(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.SingleLiteral)); }
public void ShouldReturnFalseIfNotNumericToken() { Assert.False(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Colon)); Assert.False(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.DateTimeLiteral)); Assert.False(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.GeographyLiteral)); Assert.False(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Identifier)); Assert.False(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Unknown)); }
public void ShouldReturnFalseIfNotNumericToken() { ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Colon).Should().BeFalse(); ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.DateTimeLiteral).Should().BeFalse(); ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.GeographyLiteral).Should().BeFalse(); ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Identifier).Should().BeFalse(); ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Unknown).Should().BeFalse(); }
public void ShouldReturnTrueIfNumericToken() { ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.DecimalLiteral).Should().BeTrue(); ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.IntegerLiteral).Should().BeTrue(); ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.DoubleLiteral).Should().BeTrue(); ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Int64Literal).Should().BeTrue(); ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.SingleLiteral).Should().BeTrue(); }
/// <summary> /// Parses the -, not unary operators. /// </summary> /// <returns>The lexical token representing the expression.</returns> private QueryToken ParseUnary() { this.RecurseEnter(); if (this.lexer.CurrentToken.Kind == ExpressionTokenKind.Minus || this.TokenIdentifierIs(ExpressionConstants.KeywordNot)) { ExpressionToken operatorToken = this.lexer.CurrentToken; this.lexer.NextToken(); if (operatorToken.Kind == ExpressionTokenKind.Minus && (ExpressionLexerUtils.IsNumeric(this.lexer.CurrentToken.Kind))) { ExpressionToken numberLiteral = this.lexer.CurrentToken; numberLiteral.Text = "-" + numberLiteral.Text; numberLiteral.Position = operatorToken.Position; this.lexer.CurrentToken = numberLiteral; this.RecurseLeave(); return(this.ParsePrimary()); } QueryToken operand = this.ParseUnary(); UnaryOperatorKind unaryOperatorKind; if (operatorToken.Kind == ExpressionTokenKind.Minus) { unaryOperatorKind = UnaryOperatorKind.Negate; } else { Debug.Assert(operatorToken.IdentifierIs(ExpressionConstants.KeywordNot, enableCaseInsensitiveBuiltinIdentifier), "Was a new unary operator added?"); unaryOperatorKind = UnaryOperatorKind.Not; } this.RecurseLeave(); return(new UnaryOperatorToken(unaryOperatorKind, operand)); } this.RecurseLeave(); return(this.ParsePrimary()); }
private ExpressionToken NextTokenImplementation(out Exception error) { DebugUtils.CheckNoExternalCallers(); error = null; if (this.ignoreWhitespace) { this.ParseWhitespace(); } ExpressionTokenKind t; int tokenPos = this.textPos; switch (this.ch) { case '(': this.NextChar(); t = ExpressionTokenKind.OpenParen; break; case ')': this.NextChar(); t = ExpressionTokenKind.CloseParen; break; case ',': this.NextChar(); t = ExpressionTokenKind.Comma; break; case '-': bool hasNext = this.textPos + 1 < this.textLen; if (hasNext && Char.IsDigit(this.text[this.textPos + 1])) { this.NextChar(); t = this.ParseFromDigit(); if (ExpressionLexerUtils.IsNumeric(t)) { break; } // If it looked like a numeric but wasn't (because it was a binary 0x... value for example), // we'll rewind and fall through to a simple '-' token. this.SetTextPos(tokenPos); } else if (hasNext && this.text[tokenPos + 1] == ExpressionConstants.InfinityLiteral[0]) { this.NextChar(); this.ParseIdentifier(); string currentIdentifier = this.text.Substring(tokenPos + 1, this.textPos - tokenPos - 1); if (ExpressionLexerUtils.IsInfinityLiteralDouble(currentIdentifier)) { t = ExpressionTokenKind.DoubleLiteral; break; } else if (ExpressionLexerUtils.IsInfinityLiteralSingle(currentIdentifier)) { t = ExpressionTokenKind.SingleLiteral; break; } // If it looked like '-INF' but wasn't we'll rewind and fall through to a simple '-' token. this.SetTextPos(tokenPos); } this.NextChar(); t = ExpressionTokenKind.Minus; break; case '=': this.NextChar(); t = ExpressionTokenKind.Equal; break; case '/': this.NextChar(); t = ExpressionTokenKind.Slash; break; case '?': this.NextChar(); t = ExpressionTokenKind.Question; break; case '.': this.NextChar(); t = ExpressionTokenKind.Dot; break; case '\'': char quote = this.ch.Value; do { this.AdvanceToNextOccuranceOf(quote); if (this.textPos == this.textLen) { error = ParseError(ODataErrorStrings.ExpressionLexer_UnterminatedStringLiteral(this.textPos, this.text)); } this.NextChar(); }while (this.ch.HasValue && (this.ch == quote)); t = ExpressionTokenKind.StringLiteral; break; case '*': this.NextChar(); t = ExpressionTokenKind.Star; break; case ':': this.NextChar(); t = ExpressionTokenKind.Colon; break; case '{': this.ParseBracketedExpression('{', '}'); t = ExpressionTokenKind.BracketedExpression; break; case '[': this.ParseBracketedExpression('[', ']'); t = ExpressionTokenKind.BracketedExpression; break; default: if (this.IsValidWhiteSpace) { Debug.Assert(!this.ignoreWhitespace, "should not hit ws while ignoring it"); this.ParseWhitespace(); t = ExpressionTokenKind.Unknown; break; } if (this.IsValidStartingCharForIdentifier) { this.ParseIdentifier(); t = ExpressionTokenKind.Identifier; break; } if (this.IsValidDigit) { t = this.ParseFromDigit(); break; } if (this.textPos == this.textLen) { t = ExpressionTokenKind.End; break; } if (this.useSemicolonDelimeter && this.ch == ';') { this.NextChar(); t = ExpressionTokenKind.SemiColon; break; } if (this.parsingFunctionParameters && this.ch == '@') { this.NextChar(); if (this.textPos == this.textLen) { error = ParseError(ODataErrorStrings.ExpressionLexer_SyntaxError(this.textPos, this.text)); t = ExpressionTokenKind.Unknown; break; } if (!this.IsValidStartingCharForIdentifier) { error = ParseError(ODataErrorStrings.ExpressionLexer_InvalidCharacter(this.ch, this.textPos, this.text)); t = ExpressionTokenKind.Unknown; break; } this.ParseIdentifier(); t = ExpressionTokenKind.ParameterAlias; break; } error = ParseError(ODataErrorStrings.ExpressionLexer_InvalidCharacter(this.ch, this.textPos, this.text)); t = ExpressionTokenKind.Unknown; break; } this.token.Kind = t; this.token.Text = this.text.Substring(tokenPos, this.textPos - tokenPos); this.token.Position = tokenPos; // Handle type-prefixed literals such as binary, datetime or guid. this.HandleTypePrefixedLiterals(); // Handle keywords. if (this.token.Kind == ExpressionTokenKind.Identifier) { if (ExpressionLexerUtils.IsInfinityOrNaNDouble(this.token.Text)) { this.token.Kind = ExpressionTokenKind.DoubleLiteral; } else if (ExpressionLexerUtils.IsInfinityOrNanSingle(this.token.Text)) { this.token.Kind = ExpressionTokenKind.SingleLiteral; } else if (this.token.Text == ExpressionConstants.KeywordTrue || this.token.Text == ExpressionConstants.KeywordFalse) { this.token.Kind = ExpressionTokenKind.BooleanLiteral; } else if (this.token.Text == ExpressionConstants.KeywordNull) { this.token.Kind = ExpressionTokenKind.NullLiteral; } } return(this.token); }
protected virtual ExpressionToken NextTokenImplementation(out Exception error) { error = null; if (this.ignoreWhitespace) { this.ParseWhitespace(); } ExpressionTokenKind t; int tokenPos = this.textPos; switch (this.ch) { case '(': this.NextChar(); t = ExpressionTokenKind.OpenParen; break; case ')': this.NextChar(); t = ExpressionTokenKind.CloseParen; break; case ',': this.NextChar(); t = ExpressionTokenKind.Comma; break; case '-': bool hasNext = this.textPos + 1 < this.TextLen; if (hasNext && Char.IsDigit(this.Text[this.textPos + 1])) { // don't separate '-' and its following digits : -2147483648 is valid int.MinValue, but 2147483648 is long. t = this.ParseFromDigit(); if (ExpressionLexerUtils.IsNumeric(t)) { break; } // If it looked like a numeric but wasn't (because it was a binary 0x... value for example), // we'll rewind and fall through to a simple '-' token. this.SetTextPos(tokenPos); } else if (hasNext && this.Text[tokenPos + 1] == ExpressionConstants.InfinityLiteral[0]) { this.NextChar(); this.ParseIdentifier(); string currentIdentifier = this.Text.Substring(tokenPos + 1, this.textPos - tokenPos - 1); if (ExpressionLexerUtils.IsInfinityLiteralDouble(currentIdentifier)) { t = ExpressionTokenKind.DoubleLiteral; break; } else if (ExpressionLexerUtils.IsInfinityLiteralSingle(currentIdentifier)) { t = ExpressionTokenKind.SingleLiteral; break; } // If it looked like '-INF' but wasn't we'll rewind and fall through to a simple '-' token. this.SetTextPos(tokenPos); } this.NextChar(); t = ExpressionTokenKind.Minus; break; case '=': this.NextChar(); t = ExpressionTokenKind.Equal; break; case '/': this.NextChar(); t = ExpressionTokenKind.Slash; break; case '?': this.NextChar(); t = ExpressionTokenKind.Question; break; case '.': this.NextChar(); t = ExpressionTokenKind.Dot; break; case '\'': char quote = this.ch.Value; do { this.AdvanceToNextOccuranceOf(quote); if (this.textPos == this.TextLen) { error = ParseError(ODataErrorStrings.ExpressionLexer_UnterminatedStringLiteral(this.textPos, this.Text)); } this.NextChar(); }while (this.ch.HasValue && (this.ch.Value == quote)); t = ExpressionTokenKind.StringLiteral; break; case '*': this.NextChar(); t = ExpressionTokenKind.Star; break; case ':': this.NextChar(); t = ExpressionTokenKind.Colon; break; case '{': this.NextChar(); this.AdvanceThroughBalancedExpression('{', '}'); t = ExpressionTokenKind.BracketedExpression; break; case '[': this.NextChar(); this.AdvanceThroughBalancedExpression('[', ']'); t = ExpressionTokenKind.BracketedExpression; break; default: if (this.IsValidWhiteSpace) { Debug.Assert(!this.ignoreWhitespace, "should not hit ws while ignoring it"); this.ParseWhitespace(); t = ExpressionTokenKind.Unknown; break; } if (this.IsValidStartingCharForIdentifier) { this.ParseIdentifier(); // Guids will have '-' in them // guidValue = 8HEXDIG "-" 4HEXDIG "-" 4HEXDIG "-" 4HEXDIG "-" 12HEXDIG if (this.ch == '-' && this.TryParseGuid(tokenPos)) { t = ExpressionTokenKind.GuidLiteral; break; } t = ExpressionTokenKind.Identifier; break; } if (this.IsValidDigit) { t = this.ParseFromDigit(); break; } if (this.textPos == this.TextLen) { t = ExpressionTokenKind.End; break; } if (this.useSemicolonDelimeter && this.ch == ';') { this.NextChar(); t = ExpressionTokenKind.SemiColon; break; } if (this.parsingFunctionParameters && this.ch == '@') { this.NextChar(); if (this.textPos == this.TextLen) { error = ParseError(ODataErrorStrings.ExpressionLexer_SyntaxError(this.textPos, this.Text)); t = ExpressionTokenKind.Unknown; break; } if (!this.IsValidStartingCharForIdentifier) { error = ParseError(ODataErrorStrings.ExpressionLexer_InvalidCharacter(this.ch, this.textPos, this.Text)); t = ExpressionTokenKind.Unknown; break; } this.ParseIdentifier(); t = ExpressionTokenKind.ParameterAlias; break; } error = ParseError(ODataErrorStrings.ExpressionLexer_InvalidCharacter(this.ch, this.textPos, this.Text)); t = ExpressionTokenKind.Unknown; break; } this.token.Kind = t; this.token.Text = this.Text.Substring(tokenPos, this.textPos - tokenPos); this.token.Position = tokenPos; this.HandleTypePrefixedLiterals(); return(this.token); }