public void ShouldReturnFalseIfNotNumericToken()
{
ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Colon).Should().BeFalse();
ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.DateTimeLiteral).Should().BeFalse();
ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.GeographyLiteral).Should().BeFalse();
ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Identifier).Should().BeFalse();
ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Unknown).Should().BeFalse();
}
public void ShouldReturnTrueIfNumericToken()
{
ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.DecimalLiteral).Should().BeTrue();
ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.IntegerLiteral).Should().BeTrue();
ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.DoubleLiteral).Should().BeTrue();
ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Int64Literal).Should().BeTrue();
ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.SingleLiteral).Should().BeTrue();
}
public void ShouldReturnTrueIfNumericToken()
{
Assert.True(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.DecimalLiteral));
Assert.True(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.IntegerLiteral));
Assert.True(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.DoubleLiteral));
Assert.True(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Int64Literal));
Assert.True(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.SingleLiteral));
}
public void ShouldReturnFalseIfNotNumericToken()
{
Assert.False(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Colon));
Assert.False(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.DateTimeLiteral));
Assert.False(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.GeographyLiteral));
Assert.False(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Identifier));
Assert.False(ExpressionLexerUtils.IsNumeric(ExpressionTokenKind.Unknown));
}
/// <summary>
/// Get type-prefixed literals such as double, boolean...
/// </summary>
/// <param name="tokenText">Token texk</param>
/// <returns>ExpressionTokenKind by the token text</returns>
private static ExpressionTokenKind?GetBuiltInTypesLiteralPrefix(string tokenText)
{
if (ExpressionLexerUtils.IsInfinityOrNaNDouble(tokenText))
{
return(ExpressionTokenKind.DoubleLiteral);
}
else if (ExpressionLexerUtils.IsInfinityOrNanSingle(tokenText))
{
return(ExpressionTokenKind.SingleLiteral);
}
else if (tokenText == ExpressionConstants.KeywordTrue || tokenText == ExpressionConstants.KeywordFalse)
{
return(ExpressionTokenKind.BooleanLiteral);
}
else if (tokenText == ExpressionConstants.KeywordNull)
{
return(ExpressionTokenKind.NullLiteral);
}
return(null);
}
/// <summary>
/// Parses the -, not unary operators.
/// </summary>
/// <returns>The lexical token representing the expression.</returns>
private QueryToken ParseUnary()
{
this.RecurseEnter();
if (this.lexer.CurrentToken.Kind == ExpressionTokenKind.Minus || this.TokenIdentifierIs(ExpressionConstants.KeywordNot))
{
ExpressionToken operatorToken = this.lexer.CurrentToken;
this.lexer.NextToken();
if (operatorToken.Kind == ExpressionTokenKind.Minus && (ExpressionLexerUtils.IsNumeric(this.lexer.CurrentToken.Kind)))
{
ExpressionToken numberLiteral = this.lexer.CurrentToken;
numberLiteral.Text = "-" + numberLiteral.Text;
numberLiteral.Position = operatorToken.Position;
this.lexer.CurrentToken = numberLiteral;
this.RecurseLeave();
return(this.ParsePrimary());
}
QueryToken operand = this.ParseUnary();
UnaryOperatorKind unaryOperatorKind;
if (operatorToken.Kind == ExpressionTokenKind.Minus)
{
unaryOperatorKind = UnaryOperatorKind.Negate;
}
else
{
Debug.Assert(operatorToken.IdentifierIs(ExpressionConstants.KeywordNot, enableCaseInsensitiveBuiltinIdentifier), "Was a new unary operator added?");
unaryOperatorKind = UnaryOperatorKind.Not;
}
this.RecurseLeave();
return(new UnaryOperatorToken(unaryOperatorKind, operand));
}
this.RecurseLeave();
return(this.ParsePrimary());
}
protected virtual ExpressionToken NextTokenImplementation(out Exception error)
{
error = null;
if (this.ignoreWhitespace)
{
this.ParseWhitespace();
}
ExpressionTokenKind t;
int tokenPos = this.textPos;
switch (this.ch)
{
case '(':
this.NextChar();
t = ExpressionTokenKind.OpenParen;
break;
case ')':
this.NextChar();
t = ExpressionTokenKind.CloseParen;
break;
case ',':
this.NextChar();
t = ExpressionTokenKind.Comma;
break;
case '-':
bool hasNext = this.textPos + 1 < this.TextLen;
if (hasNext && Char.IsDigit(this.Text[this.textPos + 1]))
{
// don't separate '-' and its following digits : -2147483648 is valid int.MinValue, but 2147483648 is long.
t = this.ParseFromDigit();
if (ExpressionLexerUtils.IsNumeric(t))
{
break;
}
// If it looked like a numeric but wasn't (because it was a binary 0x... value for example),
// we'll rewind and fall through to a simple '-' token.
this.SetTextPos(tokenPos);
}
else if (hasNext && this.Text[tokenPos + 1] == ExpressionConstants.InfinityLiteral[0])
{
this.NextChar();
this.ParseIdentifier();
string currentIdentifier = this.Text.Substring(tokenPos + 1, this.textPos - tokenPos - 1);
if (ExpressionLexerUtils.IsInfinityLiteralDouble(currentIdentifier))
{
t = ExpressionTokenKind.DoubleLiteral;
break;
}
else if (ExpressionLexerUtils.IsInfinityLiteralSingle(currentIdentifier))
{
t = ExpressionTokenKind.SingleLiteral;
break;
}
// If it looked like '-INF' but wasn't we'll rewind and fall through to a simple '-' token.
this.SetTextPos(tokenPos);
}
this.NextChar();
t = ExpressionTokenKind.Minus;
break;
case '=':
this.NextChar();
t = ExpressionTokenKind.Equal;
break;
case '/':
this.NextChar();
t = ExpressionTokenKind.Slash;
break;
case '?':
this.NextChar();
t = ExpressionTokenKind.Question;
break;
case '.':
this.NextChar();
t = ExpressionTokenKind.Dot;
break;
case '\'':
char quote = this.ch.Value;
do
{
this.AdvanceToNextOccuranceOf(quote);
if (this.textPos == this.TextLen)
{
error = ParseError(ODataErrorStrings.ExpressionLexer_UnterminatedStringLiteral(this.textPos, this.Text));
}
this.NextChar();
}while (this.ch.HasValue && (this.ch.Value == quote));
t = ExpressionTokenKind.StringLiteral;
break;
case '*':
this.NextChar();
t = ExpressionTokenKind.Star;
break;
case ':':
this.NextChar();
t = ExpressionTokenKind.Colon;
break;
case '{':
this.NextChar();
this.AdvanceThroughBalancedExpression('{', '}');
t = ExpressionTokenKind.BracketedExpression;
break;
case '[':
this.NextChar();
this.AdvanceThroughBalancedExpression('[', ']');
t = ExpressionTokenKind.BracketedExpression;
break;
default:
if (this.IsValidWhiteSpace)
{
Debug.Assert(!this.ignoreWhitespace, "should not hit ws while ignoring it");
this.ParseWhitespace();
t = ExpressionTokenKind.Unknown;
break;
}
if (this.IsValidStartingCharForIdentifier)
{
this.ParseIdentifier();
// Guids will have '-' in them
// guidValue = 8HEXDIG "-" 4HEXDIG "-" 4HEXDIG "-" 4HEXDIG "-" 12HEXDIG
if (this.ch == '-' &&
this.TryParseGuid(tokenPos))
{
t = ExpressionTokenKind.GuidLiteral;
break;
}
t = ExpressionTokenKind.Identifier;
break;
}
if (this.IsValidDigit)
{
t = this.ParseFromDigit();
break;
}
if (this.textPos == this.TextLen)
{
t = ExpressionTokenKind.End;
break;
}
if (this.useSemicolonDelimeter && this.ch == ';')
{
this.NextChar();
t = ExpressionTokenKind.SemiColon;
break;
}
if (this.parsingFunctionParameters && this.ch == '@')
{
this.NextChar();
if (this.textPos == this.TextLen)
{
error = ParseError(ODataErrorStrings.ExpressionLexer_SyntaxError(this.textPos, this.Text));
t = ExpressionTokenKind.Unknown;
break;
}
if (!this.IsValidStartingCharForIdentifier)
{
error = ParseError(ODataErrorStrings.ExpressionLexer_InvalidCharacter(this.ch, this.textPos, this.Text));
t = ExpressionTokenKind.Unknown;
break;
}
this.ParseIdentifier();
t = ExpressionTokenKind.ParameterAlias;
break;
}
error = ParseError(ODataErrorStrings.ExpressionLexer_InvalidCharacter(this.ch, this.textPos, this.Text));
t = ExpressionTokenKind.Unknown;
break;
}
this.token.Kind = t;
this.token.Text = this.Text.Substring(tokenPos, this.textPos - tokenPos);
this.token.Position = tokenPos;
this.HandleTypePrefixedLiterals();
return(this.token);
}
public void IsInfinityLiteralSingleShouldReturnTrueForINz()
{
bool result = ExpressionLexerUtils.IsInfinityLiteralSingle("INzf");
Assert.False(result);
}
public void IsInfinityOrNaNDoubleShouldReturnFalseForNaB()
{
bool result = ExpressionLexerUtils.IsInfinityOrNaNDouble("NaB");
result.Should().BeFalse();
}
public void IsInfinityOrNaNSingleShouldReturnTrueForNaNF()
{
bool result = ExpressionLexerUtils.IsInfinityOrNanSingle("NaNF");
Assert.True(result);
}
public void IsInfinityOrNaNSingleShouldReturnFalseForBlarg()
{
bool result = ExpressionLexerUtils.IsInfinityOrNanSingle("Blarg");
Assert.False(result);
}
public void IsInfinityOrNaNDoubleShouldReturnFalseForNaB()
{
bool result = ExpressionLexerUtils.IsInfinityOrNaNDouble("NaB");
Assert.False(result);
}
public void IsInfinityLiteralDoubleShouldReturnTrueForINF()
{
bool result = ExpressionLexerUtils.IsInfinityLiteralDouble("INF");
Assert.True(result);
}
private ExpressionToken NextTokenImplementation(out Exception error)
{
DebugUtils.CheckNoExternalCallers();
error = null;
if (this.ignoreWhitespace)
{
this.ParseWhitespace();
}
ExpressionTokenKind t;
int tokenPos = this.textPos;
switch (this.ch)
{
case '(':
this.NextChar();
t = ExpressionTokenKind.OpenParen;
break;
case ')':
this.NextChar();
t = ExpressionTokenKind.CloseParen;
break;
case ',':
this.NextChar();
t = ExpressionTokenKind.Comma;
break;
case '-':
bool hasNext = this.textPos + 1 < this.textLen;
if (hasNext && Char.IsDigit(this.text[this.textPos + 1]))
{
this.NextChar();
t = this.ParseFromDigit();
if (ExpressionLexerUtils.IsNumeric(t))
{
break;
}
// If it looked like a numeric but wasn't (because it was a binary 0x... value for example),
// we'll rewind and fall through to a simple '-' token.
this.SetTextPos(tokenPos);
}
else if (hasNext && this.text[tokenPos + 1] == ExpressionConstants.InfinityLiteral[0])
{
this.NextChar();
this.ParseIdentifier();
string currentIdentifier = this.text.Substring(tokenPos + 1, this.textPos - tokenPos - 1);
if (ExpressionLexerUtils.IsInfinityLiteralDouble(currentIdentifier))
{
t = ExpressionTokenKind.DoubleLiteral;
break;
}
else if (ExpressionLexerUtils.IsInfinityLiteralSingle(currentIdentifier))
{
t = ExpressionTokenKind.SingleLiteral;
break;
}
// If it looked like '-INF' but wasn't we'll rewind and fall through to a simple '-' token.
this.SetTextPos(tokenPos);
}
this.NextChar();
t = ExpressionTokenKind.Minus;
break;
case '=':
this.NextChar();
t = ExpressionTokenKind.Equal;
break;
case '/':
this.NextChar();
t = ExpressionTokenKind.Slash;
break;
case '?':
this.NextChar();
t = ExpressionTokenKind.Question;
break;
case '.':
this.NextChar();
t = ExpressionTokenKind.Dot;
break;
case '\'':
char quote = this.ch.Value;
do
{
this.AdvanceToNextOccuranceOf(quote);
if (this.textPos == this.textLen)
{
error = ParseError(ODataErrorStrings.ExpressionLexer_UnterminatedStringLiteral(this.textPos, this.text));
}
this.NextChar();
}while (this.ch.HasValue && (this.ch == quote));
t = ExpressionTokenKind.StringLiteral;
break;
case '*':
this.NextChar();
t = ExpressionTokenKind.Star;
break;
case ':':
this.NextChar();
t = ExpressionTokenKind.Colon;
break;
case '{':
this.ParseBracketedExpression('{', '}');
t = ExpressionTokenKind.BracketedExpression;
break;
case '[':
this.ParseBracketedExpression('[', ']');
t = ExpressionTokenKind.BracketedExpression;
break;
default:
if (this.IsValidWhiteSpace)
{
Debug.Assert(!this.ignoreWhitespace, "should not hit ws while ignoring it");
this.ParseWhitespace();
t = ExpressionTokenKind.Unknown;
break;
}
if (this.IsValidStartingCharForIdentifier)
{
this.ParseIdentifier();
t = ExpressionTokenKind.Identifier;
break;
}
if (this.IsValidDigit)
{
t = this.ParseFromDigit();
break;
}
if (this.textPos == this.textLen)
{
t = ExpressionTokenKind.End;
break;
}
if (this.useSemicolonDelimeter && this.ch == ';')
{
this.NextChar();
t = ExpressionTokenKind.SemiColon;
break;
}
if (this.parsingFunctionParameters && this.ch == '@')
{
this.NextChar();
if (this.textPos == this.textLen)
{
error = ParseError(ODataErrorStrings.ExpressionLexer_SyntaxError(this.textPos, this.text));
t = ExpressionTokenKind.Unknown;
break;
}
if (!this.IsValidStartingCharForIdentifier)
{
error = ParseError(ODataErrorStrings.ExpressionLexer_InvalidCharacter(this.ch, this.textPos, this.text));
t = ExpressionTokenKind.Unknown;
break;
}
this.ParseIdentifier();
t = ExpressionTokenKind.ParameterAlias;
break;
}
error = ParseError(ODataErrorStrings.ExpressionLexer_InvalidCharacter(this.ch, this.textPos, this.text));
t = ExpressionTokenKind.Unknown;
break;
}
this.token.Kind = t;
this.token.Text = this.text.Substring(tokenPos, this.textPos - tokenPos);
this.token.Position = tokenPos;
// Handle type-prefixed literals such as binary, datetime or guid.
this.HandleTypePrefixedLiterals();
// Handle keywords.
if (this.token.Kind == ExpressionTokenKind.Identifier)
{
if (ExpressionLexerUtils.IsInfinityOrNaNDouble(this.token.Text))
{
this.token.Kind = ExpressionTokenKind.DoubleLiteral;
}
else if (ExpressionLexerUtils.IsInfinityOrNanSingle(this.token.Text))
{
this.token.Kind = ExpressionTokenKind.SingleLiteral;
}
else if (this.token.Text == ExpressionConstants.KeywordTrue || this.token.Text == ExpressionConstants.KeywordFalse)
{
this.token.Kind = ExpressionTokenKind.BooleanLiteral;
}
else if (this.token.Text == ExpressionConstants.KeywordNull)
{
this.token.Kind = ExpressionTokenKind.NullLiteral;
}
}
return(this.token);
}
public void IsInfinityLiteralDoubleShouldReturnTrueForINF()
{
bool result = ExpressionLexerUtils.IsInfinityLiteralDouble("INF");
result.Should().BeTrue();
}
public void IsInfinityOrNaNSingleShouldReturnTrueForNaNF()
{
bool result = ExpressionLexerUtils.IsInfinityOrNanSingle("NaNF");
result.Should().BeTrue();
}
public void IsInfinityOrNaNSingleShouldReturnFalseForBlarg()
{
bool result = ExpressionLexerUtils.IsInfinityOrNanSingle("Blarg");
result.Should().BeFalse();
}
public void IsInfinityLiteralSingleShouldReturnTrueForINz()
{
bool result = ExpressionLexerUtils.IsInfinityLiteralSingle("INzf");
result.Should().BeFalse();
}
