public void Tokenize (string s)
{
if (s == null)
throw new ArgumentNullException ("s");
this.inputString = s;
this.position = 0;
this.token = new Token (null, TokenType.BOF, 0);
GetNextToken ();
}
public ConditionFactorExpression (Token token)
{
this.token = token;
}
// FIXME: in some situations items might not be allowed
static Token EvaluateToken (Token token, Project context)
{
Expression oe = new Expression ();
oe.Parse (token.Value, ParseOptions.AllowItemsMetadataAndSplit);
return new Token ((string) oe.ConvertTo (context, typeof (string)), token.Type);
}
private bool Same(string expression, Token.TokenType token)
{
if (lexer.IsNext(token))
{
if (!lexer.Advance())
{
errorPosition = lexer.GetErrorPosition();
if (null != lexer.UnexpectedlyFound)
{
ProjectErrorUtilities.VerifyThrowInvalidProject(false, this.conditionAttribute, lexer.GetErrorResource(), expression, errorPosition, lexer.UnexpectedlyFound);
}
else
{
ProjectErrorUtilities.VerifyThrowInvalidProject(false, this.conditionAttribute, lexer.GetErrorResource(), expression, errorPosition);
}
}
return true;
}
else
return false;
}
public void GetNextToken ()
{
if (putback != null) {
token = putback;
putback = null;
return;
}
if (token.Type == TokenType.EOF)
throw new ExpressionParseException (String.Format (
"Error while parsing condition \"{0}\", ended abruptly.",
inputString));
SkipWhiteSpace ();
tokenPosition = position;
// int i = PeekChar ();
int i = ReadChar ();
if (i == -1) {
token = new Token (null, TokenType.EOF, tokenPosition);
return;
}
char ch = (char) i;
// FIXME: looks like a hack: if '-' is here '->' won't be tokenized
// maybe we should treat item reference as a token
if (ch == '-' && PeekChar () == '>') {
ReadChar ();
token = new Token ("->", TokenType.Transform, tokenPosition);
} else if (Char.IsDigit (ch) || ch == '-') {
StringBuilder sb = new StringBuilder ();
sb.Append (ch);
while ((i = PeekChar ()) != -1) {
ch = (char) i;
if (Char.IsDigit (ch) || ch == '.')
sb.Append ((char) ReadChar ());
else
break;
}
token = new Token (sb.ToString (), TokenType.Number, tokenPosition);
} else if (ch == '\'' && position < inputString.Length) {
StringBuilder sb = new StringBuilder ();
string temp;
sb.Append (ch);
bool is_itemref = (PeekChar () == '@');
int num_open_braces = 0;
bool in_literal = false;
while ((i = PeekChar ()) != -1) {
ch = (char) i;
if (ch == '(' && !in_literal && is_itemref)
num_open_braces ++;
if (ch == ')' && !in_literal && is_itemref)
num_open_braces --;
sb.Append ((char) ReadChar ());
if (ch == '\'') {
if (num_open_braces == 0)
break;
in_literal = !in_literal;
}
}
temp = sb.ToString ();
token = new Token (temp.Substring (1, temp.Length - 2), TokenType.String, tokenPosition);
} else if (ch == '_' || Char.IsLetter (ch)) {
StringBuilder sb = new StringBuilder ();
sb.Append ((char) ch);
while ((i = PeekChar ()) != -1) {
if ((char) i == '_' || Char.IsLetterOrDigit ((char) i))
sb.Append ((char) ReadChar ());
else
break;
}
string temp = sb.ToString ();
if (keywords.ContainsKey (temp))
token = new Token (temp, keywords [temp], tokenPosition);
else
token = new Token (temp, TokenType.String, tokenPosition);
} else if (ch == '!' && PeekChar () == (int) '=') {
token = new Token ("!=", TokenType.NotEqual, tokenPosition);
ReadChar ();
} else if (ch == '<' && PeekChar () == (int) '=') {
token = new Token ("<=", TokenType.LessOrEqual, tokenPosition);
ReadChar ();
} else if (ch == '>' && PeekChar () == (int) '=') {
token = new Token (">=", TokenType.GreaterOrEqual, tokenPosition);
ReadChar ();
} else if (ch == '=' && PeekChar () == (int) '=') {
token = new Token ("==", TokenType.Equal, tokenPosition);
ReadChar ();
} else if (ch >= 32 && ch < 128) {
if (charIndexToTokenType [ch] != TokenType.Invalid) {
token = new Token (new String (ch, 1), charIndexToTokenType [ch], tokenPosition);
return;
} else
throw new ExpressionParseException (String.Format ("Invalid punctuation: {0}", ch));
} else
throw new ExpressionParseException (String.Format ("Invalid token: {0}", ch));
}
private bool ParseNumeric( int start )
{
if ((expression.Length-parsePoint) > 2 && expression[parsePoint] == '0' && (expression[parsePoint + 1] == 'x' || expression[parsePoint + 1] == 'X'))
{
// Hex number
parsePoint += 2;
SkipHexDigits();
lookahead = new Token(Token.TokenType.Numeric, expression.Substring(start, parsePoint - start));
}
else if ( CharacterUtilities.IsNumberStart(expression[parsePoint]))
{
// Decimal number
if (expression[parsePoint] == '+')
{
parsePoint++;
}
else if (expression[parsePoint] == '-')
{
parsePoint++;
}
SkipDigits();
if (parsePoint < expression.Length && expression[parsePoint] == '.')
{
parsePoint++;
}
if (parsePoint < expression.Length)
{
SkipDigits();
}
// Do we need to error on malformed input like 0.00.00)? or will the conversion handle it?
// For now, let the conversion generate the error.
lookahead = new Token(Token.TokenType.Numeric, expression.Substring(start, parsePoint - start));
}
else
{
// Unreachable
errorState = true;
errorPosition = start + 1;
return false;
}
return true;
}
// FIXME test this
public void Putback (Token token)
{
putback = token;
}
private bool ParseQuotedString()
{
parsePoint++;
int start = parsePoint;
while (parsePoint < expression.Length && expression[parsePoint] != '\'')
{
// Standalone percent-sign must be allowed within a condition because it's
// needed to escape special characters. However, percent-sign followed
// by open-parenthesis is an indication of an item metadata reference, and
// that is only allowed in certain contexts.
if ((expression[parsePoint] == '%') && ((parsePoint + 1) < expression.Length) && (expression[parsePoint + 1] == '('))
{
// If the caller specified that he DOESN'T want to allow item metadata...
if ((this.options & ParserOptions.AllowItemMetadata) == 0)
{
errorPosition = start + 1;
errorState = true;
errorResource = "UnexpectedCharacterInCondition";
unexpectedlyFound = "%";
return false;
}
}
else if (expression[parsePoint] == '@' && ((parsePoint + 1) < expression.Length) && (expression[parsePoint + 1] == '('))
{
// If the caller specified that he DOESN'T want to allow item lists ...
if ((this.options & ParserOptions.AllowItemLists) == 0)
{
errorPosition = start + 1;
errorState = true;
errorResource = "ItemListNotAllowedInThisConditional";
return false;
}
// Item lists have to be parsed because of the replacement syntax e.g. @(Foo,'_').
// I have to know how to parse those so I can skip over the tic marks. I don't
// have to do that with other things like propertygroups, hence itemlists are
// treated specially.
ParseInternalItemList();
continue;
}
parsePoint++;
}
if (parsePoint >= expression.Length)
{
// Quoted string wasn't closed
errorState = true;
errorPosition = start; // The message is going to say "expected after position n" so don't add 1 here.
errorResource = "IllFormedQuotedStringInCondition";
// Not useful to set unexpectedlyFound here. By definition it got to the end of the string.
return false;
}
string originalTokenString = expression.Substring(start, parsePoint - start);
lookahead = new Token(Token.TokenType.String, originalTokenString);
parsePoint++;
return true;
}
private bool ParseSimpleStringOrFunction( int start )
{
SkipSimpleStringChars();
if (0 == string.Compare(expression.Substring(start, parsePoint - start), "and", StringComparison.OrdinalIgnoreCase))
{
lookahead = new Token(Token.TokenType.And, expression.Substring(start, parsePoint - start));
}
else if (0 == string.Compare(expression.Substring(start, parsePoint - start), "or", StringComparison.OrdinalIgnoreCase))
{
lookahead = new Token(Token.TokenType.Or, expression.Substring(start, parsePoint - start));
}
else
{
int end = parsePoint;
SkipWhiteSpace();
if (parsePoint < expression.Length && expression[parsePoint] == '(')
{
lookahead = new Token(Token.TokenType.Function, expression.Substring(start, end - start));
}
else
{
string tokenValue = expression.Substring(start, end - start);
lookahead = new Token(Token.TokenType.String, tokenValue);
}
}
return true;
}
/// <summary>
/// Parses a string of the form %(itemmetadataname).
/// </summary>
/// <returns></returns>
/// <owner>RGoel</owner>
private bool ParseItemMetadata()
{
string itemMetadataExpression = this.ParsePropertyOrItemMetadata();
if (itemMetadataExpression == null)
{
// The ParsePropertyOrItemMetadata method returns the correct error resources
// for parsing properties such as $(propertyname). At this stage in the Whidbey
// cycle, we're not allowed to add new string resources, so I can't add a new
// resource specific to item metadata, so here, we just change the error to
// the generic "UnexpectedCharacter".
errorResource = "UnexpectedCharacterInCondition";
return false;
}
else
{
this.lookahead = new Token(Token.TokenType.ItemMetadata, itemMetadataExpression);
return true;
}
}
private bool ParseItemList()
{
int start = parsePoint;
if (!ParseInternalItemList())
{
return false;
}
lookahead = new Token(Token.TokenType.ItemList, expression.Substring(start, parsePoint - start));
return true;
}
/// <summary>
/// Parses a string of the form $(propertyname).
/// </summary>
/// <returns></returns>
/// <owner>RGoel, DavidLe</owner>
private bool ParseProperty()
{
string propertyExpression = this.ParsePropertyOrItemMetadata();
if (propertyExpression == null)
{
return false;
}
else
{
this.lookahead = new Token(Token.TokenType.Property, propertyExpression);
return true;
}
}
/// <summary>
/// Advance
/// returns true on successful advance
/// and false on an erroneous token
///
/// Doesn't return error until the bogus input is encountered.
/// Advance() returns true even after EndOfInput is encountered.
/// </summary>
internal bool Advance()
{
if (errorState)
return false;
if (lookahead != null && lookahead.IsToken(Token.TokenType.EndOfInput))
return true;
SkipWhiteSpace();
// Update error position after skipping whitespace
errorPosition = parsePoint + 1;
if (parsePoint >= expression.Length)
{
lookahead = new Token(Token.TokenType.EndOfInput, null /* end of input */);
}
else
{
switch (expression[parsePoint])
{
case ',':
lookahead = new Token(Token.TokenType.Comma, comma);
parsePoint++;
break;
case '(':
lookahead = new Token(Token.TokenType.LeftParenthesis, leftParenthesis);
parsePoint++;
break;
case ')':
lookahead = new Token(Token.TokenType.RightParenthesis, rightParenthesis);
parsePoint++;
break;
case '$':
if (!ParseProperty())
return false;
break;
case '%':
// If the caller specified that he DOESN'T want to allow item metadata ...
if ((this.options & ParserOptions.AllowItemMetadata) == 0)
{
errorPosition = this.parsePoint;
errorState = true;
errorResource = "UnexpectedCharacterInCondition";
unexpectedlyFound = "%";
return false;
}
if (!ParseItemMetadata())
return false;
break;
case '@':
int start = this.parsePoint;
// If the caller specified that he DOESN'T want to allow item lists ...
if ((this.options & ParserOptions.AllowItemLists) == 0)
{
if ((parsePoint + 1) < expression.Length && expression[parsePoint + 1] == '(')
{
errorPosition = start + 1;
errorState = true;
errorResource = "ItemListNotAllowedInThisConditional";
return false;
}
}
if (!ParseItemList())
return false;
break;
case '!':
// negation and not-equal
if ((parsePoint + 1) < expression.Length && expression[parsePoint + 1] == '=')
{
lookahead = new Token(Token.TokenType.NotEqualTo, notEqualTo);
parsePoint += 2;
}
else
{
lookahead = new Token(Token.TokenType.Not, not);
parsePoint++;
}
break;
case '>':
// gt and gte
if ((parsePoint + 1) < expression.Length && expression[parsePoint + 1] == '=')
{
lookahead = new Token(Token.TokenType.GreaterThanOrEqualTo, greaterThanOrEqualTo);
parsePoint += 2;
}
else
{
lookahead = new Token(Token.TokenType.GreaterThan, greaterThan);
parsePoint++;
}
break;
case '<':
// lt and lte
if ((parsePoint + 1) < expression.Length && expression[parsePoint + 1] == '=')
{
lookahead = new Token(Token.TokenType.LessThanOrEqualTo, lessThanOrEqualTo);
parsePoint += 2;
}
else
{
lookahead = new Token(Token.TokenType.LessThan, lessThan);
parsePoint++;
}
break;
case '=':
if ((parsePoint + 1) < expression.Length && expression[parsePoint + 1] == '=')
{
lookahead = new Token(Token.TokenType.EqualTo, equalTo);
parsePoint += 2;
}
else
{
errorPosition = parsePoint + 2; // expression[parsePoint + 1], counting from 1
errorResource = "IllFormedEqualsInCondition";
if ((parsePoint + 1) < expression.Length)
{
// store the char we found instead
unexpectedlyFound = Convert.ToString(expression[parsePoint + 1], CultureInfo.InvariantCulture);
}
else
{
unexpectedlyFound = EndOfInput;
}
parsePoint++;
errorState = true;
return false;
}
break;
case '\'':
if (!ParseQuotedString())
return false;
break;
default:
// Simple strings, function calls, decimal numbers, hex numbers
if (!ParseRemaining())
return false;
break;
}
}
return true;
}
internal bool IsNext( Token.TokenType type )
{
return lookahead.IsToken(type);
}
// FIXME: in some situations items might not be allowed
static Token EvaluateToken (Token token, IExpressionContext context)
{
string val = context.EvaluateString (token.Value);
return new Token (val, TokenType.String);
}
public void ScanForClosingParens (int parensCounter = 1)
{
tokenPosition = position;
int start = position;
int ch;
while ((ch = ReadChar ()) >= 0) {
switch (ch) {
case ')':
if (--parensCounter == 0) {
--position;
token = new Token (inputString.Substring (start, position - start), TokenType.String, tokenPosition);
return;
}
break;
case '(':
++parensCounter;
break;
}
}
token = new Token (null, TokenType.EOF, tokenPosition);
}
