public void Tokenize (string s) { if (s == null) throw new ArgumentNullException ("s"); this.inputString = s; this.position = 0; this.token = new Token (null, TokenType.BOF, 0); GetNextToken (); }
public ConditionFactorExpression (Token token) { this.token = token; }
// FIXME: in some situations items might not be allowed static Token EvaluateToken (Token token, Project context) { Expression oe = new Expression (); oe.Parse (token.Value, ParseOptions.AllowItemsMetadataAndSplit); return new Token ((string) oe.ConvertTo (context, typeof (string)), token.Type); }
private bool Same(string expression, Token.TokenType token) { if (lexer.IsNext(token)) { if (!lexer.Advance()) { errorPosition = lexer.GetErrorPosition(); if (null != lexer.UnexpectedlyFound) { ProjectErrorUtilities.VerifyThrowInvalidProject(false, this.conditionAttribute, lexer.GetErrorResource(), expression, errorPosition, lexer.UnexpectedlyFound); } else { ProjectErrorUtilities.VerifyThrowInvalidProject(false, this.conditionAttribute, lexer.GetErrorResource(), expression, errorPosition); } } return true; } else return false; }
public void GetNextToken () { if (putback != null) { token = putback; putback = null; return; } if (token.Type == TokenType.EOF) throw new ExpressionParseException (String.Format ( "Error while parsing condition \"{0}\", ended abruptly.", inputString)); SkipWhiteSpace (); tokenPosition = position; // int i = PeekChar (); int i = ReadChar (); if (i == -1) { token = new Token (null, TokenType.EOF, tokenPosition); return; } char ch = (char) i; // FIXME: looks like a hack: if '-' is here '->' won't be tokenized // maybe we should treat item reference as a token if (ch == '-' && PeekChar () == '>') { ReadChar (); token = new Token ("->", TokenType.Transform, tokenPosition); } else if (Char.IsDigit (ch) || ch == '-') { StringBuilder sb = new StringBuilder (); sb.Append (ch); while ((i = PeekChar ()) != -1) { ch = (char) i; if (Char.IsDigit (ch) || ch == '.') sb.Append ((char) ReadChar ()); else break; } token = new Token (sb.ToString (), TokenType.Number, tokenPosition); } else if (ch == '\'' && position < inputString.Length) { StringBuilder sb = new StringBuilder (); string temp; sb.Append (ch); bool is_itemref = (PeekChar () == '@'); int num_open_braces = 0; bool in_literal = false; while ((i = PeekChar ()) != -1) { ch = (char) i; if (ch == '(' && !in_literal && is_itemref) num_open_braces ++; if (ch == ')' && !in_literal && is_itemref) num_open_braces --; sb.Append ((char) ReadChar ()); if (ch == '\'') { if (num_open_braces == 0) break; in_literal = !in_literal; } } temp = sb.ToString (); token = new Token (temp.Substring (1, temp.Length - 2), TokenType.String, tokenPosition); } else if (ch == '_' || Char.IsLetter (ch)) { StringBuilder sb = new StringBuilder (); sb.Append ((char) ch); while ((i = PeekChar ()) != -1) { if ((char) i == '_' || Char.IsLetterOrDigit ((char) i)) sb.Append ((char) ReadChar ()); else break; } string temp = sb.ToString (); if (keywords.ContainsKey (temp)) token = new Token (temp, keywords [temp], tokenPosition); else token = new Token (temp, TokenType.String, tokenPosition); } else if (ch == '!' && PeekChar () == (int) '=') { token = new Token ("!=", TokenType.NotEqual, tokenPosition); ReadChar (); } else if (ch == '<' && PeekChar () == (int) '=') { token = new Token ("<=", TokenType.LessOrEqual, tokenPosition); ReadChar (); } else if (ch == '>' && PeekChar () == (int) '=') { token = new Token (">=", TokenType.GreaterOrEqual, tokenPosition); ReadChar (); } else if (ch == '=' && PeekChar () == (int) '=') { token = new Token ("==", TokenType.Equal, tokenPosition); ReadChar (); } else if (ch >= 32 && ch < 128) { if (charIndexToTokenType [ch] != TokenType.Invalid) { token = new Token (new String (ch, 1), charIndexToTokenType [ch], tokenPosition); return; } else throw new ExpressionParseException (String.Format ("Invalid punctuation: {0}", ch)); } else throw new ExpressionParseException (String.Format ("Invalid token: {0}", ch)); }
private bool ParseNumeric( int start ) { if ((expression.Length-parsePoint) > 2 && expression[parsePoint] == '0' && (expression[parsePoint + 1] == 'x' || expression[parsePoint + 1] == 'X')) { // Hex number parsePoint += 2; SkipHexDigits(); lookahead = new Token(Token.TokenType.Numeric, expression.Substring(start, parsePoint - start)); } else if ( CharacterUtilities.IsNumberStart(expression[parsePoint])) { // Decimal number if (expression[parsePoint] == '+') { parsePoint++; } else if (expression[parsePoint] == '-') { parsePoint++; } SkipDigits(); if (parsePoint < expression.Length && expression[parsePoint] == '.') { parsePoint++; } if (parsePoint < expression.Length) { SkipDigits(); } // Do we need to error on malformed input like 0.00.00)? or will the conversion handle it? // For now, let the conversion generate the error. lookahead = new Token(Token.TokenType.Numeric, expression.Substring(start, parsePoint - start)); } else { // Unreachable errorState = true; errorPosition = start + 1; return false; } return true; }
// FIXME test this public void Putback (Token token) { putback = token; }
private bool ParseQuotedString() { parsePoint++; int start = parsePoint; while (parsePoint < expression.Length && expression[parsePoint] != '\'') { // Standalone percent-sign must be allowed within a condition because it's // needed to escape special characters. However, percent-sign followed // by open-parenthesis is an indication of an item metadata reference, and // that is only allowed in certain contexts. if ((expression[parsePoint] == '%') && ((parsePoint + 1) < expression.Length) && (expression[parsePoint + 1] == '(')) { // If the caller specified that he DOESN'T want to allow item metadata... if ((this.options & ParserOptions.AllowItemMetadata) == 0) { errorPosition = start + 1; errorState = true; errorResource = "UnexpectedCharacterInCondition"; unexpectedlyFound = "%"; return false; } } else if (expression[parsePoint] == '@' && ((parsePoint + 1) < expression.Length) && (expression[parsePoint + 1] == '(')) { // If the caller specified that he DOESN'T want to allow item lists ... if ((this.options & ParserOptions.AllowItemLists) == 0) { errorPosition = start + 1; errorState = true; errorResource = "ItemListNotAllowedInThisConditional"; return false; } // Item lists have to be parsed because of the replacement syntax e.g. @(Foo,'_'). // I have to know how to parse those so I can skip over the tic marks. I don't // have to do that with other things like propertygroups, hence itemlists are // treated specially. ParseInternalItemList(); continue; } parsePoint++; } if (parsePoint >= expression.Length) { // Quoted string wasn't closed errorState = true; errorPosition = start; // The message is going to say "expected after position n" so don't add 1 here. errorResource = "IllFormedQuotedStringInCondition"; // Not useful to set unexpectedlyFound here. By definition it got to the end of the string. return false; } string originalTokenString = expression.Substring(start, parsePoint - start); lookahead = new Token(Token.TokenType.String, originalTokenString); parsePoint++; return true; }
private bool ParseSimpleStringOrFunction( int start ) { SkipSimpleStringChars(); if (0 == string.Compare(expression.Substring(start, parsePoint - start), "and", StringComparison.OrdinalIgnoreCase)) { lookahead = new Token(Token.TokenType.And, expression.Substring(start, parsePoint - start)); } else if (0 == string.Compare(expression.Substring(start, parsePoint - start), "or", StringComparison.OrdinalIgnoreCase)) { lookahead = new Token(Token.TokenType.Or, expression.Substring(start, parsePoint - start)); } else { int end = parsePoint; SkipWhiteSpace(); if (parsePoint < expression.Length && expression[parsePoint] == '(') { lookahead = new Token(Token.TokenType.Function, expression.Substring(start, end - start)); } else { string tokenValue = expression.Substring(start, end - start); lookahead = new Token(Token.TokenType.String, tokenValue); } } return true; }
/// <summary> /// Parses a string of the form %(itemmetadataname). /// </summary> /// <returns></returns> /// <owner>RGoel</owner> private bool ParseItemMetadata() { string itemMetadataExpression = this.ParsePropertyOrItemMetadata(); if (itemMetadataExpression == null) { // The ParsePropertyOrItemMetadata method returns the correct error resources // for parsing properties such as $(propertyname). At this stage in the Whidbey // cycle, we're not allowed to add new string resources, so I can't add a new // resource specific to item metadata, so here, we just change the error to // the generic "UnexpectedCharacter". errorResource = "UnexpectedCharacterInCondition"; return false; } else { this.lookahead = new Token(Token.TokenType.ItemMetadata, itemMetadataExpression); return true; } }
private bool ParseItemList() { int start = parsePoint; if (!ParseInternalItemList()) { return false; } lookahead = new Token(Token.TokenType.ItemList, expression.Substring(start, parsePoint - start)); return true; }
/// <summary> /// Parses a string of the form $(propertyname). /// </summary> /// <returns></returns> /// <owner>RGoel, DavidLe</owner> private bool ParseProperty() { string propertyExpression = this.ParsePropertyOrItemMetadata(); if (propertyExpression == null) { return false; } else { this.lookahead = new Token(Token.TokenType.Property, propertyExpression); return true; } }
/// <summary> /// Advance /// returns true on successful advance /// and false on an erroneous token /// /// Doesn't return error until the bogus input is encountered. /// Advance() returns true even after EndOfInput is encountered. /// </summary> internal bool Advance() { if (errorState) return false; if (lookahead != null && lookahead.IsToken(Token.TokenType.EndOfInput)) return true; SkipWhiteSpace(); // Update error position after skipping whitespace errorPosition = parsePoint + 1; if (parsePoint >= expression.Length) { lookahead = new Token(Token.TokenType.EndOfInput, null /* end of input */); } else { switch (expression[parsePoint]) { case ',': lookahead = new Token(Token.TokenType.Comma, comma); parsePoint++; break; case '(': lookahead = new Token(Token.TokenType.LeftParenthesis, leftParenthesis); parsePoint++; break; case ')': lookahead = new Token(Token.TokenType.RightParenthesis, rightParenthesis); parsePoint++; break; case '$': if (!ParseProperty()) return false; break; case '%': // If the caller specified that he DOESN'T want to allow item metadata ... if ((this.options & ParserOptions.AllowItemMetadata) == 0) { errorPosition = this.parsePoint; errorState = true; errorResource = "UnexpectedCharacterInCondition"; unexpectedlyFound = "%"; return false; } if (!ParseItemMetadata()) return false; break; case '@': int start = this.parsePoint; // If the caller specified that he DOESN'T want to allow item lists ... if ((this.options & ParserOptions.AllowItemLists) == 0) { if ((parsePoint + 1) < expression.Length && expression[parsePoint + 1] == '(') { errorPosition = start + 1; errorState = true; errorResource = "ItemListNotAllowedInThisConditional"; return false; } } if (!ParseItemList()) return false; break; case '!': // negation and not-equal if ((parsePoint + 1) < expression.Length && expression[parsePoint + 1] == '=') { lookahead = new Token(Token.TokenType.NotEqualTo, notEqualTo); parsePoint += 2; } else { lookahead = new Token(Token.TokenType.Not, not); parsePoint++; } break; case '>': // gt and gte if ((parsePoint + 1) < expression.Length && expression[parsePoint + 1] == '=') { lookahead = new Token(Token.TokenType.GreaterThanOrEqualTo, greaterThanOrEqualTo); parsePoint += 2; } else { lookahead = new Token(Token.TokenType.GreaterThan, greaterThan); parsePoint++; } break; case '<': // lt and lte if ((parsePoint + 1) < expression.Length && expression[parsePoint + 1] == '=') { lookahead = new Token(Token.TokenType.LessThanOrEqualTo, lessThanOrEqualTo); parsePoint += 2; } else { lookahead = new Token(Token.TokenType.LessThan, lessThan); parsePoint++; } break; case '=': if ((parsePoint + 1) < expression.Length && expression[parsePoint + 1] == '=') { lookahead = new Token(Token.TokenType.EqualTo, equalTo); parsePoint += 2; } else { errorPosition = parsePoint + 2; // expression[parsePoint + 1], counting from 1 errorResource = "IllFormedEqualsInCondition"; if ((parsePoint + 1) < expression.Length) { // store the char we found instead unexpectedlyFound = Convert.ToString(expression[parsePoint + 1], CultureInfo.InvariantCulture); } else { unexpectedlyFound = EndOfInput; } parsePoint++; errorState = true; return false; } break; case '\'': if (!ParseQuotedString()) return false; break; default: // Simple strings, function calls, decimal numbers, hex numbers if (!ParseRemaining()) return false; break; } } return true; }
internal bool IsNext( Token.TokenType type ) { return lookahead.IsToken(type); }
// FIXME: in some situations items might not be allowed static Token EvaluateToken (Token token, IExpressionContext context) { string val = context.EvaluateString (token.Value); return new Token (val, TokenType.String); }
public void ScanForClosingParens (int parensCounter = 1) { tokenPosition = position; int start = position; int ch; while ((ch = ReadChar ()) >= 0) { switch (ch) { case ')': if (--parensCounter == 0) { --position; token = new Token (inputString.Substring (start, position - start), TokenType.String, tokenPosition); return; } break; case '(': ++parensCounter; break; } } token = new Token (null, TokenType.EOF, tokenPosition); }