public void Tokenize (string s)
{
if (s == null)
throw new ArgumentNullException ("s");
this.inputString = s;
this.position = 0;
this.token = new Token (null, TokenType.BOF, 0);
GetNextToken ();
}
public void GetNextToken ()
{
if (putback != null) {
token = putback;
putback = null;
return;
}
if (token.Type == TokenType.EOF)
throw new ExpressionParseException (String.Format (
"Error while parsing condition \"{0}\", ended abruptly.",
inputString));
SkipWhiteSpace ();
tokenPosition = position;
// int i = PeekChar ();
int i = ReadChar ();
if (i == -1) {
token = new Token (null, TokenType.EOF, tokenPosition);
return;
}
char ch = (char) i;
// FIXME: looks like a hack: if '-' is here '->' won't be tokenized
// maybe we should treat item reference as a token
if (ch == '-' && PeekChar () == '>') {
ReadChar ();
token = new Token ("->", TokenType.Transform, tokenPosition);
} else if (Char.IsDigit (ch) || ch == '-') {
StringBuilder sb = new StringBuilder ();
sb.Append (ch);
while ((i = PeekChar ()) != -1) {
ch = (char) i;
if (Char.IsDigit (ch) || ch == '.')
sb.Append ((char) ReadChar ());
else
break;
}
token = new Token (sb.ToString (), TokenType.Number, tokenPosition);
} else if (ch == '\'' && position < inputString.Length) {
StringBuilder sb = new StringBuilder ();
string temp;
sb.Append (ch);
bool is_itemref = (PeekChar () == '@');
int num_open_braces = 0;
bool in_literal = false;
while ((i = PeekChar ()) != -1) {
ch = (char) i;
if (ch == '(' && !in_literal && is_itemref)
num_open_braces ++;
if (ch == ')' && !in_literal && is_itemref)
num_open_braces --;
sb.Append ((char) ReadChar ());
if (ch == '\'') {
if (num_open_braces == 0)
break;
in_literal = !in_literal;
}
}
temp = sb.ToString ();
token = new Token (temp.Substring (1, temp.Length - 2), TokenType.String, tokenPosition);
} else if (ch == '_' || Char.IsLetter (ch)) {
StringBuilder sb = new StringBuilder ();
sb.Append ((char) ch);
while ((i = PeekChar ()) != -1) {
if ((char) i == '_' || Char.IsLetterOrDigit ((char) i))
sb.Append ((char) ReadChar ());
else
break;
}
string temp = sb.ToString ();
if (keywords.ContainsKey (temp))
token = new Token (temp, keywords [temp], tokenPosition);
else
token = new Token (temp, TokenType.String, tokenPosition);
} else if (ch == '!' && PeekChar () == (int) '=') {
token = new Token ("!=", TokenType.NotEqual, tokenPosition);
ReadChar ();
} else if (ch == '<' && PeekChar () == (int) '=') {
token = new Token ("<=", TokenType.LessOrEqual, tokenPosition);
ReadChar ();
} else if (ch == '>' && PeekChar () == (int) '=') {
token = new Token (">=", TokenType.GreaterOrEqual, tokenPosition);
ReadChar ();
} else if (ch == '=' && PeekChar () == (int) '=') {
token = new Token ("==", TokenType.Equal, tokenPosition);
ReadChar ();
} else if (ch >= 32 && ch < 128) {
if (charIndexToTokenType [ch] != TokenType.Invalid) {
token = new Token (new String (ch, 1), charIndexToTokenType [ch], tokenPosition);
return;
} else
throw new ExpressionParseException (String.Format ("Invalid punctuation: {0}", ch));
} else
throw new ExpressionParseException (String.Format ("Invalid token: {0}", ch));
}
public ConditionFactorExpression (Token token)
{
this.token = token;
}
// FIXME test this
public void Putback (Token token)
{
putback = token;
}
// FIXME: in some situations items might not be allowed
static Token EvaluateToken (Token token, IExpressionContext context)
{
string val = context.EvaluateString (token.Value);
return new Token (val, TokenType.String, 0);
}
public void GetNextToken()
{
if (hasPutback)
{
token = putback;
hasPutback = false;
return;
}
if (token.Type == TokenType.EOF)
{
throw new ExpressionParseException(String.Format(
"Error while parsing condition \"{0}\", ended abruptly.",
inputString));
}
SkipWhiteSpace();
tokenPosition = position;
tokenLength = 0;
// int i = PeekChar ();
int i = ReadChar();
if (i == -1)
{
token = new Token(null, TokenType.EOF, tokenPosition);
return;
}
char ch = (char)i;
// FIXME: looks like a hack: if '-' is here '->' won't be tokenized
// maybe we should treat item reference as a token
if (ch == '-' && PeekChar() == '>')
{
ReadChar();
token = new Token("->", TokenType.Transform, tokenPosition);
}
else if (Char.IsDigit(ch) || ch == '-')
{
tokenLength++;
while ((i = PeekChar()) != -1)
{
ch = (char)i;
if (Char.IsDigit(ch) || ch == '.')
{
ReadChar();
tokenLength++;
}
else
{
break;
}
}
token = new Token(inputString.Substring(tokenPosition, tokenLength), TokenType.Number, tokenPosition);
}
else if (ch == '\'' && position < inputString.Length)
{
tokenLength++;
bool is_itemref = (PeekChar() == '@');
int num_open_braces = 0;
bool in_literal = false;
while ((i = PeekChar()) != -1)
{
ch = (char)i;
if (ch == '(' && !in_literal && is_itemref)
{
num_open_braces++;
}
if (ch == ')' && !in_literal && is_itemref)
{
num_open_braces--;
}
ReadChar();
tokenLength++;
if (ch == '\'')
{
if (num_open_braces == 0)
{
break;
}
in_literal = !in_literal;
}
}
token = new Token(inputString.Substring(tokenPosition + 1, tokenLength - 2), TokenType.String, tokenPosition);
}
else if (ch == '_' || Char.IsLetter(ch))
{
tokenLength++;
while ((i = PeekChar()) != -1)
{
if ((char)i == '_' || Char.IsLetterOrDigit((char)i))
{
ReadChar();
tokenLength++;
}
else
{
break;
}
}
string temp = inputString.Substring(tokenPosition, tokenLength);
if (keywords.ContainsKey(temp))
{
token = new Token(temp, keywords [temp], tokenPosition);
}
else
{
token = new Token(temp, TokenType.String, tokenPosition);
}
}
else if (ch == '!' && PeekChar() == (int)'=')
{
token = new Token("!=", TokenType.NotEqual, tokenPosition);
ReadChar();
}
else if (ch == '<' && PeekChar() == (int)'=')
{
token = new Token("<=", TokenType.LessOrEqual, tokenPosition);
ReadChar();
}
else if (ch == '>' && PeekChar() == (int)'=')
{
token = new Token(">=", TokenType.GreaterOrEqual, tokenPosition);
ReadChar();
}
else if (ch == '=' && PeekChar() == (int)'=')
{
token = new Token("==", TokenType.Equal, tokenPosition);
ReadChar();
}
else if (ch >= 32 && ch < 128)
{
if (charIndexToTokenType [ch] != TokenType.Invalid)
{
token = new Token(new String(ch, 1), charIndexToTokenType [ch], tokenPosition);
return;
}
else
{
throw new ExpressionParseException(String.Format("Invalid punctuation: {0}", ch));
}
}
else
{
throw new ExpressionParseException(String.Format("Invalid token: {0}", ch));
}
}
// FIXME test this
public void Putback(Token token)
{
hasPutback = true;
putback = token;
}
public void ScanForClosingParens (int parensCounter = 1)
{
tokenPosition = position;
int start = position;
int ch;
while ((ch = ReadChar ()) >= 0) {
switch (ch) {
case ')':
if (--parensCounter == 0) {
--position;
token = new Token (inputString.Substring (start, position - start), TokenType.String, tokenPosition);
return;
}
break;
case '(':
++parensCounter;
break;
}
}
token = new Token (null, TokenType.EOF, tokenPosition);
}
// FIXME test this
public void Putback (Token token)
{
hasPutback = true;
putback = token;
}
