public ChoExpressionTokenizerState(ChoTokenType currentToken, string tokenText, ChoPosition currentPosition)
{
_currentToken = currentToken;
_tokenText = tokenText;
_currentPosition = currentPosition;
}
internal ChoExpressionTokenizerState(eSquare.Core.ExpresssionEvaluator.ChoExpressionTokenizer.ChoTokenType currentToken, string tokenText, ChoPosition currentPosition)
{
_currentToken = currentToken;
_tokenText = tokenText;
_currentPosition = currentPosition;
}
public void GetNextToken()
{
if (_tokenType == ChoTokenType.EOF)
{
throw new ChoExpressionParseException("Cannot read past end of stream.");
}
if (IgnoreWhitespace)
{
SkipWhitespace();
}
_tokenStartPosition = new ChoPosition(_position);
int nextChar = PeekChar();
if (nextChar == -1)
{
_tokenType = ChoTokenType.EOF;
return;
}
char ch = (char)nextChar;
if (!SingleCharacterMode)
{
#region Handle WhiteSpace chars
if (!IgnoreWhitespace && Char.IsWhiteSpace(ch))
{
StringBuilder sb = new StringBuilder();
int ch2;
while ((ch2 = PeekChar()) != -1)
{
if (!Char.IsWhiteSpace((char)ch2))
{
break;
}
sb.Append((char)ch2);
ReadChar();
}
;
_tokenType = ChoTokenType.Whitespace;
_tokenText = sb.ToString();
return;
}
#endregion Handle WhiteSpace chars
#region Handle Digits
if (Char.IsDigit(ch))
{
_tokenType = ChoTokenType.Number;
StringBuilder sb = new StringBuilder();
//sb.Append(ch);
while ((nextChar = PeekChar()) != -1)
{
ch = (char)nextChar;
if (!Char.IsDigit(ch))
{
break;
}
sb.Append(ch);
ReadChar();
}
;
_tokenText = sb.ToString();
return;
}
#endregion Handle Digits
#region Handle String in quotes
if (ch == '\'')
{
_tokenType = ChoTokenType.String;
StringBuilder sb = new StringBuilder();
ReadChar();
while ((nextChar = ReadChar()) != -1)
{
ch = (char)nextChar;
if (ch == '\'')
{
if (PeekChar() == (int)'\'')
{
ReadChar();
}
else
{
break;
}
}
sb.Append(ch);
}
;
_tokenText = sb.ToString();
return;
}
#endregion Handle String in quotes
#region Handle Keywords
if (ch == '_' || Char.IsLetter(ch))
{
_tokenType = ChoTokenType.Keyword;
StringBuilder sb = new StringBuilder();
sb.Append((char)ch);
ReadChar();
while ((nextChar = PeekChar()) != -1)
{
if ((char)nextChar == '_' || (char)nextChar == '-' || Char.IsLetterOrDigit((char)nextChar))
{
sb.Append((char)ReadChar());
}
else
{
break;
}
}
;
_tokenText = sb.ToString();
if (_tokenText.EndsWith("-"))
{
throw new ChoExpressionParseException(String.Format(CultureInfo.InvariantCulture,
"Identifier cannot end with a dash: {0}", _tokenText), CurrentPosition.CharIndex);
}
return;
}
#endregion Handle Keywords
ReadChar();
if (ch == ':' && PeekChar() == (int)':')
{
_tokenType = ChoTokenType.DoubleColon;
_tokenText = "::";
ReadChar();
return;
}
if (ch == '!' && PeekChar() == (int)'=')
{
_tokenType = ChoTokenType.NE;
_tokenText = "!=";
ReadChar();
return;
}
if (ch == '=' && PeekChar() == (int)'=')
{
_tokenType = ChoTokenType.EQ;
_tokenText = "==";
ReadChar();
return;
}
if (ch == '<' && PeekChar() == (int)'=')
{
_tokenType = ChoTokenType.LE;
_tokenText = "<=";
ReadChar();
return;
}
if (ch == '>' && PeekChar() == (int)'=')
{
_tokenType = ChoTokenType.GE;
_tokenText = ">=";
ReadChar();
return;
}
}
else
{
ReadChar();
}
_tokenText = new String(ch, 1);
_tokenType = ChoTokenType.Punctuation;
if (ch >= 32 && ch < 128)
{
_tokenType = charIndexToTokenType[ch];
}
}
