public void InitTokenizer(string s)
{
_inputString = s;
_position = 0;
_tokenType = SoqlTokenType.BOF;
GetNextToken();
}
public void Expect(SoqlTokenType type)
{
if (_tokenType != type)
{
throw new SoqlException("Expected token of type: " + type + ", got " + _tokenType + " (" + _tokenValue + ").", TokenPosition);
}
GetNextToken();
}
public CharToTokenType(char ch, SoqlTokenType tokenType)
{
this.ch = ch;
this.tokenType = tokenType;
}
public bool IsToken(SoqlTokenType token)
{
return _tokenType == token;
}
public void Expect(SoqlTokenType type)
{
if (_tokenType != type)
throw new SoqlException("Expected token of type: " + type + ", got " + _tokenType + " (" + _tokenValue + ").", TokenPosition);
GetNextToken();
}
public void InitTokenizer(string s)
{
_inputString = s;
_position = 0;
_tokenType = SoqlTokenType.BOF;
GetNextToken();
}
public void GetNextToken()
{
if (_tokenType == SoqlTokenType.EOF)
{
throw new Exception("Cannot read past end of stream.");
}
if (IgnoreWhiteSpace)
{
SkipWhitespace();
}
;
_tokenPosition = _position;
int i = PeekChar();
if (i == -1)
{
TokenType = SoqlTokenType.EOF;
return;
}
char ch = (char)i;
if (!IgnoreWhiteSpace && Char.IsWhiteSpace(ch))
{
StringBuilder sb = new StringBuilder();
int ch2;
while ((ch2 = PeekChar()) != -1)
{
if (!Char.IsWhiteSpace((char)ch2))
{
break;
}
sb.Append((char)ch2);
ReadChar();
}
;
TokenType = SoqlTokenType.Whitespace;
_tokenValue = sb.ToString();
return;
}
if (Char.IsDigit(ch))
{
TokenType = SoqlTokenType.Number;
string s = "";
s += ch;
ReadChar();
while ((i = PeekChar()) != -1)
{
ch = (char)i;
if (Char.IsDigit(ch) || (ch == '.'))
{
s += (char)ReadChar();
}
else
{
break;
};
}
;
_tokenValue = s;
return;
}
if (ch == '\'')
{
TokenType = SoqlTokenType.String;
string s = "";
s += ch;
ReadChar();
while ((i = PeekChar()) != -1)
{
ch = (char)i;
s += (char)ReadChar();
if (ch == '\'')
{
if (PeekChar() == (int)'\'')
{
s += '\'';
ReadChar();
}
else
{
break;
}
}
}
;
_tokenValue = s;
return;
}
if (ch == '_' || Char.IsLetter(ch))
{
TokenType = SoqlTokenType.Keyword;
StringBuilder sb = new StringBuilder();
sb.Append((char)ch);
ReadChar();
while ((i = PeekChar()) != -1)
{
if ((char)i == '_' || Char.IsLetterOrDigit((char)i))
{
sb.Append((char)ReadChar());
}
else
{
break;
};
}
;
_tokenValue = sb.ToString();
_tokenValueLowercase = _tokenValue.ToLower();
return;
}
ReadChar();
_tokenValue = ch.ToString();
if (ch == '<' && PeekChar() == (int)'>')
{
TokenType = SoqlTokenType.NE;
_tokenValue = "<>";
ReadChar();
return;
}
if (ch == '!' && PeekChar() == (int)'=')
{
TokenType = SoqlTokenType.NE;
_tokenValue = "!=";
ReadChar();
return;
}
if (ch == '&' && PeekChar() == (int)'&')
{
TokenType = SoqlTokenType.And;
_tokenValue = "&&";
ReadChar();
return;
}
if (ch == '|' && PeekChar() == (int)'|')
{
TokenType = SoqlTokenType.Or;
_tokenValue = "||";
ReadChar();
return;
}
if (ch == '<' && PeekChar() == (int)'=')
{
TokenType = SoqlTokenType.LE;
_tokenValue = "<=";
ReadChar();
return;
}
if (ch == '>' && PeekChar() == (int)'=')
{
TokenType = SoqlTokenType.GE;
_tokenValue = ">=";
ReadChar();
return;
}
if (ch == '=' && PeekChar() == (int)'=')
{
TokenType = SoqlTokenType.EQ;
_tokenValue = "==";
ReadChar();
return;
}
if (ch >= 32 && ch < 128)
{
SoqlTokenType tt = charIndexToTokenType[ch];
if (tt != SoqlTokenType.Invalid)
{
TokenType = tt;
_tokenValue = new String(ch, 1);
return;
}
else
{
throw new Exception("Invalid punctuation: " + ch);
}
}
throw new Exception("Invalid token: " + ch);
}
public CharToTokenType(char ch, SoqlTokenType tokenType)
{
this.ch = ch;
this.tokenType = tokenType;
}
public bool IsToken(SoqlTokenType token)
{
return(_tokenType == token);
}
