public bool Read()
{
while (_pos < _length)
{
_tokenStartPos = Position;
_tokenText.Clear();
var ch = _source[_pos];
if (char.IsWhiteSpace(ch))
{
_tokenText.Append(ch);
MoveNext();
while (_pos < _length && char.IsWhiteSpace(_source[_pos]))
{
_tokenText.Append(_source[_pos]);
MoveNext();
}
if (_returnWhiteSpace)
{
_tokenType = TokenType.WhiteSpace;
return true;
}
else continue;
}
if (char.IsLetter(ch) || ch == '_')
{
while (_pos < _length && (char.IsLetterOrDigit(_source[_pos]) || _source[_pos] == '_'))
{
_tokenText.Append(_source[_pos]);
MoveNext();
}
if ((_tokenText.Length == 3 && _tokenText.ToString() == "and") ||
(_tokenText.Length == 2 && _tokenText.ToString() == "or"))
{
_tokenType = TokenParser.TokenType.Operator;
}
else
{
_tokenType = TokenType.Word;
}
return true;
}
if (char.IsDigit(ch))
{
var gotDot = false;
while (_pos < _length)
{
ch = _source[_pos];
if (char.IsDigit(ch))
{
_tokenText.Append(ch);
MoveNext();
}
else if (ch == '.' && !gotDot)
{
_tokenText.Append(".");
MoveNext();
gotDot = true;
}
else break;
}
_tokenType = TokenType.Number;
return true;
}
if (ch == '\"' || ch == '\'')
{
var startCh = ch;
_tokenText.Append(ch);
MoveNext();
while (_pos < _length)
{
ch = _source[_pos];
if (ch == '\\' && _pos + 1 < _length)
{
_tokenText.Append(ch);
_tokenText.Append(_source[_pos + 1]);
MoveNext(2);
}
else if (ch == startCh)
{
_tokenText.Append(ch);
MoveNext();
break;
}
else
{
_tokenText.Append(ch);
MoveNext();
}
}
_tokenType = TokenType.StringLiteral;
return true;
}
if (ch == '/')
{
if (_pos + 1 < _length && _source[_pos + 1] == '/')
{
while (_pos < _length && _source[_pos] != '\r' && _source[_pos] != '\n')
{
_tokenText.Append(_source[_pos]);
MoveNext();
}
if (_returnComments)
{
_tokenType = TokenType.Comment;
return true;
}
else continue;
}
if (_pos + 1 < _length && _source[_pos + 1] == '*')
{
var index = _source.IndexOf("*/", _pos);
if (index < 0) index = _length;
else index += "*/".Length;
while (_pos < index)
{
_tokenText.Append(_source[_pos]);
MoveNext();
}
if (_returnComments)
{
_tokenType = TokenType.Comment;
return true;
}
else continue;
}
if (_pos + 1 < _length && _source[_pos + 1] == '=')
{
_tokenText.Append("/=");
MoveNext(2);
_tokenType = TokenParser.TokenType.Operator;
return true;
}
_tokenText.Append("/");
MoveNext();
_tokenType = TokenParser.TokenType.Operator;
return true;
}
if (ch == '+' || ch == '-' || ch == '*' || ch == '%' || ch == '=')
{
_tokenText.Append(ch);
MoveNext();
_tokenType = TokenParser.TokenType.Operator;
if (_pos < _length && _source[_pos] == '=')
{
_tokenText.Append("=");
MoveNext();
}
return true;
}
if (ch == '?' || ch == ':' || ch == ',' || ch == '.' || ch == '(' || ch == ')' || ch == '{' || ch == '}' || ch == '[' || ch == ']' || ch == '&')
{
_tokenText.Append(ch);
MoveNext();
_tokenType = TokenParser.TokenType.Operator;
return true;
}
if (ch == '#')
{
_tokenText.Append(ch);
MoveNext();
if (_pos < _length && char.IsLetter(_source[_pos]))
{
while (_pos < _length && char.IsLetter(_source[_pos]))
{
_tokenText.Append(_source[_pos]);
MoveNext();
}
_tokenType = TokenType.Preprocessor;
return true;
}
else
{
_tokenType = TokenType.Operator;
return true;
}
}
_tokenText.Append(ch);
MoveNext();
_tokenType = TokenType.Unknown;
return true;
}
// End of file
_tokenType = TokenType.WhiteSpace;
_tokenText.Clear();
return false;
}
public bool ReadNestable()
{
var startPos = Position;
if (!Read()) return false;
var firstTokenType = _tokenType;
if (_tokenType == TokenParser.TokenType.Operator)
{
switch (_tokenText.ToString())
{
case "(":
if (ReadNestable_Inner(")"))
{
_tokenStartPos = startPos;
_tokenText.Clear();
_tokenText.Append(_source.Substring(_tokenStartPos.Offset, _pos - _tokenStartPos.Offset));
_tokenType = TokenParser.TokenType.Nested;
return true;
}
break;
case "{":
if (ReadNestable_Inner("}"))
{
_tokenStartPos = startPos;
_tokenText.Clear();
_tokenText.Append(_source.Substring(_tokenStartPos.Offset, _pos - _tokenStartPos.Offset));
_tokenType = TokenParser.TokenType.Nested;
return true;
}
break;
case "[":
if (ReadNestable_Inner("]"))
{
_tokenStartPos = startPos;
_tokenText.Clear();
_tokenText.Append(_source.Substring(_tokenStartPos.Offset, _pos - _tokenStartPos.Offset));
_tokenType = TokenParser.TokenType.Nested;
return true;
}
break;
}
}
_tokenStartPos = startPos;
_tokenType = firstTokenType;
return true;
}
public bool Peek(out Position tokenEndPos)
{
var pos = Position;
if (!Read())
{
tokenEndPos = Position;
return false;
}
tokenEndPos = Position;
Position = pos;
return true;
}
