public bool Read() { while (_pos < _length) { _tokenStartPos = Position; _tokenText.Clear(); var ch = _source[_pos]; if (char.IsWhiteSpace(ch)) { _tokenText.Append(ch); MoveNext(); while (_pos < _length && char.IsWhiteSpace(_source[_pos])) { _tokenText.Append(_source[_pos]); MoveNext(); } if (_returnWhiteSpace) { _tokenType = TokenType.WhiteSpace; return true; } else continue; } if (char.IsLetter(ch) || ch == '_') { while (_pos < _length && (char.IsLetterOrDigit(_source[_pos]) || _source[_pos] == '_')) { _tokenText.Append(_source[_pos]); MoveNext(); } if ((_tokenText.Length == 3 && _tokenText.ToString() == "and") || (_tokenText.Length == 2 && _tokenText.ToString() == "or")) { _tokenType = TokenParser.TokenType.Operator; } else { _tokenType = TokenType.Word; } return true; } if (char.IsDigit(ch)) { var gotDot = false; while (_pos < _length) { ch = _source[_pos]; if (char.IsDigit(ch)) { _tokenText.Append(ch); MoveNext(); } else if (ch == '.' && !gotDot) { _tokenText.Append("."); MoveNext(); gotDot = true; } else break; } _tokenType = TokenType.Number; return true; } if (ch == '\"' || ch == '\'') { var startCh = ch; _tokenText.Append(ch); MoveNext(); while (_pos < _length) { ch = _source[_pos]; if (ch == '\\' && _pos + 1 < _length) { _tokenText.Append(ch); _tokenText.Append(_source[_pos + 1]); MoveNext(2); } else if (ch == startCh) { _tokenText.Append(ch); MoveNext(); break; } else { _tokenText.Append(ch); MoveNext(); } } _tokenType = TokenType.StringLiteral; return true; } if (ch == '/') { if (_pos + 1 < _length && _source[_pos + 1] == '/') { while (_pos < _length && _source[_pos] != '\r' && _source[_pos] != '\n') { _tokenText.Append(_source[_pos]); MoveNext(); } if (_returnComments) { _tokenType = TokenType.Comment; return true; } else continue; } if (_pos + 1 < _length && _source[_pos + 1] == '*') { var index = _source.IndexOf("*/", _pos); if (index < 0) index = _length; else index += "*/".Length; while (_pos < index) { _tokenText.Append(_source[_pos]); MoveNext(); } if (_returnComments) { _tokenType = TokenType.Comment; return true; } else continue; } if (_pos + 1 < _length && _source[_pos + 1] == '=') { _tokenText.Append("/="); MoveNext(2); _tokenType = TokenParser.TokenType.Operator; return true; } _tokenText.Append("/"); MoveNext(); _tokenType = TokenParser.TokenType.Operator; return true; } if (ch == '+' || ch == '-' || ch == '*' || ch == '%' || ch == '=') { _tokenText.Append(ch); MoveNext(); _tokenType = TokenParser.TokenType.Operator; if (_pos < _length && _source[_pos] == '=') { _tokenText.Append("="); MoveNext(); } return true; } if (ch == '?' || ch == ':' || ch == ',' || ch == '.' || ch == '(' || ch == ')' || ch == '{' || ch == '}' || ch == '[' || ch == ']' || ch == '&') { _tokenText.Append(ch); MoveNext(); _tokenType = TokenParser.TokenType.Operator; return true; } if (ch == '#') { _tokenText.Append(ch); MoveNext(); if (_pos < _length && char.IsLetter(_source[_pos])) { while (_pos < _length && char.IsLetter(_source[_pos])) { _tokenText.Append(_source[_pos]); MoveNext(); } _tokenType = TokenType.Preprocessor; return true; } else { _tokenType = TokenType.Operator; return true; } } _tokenText.Append(ch); MoveNext(); _tokenType = TokenType.Unknown; return true; } // End of file _tokenType = TokenType.WhiteSpace; _tokenText.Clear(); return false; }
public bool ReadNestable() { var startPos = Position; if (!Read()) return false; var firstTokenType = _tokenType; if (_tokenType == TokenParser.TokenType.Operator) { switch (_tokenText.ToString()) { case "(": if (ReadNestable_Inner(")")) { _tokenStartPos = startPos; _tokenText.Clear(); _tokenText.Append(_source.Substring(_tokenStartPos.Offset, _pos - _tokenStartPos.Offset)); _tokenType = TokenParser.TokenType.Nested; return true; } break; case "{": if (ReadNestable_Inner("}")) { _tokenStartPos = startPos; _tokenText.Clear(); _tokenText.Append(_source.Substring(_tokenStartPos.Offset, _pos - _tokenStartPos.Offset)); _tokenType = TokenParser.TokenType.Nested; return true; } break; case "[": if (ReadNestable_Inner("]")) { _tokenStartPos = startPos; _tokenText.Clear(); _tokenText.Append(_source.Substring(_tokenStartPos.Offset, _pos - _tokenStartPos.Offset)); _tokenType = TokenParser.TokenType.Nested; return true; } break; } } _tokenStartPos = startPos; _tokenType = firstTokenType; return true; }
public bool Peek(out Position tokenEndPos) { var pos = Position; if (!Read()) { tokenEndPos = Position; return false; } tokenEndPos = Position; Position = pos; return true; }