private void ClearState() { CurrentToken = null; this.tokenType = TokenType.Unknown; buffer.Clear(); readFilter = null; }
private void ReadWhiteSpace() { CurrentToken = new WhiteSpaceToken(startPosition, ReadRaw((c) => char.IsWhiteSpace(c))); }
private void ReadText() { CurrentToken = new TextToken(startPosition, ReadRaw((c) => !char.IsWhiteSpace(c))); }