/// <summary>
/// Reads the next <see cref="StringToken"/> object.
/// </summary>
/// <returns>The next <see cref="StringToken"/> object.</returns>
public StringToken Read() {
if (peeked != null) {
var tok = peeked;
peeked = null;
return tok;
}
ReadNext:
char? chr;
if (!Peek(out chr) || !chr.HasValue)
return CreateToken(StringTokenKind.EndOfFile, string.Empty);
switch (chr) {
case ' ':
case '\t':
if (IgnoreWhitespace) {
ReadWhitespace();
goto ReadNext;
}
return ReadWhitespace();
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
return ReadNumber();
case '\r':
Start();
SkipPos();
if (Peek(out chr) && chr == '\n') SkipPos();
Line++;
Column = 1;
return CreateToken(StringTokenKind.EndOfLine);
case '\n':
Start();
SkipPos();
Line++;
Column = 1;
return CreateToken(StringTokenKind.EndOfLine);
case '"':
case '\'':
return ReadString();
case '(':
case ')':
case '[':
case ']':
case '{':
case '}':
Start();
SkipPos();
return CreateToken(StringTokenKind.Brackets);
default:
if (char.IsLetter(chr.Value) || chr == '_')
return ReadWord();
if (char.IsSymbol(chr.Value)) {
Start();
SkipPos();
return CreateToken(StringTokenKind.Symbol);
}
Start();
SkipPos();
return CreateToken(StringTokenKind.Unknown);
}
}
/// <summary>
/// Skips the current token.
/// </summary>
/// <returns><c>true</c> if token was skipped, <c>false</c> otherwise.</returns>
public bool Skip() {
if (peeked == null)
return Read() != null;
peeked = null;
return true;
}
/// <summary>
/// Returns the <see cref="StringTokenizer"/> at the beginning of the <see cref="StringTokenizer"/> without removing it.
/// </summary>
/// <returns>The <see cref="StringToken"/> at the beginning of the <see cref="StringToken"/> without removing it.</returns>
public StringToken Peek() {
if (peeked != null)
return peeked;
return (peeked = Read());
}
/// <summary>
/// Reads the next <see cref="StringToken"/> object.
/// </summary>
/// <returns>The next <see cref="StringToken"/> object.</returns>
public StringToken Read()
{
if (peeked != null)
{
var tok = peeked;
peeked = null;
return(tok);
}
ReadNext:
char?chr;
if (!Peek(out chr) || !chr.HasValue)
{
return(CreateToken(StringTokenKind.EndOfFile, string.Empty));
}
switch (chr)
{
case ' ':
case '\t':
if (IgnoreWhitespace)
{
ReadWhitespace();
goto ReadNext;
}
return(ReadWhitespace());
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9':
return(ReadNumber());
case '\r':
Start();
SkipPos();
if (Peek(out chr) && chr == '\n')
{
SkipPos();
}
Line++;
Column = 1;
return(CreateToken(StringTokenKind.EndOfLine));
case '\n':
Start();
SkipPos();
Line++;
Column = 1;
return(CreateToken(StringTokenKind.EndOfLine));
case '"':
case '\'':
return(ReadString());
case '(':
case ')':
case '[':
case ']':
case '{':
case '}':
Start();
SkipPos();
return(CreateToken(StringTokenKind.Brackets));
default:
if (char.IsLetter(chr.Value) || chr == '_')
{
return(ReadWord());
}
if (char.IsSymbol(chr.Value))
{
Start();
SkipPos();
return(CreateToken(StringTokenKind.Symbol));
}
Start();
SkipPos();
return(CreateToken(StringTokenKind.Unknown));
}
}