public Token GetNextToken()
{
var token = new Token();
do
{
if (_cursor == _data.Length)
{
token.Type = TokenType.EOF;
return(token);
}
if (_data[_cursor] == '\r')
{
_cursor++;
_lineCursor++;
_colCursor = 1;
if (_cursor < _data.Length && _data[_cursor] == '\n')
{
_cursor++;
}
}
else if (_data[_cursor] == '\n')
{
_cursor++;
_lineCursor++;
_colCursor = 1;
}
else if (char.IsWhiteSpace(_data[_cursor]))
{
_cursor++;
_colCursor++;
}
else
{
break;
}
} while (true);
token.Start = _cursor;
token.Line = _lineCursor;
token.Column = _colCursor;
switch (_data[_cursor])
{
case '%': {
var line = SegmentUntilEOL(_cursor + 1);
if (line.Count > 0 && line[0] == '!')
{
token.Type = TokenType.PDF_TAG;
}
else
{
token.Type = TokenType.COMMENT;
}
token.Content = line;
token.Length = line.Count + 1;
_cursor += line.Count + 1;
} break;
case '+':
case '-':
case '.':
case '0':
case '1':
case '2':
case '3':
case '4':
case '5':
case '6':
case '7':
case '8':
case '9': {
var number = SegmentUntilDelimiter(_cursor);
if (Ext.FirstIndexOf(number, '.') > -1)
{
token.Type = TokenType.LITERAL_DECIMAL;
}
else
{
token.Type = TokenType.LITERAL_INTEGER;
}
token.Content = number;
token.Length = number.Count;
_cursor += token.Length;
_colCursor += token.Length;
}
break;
case '{':
case '}': {
token.Type = TokenType.BRACE;
token.Content = new ArraySegment <char>(_data, _cursor, 1);
token.Length = 1;
_cursor++;
_colCursor++;
} break;
case '[':
case ']': {
token.Type = TokenType.BRACKET;
token.Content = new ArraySegment <char>(_data, _cursor, 1);
token.Length = 1;
_cursor++;
_colCursor++;
} break;
case '(': {
token.Type = TokenType.LITERAL_STRING;
token.Content = SegmentUntil(_cursor + 1, ')');
token.Length = token.Content.Count + 2;
_cursor += token.Length;
_colCursor += token.Length;
} break;
case '<': {
token.Type = TokenType.LITERAL_STRING_HEX;
token.Content = SegmentUntil(_cursor + 1, '>');
token.Length = token.Content.Count + 2;
_cursor += token.Length;
_colCursor += token.Length;
} break;
case '/': {
token.Type = TokenType.LITERAL_NAME;
token.Content = SegmentUntilDelimiter(_cursor + 1);
token.Length = token.Content.Count + 1;
_cursor += token.Length;
_colCursor += token.Length;
} break;
default: {
token.Content = SegmentUntilDelimiter(_cursor);
var contentSpan = (ReadOnlySpan <char>)token.Content;
if (contentSpan.Equals("true", StringComparison.Ordinal))
{
token.Type = TokenType.LITERAL_BOOLEAN;
token.Keyword = Keyword.LITERAL_TRUE;
}
else if (contentSpan.Equals("false", StringComparison.Ordinal))
{
token.Type = TokenType.LITERAL_BOOLEAN;
token.Keyword = Keyword.LITERAL_TRUE;
}
else if (KeywordMap.TryGetValue(contentSpan, out var keyword))
{
token.Type = TokenType.KEYWORD;
token.Keyword = keyword;
}
token.Length = token.Content.Count;
_cursor += token.Length;
_colCursor += token.Length;
} break;
}
return(token);
}
public override string ToString()
{
return($"[{Line:D4}:{Column:D3}] Type: {Type}, Keyword: {Keyword}, Content: {Ext.Dump(Content)}");
}