private Token NextLineTermination()
{
int start = Index;
int line = CurrentLine;
bool isColon = false;
while (!IsEof())
{
char c = Code.GetChar(Index);
if (CharUtils.IsLineTerminator(c))
{
if (c == '\r' && Code.GetChar(Index + 1) == '\n')
{
++Index;
}
++Index;
isColon |= (c == ':');
if (c != ':')
{
++CurrentLine;
CurrentLineStart = Index;
}
}
else
{
break;
}
SkipWhitespaces();
}
Token token = isColon && line == CurrentLine
? new ColonLineTerminationToken()
: new LineTerminationToken();
token.Start = start;
token.End = Index;
token.LineNumber = CurrentLine - 1;
token.LineStart = CurrentLineStart;
return(token);
}
public Token NextToken()
{
SkipWhitespaces();
if (IsEof())
{
return(new EofToken
{
Start = Index,
End = Index,
LineNumber = CurrentLine,
LineStart = CurrentLineStart,
});
}
char c = Code.GetChar(Index);
char next = Code.GetChar(Index + 1);
if (CharUtils.IsLineTerminator(c))
{
return(NextLineTermination());
}
var comment = NextComment();
if (comment != null)
{
return(comment);
}
if (CharUtils.IsIdentifierStart(c))
{
return(NextIdentifier());
}
if (c == '"')
{
return(NextStringLiteral());
}
if (c == '.')
{
if (CharUtils.IsDecDigit(next))
{
return(NextNumericLiteral());
}
return(NextPunctuation());
}
if (CharUtils.IsDecDigit(c))
{
return(NextNumericLiteral());
}
if (c == '&')
{
if (CharUtils.Equals(next, 'h') || CharUtils.IsDecDigit(next))
{
return(NextNumericLiteral());
}
return(NextPunctuation());
}
if (c == '#')
{
return(NextDateLiteral());
}
if (c == '[')
{
return(NextExtendedIdentifier());
}
return(NextPunctuation());
}
