private Token NextLineTermination() { int start = Index; int line = CurrentLine; bool isColon = false; while (!IsEof()) { char c = Code.GetChar(Index); if (CharUtils.IsLineTerminator(c)) { if (c == '\r' && Code.GetChar(Index + 1) == '\n') { ++Index; } ++Index; isColon |= (c == ':'); if (c != ':') { ++CurrentLine; CurrentLineStart = Index; } } else { break; } SkipWhitespaces(); } Token token = isColon && line == CurrentLine ? new ColonLineTerminationToken() : new LineTerminationToken(); token.Start = start; token.End = Index; token.LineNumber = CurrentLine - 1; token.LineStart = CurrentLineStart; return(token); }
public Token NextToken() { SkipWhitespaces(); if (IsEof()) { return(new EofToken { Start = Index, End = Index, LineNumber = CurrentLine, LineStart = CurrentLineStart, }); } char c = Code.GetChar(Index); char next = Code.GetChar(Index + 1); if (CharUtils.IsLineTerminator(c)) { return(NextLineTermination()); } var comment = NextComment(); if (comment != null) { return(comment); } if (CharUtils.IsIdentifierStart(c)) { return(NextIdentifier()); } if (c == '"') { return(NextStringLiteral()); } if (c == '.') { if (CharUtils.IsDecDigit(next)) { return(NextNumericLiteral()); } return(NextPunctuation()); } if (CharUtils.IsDecDigit(c)) { return(NextNumericLiteral()); } if (c == '&') { if (CharUtils.Equals(next, 'h') || CharUtils.IsDecDigit(next)) { return(NextNumericLiteral()); } return(NextPunctuation()); } if (c == '#') { return(NextDateLiteral()); } if (c == '[') { return(NextExtendedIdentifier()); } return(NextPunctuation()); }