private Token NextToken()
{
if (positionInTokenList + 1 < tokenList.Count)
{
currentToken = tokenList[++positionInTokenList];
this.textPosition += currentToken.Start - currentToken.FullStart + currentToken.Length;
}
return currentToken;
}
private bool ParseExpected(SyntaxKind type)
{
int tempIndex = positionInTokenList + 1;
if (tempIndex < tokenList.Count)
{
if (Peek().Kind == type)
{
currentToken = NextToken();
return true;
}
}
return false;
}
private SyntaxTree CreateSyntaxTreeInner(TextReader luaReader)
{
Validate.IsNotNull(luaReader, nameof(luaReader));
positionInTokenList = -1; //Make sure that internal state is at "beginning"
tokenList = Lexer.Tokenize(luaReader);
if (tokenList.Count == 1)
{
currentToken = Peek();
}
statementNodeList = new List<StatementNode>();
ChunkNode root = ParseChunkNode();
return new SyntaxTree(root, tokenList, statementNodeList, errorList.ToImmutableList());
}
private static readonly char[] longCommentID2 = { '-', '[', '=' }; //TODO: flawed approach? what if --[=asdfadf]?
public static List<Token> Tokenize(TextReader textReader)
{
Validation.Requires.NotNull(textReader, nameof(textReader));
TrackableTextReader trackableTextReader = new TrackableTextReader(textReader);
Token nextToken;
List<Trivia> trivia;
List<Token> tokenList = new List<Token>();
while (!trackableTextReader.EndOfStream())
{
int fullStart = (int)trackableTextReader.Position;
trivia = ConsumeTrivia(trackableTextReader);
nextToken = ReadNextToken(trackableTextReader, trivia, fullStart);
tokenList.Add(nextToken);
if (trackableTextReader.EndOfStream() && nextToken.Kind != SyntaxKind.EndOfFile)
{
nextToken = new Token(SyntaxKind.EndOfFile, string.Empty, new List<Trivia>(), nextToken.End, nextToken.End);
tokenList.Add(nextToken);
}
}
if (tokenList.Count == 0)
{
//If there is an empty program send back an end of file token.
tokenList.Add(new Token(SyntaxKind.EndOfFile, "", new List<Trivia>(), 0, 0));
}
return tokenList;
}
private void SkipCurrentToken(string message = null)
{
NextToken();
var tempTriviaList = currentToken.LeadingTrivia;
tempTriviaList.Add(new Trivia(currentToken.Kind, currentToken.Text));
foreach (var triv in Peek().LeadingTrivia)
{
tempTriviaList.Add(triv);
}
var tokenWithAddedTrivia = new Token(Peek().Kind, Peek().Text, tempTriviaList, currentToken.FullStart, this.textPosition);
tokenList.RemoveAt(positionInTokenList);
if (positionInTokenList > tokenList.Count)
{
tokenList.Add(tokenWithAddedTrivia);
}
else
{
tokenList[positionInTokenList] = tokenWithAddedTrivia;
}
this.SetPositionInTokenList(positionInTokenList - 1);
currentToken = (positionInTokenList >= 0) ? currentToken = tokenList[positionInTokenList] : null;
}