private void NextToken()
{
_previousToken = _token;
bool result;
while (_tokensPreviewStart < _tokensPreview.Count)
{
_token = _tokensPreview[_tokensPreviewStart];
_tokensPreviewStart++;
// We can reset the tokens if we hit the upper limit of the preview
if (_tokensPreviewStart == _tokensPreview.Count)
{
_tokensPreviewStart = 0;
_tokensPreview.Clear();
}
if (IsHidden(_token.Type))
{
if (_isKeepTrivia)
{
PushTrivia(_token);
}
}
else
{
return;
}
}
// Skip Comments
while ((result = _tokenIt.MoveNext()))
{
if (IsHidden(_tokenIt.Current.Type))
{
if (_isKeepTrivia)
{
PushTrivia(_tokenIt.Current);
}
}
else
{
break;
}
}
_token = result ? _tokenIt.Current : Token.Eof;
}
private void NextToken()
{
_previousToken = _token;
bool result;
if (_tokensPreview.Count > 0)
{
_token = _tokensPreview.First.Value;
_tokensPreview.RemoveFirst();
return;
}
// Skip Comments
while ((result = _tokenIt.MoveNext()) && IsHidden(_tokenIt.Current.Type))
{
}
_token = result ? _tokenIt.Current : Token.Eof;
}
