protected void Reset(ref ResetPoint point)
{
_mode = point.Mode;
var offset = point.Position - _firstToken;
Debug.Assert(offset >= 0 && offset < _tokenCount);
_tokenOffset = offset;
_currentToken = default(SyntaxToken);
_currentNode = default(BlendedNode);
_prevTokenTrailingTrivia = point.PrevTokenTrailingTrivia;
if (_blendedTokens != null)
{
// look forward for slots not holding a token
for (int i = _tokenOffset; i < _tokenCount; i++)
{
if (_blendedTokens[i].Token == null)
{
// forget anything after and including any slot not holding a token
_tokenCount = i;
if (_tokenCount == _tokenOffset)
{
FetchCurrentToken();
}
break;
}
}
}
}
protected void Release(ref ResetPoint point)
{
RoslynDebug.Assert(_resetCount == point.ResetCount);
_resetCount--;
if (_resetCount == 0)
{
_resetStart = -1;
}
}
private void Reset(ref ResetPoint state)
{
_scanStack.Pop();
_mode = state.Mode;
_tokenIndex = state.TokenIndex;
_greaterThanTokenIsNotOperator = state.GreaterThanTokenIsNotOperator;
_termState = state.TermState;
}
protected void Release(ref ResetPoint point)
{
Debug.Assert(this.resetCount == point.ResetCount);
this.resetCount--;
if (this.resetCount == 0)
{
this.resetStart = -1;
}
}
protected void Reset(ref ResetPoint point)
{
_mode = point.Mode;
var offset = point.Position - _firstToken;
Debug.Assert(offset >= 0 && offset < _tokenCount);
_tokenOffset = offset;
_currentToken = default(SyntaxToken);
_prevTokenTrailingTrivia = point.PrevTokenTrailingTrivia;
}
private void Reset(ref ResetPoint state)
{
_scanStack.Pop();
_mode = state.Mode;
_tokenIndex = state.TokenIndex;
_greaterThanTokenIsNotOperator = state.GreaterThanTokenIsNotOperator;
_allowLinearAndPointAsIdentifiers = state.AllowLinearAndPointAsIdentifiers;
CommaIsSeparatorStack = state.CommaIsSeparatorStack;
_termState = state.TermState;
}
public TokenizerResult Tokenize(ITracker tracker)
{
tokenizers ??= StandardList.List;
if (PreviousType != Token.TokenType.NA && ((tracker.Before?.Type ?? Token.TokenType.None) & PreviousType) == Token.TokenType.NA)
{
return(TokenizerResult.Failure);
}
int level = 0;
int textRegionStart = tracker.Index;
string readString = "";
while (true)
{
if (!tracker.ReadWithOffset(readString.Length, out char c))
{
tracker.AddError("No closing bracket in level: " + level, 2, new TextRegion(tracker.InitialIndex));
return(TokenizerResult.Failure);
}
readString += c;
if (c == OpeningBracket)
{
level++;
}
else if (c == ClosingBracket)
{
level--;
if (level == 0)
{
List <Token> tokens;
using (var point = new ResetPoint(tracker))
{
// Usage of PseudoToken to ensure that tracker.Last returns TokenType.None for further checks
var pseudoToken = new Token(new TextRegion(tracker.Index, tracker.Index + 1), OpeningBracket.ToString(), "PseudoToken", Token.TokenType.None);
tracker.AddToken(pseudoToken, 1);
tokens = new Tokenizer(tokenizers).TokenizeFrom(tracker, point.resetPoint, tracker.Index - 1 + readString.Length - 1);
tokens.Remove(pseudoToken); // should equal tokens.RemoveAt(0)
}
tracker.AddToken(new CompoundToken(new TextRegion(textRegionStart, textRegionStart + readString.Length), tokens, OpeningBracket, ClosingBracket, Type, Name), readString.Length);
return(TokenizerResult.Success);
}
}
}
}
public TokenizerResult Tokenize(ITracker tracker)
{
ConstructToken token;
using (var point = new ResetPoint(tracker))
token = TokenizeConstruct(tracker);
if (token is null)
{
return(TokenizerResult.Failure);
}
else
{
tracker.AddToken(token, token.Range.End - tracker.Index);
return(TokenizerResult.Success);
}
}
private bool LooksLikeTupleArrayType()
{
if (this.CurrentToken.Kind != SyntaxKind.OpenParenToken)
{
return(false);
}
ResetPoint resetPoint = GetResetPoint();
try
{
return(ScanType(forPattern: true) != ScanTypeFlags.NotType);
}
finally
{
this.Reset(ref resetPoint);
this.Release(ref resetPoint);
}
}
protected override ConstructToken TokenizeConstruct(ITracker tracker)
{
int temp = tracker.Index;
var tokens = new List <Token>();
if (!TryTokenize(tracker, ifString, tokens))
{
return(null);
}
whitespaces.Tokenize(tracker);
if (!TryTokenize(tracker, roundBrackets, tokens, "condition"))
{
return(null);
}
whitespaces.Tokenize(tracker);
if (!TryTokenize(tracker, curlyBrackets, tokens, "ifBlock"))
{
return(null);
}
using (var point = new ResetPoint(tracker))
{
whitespaces.Tokenize(tracker);
if (TryTokenize(tracker, elseString, tokens))
{
whitespaces.Tokenize(tracker);
if (!TryTokenize(tracker, curlyBrackets, tokens, "elseBlock"))
{
return(null);
}
point.Disable();
}
}
return(new ConstructToken(new TextRegion(temp, tracker.Index), tokens, "if"));
}
protected void Reset(ref ResetPoint point)
{
var offset = point.Position - _firstToken;
Debug.Assert(offset >= 0);
if (offset >= _tokenCount)
{
// Re-fetch tokens to the position in the reset point
PeekToken(offset - _tokenOffset);
// Re-calculate new offset in case tokens got shifted to the left while we were peeking.
offset = point.Position - _firstToken;
}
_mode = point.Mode;
Debug.Assert(offset >= 0 && offset < _tokenCount);
_tokenOffset = offset;
_currentToken = null;
_currentNode = default(BlendedNode);
_prevTokenTrailingTrivia = point.PrevTokenTrailingTrivia;
if (_blendedTokens != null)
{
// look forward for slots not holding a token
for (int i = _tokenOffset; i < _tokenCount; i++)
{
if (_blendedTokens[i].Token == null)
{
// forget anything after and including any slot not holding a token
_tokenCount = i;
if (_tokenCount == _tokenOffset)
{
FetchCurrentToken();
}
break;
}
}
}
}
protected void Reset(ref ResetPoint point)
{
this.mode = point.Mode;
var offset = point.Position - this.firstToken;
Debug.Assert(offset >= 0 && offset < this.tokenCount);
this.tokenOffset = offset;
this.currentToken = default(SyntaxToken);
this.currentNode = default(BlendedNode);
this.prevTokenTrailingTrivia = point.PrevTokenTrailingTrivia;
if (this.blendedTokens != null)
{
// look forward for slots not holding a token
for (int i = this.tokenOffset; i < this.tokenCount; i++)
{
if (this.blendedTokens[i].Token == null)
{
// forget anything after and including any slot not holding a token
this.tokenCount = i;
break;
}
}
}
}
private StatementSyntax ParsePossibleBadAwaitStatement()
{
ResetPoint resetPointBeforeStatement = this.GetResetPoint();
//StatementSyntax result = ParsePossibleBadAwaitStatement(ref resetPointBeforeStatement);
// Precondition: We have already attempted to parse the statement as a non-declaration and failed.
//
// That means that we are in one of the following cases:
//
// 1) This is a perfectly mundane and correct local declaration statement like "int x;"
// 2) This is a perfectly mundane but erroneous local declaration statement, like "int X();"
// 3) We are in the rare case of the code containing "await x;" and the intention is that
// "await" is the type of "x". This only works in a non-async method.
// 4) We have what would be a legal await statement, like "await X();", but we are not in
// an async method, so the parse failed. (Had we been in an async method then the parse
// attempt done by our caller would have succeeded.)
// 5) The statement begins with "await" but is not a legal local declaration and not a legal
// await expression regardless of whether the method is marked as "async".
StatementSyntax result = ParseLocalDeclarationStatement();
this.Release(ref resetPointBeforeStatement);
return(result);
}
private void Reset(ref ResetPoint state)
{
this._termState = state.TerminatorState;
this._isInTry = state.IsInTry;
base.Reset(ref state.BaseResetPoint);
}
private void ParseUsingExpression(ref VariableDeclarationSyntax declaration, ref ExpressionSyntax expression, ref ResetPoint resetPoint)
{
TypeSyntax type;
// Now, this can be either an expression or a decl list
ScanTypeFlags st;
st = this.ScanType();
if (st == ScanTypeFlags.NullableType)
{
// We need to handle:
// * using (f ? x = a : x = b)
// * using (f ? x = a)
// * using (f ? x, y)
if (this.CurrentToken.Kind != SyntaxKind.IdentifierToken)
{
this.Reset(ref resetPoint);
expression = this.ParseExpression();
}
else
{
SeparatedSyntaxListBuilder <VariableDeclaratorSyntax> variables;
switch (this.PeekToken(1).Kind)
{
default:
this.Reset(ref resetPoint);
expression = this.ParseExpression();
break;
case SyntaxKind.CommaToken:
case SyntaxKind.CloseParenToken:
this.Reset(ref resetPoint);
variables = this._pool.AllocateSeparated <VariableDeclaratorSyntax>();
this.ParseDeclaration(false, out type, variables);
declaration = _syntaxFactory.VariableDeclaration(type, variables.ToList());
this._pool.Free(variables);
break;
case SyntaxKind.EqualsToken:
// Parse it as a decl. If the next token is a : and only one variable was parsed,
// convert the whole thing to ?: expression.
this.Reset(ref resetPoint);
variables = this._pool.AllocateSeparated <VariableDeclaratorSyntax>();
this.ParseDeclaration(false, out type, variables);
//// We may have non-nullable types in error scenarios.
//if (this.CurrentToken.Kind == SyntaxKind.ColonToken &&
// type.Kind == SyntaxKind.NullableType &&
// SyntaxKindFacts.IsName(((NullableTypeSyntax)type).ElementType.Kind) &&
// variables.Count == 1)
//{
// // We have "name? id = expr :" so need to convert to a ?: expression.
// this.Reset(ref resetPoint);
// expression = this.ParseExpression();
//}
//else
//{
declaration = _syntaxFactory.VariableDeclaration(type, variables.ToList());
//}
this._pool.Free(variables);
break;
}
}
}
else if (IsUsingStatementVariableDeclaration(st))
{
this.Reset(ref resetPoint);
var variables = this._pool.AllocateSeparated <VariableDeclaratorSyntax>();
this.ParseDeclaration(false, out type, variables);
declaration = _syntaxFactory.VariableDeclaration(type, variables);
this._pool.Free(variables);
}
else
{
// Must be an expression statement
this.Reset(ref resetPoint);
expression = this.ParseExpression();
}
}
private void Release(ref ResetPoint state)
{
base.Release(ref state.BaseResetPoint);
}
private void ParseMemberName(
out SyntaxToken identifierOrThisOpt /*,
* out TypeParameterListSyntax typeParameterListOpt*/)
{
identifierOrThisOpt = null;
//typeParameterListOpt = null;
if (!IsPossibleMemberName())
{
// No clue what this is. Just bail. Our caller will have to
// move forward and try again.
return;
}
NameSyntax explicitInterfaceName = null;
SyntaxToken separator = null;
ResetPoint beforeIdentifierPoint = default(ResetPoint);
bool beforeIdentifierPointSet = false;
try
{
while (true)
{
//// Check if we got 'this'. If so, then we have an indexer.
//// Note: we parse out type parameters here as well so that
//// we can give a useful error about illegal generic indexers.
//if (this.CurrentToken.Kind == SyntaxKind.ThisKeyword)
//{
// beforeIdentifierPoint = GetResetPoint();
// beforeIdentifierPointSet = true;
// identifierOrThisOpt = this.EatToken();
// typeParameterListOpt = this.ParseTypeParameterList();
// break;
//}
// now, scan past the next name. if it's followed by a dot then
// it's part of the explicit name we're building up. Otherwise,
// it's the name of the member.
var point = GetResetPoint();
bool isMemberName;
try
{
ScanNamedTypePart();
isMemberName = !IsDotOrColonColon();
}
finally
{
this.Reset(ref point);
this.Release(ref point);
}
if (isMemberName)
{
// We're past any explicit interface portion and We've
// gotten to the member name.
beforeIdentifierPoint = GetResetPoint();
beforeIdentifierPointSet = true;
if (separator != null && separator.Kind == SyntaxKind.ColonColonToken)
{
separator = this.AddError(separator, ErrorCode.ERR_AliasQualAsExpression);
separator = this.ConvertToMissingWithTrailingTrivia(separator, SyntaxKind.DotToken);
}
identifierOrThisOpt = this.ParseIdentifierToken();
//typeParameterListOpt = this.ParseTypeParameterList();
break;
}
else
{
// If we saw a . or :: then we must have something explicit.
// first parse the upcoming name portion.
var saveTerm = _termState;
_termState |= TerminatorState.IsEndOfNameInExplicitInterface;
if (explicitInterfaceName == null)
{
// If this is the first time, then just get the next simple
// name and store it as the explicit interface name.
explicitInterfaceName = this.ParseSimpleName(NameOptions.InTypeList);
// Now, get the next separator.
separator = this.CurrentToken.Kind == SyntaxKind.ColonColonToken
? this.EatToken() // fine after the first identifier
: this.EatToken(SyntaxKind.DotToken);
}
else
{
// Parse out the next part and combine it with the
// current explicit name to form the new explicit name.
var tmp = this.ParseQualifiedNameRight(NameOptions.InTypeList, explicitInterfaceName, separator);
Debug.Assert(!ReferenceEquals(tmp, explicitInterfaceName), "We should have consumed something and updated explicitInterfaceName");
explicitInterfaceName = tmp;
// Now, get the next separator.
separator = this.CurrentToken.Kind == SyntaxKind.ColonColonToken
? this.ConvertToMissingWithTrailingTrivia(this.EatToken(), SyntaxKind.DotToken)
: this.EatToken(SyntaxKind.DotToken);
}
_termState = saveTerm;
}
}
}
finally
{
if (beforeIdentifierPointSet)
{
Release(ref beforeIdentifierPoint);
}
}
}
