protected void Reset(ref ResetPoint point) { _mode = point.Mode; var offset = point.Position - _firstToken; Debug.Assert(offset >= 0 && offset < _tokenCount); _tokenOffset = offset; _currentToken = default(SyntaxToken); _currentNode = default(BlendedNode); _prevTokenTrailingTrivia = point.PrevTokenTrailingTrivia; if (_blendedTokens != null) { // look forward for slots not holding a token for (int i = _tokenOffset; i < _tokenCount; i++) { if (_blendedTokens[i].Token == null) { // forget anything after and including any slot not holding a token _tokenCount = i; if (_tokenCount == _tokenOffset) { FetchCurrentToken(); } break; } } } }
protected void Release(ref ResetPoint point) { RoslynDebug.Assert(_resetCount == point.ResetCount); _resetCount--; if (_resetCount == 0) { _resetStart = -1; } }
private void Reset(ref ResetPoint state) { _scanStack.Pop(); _mode = state.Mode; _tokenIndex = state.TokenIndex; _greaterThanTokenIsNotOperator = state.GreaterThanTokenIsNotOperator; _termState = state.TermState; }
protected void Release(ref ResetPoint point) { Debug.Assert(this.resetCount == point.ResetCount); this.resetCount--; if (this.resetCount == 0) { this.resetStart = -1; } }
protected void Reset(ref ResetPoint point) { _mode = point.Mode; var offset = point.Position - _firstToken; Debug.Assert(offset >= 0 && offset < _tokenCount); _tokenOffset = offset; _currentToken = default(SyntaxToken); _prevTokenTrailingTrivia = point.PrevTokenTrailingTrivia; }
private void Reset(ref ResetPoint state) { _scanStack.Pop(); _mode = state.Mode; _tokenIndex = state.TokenIndex; _greaterThanTokenIsNotOperator = state.GreaterThanTokenIsNotOperator; _allowLinearAndPointAsIdentifiers = state.AllowLinearAndPointAsIdentifiers; CommaIsSeparatorStack = state.CommaIsSeparatorStack; _termState = state.TermState; }
public TokenizerResult Tokenize(ITracker tracker) { tokenizers ??= StandardList.List; if (PreviousType != Token.TokenType.NA && ((tracker.Before?.Type ?? Token.TokenType.None) & PreviousType) == Token.TokenType.NA) { return(TokenizerResult.Failure); } int level = 0; int textRegionStart = tracker.Index; string readString = ""; while (true) { if (!tracker.ReadWithOffset(readString.Length, out char c)) { tracker.AddError("No closing bracket in level: " + level, 2, new TextRegion(tracker.InitialIndex)); return(TokenizerResult.Failure); } readString += c; if (c == OpeningBracket) { level++; } else if (c == ClosingBracket) { level--; if (level == 0) { List <Token> tokens; using (var point = new ResetPoint(tracker)) { // Usage of PseudoToken to ensure that tracker.Last returns TokenType.None for further checks var pseudoToken = new Token(new TextRegion(tracker.Index, tracker.Index + 1), OpeningBracket.ToString(), "PseudoToken", Token.TokenType.None); tracker.AddToken(pseudoToken, 1); tokens = new Tokenizer(tokenizers).TokenizeFrom(tracker, point.resetPoint, tracker.Index - 1 + readString.Length - 1); tokens.Remove(pseudoToken); // should equal tokens.RemoveAt(0) } tracker.AddToken(new CompoundToken(new TextRegion(textRegionStart, textRegionStart + readString.Length), tokens, OpeningBracket, ClosingBracket, Type, Name), readString.Length); return(TokenizerResult.Success); } } } }
public TokenizerResult Tokenize(ITracker tracker) { ConstructToken token; using (var point = new ResetPoint(tracker)) token = TokenizeConstruct(tracker); if (token is null) { return(TokenizerResult.Failure); } else { tracker.AddToken(token, token.Range.End - tracker.Index); return(TokenizerResult.Success); } }
private bool LooksLikeTupleArrayType() { if (this.CurrentToken.Kind != SyntaxKind.OpenParenToken) { return(false); } ResetPoint resetPoint = GetResetPoint(); try { return(ScanType(forPattern: true) != ScanTypeFlags.NotType); } finally { this.Reset(ref resetPoint); this.Release(ref resetPoint); } }
protected override ConstructToken TokenizeConstruct(ITracker tracker) { int temp = tracker.Index; var tokens = new List <Token>(); if (!TryTokenize(tracker, ifString, tokens)) { return(null); } whitespaces.Tokenize(tracker); if (!TryTokenize(tracker, roundBrackets, tokens, "condition")) { return(null); } whitespaces.Tokenize(tracker); if (!TryTokenize(tracker, curlyBrackets, tokens, "ifBlock")) { return(null); } using (var point = new ResetPoint(tracker)) { whitespaces.Tokenize(tracker); if (TryTokenize(tracker, elseString, tokens)) { whitespaces.Tokenize(tracker); if (!TryTokenize(tracker, curlyBrackets, tokens, "elseBlock")) { return(null); } point.Disable(); } } return(new ConstructToken(new TextRegion(temp, tracker.Index), tokens, "if")); }
protected void Reset(ref ResetPoint point) { var offset = point.Position - _firstToken; Debug.Assert(offset >= 0); if (offset >= _tokenCount) { // Re-fetch tokens to the position in the reset point PeekToken(offset - _tokenOffset); // Re-calculate new offset in case tokens got shifted to the left while we were peeking. offset = point.Position - _firstToken; } _mode = point.Mode; Debug.Assert(offset >= 0 && offset < _tokenCount); _tokenOffset = offset; _currentToken = null; _currentNode = default(BlendedNode); _prevTokenTrailingTrivia = point.PrevTokenTrailingTrivia; if (_blendedTokens != null) { // look forward for slots not holding a token for (int i = _tokenOffset; i < _tokenCount; i++) { if (_blendedTokens[i].Token == null) { // forget anything after and including any slot not holding a token _tokenCount = i; if (_tokenCount == _tokenOffset) { FetchCurrentToken(); } break; } } } }
protected void Reset(ref ResetPoint point) { this.mode = point.Mode; var offset = point.Position - this.firstToken; Debug.Assert(offset >= 0 && offset < this.tokenCount); this.tokenOffset = offset; this.currentToken = default(SyntaxToken); this.currentNode = default(BlendedNode); this.prevTokenTrailingTrivia = point.PrevTokenTrailingTrivia; if (this.blendedTokens != null) { // look forward for slots not holding a token for (int i = this.tokenOffset; i < this.tokenCount; i++) { if (this.blendedTokens[i].Token == null) { // forget anything after and including any slot not holding a token this.tokenCount = i; break; } } } }
private StatementSyntax ParsePossibleBadAwaitStatement() { ResetPoint resetPointBeforeStatement = this.GetResetPoint(); //StatementSyntax result = ParsePossibleBadAwaitStatement(ref resetPointBeforeStatement); // Precondition: We have already attempted to parse the statement as a non-declaration and failed. // // That means that we are in one of the following cases: // // 1) This is a perfectly mundane and correct local declaration statement like "int x;" // 2) This is a perfectly mundane but erroneous local declaration statement, like "int X();" // 3) We are in the rare case of the code containing "await x;" and the intention is that // "await" is the type of "x". This only works in a non-async method. // 4) We have what would be a legal await statement, like "await X();", but we are not in // an async method, so the parse failed. (Had we been in an async method then the parse // attempt done by our caller would have succeeded.) // 5) The statement begins with "await" but is not a legal local declaration and not a legal // await expression regardless of whether the method is marked as "async". StatementSyntax result = ParseLocalDeclarationStatement(); this.Release(ref resetPointBeforeStatement); return(result); }
private void Reset(ref ResetPoint state) { this._termState = state.TerminatorState; this._isInTry = state.IsInTry; base.Reset(ref state.BaseResetPoint); }
private void ParseUsingExpression(ref VariableDeclarationSyntax declaration, ref ExpressionSyntax expression, ref ResetPoint resetPoint) { TypeSyntax type; // Now, this can be either an expression or a decl list ScanTypeFlags st; st = this.ScanType(); if (st == ScanTypeFlags.NullableType) { // We need to handle: // * using (f ? x = a : x = b) // * using (f ? x = a) // * using (f ? x, y) if (this.CurrentToken.Kind != SyntaxKind.IdentifierToken) { this.Reset(ref resetPoint); expression = this.ParseExpression(); } else { SeparatedSyntaxListBuilder <VariableDeclaratorSyntax> variables; switch (this.PeekToken(1).Kind) { default: this.Reset(ref resetPoint); expression = this.ParseExpression(); break; case SyntaxKind.CommaToken: case SyntaxKind.CloseParenToken: this.Reset(ref resetPoint); variables = this._pool.AllocateSeparated <VariableDeclaratorSyntax>(); this.ParseDeclaration(false, out type, variables); declaration = _syntaxFactory.VariableDeclaration(type, variables.ToList()); this._pool.Free(variables); break; case SyntaxKind.EqualsToken: // Parse it as a decl. If the next token is a : and only one variable was parsed, // convert the whole thing to ?: expression. this.Reset(ref resetPoint); variables = this._pool.AllocateSeparated <VariableDeclaratorSyntax>(); this.ParseDeclaration(false, out type, variables); //// We may have non-nullable types in error scenarios. //if (this.CurrentToken.Kind == SyntaxKind.ColonToken && // type.Kind == SyntaxKind.NullableType && // SyntaxKindFacts.IsName(((NullableTypeSyntax)type).ElementType.Kind) && // variables.Count == 1) //{ // // We have "name? id = expr :" so need to convert to a ?: expression. // this.Reset(ref resetPoint); // expression = this.ParseExpression(); //} //else //{ declaration = _syntaxFactory.VariableDeclaration(type, variables.ToList()); //} this._pool.Free(variables); break; } } } else if (IsUsingStatementVariableDeclaration(st)) { this.Reset(ref resetPoint); var variables = this._pool.AllocateSeparated <VariableDeclaratorSyntax>(); this.ParseDeclaration(false, out type, variables); declaration = _syntaxFactory.VariableDeclaration(type, variables); this._pool.Free(variables); } else { // Must be an expression statement this.Reset(ref resetPoint); expression = this.ParseExpression(); } }
private void Release(ref ResetPoint state) { base.Release(ref state.BaseResetPoint); }
private void ParseMemberName( out SyntaxToken identifierOrThisOpt /*, * out TypeParameterListSyntax typeParameterListOpt*/) { identifierOrThisOpt = null; //typeParameterListOpt = null; if (!IsPossibleMemberName()) { // No clue what this is. Just bail. Our caller will have to // move forward and try again. return; } NameSyntax explicitInterfaceName = null; SyntaxToken separator = null; ResetPoint beforeIdentifierPoint = default(ResetPoint); bool beforeIdentifierPointSet = false; try { while (true) { //// Check if we got 'this'. If so, then we have an indexer. //// Note: we parse out type parameters here as well so that //// we can give a useful error about illegal generic indexers. //if (this.CurrentToken.Kind == SyntaxKind.ThisKeyword) //{ // beforeIdentifierPoint = GetResetPoint(); // beforeIdentifierPointSet = true; // identifierOrThisOpt = this.EatToken(); // typeParameterListOpt = this.ParseTypeParameterList(); // break; //} // now, scan past the next name. if it's followed by a dot then // it's part of the explicit name we're building up. Otherwise, // it's the name of the member. var point = GetResetPoint(); bool isMemberName; try { ScanNamedTypePart(); isMemberName = !IsDotOrColonColon(); } finally { this.Reset(ref point); this.Release(ref point); } if (isMemberName) { // We're past any explicit interface portion and We've // gotten to the member name. beforeIdentifierPoint = GetResetPoint(); beforeIdentifierPointSet = true; if (separator != null && separator.Kind == SyntaxKind.ColonColonToken) { separator = this.AddError(separator, ErrorCode.ERR_AliasQualAsExpression); separator = this.ConvertToMissingWithTrailingTrivia(separator, SyntaxKind.DotToken); } identifierOrThisOpt = this.ParseIdentifierToken(); //typeParameterListOpt = this.ParseTypeParameterList(); break; } else { // If we saw a . or :: then we must have something explicit. // first parse the upcoming name portion. var saveTerm = _termState; _termState |= TerminatorState.IsEndOfNameInExplicitInterface; if (explicitInterfaceName == null) { // If this is the first time, then just get the next simple // name and store it as the explicit interface name. explicitInterfaceName = this.ParseSimpleName(NameOptions.InTypeList); // Now, get the next separator. separator = this.CurrentToken.Kind == SyntaxKind.ColonColonToken ? this.EatToken() // fine after the first identifier : this.EatToken(SyntaxKind.DotToken); } else { // Parse out the next part and combine it with the // current explicit name to form the new explicit name. var tmp = this.ParseQualifiedNameRight(NameOptions.InTypeList, explicitInterfaceName, separator); Debug.Assert(!ReferenceEquals(tmp, explicitInterfaceName), "We should have consumed something and updated explicitInterfaceName"); explicitInterfaceName = tmp; // Now, get the next separator. separator = this.CurrentToken.Kind == SyntaxKind.ColonColonToken ? this.ConvertToMissingWithTrailingTrivia(this.EatToken(), SyntaxKind.DotToken) : this.EatToken(SyntaxKind.DotToken); } _termState = saveTerm; } } } finally { if (beforeIdentifierPointSet) { Release(ref beforeIdentifierPoint); } } }