Esempio n. 1
0
        internal ContinuationToken(Token virtualConcatenatedToken, int startIndex, int stopIndex, ITokensLine tokensLine, bool isContinuationFromPreviousLine, bool isContinuedOnNextLine)
            : base(virtualConcatenatedToken.TokenType, startIndex, stopIndex, virtualConcatenatedToken.UsesVirtualSpaceAtEndOfLine, tokensLine)
        {
            //  Store the concatenated source text
            MultilineContinuationText = virtualConcatenatedToken.Text;
            IsContinuationFromPreviousLine = isContinuationFromPreviousLine;
            IsContinuedOnNextLine = isContinuedOnNextLine;

            // Copy the delimiter properties
            if (virtualConcatenatedToken.UsesDelimiters)
            {
                UsesDelimiters = virtualConcatenatedToken.UsesDelimiters;
                HasOpeningDelimiter = virtualConcatenatedToken.HasOpeningDelimiter;
                HasClosingDelimiter = virtualConcatenatedToken.HasClosingDelimiter;
                ExpectedClosingDelimiter = virtualConcatenatedToken.ExpectedClosingDelimiter;
            }

            // Copy the literal value
            LiteralValue = virtualConcatenatedToken.LiteralValue;

            // Set specific Channel to enable filtering of all tokens participating in the continuation, except the first one
            if (IsContinuationFromPreviousLine)
            {
                Channel = CHANNEL_ContinuationTokens;
            }
        }
Esempio n. 2
0
 /// <summary>
 /// Sets the current iterator position to point to a specific token.
 /// After a call to this method, GetNextToken returns the token FOLLOWING startToken.
 /// </summary>
 public void SeekToToken(Token startToken)
 {
     // Find line for the start token
     currentPosition.LineIndex = tokensLines.IndexOf(startToken.TokensLine, startToken.TokensLine.LineIndex);
     currentLine = startToken.TokensLine;
     // Find index in line for the start token
     currentPosition.TokenIndexInLine = currentLine.SourceTokens.IndexOf(startToken);
     currentToken = startToken;
 }
Esempio n. 3
0
 /// <summary>
 /// Resets the iterator position : before the first token of the document
 /// </summary>
 public virtual void Reset()
 {
     currentPosition.LineIndex = 0;
     if (tokensLines.Count > 0)
     {
         currentLine = tokensLines[currentPosition.LineIndex];
     }
     currentPosition.TokenIndexInLine = -1;
     currentToken = null;
 }
Esempio n. 4
0
        /// <summary>
        /// Constructor for tokens without delimiters, using the virtual space at the end of the line
        /// </summary>
        internal Token(TokenType tokenType, int startIndex, int stopIndex, bool usesVirtualSpaceAtEndOfLine, ITokensLine tokensLine)
        {
            TokenType   = tokenType;
            TokenFamily = TokenUtils.GetTokenFamilyFromTokenType(tokenType);
            HasError    = false;
            SetInitialChannelFromTokenFamily();

            this.startIndex = startIndex;
            this.stopIndex  = stopIndex;
            this.tokensLine = tokensLine;

            UsesDelimiters      = false;
            HasClosingDelimiter = false;

            UsesVirtualSpaceAtEndOfLine = usesVirtualSpaceAtEndOfLine;
        }
Esempio n. 5
0
        /// <summary>
        /// Constructor for tokens without delimiters, using the virtual space at the end of the line
        /// </summary>
        internal Token(TokenType tokenType, int startIndex, int stopIndex, bool usesVirtualSpaceAtEndOfLine, ITokensLine tokensLine)
        {
            TokenType = tokenType;
            TokenFamily = TokenUtils.GetTokenFamilyFromTokenType(tokenType);
            HasError = false;
            SetInitialChannelFromTokenFamily();

            this.startIndex = startIndex;
            this.stopIndex = stopIndex;
            this.tokensLine = tokensLine;

            UsesDelimiters = false;
            HasClosingDelimiter = false;

            UsesVirtualSpaceAtEndOfLine = usesVirtualSpaceAtEndOfLine;
        }
Esempio n. 6
0
        public static string BuildResultString(ITokensLine tokensLine)
        {
            StringBuilder tokensText = new StringBuilder();
            foreach (Token token in tokensLine.SourceTokens)
            {
                tokensText.AppendLine(token.ToString());
            }

            StringBuilder diagnosticsText = new StringBuilder();
            foreach (Diagnostic diagnostic in tokensLine.ScannerDiagnostics)
            {
                diagnosticsText.AppendLine(diagnostic.ToString());
            }

            return tokensText.ToString() + diagnosticsText.ToString();
        }
Esempio n. 7
0
        protected override void ColorizeLine(DocumentLine line)
        {
            TypeCobolCompilerService compilerService = CurrentContext.TextView.GetService(typeof(TypeCobolCompilerService)) as TypeCobolCompilerService;

            if (compilerService != null)
            {
                int lineIndex = line.LineNumber - 1;
                if (compilerService.CompilationUnit.TokensDocumentSnapshot.Lines.Count > lineIndex)
                {
                    ITokensLine tokensLine      = compilerService.CompilationUnit.TokensDocumentSnapshot.Lines[lineIndex];
                    int         lineStartOffset = line.Offset;

                    // Areas
                    if (!tokensLine.SequenceNumber.IsEmpty)
                    {
                        ApplyTextAreaStyle(lineStartOffset, tokensLine.SequenceNumber, TokenStyles.GetAreaStyle(TextAreaType.SequenceNumber));
                    }
                    if (!tokensLine.Indicator.IsEmpty)
                    {
                        if (tokensLine.Type == CobolTextLineType.Comment)
                        {
                            ApplyTextAreaStyle(lineStartOffset, tokensLine.Indicator, TokenStyles.GetAreaStyle(TextAreaType.Comment));
                        }
                        else if (tokensLine.Type == CobolTextLineType.Debug || tokensLine.Type == CobolTextLineType.Continuation)
                        {
                            ApplyTextAreaStyle(lineStartOffset, tokensLine.Indicator, TokenStyles.GetAreaStyle(TextAreaType.Indicator));
                        }
                    }
                    if (!tokensLine.Comment.IsEmpty)
                    {
                        ApplyTextAreaStyle(lineStartOffset, tokensLine.Comment, TokenStyles.GetAreaStyle(TextAreaType.Comment));
                    }

                    // Tokens
                    foreach (Token token in tokensLine.SourceTokens)
                    {
                        HighlightingColor tokenStyle = TokenStyles.GetTokenStyle(token);
                        if (tokenStyle != null && token.Length > 0)
                        {
                            ApplyTokenStyle(lineStartOffset, token, tokenStyle);
                        }
                    }
                }
            }
        }
Esempio n. 8
0
        public static string BuildResultString(ITokensLine tokensLine)
        {
            StringBuilder tokensText = new StringBuilder();

            foreach (Token token in tokensLine.SourceTokens)
            {
                tokensText.AppendLine(token.ToString());
            }

            StringBuilder diagnosticsText = new StringBuilder();

            foreach (Diagnostic diagnostic in tokensLine.ScannerDiagnostics)
            {
                diagnosticsText.AppendLine(diagnostic.ToString());
            }

            return(tokensText.ToString() + diagnosticsText.ToString());
        }
Esempio n. 9
0
        /// <summary>
        /// Constructor for tokens with delimiters, using the virtual space at the end of the line
        /// </summary>
        internal Token(TokenType tokenType, int startIndex, int stopIndex, bool usesVirtualSpaceAtEndOfLine, ITokensLine tokensLine, bool hasOpeningDelimiter, bool hasClosingDelimiter, char expectedClosingDelimiter)
        {
            TokenType   = tokenType;
            TokenFamily = TokenUtils.GetTokenFamilyFromTokenType(tokenType);
            SetInitialChannelFromTokenFamily();

            HasError = false;

            this.startIndex = startIndex;
            this.stopIndex  = stopIndex;
            this.tokensLine = tokensLine;

            UsesDelimiters           = true;
            HasOpeningDelimiter      = hasOpeningDelimiter;
            HasClosingDelimiter      = hasClosingDelimiter;
            ExpectedClosingDelimiter = expectedClosingDelimiter;

            UsesVirtualSpaceAtEndOfLine = usesVirtualSpaceAtEndOfLine;
        }
Esempio n. 10
0
        /// <summary>
        /// Get the next token, with or without applying the channel filter
        /// </summary>
        public Token NextToken(bool applyChannelFilter)
        {
            // If document is empty, immediately return EndOfFile
            if (currentLine == null)
            {
                return(Token.END_OF_FILE);
            }

            // While we can find a next token
            currentToken = null;
            while (currentToken == null)
            {
                // try to find the next token on the same line
                currentPosition.TokenIndexInLine++;
                // but if we reached the end of the current line ...
                while (currentPosition.TokenIndexInLine >= currentLine.SourceTokens.Count)
                {
                    // .. advance to next line
                    currentPosition.LineIndex++;
                    currentPosition.TokenIndexInLine = 0;
                    if (currentPosition.LineIndex < tokensLines.Count)
                    {
                        // TO DO - OPTIMIZATION : should we use here an Enumerator on tokensLines ?
                        currentLine = tokensLines[currentPosition.LineIndex];
                    }
                    // and if we reached the last line of the document ...
                    else
                    {
                        // return EndOfFile
                        currentLine = null;
                        return(Token.END_OF_FILE);
                    }
                }
                // Check if the next token found matches the filter criteria
                Token nextTokenCandidate = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
                if (!applyChannelFilter || nextTokenCandidate.Channel == channelFilter)
                {
                    currentToken = nextTokenCandidate;
                }
            }
            // found a next token matching the filter criteria
            return(currentToken);
        }
Esempio n. 11
0
 public void BeginParsingFile(TextSourceInfo textSourceInfo, ITokensLinesIterator tokensCountIterator)
 {
     CurrentFileInfo = new ParsedFileInfo(textSourceInfo.Name, parserRulesCount, parserDecisionsCount);
     // Only for CodeElementsParser
     if (tokensCountIterator != null)
     {
         ITokensLine lastLine = null;
         Token       token    = null;
         while ((token = tokensCountIterator.NextToken()) != Token.END_OF_FILE)
         {
             CurrentFileInfo.TokensCount++;
             if (token.TokensLine != lastLine)
             {
                 CurrentFileInfo.LinesCount++;
                 lastLine = token.TokensLine;
             }
         }
     }
 }
Esempio n. 12
0
        /// <summary>
        /// Constructor for tokens with delimiters, using the virtual space at the end of the line
        /// </summary>
        internal Token(TokenType tokenType, int startIndex, int stopIndex, bool usesVirtualSpaceAtEndOfLine, ITokensLine tokensLine, bool hasOpeningDelimiter, bool hasClosingDelimiter, char expectedClosingDelimiter)
        {
            TokenType = tokenType;
            TokenFamily = TokenUtils.GetTokenFamilyFromTokenType(tokenType);
            SetInitialChannelFromTokenFamily();

            HasError = false;

            this.startIndex = startIndex;
            this.stopIndex = stopIndex;
            this.tokensLine = tokensLine;

            UsesDelimiters = true;
            HasOpeningDelimiter = hasOpeningDelimiter;
            HasClosingDelimiter = hasClosingDelimiter;
            ExpectedClosingDelimiter = expectedClosingDelimiter;

            UsesVirtualSpaceAtEndOfLine = usesVirtualSpaceAtEndOfLine;
        }
Esempio n. 13
0
 /// <summary>
 /// Sets the current iterator position to a previous position returned by GetCurrentPosition.
 /// After a call to this method, GetNextToken returns the token FOLLOWING the current position.
 /// </summary>
 public void SeekToPosition(object iteratorPosition)
 {
     currentPosition = (TokensLineIteratorPosition)iteratorPosition;
     if (currentPosition.LineIndex >= 0 && currentPosition.LineIndex < tokensLines.Count)
     {
         currentLine = tokensLines[currentPosition.LineIndex];
     }
     else
     {
         currentLine = null;
     }
     if (currentPosition.TokenIndexInLine >= 0 && currentLine != null)
     {
         currentToken = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
     }
     else
     {
         currentToken = null;
     }
 }
Esempio n. 14
0
        /// <summary>
        /// Returns all tokens (no channel filter), line per line, selected between startToken and stopToken.
        /// This method takes a snapshot of the current position before its execution and restores it before returning.
        /// </summary>
        public MultilineTokensGroupSelection SelectAllTokensBetween(Token startToken, Token stopToken)
        {
            // Check parameters
            ITokensLine startLine      = startToken.TokensLine;
            int         startLineIndex = tokensLines.IndexOf(startLine, startLine.LineIndex);
            ITokensLine stopLine       = stopToken.TokensLine;
            int         stopLineIndex  = tokensLines.IndexOf(stopLine, stopLine.LineIndex);

            if (startLineIndex < 0 || stopLineIndex < 0 || stopLineIndex < startLineIndex ||
                ((startLineIndex == stopLineIndex) && (stopToken.StartIndex < startToken.StartIndex)))
            {
                throw new InvalidOperationException("Invalid start or stop token : line or columns number do not define a valid selection interval");
            }

            // Save iterator position (to restore it before return)
            SaveCurrentPositionSnapshot();

            // Seek to the first token at the start of the line where startToken appears
            currentPosition.LineIndex = startLineIndex;
            currentLine = startToken.TokensLine;
            currentPosition.TokenIndexInLine = 0;
            currentToken = currentLine.SourceTokens[0];

            // List of tokens not selected on the first line before startToken
            IList <Token> tokensOnFirstLineBeforeStartToken = null;

            if (currentToken != startToken)
            {
                tokensOnFirstLineBeforeStartToken = new List <Token>();
                do
                {
                    tokensOnFirstLineBeforeStartToken.Add(currentToken);
                }while (NextToken(false) != startToken);
            }

            // List of tokens selected ...
            int numberOfSelectedLines = stopLineIndex - startLineIndex + 1;

            IList <Token>[] selectedTokensOnSeveralLines = new IList <Token> [numberOfSelectedLines];
            // ... on the first line
            IList <Token> tokensSelectedOnFirstLine = new List <Token>();

            selectedTokensOnSeveralLines[0] = tokensSelectedOnFirstLine;
            if (numberOfSelectedLines == 1)
            {
                if (currentToken == stopToken)
                {
                    tokensSelectedOnFirstLine.Add(currentToken);
                }
                else
                {
                    do
                    {
                        tokensSelectedOnFirstLine.Add(currentToken);
                    }while (NextToken(false) != stopToken);
                    tokensSelectedOnFirstLine.Add(currentToken);
                }
                NextToken(false);
            }
            else
            {
                do
                {
                    tokensSelectedOnFirstLine.Add(currentToken);
                }while (NextToken(false).TokensLine == startLine);
            }
            // ... on intermediate lines
            // => optimization : reuse the complete lists of the source lines
            if (numberOfSelectedLines > 2)
            {
                for (int intermediateLineIndex = startLineIndex + 1; intermediateLineIndex < stopLineIndex; intermediateLineIndex++)
                {
                    // TO DO - OPTIMIZATION - IMPORTANT : here, we should use an Enumerator on tokensLines
                    selectedTokensOnSeveralLines[intermediateLineIndex - startLineIndex] = tokensLines[intermediateLineIndex].SourceTokens;
                }
                currentPosition.LineIndex = stopLineIndex;
                currentLine = tokensLines[currentPosition.LineIndex];
                currentPosition.TokenIndexInLine = 0;
                currentToken = currentLine.SourceTokens[0];
            }
            // ... on the last line
            if (numberOfSelectedLines > 1)
            {
                IList <Token> tokensSelectedOnLastLine = new List <Token>();
                selectedTokensOnSeveralLines[numberOfSelectedLines - 1] = tokensSelectedOnLastLine;
                if (currentToken == stopToken)
                {
                    tokensSelectedOnLastLine.Add(currentToken);
                }
                else
                {
                    do
                    {
                        tokensSelectedOnLastLine.Add(currentToken);
                    }while (NextToken(false) != stopToken);
                    tokensSelectedOnLastLine.Add(currentToken);
                }
                NextToken(false);
            }

            // List of tokens not selected on the last line after stopToken
            IList <Token> tokensOnLastLineAfterStopToken = null;

            if (currentToken != null && currentToken.TokensLine == stopToken.TokensLine)
            {
                Token lastTokenOfLastLine = currentLine.SourceTokens[currentLine.SourceTokens.Count - 1];
                if (stopToken != lastTokenOfLastLine)
                {
                    tokensOnLastLineAfterStopToken = new List <Token>();
                    if (currentToken == lastTokenOfLastLine)
                    {
                        tokensOnLastLineAfterStopToken.Add(currentToken);
                    }
                    else
                    {
                        do
                        {
                            tokensOnLastLineAfterStopToken.Add(currentToken);
                        }while (NextToken(false) != lastTokenOfLastLine);
                        tokensOnLastLineAfterStopToken.Add(currentToken);
                    }
                }
            }

            // Restore initial iterator position before return
            ReturnToLastPositionSnapshot();

            return(new MultilineTokensGroupSelection(startLineIndex,
                                                     tokensOnFirstLineBeforeStartToken,
                                                     selectedTokensOnSeveralLines,
                                                     tokensOnLastLineAfterStopToken));
        }
Esempio n. 15
0
        /// <summary>
        /// Get the previous token, with or without applying the channel filter
        /// </summary>
        public Token PreviousToken(bool applyChannelFilter)
        {
            // If the iterator already points before the first token :
            // impossible to get previous token, do nothing
            if (currentPosition.LineIndex == 0 && currentPosition.TokenIndexInLine == -1)
            {
                return(null);
            }

            // Look for previous tokens until we find one that matches the filter
            for (; ;)
            {
                // Try to get the previous token on the same line
                currentPosition.TokenIndexInLine -= 1;
                // Current token was not the first token on his line
                if (currentPosition.TokenIndexInLine >= 0)
                {
                    // => simply return previous token on the same line
                    currentToken = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
                }
                // Current token was the first token on the line
                // => look for a previous token on the preceding line
                else
                {
                    // If there is a preceding line
                    if (currentPosition.LineIndex > 0)
                    {
                        // => find the first preceding line which is not empty
                        while (currentPosition.LineIndex > 0)
                        {
                            currentPosition.LineIndex = currentPosition.LineIndex - 1;
                            // TO DO - OPTIMIZATION - IMPORTANT : here, we should use an Enumerator on tokensLines
                            currentLine = tokensLines[currentPosition.LineIndex];
                            if (currentLine.SourceTokens.Count > 0)
                            {
                                break;
                            }
                        }
                        // If such a line was found :
                        if (currentLine.SourceTokens.Count > 0)
                        {
                            // Select the last token on this line
                            currentPosition.TokenIndexInLine = currentLine.SourceTokens.Count - 1;
                            currentToken = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
                        }
                        else
                        {
                            // => position the iterator before the first token of the document
                            currentPosition.TokenIndexInLine = -1;
                            currentToken = null;
                            break;
                        }
                    }
                    // If there is no preceding line
                    else
                    {
                        // => position the iterator before the first token of the document
                        currentPosition.TokenIndexInLine = -1;
                        currentToken = null;
                        break;
                    }
                }
                // Check if the previous token found matches the filter criteria
                if (!applyChannelFilter || currentToken == null | currentToken?.Channel == channelFilter)
                {
                    break;
                }
            }
            return(currentToken);
        }
Esempio n. 16
0
 internal void CorrectTokensLine(ITokensLine tokensLine, int startIndex, int stopIndex)
 {
     this.tokensLine = tokensLine;
     this.startIndex = startIndex;
     this.stopIndex  = stopIndex;
 }
Esempio n. 17
0
 /// <summary>
 /// Constructor for tokens without delimiters
 /// </summary>
 public Token(TokenType tokenType, int startIndex, int stopIndex, ITokensLine tokensLine) :
     this(tokenType, startIndex, stopIndex, false, tokensLine)
 {
 }
Esempio n. 18
0
        /// <summary>
        /// Get the next token, with or without applying the channel filter
        /// </summary>
        public Token NextToken(bool applyChannelFilter)
        {
            // If document is empty, immediately return EndOfFile
            if (currentLine == null)
            {
                return Token.END_OF_FILE;
            }

            // While we can find a next token
            currentToken = null;
            while (currentToken == null)
            {
                // try to find the next token on the same line
                currentPosition.TokenIndexInLine++;
                // but if we reached the end of the current line ...
                while (currentPosition.TokenIndexInLine >= currentLine.SourceTokens.Count)
                {
                    // .. advance to next line
                    currentPosition.LineIndex++;
                    currentPosition.TokenIndexInLine = 0;
                    if (currentPosition.LineIndex < tokensLines.Count)
                    {
                        // TO DO - OPTIMIZATION : should we use here an Enumerator on tokensLines ?
                        currentLine = tokensLines[currentPosition.LineIndex];
                    }
                    // and if we reached the last line of the document ...
                    else
                    {
                        // return EndOfFile
                        currentLine = null;
                        return Token.END_OF_FILE;
                    }
                }
                // Check if the next token found matches the filter criteria
                Token nextTokenCandidate = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
                if (!applyChannelFilter || nextTokenCandidate.Channel == channelFilter)
                {
                    currentToken = nextTokenCandidate;
                }
            }
            // found a next token matching the filter criteria
            return currentToken;
        }
Esempio n. 19
0
 internal void CorrectTokensLine(ITokensLine tokensLine, int startIndex, int stopIndex)
 {
     this.tokensLine = tokensLine;
     this.startIndex = startIndex;
     this.stopIndex = stopIndex;
 }
Esempio n. 20
0
        /// <summary>
        /// Returns all tokens (no channel filter), line per line, selected between startToken and stopToken.
        /// This method takes a snapshot of the current position before its execution and restores it before returning.
        /// </summary>
        public MultilineTokensGroupSelection SelectAllTokensBetween(Token startToken, Token stopToken)
        {
            // Check parameters
            ITokensLine startLine = startToken.TokensLine;
            int startLineIndex = tokensLines.IndexOf(startLine, startLine.InitialLineIndex);
            ITokensLine stopLine = stopToken.TokensLine;
            int stopLineIndex = tokensLines.IndexOf(stopLine, stopLine.InitialLineIndex);
            if (startLineIndex < 0 || stopLineIndex < 0 || stopLineIndex < startLineIndex ||
                ((startLineIndex == stopLineIndex) && (stopToken.StartIndex < startToken.StartIndex)))
            {
                throw new InvalidOperationException("Invalid start or stop token : line or columns number do not define a valid selection interval");
            }

            // Save iterator position (to restore it before return)
            SaveCurrentPositionSnapshot();

            // Seek to the first token at the start of the line where startToken appears
            currentPosition.LineIndex = startLineIndex;
            currentLine = startToken.TokensLine;
            currentPosition.TokenIndexInLine = 0;
            currentToken = currentLine.SourceTokens[0];

            // List of tokens not selected on the first line before startToken
            IList<Token> tokensOnFirstLineBeforeStartToken = null;
            if(currentToken != startToken)
            {
                tokensOnFirstLineBeforeStartToken = new List<Token>();
                do
                {
                    tokensOnFirstLineBeforeStartToken.Add(currentToken);
                }
                while (NextToken(false) != startToken);
            }

            // List of tokens selected ...
            int numberOfSelectedLines = stopLineIndex - startLineIndex + 1;
            IList<Token>[] selectedTokensOnSeveralLines = new IList<Token>[numberOfSelectedLines];
            // ... on the first line
            IList<Token> tokensSelectedOnFirstLine = new List<Token>();
            selectedTokensOnSeveralLines[0] = tokensSelectedOnFirstLine;
            if(numberOfSelectedLines == 1)
            {
                if (currentToken == stopToken)
                {
                    tokensSelectedOnFirstLine.Add(currentToken);
                }
                else
                {
                    do
                    {
                        tokensSelectedOnFirstLine.Add(currentToken);
                    }
                    while (NextToken(false) != stopToken);
                    tokensSelectedOnFirstLine.Add(currentToken);
                }
                NextToken(false);
            }
            else
            {
                do
                {
                    tokensSelectedOnFirstLine.Add(currentToken);
                }
                while (NextToken(false).TokensLine == startLine);
            }
            // ... on intermediate lines
            // => optimization : reuse the complete lists of the source lines
            if(numberOfSelectedLines > 2)
            {
                for(int intermediateLineIndex = startLineIndex+1 ; intermediateLineIndex < stopLineIndex ; intermediateLineIndex++)
                {
                    // TO DO - OPTIMIZATION - IMPORTANT : here, we should use an Enumerator on tokensLines
                    selectedTokensOnSeveralLines[intermediateLineIndex - startLineIndex] = tokensLines[intermediateLineIndex].SourceTokens;
                }
                currentPosition.LineIndex = stopLineIndex;
                currentLine = tokensLines[currentPosition.LineIndex];
                currentPosition.TokenIndexInLine = 0;
                currentToken = currentLine.SourceTokens[0];
            }
            // ... on the last line
            if (numberOfSelectedLines > 1)
            {
                IList<Token> tokensSelectedOnLastLine = new List<Token>();
                selectedTokensOnSeveralLines[numberOfSelectedLines - 1] = tokensSelectedOnLastLine;
                if (currentToken == stopToken)
                {
                    tokensSelectedOnLastLine.Add(currentToken);
                }
                else
                {
                    do
                    {
                        tokensSelectedOnLastLine.Add(currentToken);
                    }
                    while (NextToken(false) != stopToken);
                    tokensSelectedOnLastLine.Add(currentToken);
                }
                NextToken(false);
            }

            // List of tokens not selected on the last line after stopToken
            IList<Token> tokensOnLastLineAfterStopToken = null;
            if (currentToken != null && currentToken.TokensLine == stopToken.TokensLine)
            {
                Token lastTokenOfLastLine = currentLine.SourceTokens[currentLine.SourceTokens.Count - 1];
                if (stopToken != lastTokenOfLastLine)
                {
                    tokensOnLastLineAfterStopToken = new List<Token>();
                    if (currentToken == lastTokenOfLastLine)
                    {
                        tokensOnLastLineAfterStopToken.Add(currentToken);
                    }
                    else
                    {
                        do
                        {
                            tokensOnLastLineAfterStopToken.Add(currentToken);
                        }
                        while (NextToken(false) != lastTokenOfLastLine);
                        tokensOnLastLineAfterStopToken.Add(currentToken);
                    }
                }
            }

            // Restore initial iterator position before return
            ReturnToLastPositionSnapshot();

            return new MultilineTokensGroupSelection(startLineIndex,
                tokensOnFirstLineBeforeStartToken,
                selectedTokensOnSeveralLines,
                tokensOnLastLineAfterStopToken);
        }
Esempio n. 21
0
 /// <summary>
 /// Sets the current iterator position to point to a specific token.
 /// After a call to this method, GetNextToken returns the token FOLLOWING startToken.
 /// </summary>
 public void SeekToToken(Token startToken)
 {
     // Find line for the start token
     currentPosition.LineIndex = tokensLines.IndexOf(startToken.TokensLine, startToken.TokensLine.InitialLineIndex);
     currentLine = startToken.TokensLine;
     // Find index in line for the start token
     currentPosition.TokenIndexInLine = currentLine.SourceTokens.IndexOf(startToken);
     currentToken = startToken;
 }
Esempio n. 22
0
 /// <summary>
 /// Sets the current iterator position to a previous position returned by GetCurrentPosition.
 /// After a call to this method, GetNextToken returns the token FOLLOWING the current position.
 /// </summary>
 public void SeekToPosition(object iteratorPosition)
 {
     currentPosition = (TokensLineIteratorPosition)iteratorPosition;
     if (currentPosition.LineIndex >= 0 && currentPosition.LineIndex < tokensLines.Count)
     {
         currentLine = tokensLines[currentPosition.LineIndex];
     }
     else
     {
         currentLine = null;
     }
     if (currentPosition.TokenIndexInLine >= 0 && currentLine != null)
     {
         currentToken = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
     }
     else
     {
         currentToken = null;
     }
 }
Esempio n. 23
0
 /// <summary>
 /// Resets the iterator position : before the first token of the document
 /// </summary>
 public void Reset()
 {
     currentPosition.LineIndex = 0;
     if (tokensLines.Count > 0)
     {
         currentLine = tokensLines[currentPosition.LineIndex];
     }
     currentPosition.TokenIndexInLine = -1;
     currentToken = null;
 }
Esempio n. 24
0
        /// <summary>
        /// Get the previous token, with or without applying the channel filter
        /// </summary>
        public Token PreviousToken(bool applyChannelFilter)
        {
            // If the iterator already points before the first token :
            // impossible to get previous token, do nothing
            if (currentPosition.LineIndex == 0 && currentPosition.TokenIndexInLine == -1)
            {
                return null;
            }

            // Look for previous tokens until we find one that matches the filter
            for (; ; )
            {
                // Try to get the previous token on the same line
                currentPosition.TokenIndexInLine -= 1;
                // Current token was not the first token on his line
                if (currentPosition.TokenIndexInLine >= 0)
                {
                    // => simply return previous token on the same line
                    currentToken = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
                }
                // Current token was the first token on the line
                // => look for a previous token on the preceding line
                else
                {
                    // If there is a preceding line
                    if (currentPosition.LineIndex > 0)
                    {
                        // => find the first preceding line which is not empty
                        while (currentPosition.LineIndex > 0)
                        {
                            currentPosition.LineIndex = currentPosition.LineIndex - 1;
                            // TO DO - OPTIMIZATION - IMPORTANT : here, we should use an Enumerator on tokensLines
                            currentLine = tokensLines[currentPosition.LineIndex];
                            if (currentLine.SourceTokens.Count > 0) break;
                        }
                        // If such a line was found :
                        if (currentLine.SourceTokens.Count > 0)
                        {
                            // Select the last token on this line
                            currentPosition.TokenIndexInLine = currentLine.SourceTokens.Count - 1;
                            currentToken = currentLine.SourceTokens[currentPosition.TokenIndexInLine];
                        }
                        else
                        {
                            // => position the iterator before the first token of the document
                            currentPosition.TokenIndexInLine = -1;
                            currentToken = null;
                            break;
                        }
                    }
                    // If there is no preceding line
                    else
                    {
                        // => position the iterator before the first token of the document
                        currentPosition.TokenIndexInLine = -1;
                        currentToken = null;
                        break;
                    }
                }
                // Check if the previous token found matches the filter criteria
                if (!applyChannelFilter || currentToken == null | currentToken.Channel == channelFilter)
                {
                    break;
                }
            }
            return currentToken;
        }
Esempio n. 25
0
 /// <summary>
 /// Constructor for tokens without delimiters
 /// </summary>
 internal Token(TokenType tokenType, int startIndex, int stopIndex, ITokensLine tokensLine)
     : this(tokenType, startIndex, stopIndex, false, tokensLine)
 {
 }
Esempio n. 26
0
 /// <summary>
 /// Constructor for tokens with delimiters
 /// </summary>
 internal Token(TokenType tokenType, int startIndex, int stopIndex, ITokensLine tokensLine, bool hasOpeningDelimiter, bool hasClosingDelimiter, char expectedClosingDelimiter) :
     this(tokenType, startIndex, stopIndex, false, tokensLine, hasOpeningDelimiter, hasClosingDelimiter, expectedClosingDelimiter)
 {
 }
Esempio n. 27
0
 /// <summary>
 /// Constructor for tokens with delimiters
 /// </summary>
 internal Token(TokenType tokenType, int startIndex, int stopIndex, ITokensLine tokensLine, bool hasOpeningDelimiter, bool hasClosingDelimiter, char expectedClosingDelimiter)
     : this(tokenType, startIndex, stopIndex, false, tokensLine, hasOpeningDelimiter, hasClosingDelimiter, expectedClosingDelimiter)
 {
 }
Esempio n. 28
0
        internal ContinuationToken(Token virtualConcatenatedToken, int startIndex, int stopIndex, ITokensLine tokensLine, bool isContinuationFromPreviousLine, bool isContinuedOnNextLine) :
            base(virtualConcatenatedToken.TokenType, startIndex, stopIndex, virtualConcatenatedToken.UsesVirtualSpaceAtEndOfLine, tokensLine)
        {
            //  Store the concatenated source text
            MultilineContinuationText      = virtualConcatenatedToken.Text;
            IsContinuationFromPreviousLine = isContinuationFromPreviousLine;
            IsContinuedOnNextLine          = isContinuedOnNextLine;

            // Copy the delimiter properties
            if (virtualConcatenatedToken.UsesDelimiters)
            {
                UsesDelimiters           = virtualConcatenatedToken.UsesDelimiters;
                HasOpeningDelimiter      = virtualConcatenatedToken.HasOpeningDelimiter;
                HasClosingDelimiter      = virtualConcatenatedToken.HasClosingDelimiter;
                ExpectedClosingDelimiter = virtualConcatenatedToken.ExpectedClosingDelimiter;
            }

            // Copy the literal value
            LiteralValue = virtualConcatenatedToken.LiteralValue;

            // Set specific Channel to enable filtering of all tokens participating in the continuation, except the first one
            if (IsContinuationFromPreviousLine)
            {
                Channel = CHANNEL_ContinuationTokens;
            }
        }
Esempio n. 29
0
 private string GetIndent(ITokensLine line, int firstTokenStartIndex)
 {
     var lineStartIndex = line.SequenceNumberText.Length + 1;// +1 for line.IndicatorChar
     return line.SourceText.Substring(0, firstTokenStartIndex-lineStartIndex);
 }
Esempio n. 30
0
        private string GetIndent(ITokensLine line, int firstTokenStartIndex)
        {
            var lineStartIndex = line.SequenceNumberText.Length + 1;            // +1 for line.IndicatorChar

            return(line.SourceText.Substring(0, firstTokenStartIndex - lineStartIndex));
        }
Esempio n. 31
0
 private int GetLine(ITokensLine line)
 {
     if (line != null) return Lines.IndexOf(line, line.InitialLineIndex)+1;
     throw new System.ArgumentNullException("this.Line must be set from the source document snapshot");
 }
Esempio n. 32
0
 public MissingToken(TokenType missingTokenType, string informationTextForMissingToken, ITokensLine lineWhereTokenIsMissing, int charPositionWhereTokenIsMissing) :
     base(missingTokenType, charPositionWhereTokenIsMissing, charPositionWhereTokenIsMissing - 1, lineWhereTokenIsMissing)
 {
     InformationText = informationTextForMissingToken;
 }