Ejemplo n.º 1
0
        /// <summary>
        /// Incremental preprocessing of a set of tokens lines changes
        /// </summary>
        internal static IList <DocumentChange <IProcessedTokensLine> > ProcessTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <ProcessedTokensLine> documentLines, IList <DocumentChange <ITokensLine> > tokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider, List <RemarksDirective.TextNameVariation> copyTextNameVariations, PerfStatsForParserInvocation perfStatsForParserInvocation, List <CopyDirective> missingCopies)
        {
            // Collect all changes applied to the processed tokens lines during the incremental scan
            IList <DocumentChange <IProcessedTokensLine> > processedTokensLinesChanges = new List <DocumentChange <IProcessedTokensLine> >();

            // There are 2 reasons to a preprocess a tokens line after a change :
            // 1. A tokens line changed : these lines were already reset during the previous steps
            // 2. If a tokens line that changed was involved in the parsing of a multiline compiler directive, the whole group of lines must be parsed again
            // Then, if a new COPY directive was parsed : the CompilationDocument to include must be prepared

            // --- PREPARATION PHASE : reset all processed tokens lines which were involved in a multiline compiler directive where at least one line changed  ---

            // Iterate over all tokens changes detected by the ScannerStep :
            // refresh all the adjacent lines participating in a ContinuationTokensGroup
            if (tokensLinesChanges != null)
            {
                int lastLineIndexReset = -1;
                foreach (DocumentChange <ITokensLine> tokensChange in tokensLinesChanges)
                {
                    processedTokensLinesChanges.Add(new DocumentChange <IProcessedTokensLine>(tokensChange.Type, tokensChange.LineIndex, (IProcessedTokensLine)tokensChange.NewLine));
                    if (tokensChange.LineIndex > lastLineIndexReset)
                    {
                        lastLineIndexReset = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, processedTokensLinesChanges, lastLineIndexReset);
                    }
                }
            }

            // --- COMPILER DIRECTIVES PHASE : Find and parse all compiler directives ---

            // Init. Prepare a compiler directive parser

            // Create a token iterator on top of tokens lines
            TokensLinesIterator tokensIterator = new TokensLinesIterator(
                textSourceInfo.Name,
                documentLines,
                null,
                Token.CHANNEL_SourceTokens);

            // Crate an Antlr compatible token source on top a the token iterator
            TokensLinesTokenSource tokenSource = new TokensLinesTokenSource(
                textSourceInfo.Name,
                tokensIterator);

            // Init a compiler directive parser
            CommonTokenStream             tokenStream      = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
            CobolCompilerDirectivesParser directivesParser = new CobolCompilerDirectivesParser(tokenStream);

            // Optionnaly activate Antlr Parser performance profiling
            // WARNING : use this in a single-treaded context only (uses static field)
            if (AntlrPerformanceProfiler == null && perfStatsForParserInvocation.ActivateDetailedAntlrPofiling)
            {
                AntlrPerformanceProfiler = new AntlrPerformanceProfiler(directivesParser);
            }
            if (AntlrPerformanceProfiler != null)
            {
                // Replace the generated parser by a subclass which traces all rules invocations
                directivesParser = new CobolCompilerDirectivesTracingParser(tokenStream);
                AntlrPerformanceProfiler.BeginParsingFile(textSourceInfo, null);
            }



            IAntlrErrorStrategy compilerDirectiveErrorStrategy = new CompilerDirectiveErrorStrategy();

            directivesParser.ErrorHandler = compilerDirectiveErrorStrategy;

            // Register all parse errors in a list in memory
            ParserDiagnosticErrorListener errorListener = new ParserDiagnosticErrorListener();

            directivesParser.RemoveErrorListeners();
            directivesParser.AddErrorListener(errorListener);

            // Prepare to analyze the parse tree
            ParseTreeWalker          walker           = new ParseTreeWalker();
            CompilerDirectiveBuilder directiveBuilder = new CompilerDirectiveBuilder(compilerOptions, copyTextNameVariations);

            // 1. Iterate over all compiler directive starting tokens found in the lines which were updated
            foreach (Token compilerDirectiveStartingToken in documentLines
                     .Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing)
                     .SelectMany(line => line.SourceTokens)
                     .Where(token => token.TokenFamily == TokenFamily.CompilerDirectiveStartingKeyword))
            {
                // 2. Reset the compiler directive parser state

                // Reset tokens iterator position before parsing
                // -> seek just before the compiler directive starting token
                tokensIterator.SeekToToken(compilerDirectiveStartingToken);
                tokensIterator.PreviousToken();
                // Special case : for efficiency reasons, in EXEC SQL INCLUDE directives
                // only the third token INCLUDE is recognized as a compiler directive
                // starting keyword by the scanner. In this case, we must rewind the
                // iterator two tokens backwards to start parsing just before the EXEC token.
                if (compilerDirectiveStartingToken.TokenType == TokenType.EXEC_SQL_INCLUDE)
                {
                    tokensIterator.PreviousToken();
                    tokensIterator.PreviousToken();
                }

                // Reset Antlr BufferedTokenStream position
                tokenStream.SetTokenSource(tokenSource);

                // Reset parsing error diagnostics
                compilerDirectiveErrorStrategy.Reset(directivesParser);

                // 3. Try to parse a compiler directive starting with the current token
                perfStatsForParserInvocation.OnStartAntlrParsing();
                if (AntlrPerformanceProfiler != null)
                {
                    AntlrPerformanceProfiler.BeginParsingSection();
                }
                CobolCompilerDirectivesParser.CompilerDirectingStatementContext directiveParseTree = directivesParser.compilerDirectingStatement();
                if (AntlrPerformanceProfiler != null)
                {
                    AntlrPerformanceProfiler.EndParsingSection(directiveParseTree.ChildCount);
                }
                perfStatsForParserInvocation.OnStopAntlrParsing(
                    AntlrPerformanceProfiler != null ? (int)AntlrPerformanceProfiler.CurrentFileInfo.DecisionTimeMs : 0,
                    AntlrPerformanceProfiler != null ? AntlrPerformanceProfiler.CurrentFileInfo.RuleInvocations.Sum() : 0);


                perfStatsForParserInvocation.OnStartTreeBuilding();
                // 4. Visit the parse tree to build a first class object representing the compiler directive
                walker.Walk(directiveBuilder, directiveParseTree);
                CompilerDirective compilerDirective  = directiveBuilder.CompilerDirective;
                bool errorFoundWhileParsingDirective = compilerDirective == null || compilerDirective.Diagnostics != null || directiveParseTree.Diagnostics != null;

                // 5. Get all tokens consumed while parsing the compiler directive
                //    and partition them line by line
                Token startToken = (Token)directiveParseTree.Start;
                Token stopToken  = (Token)directiveParseTree.Stop;
                if (stopToken == null)
                {
                    stopToken = startToken;
                }
                MultilineTokensGroupSelection tokensSelection = tokensIterator.SelectAllTokensBetween(startToken, stopToken);

                if (compilerDirective != null)
                {
                    // 6. Replace all matched tokens by :
                    // - a CompilerDirectiveToken on the first line
                    ProcessedTokensLine firstProcessedTokensLine = documentLines[tokensSelection.FirstLineIndex];
                    if (tokensSelection.SelectedTokensOnSeveralLines.Length == 1)
                    {
                        firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine(
                            tokensSelection.TokensOnFirstLineBeforeStartToken,
                            compilerDirective, errorFoundWhileParsingDirective,
                            tokensSelection.SelectedTokensOnSeveralLines[0],
                            tokensSelection.TokensOnLastLineAfterStopToken, false);
                    }
                    else
                    {
                        TokensGroup continuedTokensGroup = firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine(
                            tokensSelection.TokensOnFirstLineBeforeStartToken,
                            compilerDirective, errorFoundWhileParsingDirective,
                            tokensSelection.SelectedTokensOnSeveralLines[0],
                            null, true);

                        // - a ContinuationTokensGroup on the following lines
                        int selectionLineIndex = 1;
                        int lastLineIndex      = tokensSelection.FirstLineIndex + tokensSelection.SelectedTokensOnSeveralLines.Length - 1;
                        for (int nextLineIndex = tokensSelection.FirstLineIndex + 1; nextLineIndex <= lastLineIndex; nextLineIndex++, selectionLineIndex++)
                        {
                            IList <Token> compilerDirectiveTokensOnNextLine = tokensSelection.SelectedTokensOnSeveralLines[selectionLineIndex];
                            if (compilerDirectiveTokensOnNextLine.Count > 0)
                            {
                                ProcessedTokensLine nextProcessedTokensLine = documentLines[nextLineIndex];
                                if (nextLineIndex != lastLineIndex)
                                {
                                    continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine(
                                        continuedTokensGroup,
                                        compilerDirectiveTokensOnNextLine,
                                        null, true);
                                }
                                else
                                {
                                    continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine(
                                        continuedTokensGroup,
                                        compilerDirectiveTokensOnNextLine,
                                        tokensSelection.TokensOnLastLineAfterStopToken, false);
                                }
                            }
                        }
                    }
                }

                // 7. Register compiler directive parse errors
                if (errorFoundWhileParsingDirective)
                {
                    ProcessedTokensLine compilerDirectiveLine = documentLines[tokensSelection.FirstLineIndex];
                    if (compilerDirective != null && compilerDirective.Diagnostics != null)
                    {
                        foreach (Diagnostic directiveDiag in compilerDirective.Diagnostics)
                        {
                            compilerDirectiveLine.AddDiagnostic(directiveDiag);
                        }
                    }
                    else if (directiveParseTree.Diagnostics != null)
                    {
                        foreach (Diagnostic directiveDiag in directiveParseTree.Diagnostics)
                        {
                            if (compilerDirective != null)
                            {
                                compilerDirective.AddDiagnostic(directiveDiag);
                            }
                            compilerDirectiveLine.AddDiagnostic(directiveDiag);
                        }
                    }
                }
            }

            if (AntlrPerformanceProfiler != null)
            {
                AntlrPerformanceProfiler.EndParsingFile(directivesParser.ParseInfo.DecisionInfo, (int)(directivesParser.ParseInfo.GetTotalTimeInPrediction() / 1000000));
            }

            // 8. Advance the state off all ProcessedTokensLines :
            // NeedsCompilerDirectiveParsing => NeedsCopyDirectiveProcessing if it contains a COPY directive
            IList <ProcessedTokensLine> parsedLinesWithCopyDirectives = null;

            // NeedsCompilerDirectiveParsing => Ready otherwise
            foreach (ProcessedTokensLine parsedLine in documentLines
                     .Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing))
            {
                if (parsedLine.ImportedDocuments != null)
                {
                    if (parsedLinesWithCopyDirectives == null)
                    {
                        parsedLinesWithCopyDirectives = new List <ProcessedTokensLine>();
                    }
                    parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.NeedsCopyDirectiveProcessing;
                    parsedLinesWithCopyDirectives.Add(parsedLine);
                }
                else
                {
                    parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready;
                }
            }

            perfStatsForParserInvocation.OnStopTreeBuilding();

            // --- COPY IMPORT PHASE : Process COPY (REPLACING) directives ---

            foreach (var lineChange in processedTokensLinesChanges)
            {
                missingCopies.Remove(missingCopies.FirstOrDefault(c => c.COPYToken.Line == lineChange.LineIndex + 1));
            }

            // 1. Iterate over all updated lines containing a new COPY directive
            if (parsedLinesWithCopyDirectives != null)
            {
                foreach (ProcessedTokensLine tokensLineWithCopyDirective in parsedLinesWithCopyDirectives)
                {
                    // Iterate over all COPY directives found on one updated line
                    foreach (CopyDirective copyDirective in tokensLineWithCopyDirective.ImportedDocuments.Keys.Where(c => c.TextName != null || c.COPYToken.TokenType == TokenType.EXEC).ToArray())
                    {
                        try
                        {
                            PerfStatsForImportedDocument perfStats;

                            // Load (or retrieve in cache) the document referenced by the COPY directive
                            //Issue #315: tokensLineWithCopyDirective.ScanState must be passed because special names paragraph such as "Decimal point is comma" are declared in the enclosing program and can affect the parsing of COPY
                            ProcessedTokensDocument importedDocumentSource =
                                processedTokensDocumentProvider.GetProcessedTokensDocument(copyDirective.LibraryName,
                                                                                           copyDirective.TextName,
                                                                                           tokensLineWithCopyDirective.ScanStateBeforeCOPYToken[copyDirective.COPYToken], copyTextNameVariations, out perfStats);

                            // Store it on the current line after applying the REPLACING directive
                            ImportedTokensDocument importedDocument = new ImportedTokensDocument(copyDirective,
                                                                                                 importedDocumentSource, perfStats);
                            tokensLineWithCopyDirective.ImportedDocuments[copyDirective] = importedDocument;
                        }
                        catch (Exception e)
                        {
                            if (missingCopies != null && copyDirective != null && copyDirective.COPYToken != null && !missingCopies.Contains(copyDirective)) //If list already contains the copy directive just ignore
                            {
                                var missingCopieToReplace = missingCopies.FirstOrDefault(c => c.COPYToken.Line == copyDirective.COPYToken.Line);
                                missingCopies.Remove(missingCopieToReplace);
                                missingCopies.Add(copyDirective);
                            }


                            // Text name refenced by COPY directive was not found
                            // => register a preprocessor error on this line
                            Token failedDirectiveToken = tokensLineWithCopyDirective.TokensWithCompilerDirectives
                                                         .First(
                                token =>
                                token.TokenType == TokenType.CopyImportDirective &&
                                ((CompilerDirectiveToken)token).CompilerDirective == copyDirective);

                            Diagnostic diag = new Diagnostic(
                                MessageCode.FailedToLoadTextDocumentReferencedByCopyDirective,
                                failedDirectiveToken.Column, failedDirectiveToken.EndColumn,
                                failedDirectiveToken.Line, e.Message, e);

                            tokensLineWithCopyDirective.AddDiagnostic(diag);
                        }
                    }

                    // Advance processing status of the line
                    tokensLineWithCopyDirective.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready;
                }
            }



            // --- REPLACE PHASE : REPLACE directives are implemented in ReplaceTokensLinesIterator ---

            /* Algorithm :
             *
             * one REPLACE directive can express several replacement operations
             * one replacement operation can be of several types (distinguished for optimization purposes)
             * - SimpleTokenReplace : one source token / zero or one replacement token
             * - PartialWordReplace : one pure partial word / zero or one replacement token
             * - SimpleToMultipleTokenReplace : one source token / several replacement tokens
             * - MultipleTokenReplace : one first + several following source tokens / zero to many replacement tokens
             *
             * an iterator maintains a current set of replacement operations
             *
             * if nextToken is replace directive
             *    the current set of replacement operations is updated
             * else
             *    nextToken is compared to each replacement operation in turn
             *       if single -> single source token operation : return replacement token
             *       if single -> multiple operation : setup a secondary iterator with the list of replacement tokens
             *       if multiple -> multiple operation
             *          snapshot of the underlying iterator
             *          try to match all of the following source tokens
             *          if failure : restore snapshot and try next operation
             *          if success : setup a secondary iterator
             *
             * token comparison sourceToken / replacementCandidate :
             * 1. Compare Token type
             * 2. If same token type and for families
             *   AlphanumericLiteral
             *   NumericLiteral
             *   SyntaxLiteral
             *   Symbol
             *  => compare also Token text
             *
             * PartialCobolWord replacement :
             * p535 : The COPY statement with REPLACING phrase can be used to replace parts of words.
             * By inserting a dummy operand delimited by colons into the program text,
             * the compiler will replace the dummy operand with the required text.
             * Example 3 shows how this is used with the dummy operand :TAG:.
             * The colons serve as separators and make TAG a stand-alone operand.
             */

            return(processedTokensLinesChanges);
        }
Ejemplo n.º 2
0
        /// <summary>
        /// Incremental parsing of a set of processed tokens lines changes
        /// </summary>
        internal static IList <DocumentChange <ICodeElementsLine> > ParseProcessedTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <CodeElementsLine> documentLines, IList <DocumentChange <IProcessedTokensLine> > processedTokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, PerfStatsForParserInvocation perfStatsForParserInvocation)
        {
            // Collect all changes applied to the processed tokens lines during the incremental scan
            IList <DocumentChange <ICodeElementsLine> > codeElementsLinesChanges = new List <DocumentChange <ICodeElementsLine> >();

            // There are 2 reasons to re-parse a tokens line after a change :
            // 1. The tokens line changed : these lines were already reset during the previous steps
            // 2. If a tokens line that changed was involved in the parsing of a multiline code element, the whole group of lines must be parsed again

            // --- PREPARATION PHASE : identify all parse sections where code elements need to be refreshed ---

            IList <ParseSection> refreshParseSections       = null;
            ParseSection         largestRefreshParseSection = null;

            // Iterate over all processed tokens changes detected by the PreprocessorStep :
            // - refresh all the adjacent lines participating in a CodeElement
            // - register the start and stop token for all sections of the document which need to be parsed again
            if (processedTokensLinesChanges != null && processedTokensLinesChanges.Count > 0)
            {
                // If the document was cleared, everything must be parsed again
                if (processedTokensLinesChanges[0].Type != DocumentChangeType.DocumentCleared)
                {
                    refreshParseSections = new List <ParseSection>();

                    ParseSection lastParseSection = null;
                    foreach (DocumentChange <IProcessedTokensLine> tokensChange in processedTokensLinesChanges)
                    {
                        if (lastParseSection == null || tokensChange.LineIndex > lastParseSection.StopLineIndex)
                        {
                            lastParseSection = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, codeElementsLinesChanges, lastParseSection);
                            refreshParseSections.Add(lastParseSection);
                        }
                    }
                }
            }
            if (refreshParseSections != null)
            {
                //After getting all the parts refreshed, get the largest part that has been refreshed
                var minParseSection = refreshParseSections.OrderBy(p => p.StartLineIndex).First();
                var maxParseSection = refreshParseSections.OrderByDescending(p => p.StopLineIndex).First();
                largestRefreshParseSection = new ParseSection(minParseSection.StartLineIndex,
                                                              minParseSection.StartToken, maxParseSection.StopLineIndex, maxParseSection.StopToken,
                                                              maxParseSection.StopTokenIsFirstTokenOfTheLine);
            }


            // --- INITIALIZE ANTLR CodeElements parser ---

            // Create a token iterator on top of pre-processed tokens lines
            ITokensLinesIterator tokensIterator = ProcessedTokensDocument.GetProcessedTokensIterator(textSourceInfo, documentLines);

            // Create an Antlr compatible token source on top of the token iterator
            TokensLinesTokenSource tokenSource = new TokensLinesTokenSource(
                textSourceInfo.Name,
                tokensIterator);

            // Init parser
            TokensLinesTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
            CodeElementsParser     cobolParser = new CodeElementsParser(tokenStream);

            // REVERT TO STD PARSER ==> TracingCobolParser cobolParser = new TracingCobolParser(tokenStream);

            // Optionnaly activate Antlr Parser performance profiling
            // WARNING : use this in a single-treaded context only (uses static field)
            if (AntlrPerformanceProfiler == null && perfStatsForParserInvocation.ActivateDetailedAntlrPofiling)
            {
                AntlrPerformanceProfiler = new AntlrPerformanceProfiler(cobolParser);
            }
            if (AntlrPerformanceProfiler != null)
            {
                // Replace the generated parser by a subclass which traces all rules invocations
                cobolParser = new CodeElementsTracingParser(tokenStream);

                var tokensCountIterator = ProcessedTokensDocument.GetProcessedTokensIterator(textSourceInfo, documentLines);
                AntlrPerformanceProfiler.BeginParsingFile(textSourceInfo, tokensCountIterator);
            }

            // Customize error recovery strategy
            IAntlrErrorStrategy cobolErrorStrategy = new CodeElementErrorStrategy();

            cobolParser.ErrorHandler = cobolErrorStrategy;

            // Register all parse errors in a list in memory
            ParserDiagnosticErrorListener errorListener = new ParserDiagnosticErrorListener();

            cobolParser.RemoveErrorListeners();
            cobolParser.AddErrorListener(errorListener);

            // Prepare to analyze the parse tree
            ParseTreeWalker    walker             = new ParseTreeWalker();
            CodeElementBuilder codeElementBuilder = new CodeElementBuilder();

            codeElementBuilder.Dispatcher = new CodeElementDispatcher();
            codeElementBuilder.Dispatcher.CreateListeners();

            // --- INCREMENTAL PARSING ---

            // In case of incremental parsing, parse only the code sections we need to refresh

            if (largestRefreshParseSection != null)
            {
                // Seek just before the next code element starting token
                tokenStream.SeekToToken(largestRefreshParseSection.StartToken);
                tokenStream.StartLookingForStopToken(largestRefreshParseSection.StopToken);

                //Remove all the code elements for the future line to parse.

                for (int i = largestRefreshParseSection.StartLineIndex;
                     i < (largestRefreshParseSection.StopLineIndex == documentLines.Count - 1 && largestRefreshParseSection.StopToken == null //If the last index is equals to number of line in document, make sure to also reset the last line, otherwise, reset lines normally.
                        ? largestRefreshParseSection.StopLineIndex + 1
                        : largestRefreshParseSection.StopLineIndex);
                     i++)
                {
                    if (documentLines[i].CodeElements != null)
                    {
                        documentLines[i].ResetCodeElements();
                    }
                }
            }



            // Reset parsing error diagnostics
            cobolErrorStrategy.Reset(cobolParser);

            // Try to parse code elements :
            // - starting with the current parse section Start token
            // - ending with the current parse section Stop token
            CodeElementsParser.CobolCodeElementsContext codeElementsParseTree = null;
            try
            {
                perfStatsForParserInvocation.OnStartAntlrParsing();
                if (AntlrPerformanceProfiler != null)
                {
                    AntlrPerformanceProfiler.BeginParsingSection();
                }
                codeElementsParseTree = cobolParser.cobolCodeElements();
                if (AntlrPerformanceProfiler != null)
                {
                    AntlrPerformanceProfiler.EndParsingSection(codeElementsParseTree.ChildCount);
                }
                perfStatsForParserInvocation.OnStopAntlrParsing(
                    AntlrPerformanceProfiler != null
                        ? (int)AntlrPerformanceProfiler.CurrentFileInfo.DecisionTimeMs
                        : 0,
                    AntlrPerformanceProfiler != null
                        ? AntlrPerformanceProfiler.CurrentFileInfo.RuleInvocations.Sum()
                        : 0);
            }
            catch (Exception e)
            {
                var currentToken = (Token)cobolParser.CurrentToken;
                CodeElementsLine codeElementsLine = GetCodeElementsLineForToken(currentToken);
                if (codeElementsLine != null)
                {
                    codeElementsLine.AddParserDiagnostic(new TokenDiagnostic(MessageCode.ImplementationError,
                                                                             currentToken, currentToken.Line, e));
                }
            }

            if (codeElementsParseTree != null)
            {
                // If the parse tree is not empty
                if (codeElementsParseTree.codeElement() != null && codeElementsParseTree.codeElement().Length > 0)
                {
                    // Analyze the parse tree for each code element
                    foreach (var codeElementParseTree in codeElementsParseTree.codeElement())
                    {
                        // Get the first line that was parsed
                        var tokenStart = (Token)codeElementParseTree.Start;
                        CodeElementsLine codeElementsLine = GetCodeElementsLineForToken(tokenStart);
                        if (codeElementsLine == null)
                        {
                            continue;
                        }

                        // Register that this line was updated
                        // COMMENTED FOR THE SAKE OF PERFORMANCE -- SEE ISSUE #160
                        //int updatedLineIndex = documentLines.IndexOf(codeElementsLine, codeElementsLine.LineIndex);
                        //codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, updatedLineIndex, codeElementsLine));
                        codeElementsLinesChanges.Add(
                            new DocumentChange <ICodeElementsLine>(DocumentChangeType.LineUpdated,
                                                                   codeElementsLine.LineIndex, codeElementsLine));

                        perfStatsForParserInvocation.OnStartTreeBuilding();
                        // Visit the parse tree to build a first class object representing the code elements
                        try
                        {
                            walker.Walk(codeElementBuilder, codeElementParseTree);
                        }
                        catch (Exception ex)
                        {
                            var code  = MessageCode.ImplementationError;
                            int line  = 0;
                            int start = 0;
                            int stop  = 0;
                            if (codeElementsLine.SourceTokens != null && codeElementsLine.SourceTokens.Count > 0)
                            {
                                start = codeElementsLine.SourceTokens[0].StartIndex;
                                stop  =
                                    codeElementsLine.SourceTokens[codeElementsLine.SourceTokens.Count - 1].StopIndex;
                            }
                            codeElementsLine.AddParserDiagnostic(new ParserDiagnostic(ex.ToString(), start, stop,
                                                                                      line, null, code, ex));
                        }
                        CodeElement codeElement = codeElementBuilder.CodeElement;
                        if (codeElement != null)
                        {
                            // Attach consumed tokens and main document line numbers information to the code element
                            if (codeElement.ConsumedTokens.Count == 0)
                            {
// ISSUE #204:
                                var tempToken = tokenStream.Lt(1);
                                if (tempToken != null && tempToken != Token.END_OF_FILE)
                                {
// if not end of file,
                                    // add next token to ConsumedTokens to know where is the CodeElement in error
                                    codeElement.ConsumedTokens.Add((Token)tempToken);
                                    // this alter CodeElements semantics: in addition to matched tokens,
                                    // it includes the first token in error if no token has been matched
                                }
                            }

                            //TODO Issue #384 to discuss if this code should stay here:
                            //This should be in a Checker, but "codeElement.ConsumedTokens" is only set after all the checkers have been called
                            //Rule TCLIMITATION_NO_CE_ACROSS_SOURCES
                            if (codeElement.IsAcrossSourceFile())
                            {
                                DiagnosticUtils.AddError(codeElement,
                                                         "A Cobol statement cannot be across 2 sources files (eg. Main program and a COPY)",
                                                         MessageCode.TypeCobolParserLimitation);
                            }

                            // Add code element to the list
                            codeElementsLine.AddCodeElement(codeElement);
                        }
                    }
                }
                // If the parse tree contains errors
                if (codeElementsParseTree.Diagnostics != null)
                {
                    foreach (ParserDiagnostic d in codeElementsParseTree.Diagnostics)
                    {
                        if (d.OffendingSymbol != null)
                        {
                            CodeElementsLine codeElementsLine =
                                GetCodeElementsLineForToken((Token)d.OffendingSymbol);
                            if (codeElementsLine != null)
                            {
                                codeElementsLine.AddParserDiagnostic(d);
                            }
                        }
                    }
                }
                perfStatsForParserInvocation.OnStopTreeBuilding();
            }


            if (AntlrPerformanceProfiler != null)
            {
                AntlrPerformanceProfiler.EndParsingFile(cobolParser.ParseInfo.DecisionInfo, (int)(cobolParser.ParseInfo.GetTotalTimeInPrediction() / 1000000));
            }

            return(codeElementsLinesChanges);
        }
Ejemplo n.º 3
0
        public static void ParseProgramOrClass(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <CodeElementsLine> codeElementsLines, TypeCobolOptions compilerOptions, SymbolTable customSymbols, PerfStatsForParserInvocation perfStatsForParserInvocation, out SourceFile root, out IList <ParserDiagnostic> diagnostics, out Dictionary <CodeElement, Node> nodeCodeElementLinkers)
        {
            // Create an Antlr compatible token source on top a the token iterator
            CodeElementsLinesTokenSource tokenSource = new CodeElementsLinesTokenSource(
                textSourceInfo.Name,
                codeElementsLines);

            // Init parser
            ITokenStream       tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
            ProgramClassParser cobolParser = new ProgramClassParser(tokenStream);

            // -> activate full ambiguities detection
            //parser.Interpreter.PredictionMode = PredictionMode.LlExactAmbigDetection;

            // Optionnaly activate Antlr Parser performance profiling
            // WARNING : use this in a single-treaded context only (uses static field)
            if (AntlrPerformanceProfiler == null && perfStatsForParserInvocation.ActivateDetailedAntlrPofiling)
            {
                AntlrPerformanceProfiler = new AntlrPerformanceProfiler(cobolParser);
            }
            if (AntlrPerformanceProfiler != null)
            {
                // Replace the generated parser by a subclass which traces all rules invocations
                cobolParser = new ProgramClassTracingParser(tokenStream);
                AntlrPerformanceProfiler.BeginParsingFile(textSourceInfo, null);
            }

            // Register all parse errors in a list in memory
            ParserDiagnosticErrorListener errorListener = new ParserDiagnosticErrorListener();

            cobolParser.RemoveErrorListeners();
            cobolParser.AddErrorListener(errorListener);

            // Try to parse a Cobol program or class
            perfStatsForParserInvocation.OnStartAntlrParsing();
            if (AntlrPerformanceProfiler != null)
            {
                AntlrPerformanceProfiler.BeginParsingSection();
            }
            ProgramClassParser.CobolCompilationUnitContext programClassParseTree = cobolParser.cobolCompilationUnit();
            if (AntlrPerformanceProfiler != null)
            {
                AntlrPerformanceProfiler.EndParsingSection(programClassParseTree.ChildCount);
            }
            perfStatsForParserInvocation.OnStopAntlrParsing(
                AntlrPerformanceProfiler != null ? (int)AntlrPerformanceProfiler.CurrentFileInfo.DecisionTimeMs : 0,
                AntlrPerformanceProfiler != null ? AntlrPerformanceProfiler.CurrentFileInfo.RuleInvocations.Sum() : 0);

            if (AntlrPerformanceProfiler != null)
            {
                AntlrPerformanceProfiler.EndParsingFile(cobolParser.ParseInfo.DecisionInfo, (int)(cobolParser.ParseInfo.GetTotalTimeInPrediction() / 1000000));
            }


            // Visit the parse tree to build a first class object representing a Cobol program or class
            ParseTreeWalker  walker = new ParseTreeWalker();
            CobolNodeBuilder programClassBuilder = new CobolNodeBuilder();

            programClassBuilder.SyntaxTree    = new SyntaxTree(); //Initializie SyntaxTree for the current source file
            programClassBuilder.CustomSymbols = customSymbols;
            programClassBuilder.Dispatcher    = new NodeDispatcher();
            programClassBuilder.Dispatcher.CreateListeners();

            perfStatsForParserInvocation.OnStartTreeBuilding();

            ParserDiagnostic programClassBuilderError = null;

            try { walker.Walk(programClassBuilder, programClassParseTree); }
            catch (Exception ex)
            {
                var code = Diagnostics.MessageCode.ImplementationError;
                programClassBuilderError = new ParserDiagnostic(ex.ToString(), null, null, code, ex);
            }

            //Create link between datas
            programClassBuilder.SyntaxTree.Root.AcceptASTVisitor(new TypeCobolLinker());

            perfStatsForParserInvocation.OnStopTreeBuilding();

            //Complete some information on Node and run checker that need a full AST
            programClassBuilder.SyntaxTree.Root.AcceptASTVisitor(new Cobol85CompleteASTChecker());


            // Register compiler results
            root                   = programClassBuilder.SyntaxTree.Root; //Set output root node
            diagnostics            = programClassBuilder.GetDiagnostics(programClassParseTree);
            nodeCodeElementLinkers = programClassBuilder.NodeCodeElementLinkers;

            if (programClassBuilderError != null)
            {
                if (diagnostics == null)
                {
                    diagnostics = new List <ParserDiagnostic>();
                }
                diagnostics.Add(programClassBuilderError);
            }
        }