public static void ParseProgramOrClass(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<CodeElementsLine> codeElementsLines, TypeCobolOptions compilerOptions, SymbolTable customSymbols, out Program newProgram, out Class newClass, out IList<ParserDiagnostic> diagnostics) { // Create an Antlr compatible token source on top a the token iterator CodeElementsLinesTokenSource tokenSource = new CodeElementsLinesTokenSource( textSourceInfo.Name, codeElementsLines); // Init parser ITokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens); ProgramClassParser cobolParser = new ProgramClassParser(tokenStream); // -> activate full ambiguities detection //parser.Interpreter.PredictionMode = PredictionMode.LlExactAmbigDetection; // Register all parse errors in a list in memory DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener(); cobolParser.RemoveErrorListeners(); cobolParser.AddErrorListener(errorListener); // Try to parse a Cobol program or class ProgramClassParser.CobolCompilationUnitContext codeElementParseTree = cobolParser.cobolCompilationUnit(); // Visit the parse tree to build a first class object representing a Cobol program or class ParseTreeWalker walker = new ParseTreeWalker(); CobolNodeBuilder programClassBuilder = new CobolNodeBuilder(); programClassBuilder.CustomSymbols = customSymbols; programClassBuilder.Dispatcher = new NodeDispatcher(); programClassBuilder.Dispatcher.CreateListeners(); walker.Walk(programClassBuilder, codeElementParseTree); // Register compiler results newProgram = programClassBuilder.Program; newClass = programClassBuilder.Class; diagnostics = errorListener.Diagnostics; }
/// <summary> /// Initializes a new compilation document from a list of text lines. /// This method does not scan the inserted text lines to produce tokens. /// You must explicitely call UpdateTokensLines() to start an initial scan of the document. /// </summary> public CompilationUnit(TextSourceInfo textSourceInfo, IEnumerable<ITextLine> initialTextLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider) : base(textSourceInfo, initialTextLines, compilerOptions, processedTokensDocumentProvider) { // Initialize performance stats PerfStatsForCodeElementsParser = new PerfStatsForCompilationStep(CompilationStep.CodeElementsParser); PerfStatsForProgramClassParser = new PerfStatsForCompilationStep(CompilationStep.ProgramClassParser); }
// --- Initialization --- /// <summary> /// Initializes a new compilation document from a list of text lines. /// This method does not scan the inserted text lines to produce tokens. /// You must explicitely call UpdateTokensLines() to start an initial scan of the document. /// </summary> public CompilationDocument(TextSourceInfo textSourceInfo, IEnumerable<ITextLine> initialTextLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider) { TextSourceInfo = textSourceInfo; CompilerOptions = compilerOptions; this.processedTokensDocumentProvider = processedTokensDocumentProvider; // Initialize the compilation document lines compilationDocumentLines = ImmutableList<CodeElementsLine>.Empty.ToBuilder(); // ... with the initial list of text lines received as a parameter if (initialTextLines != null) { // Insert Cobol text lines in the internal tree structure compilationDocumentLines.AddRange(initialTextLines.Select(textLine => CreateNewDocumentLine(textLine, textSourceInfo.ColumnsLayout))); } // Initialize document views versions currentTextLinesVersion = new DocumentVersion<ICobolTextLine>(this); currentTokensLinesVersion = new DocumentVersion<ITokensLine>(this); // Initialize performance stats PerfStatsForText = new PerfStatsForCompilationStep(CompilationStep.Text); PerfStatsForScanner = new PerfStatsForCompilationStep(CompilationStep.Scanner); PerfStatsForPreprocessor = new PerfStatsForCompilationStep(CompilationStep.Preprocessor); }
/// <summary> /// Initialize a cobol document from any source of characters /// </summary> /// <param name="fileName">Name of the file the document is stored in</param> /// <param name="textSource">Sequence of unicode characters with line delimiters (Cr? Lf)</param> public ReadOnlyTextDocument(string fileName, Encoding encodingForAlphanumericLiterals, ColumnsLayout columnsLayout, IEnumerable<char> textSource) { // Document source name and text format Source = new TextSourceInfo(fileName, encodingForAlphanumericLiterals, columnsLayout); // Initialize document text lines LoadChars(textSource); }
public AvalonEditTextDocument(ICSharpCode.AvalonEdit.Document.TextDocument avalonEditTextDocument, Encoding encodingForAlphanumericLiterals, ColumnsLayout columnsLayout) { // Document source name and text format Source = new TextSourceInfo(_avalonEditTextDocument.FileName, encodingForAlphanumericLiterals, columnsLayout); _avalonEditTextDocument = avalonEditTextDocument; // Listen to all line changes in the editor _avalonEditTextDocument.VerifyAccess(); _weakLineTracker = WeakLineTracker.Register(_avalonEditTextDocument, this); }
/// <summary> /// Incremental preprocessing of a set of tokens lines changes /// </summary> internal static IList<DocumentChange<IProcessedTokensLine>> ProcessTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<ProcessedTokensLine> documentLines, IList<DocumentChange<ITokensLine>> tokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider) { // Collect all changes applied to the processed tokens lines during the incremental scan IList<DocumentChange<IProcessedTokensLine>> processedTokensLinesChanges = new List<DocumentChange<IProcessedTokensLine>>(); // There are 2 reasons to a preprocess a tokens line after a change : // 1. A tokens line changed : these lines were already reset during the previous steps // 2. If a tokens line that changed was involved in the parsing of a multiline compiler directive, the whole group of lines must be parsed again // Then, if a new COPY directive was parsed : the CompilationDocument to include must be prepared // --- PREPARATION PHASE : reset all processed tokens lines which were involved in a multiline compiler directive where at least one line changed --- // Iterate over all tokens changes detected by the ScannerStep : // refresh all the adjacent lines participating in a ContinuationTokensGroup if (tokensLinesChanges != null) { int lastLineIndexReset = -1; foreach (DocumentChange<ITokensLine> tokensChange in tokensLinesChanges) { processedTokensLinesChanges.Add(new DocumentChange<IProcessedTokensLine>(tokensChange.Type, tokensChange.LineIndex, (IProcessedTokensLine)tokensChange.NewLine)); if (tokensChange.LineIndex > lastLineIndexReset) { lastLineIndexReset = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, processedTokensLinesChanges, lastLineIndexReset); } } } // --- COMPILER DIRECTIVES PHASE : Find and parse all compiler directives --- // Init. Prepare a compiler directive parser // Create a token iterator on top of tokens lines TokensLinesIterator tokensIterator = new TokensLinesIterator( textSourceInfo.Name, documentLines, null, Token.CHANNEL_SourceTokens); // Crate an Antlr compatible token source on top a the token iterator TokensLinesTokenSource tokenSource = new TokensLinesTokenSource( textSourceInfo.Name, tokensIterator); // Init a compiler directive parser CommonTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens); CobolCompilerDirectivesParser directivesParser = new CobolCompilerDirectivesParser(tokenStream); IAntlrErrorStrategy compilerDirectiveErrorStrategy = new CompilerDirectiveErrorStrategy(); directivesParser.ErrorHandler = compilerDirectiveErrorStrategy; // Register all parse errors in a list in memory DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener(); directivesParser.RemoveErrorListeners(); directivesParser.AddErrorListener(errorListener); // Prepare to analyze the parse tree ParseTreeWalker walker = new ParseTreeWalker(); CompilerDirectiveBuilder directiveBuilder = new CompilerDirectiveBuilder(); // 1. Iterate over all compiler directive starting tokens found in the lines which were updated foreach (Token compilerDirectiveStartingToken in documentLines .Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing) .SelectMany(line => line.SourceTokens) .Where(token => token.TokenFamily == TokenFamily.CompilerDirectiveStartingKeyword)) { // 2. Reset the compiler directive parser state // Reset tokens iterator position before parsing // -> seek just before the compiler directive starting token tokensIterator.SeekToToken(compilerDirectiveStartingToken); tokensIterator.PreviousToken(); // Special case : for efficiency reasons, in EXEC SQL INCLUDE directives // only the third token INCLUDE is recognized as a compiler directive // starting keyword by the scanner. In this case, we must rewind the // iterator two tokens backwards to start parsing just before the EXEC token. if (compilerDirectiveStartingToken.TokenType == TokenType.EXEC_SQL_INCLUDE) { tokensIterator.PreviousToken(); tokensIterator.PreviousToken(); } // Reset Antlr BufferedTokenStream position tokenStream.SetTokenSource(tokenSource); // Reset parsing error diagnostics compilerDirectiveErrorStrategy.Reset(directivesParser); errorListener.Diagnostics.Clear(); // 3. Try to parse a compiler directive starting with the current token CobolCompilerDirectivesParser.CompilerDirectingStatementContext directiveParseTree = directivesParser.compilerDirectingStatement(); // 4. Visit the parse tree to build a first class object representing the compiler directive walker.Walk(directiveBuilder, directiveParseTree); CompilerDirective compilerDirective = directiveBuilder.CompilerDirective; bool errorFoundWhileParsingDirective = errorListener.Diagnostics.Count > 0 || directiveBuilder.Diagnostics.Count > 0; // 5. Get all tokens consumed while parsing the compiler directive // and partition them line by line Token startToken = (Token)directiveParseTree.Start; Token stopToken = (Token)directiveParseTree.Stop; if (stopToken == null) stopToken = startToken; MultilineTokensGroupSelection tokensSelection = tokensIterator.SelectAllTokensBetween(startToken, stopToken); if (compilerDirective != null) { // 6. Replace all matched tokens by : // - a CompilerDirectiveToken on the first line ProcessedTokensLine firstProcessedTokensLine = documentLines[tokensSelection.FirstLineIndex]; if (tokensSelection.SelectedTokensOnSeveralLines.Length == 1) { firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine( tokensSelection.TokensOnFirstLineBeforeStartToken, compilerDirective, errorFoundWhileParsingDirective, tokensSelection.SelectedTokensOnSeveralLines[0], tokensSelection.TokensOnLastLineAfterStopToken, false); } else { TokensGroup continuedTokensGroup = firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine( tokensSelection.TokensOnFirstLineBeforeStartToken, compilerDirective, errorFoundWhileParsingDirective, tokensSelection.SelectedTokensOnSeveralLines[0], null, true); // - a ContinuationTokensGroup on the following lines int selectionLineIndex = 1; int lastLineIndex = tokensSelection.FirstLineIndex + tokensSelection.SelectedTokensOnSeveralLines.Length - 1; for (int nextLineIndex = tokensSelection.FirstLineIndex + 1; nextLineIndex <= lastLineIndex; nextLineIndex++, selectionLineIndex++) { IList<Token> compilerDirectiveTokensOnNextLine = tokensSelection.SelectedTokensOnSeveralLines[selectionLineIndex]; if (compilerDirectiveTokensOnNextLine.Count > 0) { ProcessedTokensLine nextProcessedTokensLine = documentLines[nextLineIndex]; if (nextLineIndex != lastLineIndex) { continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine( continuedTokensGroup, compilerDirectiveTokensOnNextLine, null, true); } else { continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine( continuedTokensGroup, compilerDirectiveTokensOnNextLine, tokensSelection.TokensOnLastLineAfterStopToken, false); } } } } } // 7. Register compiler directive parse errors if (errorFoundWhileParsingDirective) { ProcessedTokensLine compilerDirectiveLine = documentLines[tokensSelection.FirstLineIndex]; foreach (ParserDiagnostic parserDiag in errorListener.Diagnostics) { compilerDirectiveLine.AddDiagnostic(parserDiag); } foreach (Diagnostic directiveDiag in directiveBuilder.Diagnostics) { compilerDirectiveLine.AddDiagnostic(directiveDiag); } } } // 8. Advance the state off all ProcessedTokensLines : // NeedsCompilerDirectiveParsing => NeedsCopyDirectiveProcessing if it contains a COPY directive IList<ProcessedTokensLine> parsedLinesWithCopyDirectives = null; // NeedsCompilerDirectiveParsing => Ready otherwise foreach (ProcessedTokensLine parsedLine in documentLines .Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing)) { if (parsedLine.ImportedDocuments != null) { if(parsedLinesWithCopyDirectives == null) { parsedLinesWithCopyDirectives = new List<ProcessedTokensLine>(); } parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.NeedsCopyDirectiveProcessing; parsedLinesWithCopyDirectives.Add(parsedLine); } else { parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready; } } // --- COPY IMPORT PHASE : Process COPY (REPLACING) directives --- // 1. Iterate over all updated lines containing a new COPY directive if (parsedLinesWithCopyDirectives != null) { foreach (ProcessedTokensLine tokensLineWithCopyDirective in parsedLinesWithCopyDirectives) { // Iterate over all COPY directives found on one updated line foreach (CopyDirective copyDirective in tokensLineWithCopyDirective.ImportedDocuments.Keys.ToArray<CopyDirective>()) { try { // Load (or retrieve in cache) the document referenced by the COPY directive ProcessedTokensDocument importedDocumentSource = processedTokensDocumentProvider.GetProcessedTokensDocument(copyDirective.LibraryName, copyDirective.TextName); // Store it on the current line after appying the REPLACING directive ImportedTokensDocument importedDocument = new ImportedTokensDocument(copyDirective, importedDocumentSource); tokensLineWithCopyDirective.ImportedDocuments[copyDirective] = importedDocument; } catch (Exception e) { // Text name refenced by COPY directive was not found // => register a preprocessor error on this line Token failedDirectiveToken = tokensLineWithCopyDirective.TokensWithCompilerDirectives .Where(token => token.TokenType == TokenType.CopyImportDirective && ((CompilerDirectiveToken)token).CompilerDirective == copyDirective) .First(); Diagnostic diag = new Diagnostic( MessageCode.FailedToLoadTextDocumentReferencedByCopyDirective, failedDirectiveToken.Column, failedDirectiveToken.EndColumn, e.Message); tokensLineWithCopyDirective.AddDiagnostic(diag); } } // Advance processing status of the line tokensLineWithCopyDirective.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready; } } // --- REPLACE PHASE : REPLACE directives are implemented in ReplaceTokensLinesIterator --- /* Algorithm : * * one REPLACE directive can express several replacement operations * one replacement operation can be of several types (distinguished for optimization purposes) * - SimpleTokenReplace : one source token / zero or one replacement token * - PartialWordReplace : one pure partial word / zero or one replacement token * - SimpleToMultipleTokenReplace : one source token / several replacement tokens * - MultipleTokenReplace : one first + several following source tokens / zero to many replacement tokens * * an iterator maintains a current set of replacement operations * * if nextToken is replace directive * the current set of replacement operations is updated * else * nextToken is compared to each replacement operation in turn * if single -> single source token operation : return replacement token * if single -> multiple operation : setup a secondary iterator with the list of replacement tokens * if multiple -> multiple operation * snapshot of the underlying iterator * try to match all of the following source tokens * if failure : restore snapshot and try next operation * if success : setup a secondary iterator * * token comparison sourceToken / replacementCandidate : * 1. Compare Token type * 2. If same token type and for families * AlphanumericLiteral * NumericLiteral * SyntaxLiteral * Symbol * => compare also Token text * * PartialCobolWord replacement : * p535 : The COPY statement with REPLACING phrase can be used to replace parts of words. * By inserting a dummy operand delimited by colons into the program text, * the compiler will replace the dummy operand with the required text. * Example 3 shows how this is used with the dummy operand :TAG:. * The colons serve as separators and make TAG a stand-alone operand. */ return processedTokensLinesChanges; }
/// <summary> /// Initial preprocessing of a complete document /// </summary> internal static void ProcessDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<ProcessedTokensLine> documentLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider) { ProcessTokensLinesChanges(textSourceInfo, documentLines, null, null, compilerOptions, processedTokensDocumentProvider); }
private static void ScanTokensLineWithMultilineScanState(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IList<DocumentChange<ITokensLine>> tokensLinesChanges, out int nextLineToScanIndex, out TokensLine nextLineToScan) { // Scan the current line (or continuation lines group) MultilineScanState scanState = ScanTokensLineWithContinuations(lineToScanIndex, lineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan); // Scan the following lines until we find that the scan state at the beginning of the next line has been updated while(nextLineToScan != null && nextLineToScan.InitialScanState != null && !nextLineToScan.InitialScanState.Equals(scanState)) { scanState = ScanTokensLineWithContinuations(nextLineToScanIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan); } }
private static MultilineScanState ScanTokensLineWithContinuations(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IList<DocumentChange<ITokensLine>> tokensLinesChanges, out int nextLineToScanIndex, out TokensLine nextLineToScan) { // Initialize out parameters if (lineToScanIndex < (documentLines.Count - 1)) { nextLineToScanIndex = lineToScanIndex + 1; nextLineToScan = documentLines[nextLineToScanIndex]; } else { nextLineToScanIndex = -1; nextLineToScan = null; } // Check if the line to scan participates in a multiline continuation // - because it is itself a continuation line bool partOfMultilineContinuation = lineToScan.Type == CobolTextLineType.Continuation; // - or because the following line is a continuation line if (!partOfMultilineContinuation && lineToScanIndex < (documentLines.Count - 1)) { nextLineToScanIndex = lineToScanIndex + 1; nextLineToScan = documentLines[nextLineToScanIndex]; partOfMultilineContinuation = nextLineToScan.Type == CobolTextLineType.Continuation; } // ^-- LIMITATION : we don't support the case where one or more comment lines follow the current line before a continuation line // Case 1 : the line is not part of a multiline continuation => we can parse it as as a standalone line if (!partOfMultilineContinuation) { // Create a new copy of the line before the update if necessary lineToScan = (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner); if (lineToScanIndex == 0) { // Scan the first line of the document Scanner.ScanFirstLine(lineToScan, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions); } else { // Get the scan state at the end of the previous line TokensLine previousLine = documentLines[lineToScanIndex - 1]; // Scan the current line with this initial scan state Scanner.ScanTokensLine(lineToScan, previousLine.ScanState, compilerOptions); } tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineUpdated, lineToScanIndex, lineToScan)); return lineToScan.ScanState; } // Case 2 : the line is part of a multiline continuation => we must parse all continuation lines as a group else { // Build a list of the lines we will have to scan as a group : IList<TokensLine> continuationLinesGroup = new List<TokensLine>(); int firstLineIndex = lineToScanIndex; continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); // Navigate backwards to the start of the multiline continuation if (lineToScan.Type == CobolTextLineType.Continuation && lineToScanIndex > 0) { int revLineToScanIndex = lineToScanIndex; IEnumerator<TokensLine> reversedEnumerator = documentLines.GetEnumerator(lineToScanIndex - 1, -1, true); while(reversedEnumerator.MoveNext()) { // Get the previous line until a non continuation and non comment line is encountered revLineToScanIndex--; lineToScan = reversedEnumerator.Current; if(lineToScan.Type != CobolTextLineType.Continuation /*&& <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line lineToScan.Type != CobolTextLineType.Comment && lineToScan.Type != CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines { firstLineIndex = revLineToScanIndex; continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); } else { break; } } } // Reuse our knowledge of the next line if it is available and if it is a continuation if (nextLineToScan != null && nextLineToScan.Type == CobolTextLineType.Continuation) { lineToScanIndex++; lineToScan = nextLineToScan; continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); nextLineToScanIndex = -1; nextLineToScan = null; } // Navigate forwards to the end of the multiline continuation if (lineToScanIndex < (documentLines.Count - 1)) { IEnumerator<TokensLine> enumerator = documentLines.GetEnumerator(lineToScanIndex + 1, -1, false); while (enumerator.MoveNext()) { // Get the next line until a non continuation and non comment line is encountered lineToScanIndex++; lineToScan = enumerator.Current; if (lineToScan.Type == CobolTextLineType.Continuation /*|| <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line lineToScan.Type == CobolTextLineType.Comment || lineToScan.Type == CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines { // Add this line at the end of the list of continuation lines continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); } else { // Save the knowledge of the next line and exit the loop nextLineToScanIndex = lineToScanIndex; nextLineToScan = lineToScan; break; } } } // Scan the group of continuation lines if (firstLineIndex == 0) { // Scan the first line group of the document Scanner.ScanFirstLineContinuationGroup(continuationLinesGroup, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions); } else { // Get the scan state at the end of the previous line TokensLine previousLine = documentLines[firstLineIndex - 1]; // Scan the current line group with this initial scan state Scanner.ScanTokensLineContinuationGroup(continuationLinesGroup, previousLine.ScanState, compilerOptions); } int updatedLineIndex = firstLineIndex; foreach(TokensLine updatedLine in continuationLinesGroup) { tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineUpdated, updatedLineIndex, updatedLine)); updatedLineIndex++; } return continuationLinesGroup[continuationLinesGroup.Count - 1].ScanState; } }
/// <summary> /// Initial scan of a complete document /// </summary> public static void ScanDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, TypeCobolOptions compilerOptions) { TokensLine tokensLine = null; TokensLine nextTokensLine = null; MultilineScanState lastScanState = null; // Get the first line IEnumerator<TokensLine> documentLinesEnumerator = documentLines.GetEnumerator(); if (documentLinesEnumerator.MoveNext()) { tokensLine = documentLinesEnumerator.Current; } while (tokensLine != null) { // Peek the next line to look for continuations if (documentLinesEnumerator.MoveNext()) { nextTokensLine = documentLinesEnumerator.Current; } else { nextTokensLine = null; } // If no continuation is found, scan the current line if (nextTokensLine == null || nextTokensLine.Type != CobolTextLineType.Continuation) { if (lastScanState == null) { Scanner.ScanFirstLine(tokensLine, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions); } else { Scanner.ScanTokensLine(tokensLine, lastScanState, compilerOptions); } } // If a continuation is found on the next line, scan the continuation lines as a group else { // Build a list of the lines we will have to scan as a group : IList<TokensLine> continuationLinesGroup = new List<TokensLine>(); // Add current line continuationLinesGroup.Add(tokensLine); // Add next line tokensLine = nextTokensLine; nextTokensLine = null; continuationLinesGroup.Add(tokensLine); // Navigate forwards to the end of the multiline continuation while (documentLinesEnumerator.MoveNext()) { nextTokensLine = documentLinesEnumerator.Current; if (nextTokensLine.Type == CobolTextLineType.Continuation /*|| <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line lineToScan.Type == CobolTextLineType.Comment || lineToScan.Type == CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines { // Add this line at the end of the list of continuation lines tokensLine = nextTokensLine; nextTokensLine = null; continuationLinesGroup.Add(tokensLine); } else { // Exit the loop break; } } // Scan the whole group of continuation lines if (lastScanState == null) { Scanner.ScanFirstLineContinuationGroup(continuationLinesGroup, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions); } else { Scanner.ScanTokensLineContinuationGroup(continuationLinesGroup, lastScanState, compilerOptions); } } lastScanState = tokensLine.ScanState; tokensLine = nextTokensLine; } }
/// <summary> /// Incremental scan of a set of text lines changes /// </summary> internal static IList<DocumentChange<ITokensLine>> ScanTextLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, IList<DocumentChange<ICobolTextLine>> textLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions) { // Collect all changes applied to the tokens lines during the incremental scan IList<DocumentChange<ITokensLine>> tokensLinesChanges = new List<DocumentChange<ITokensLine>>(); // There are 3 reasons to scan a line after a text change : // 1. New text lines which were just inserted or updated must be scanned for the first time // 2. Text lines must be scanned again if their initial scan state changed : a new scan of the previous line can alter the scan state at the beginning of the following line // 3. Continuation lines and multiline tokens : if a line participates in a continuation on several lines, scan the group of lines as a whole // IMPORTANT : the text changes are ordered in increasing order of line index for (int textChangeIndex = 0; textChangeIndex < textLinesChanges.Count; textChangeIndex++) { // Local variables used to optimize navigation in the document int nextLineToScanIndex = -1; TokensLine nextLineToScan = null; // Update tokens depending on the current text change DocumentChange<ICobolTextLine> textChange = textLinesChanges[textChangeIndex]; if (textChange.Type == DocumentChangeType.DocumentCleared) { tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.DocumentCleared, 0, null)); continue; } else if (textChange.Type == DocumentChangeType.LineInserted || textChange.Type == DocumentChangeType.LineUpdated) { // We update lines as a group below, but we remember here which lines were inserted if(textChange.Type == DocumentChangeType.LineInserted) { tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineInserted, textChange.LineIndex, (ITokensLine)textChange.NewLine)); } // Text lines which were inserted or updated must be scanned again ScanTokensLineWithMultilineScanState(textChange.LineIndex, (TokensLine)textChange.NewLine, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan); } else if (textChange.Type == DocumentChangeType.LineRemoved) { tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineRemoved, textChange.LineIndex, (ITokensLine)textChange.NewLine)); // Get the last line just before the line that was removed TokensLine previousLine = null; if (textChange.LineIndex > 0) { previousLine = documentLines[textChange.LineIndex - 1]; } // When a text line is removed : // - the previous line must be scanned again if the line which was removed was a member of a multiline continuation group if (previousLine != null && previousLine.HasTokenContinuedOnNextLine) { ScanTokensLineWithMultilineScanState(textChange.LineIndex - 1, previousLine, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan); } if (nextLineToScan == null && textChange.LineIndex < documentLines.Count) { nextLineToScanIndex = textChange.LineIndex; nextLineToScan = documentLines[nextLineToScanIndex]; } // - the next line must be scanned again if the scan state at the end of the previous line is different from the scan state at the beginning of the next line if (nextLineToScan != null && nextLineToScanIndex == textChange.LineIndex && previousLine != null && nextLineToScan.InitialScanState.Equals(previousLine.ScanState)) { ScanTokensLineWithMultilineScanState(textChange.LineIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan); } } // We can skip all text changes with an index smaller than the index of the last line which was already scanned if (nextLineToScan == null) { break; } else if (textChangeIndex < (textLinesChanges.Count - 1)) { int nextTextChangeIndex = textChangeIndex; DocumentChange<ICobolTextLine> nextTextChange = null; do { nextTextChangeIndex++; nextTextChange = textLinesChanges[nextTextChangeIndex]; } while (nextTextChangeIndex < (textLinesChanges.Count - 1) && nextTextChange.LineIndex <= nextLineToScanIndex); textChangeIndex = nextTextChangeIndex - 1; } } return tokensLinesChanges; }
/// <summary> /// Initial scan of a complete document /// </summary> public static void ScanDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, TypeCobolOptions compilerOptions) { ScanDocument(textSourceInfo, documentLines, compilerOptions, null); }
/// <summary> /// Incremental scan of a set of text lines changes /// </summary> internal static IList<DocumentChange<ITokensLine>> ScanTextLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, IList<DocumentChange<ICobolTextLine>> textLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions) { return ScanTextLinesChanges(textSourceInfo, documentLines, textLinesChanges, prepareDocumentLineForUpdate, compilerOptions, null); }
/// <summary> /// Incremental parsing of a set of processed tokens lines changes /// </summary> internal static IList<DocumentChange<ICodeElementsLine>> ParseProcessedTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<CodeElementsLine> documentLines, IList<DocumentChange<IProcessedTokensLine>> processedTokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions) { // Collect all changes applied to the processed tokens lines during the incremental scan IList<DocumentChange<ICodeElementsLine>> codeElementsLinesChanges = new List<DocumentChange<ICodeElementsLine>>(); // There are 2 reasons to re-parse a tokens line after a change : // 1. The tokens line changed : these lines were already reset during the previous steps // 2. If a tokens line that changed was involved in the parsing of a multiline code element, the whole group of lines must be parsed again // --- PREPARATION PHASE : identify all parse sections where code elements need to be refreshed --- IList<ParseSection> refreshParseSections = null; // Iterate over all processed tokens changes detected by the PreprocessorStep : // - refresh all the adjacent lines participating in a CodeElement // - register the start and stop token for all sections of the document which need to be parsed again if (processedTokensLinesChanges != null) { // If the document was cleared, everything must be parsed again if (processedTokensLinesChanges[0].Type != DocumentChangeType.DocumentCleared) { refreshParseSections = new List<ParseSection>(); ParseSection lastParseSection = null; foreach (DocumentChange<IProcessedTokensLine> tokensChange in processedTokensLinesChanges) { if (lastParseSection == null || tokensChange.LineIndex > lastParseSection.StopLineIndex) { lastParseSection = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, codeElementsLinesChanges, lastParseSection); refreshParseSections.Add(lastParseSection); } } } } // --- INITIALIZE ANTLR CodeElements parser --- // Create a token iterator on top of pre-processed tokens lines ITokensLinesIterator tokensIterator = ProcessedTokensDocument.GetProcessedTokensIterator(textSourceInfo, documentLines); // Create an Antlr compatible token source on top of the token iterator TokensLinesTokenSource tokenSource = new TokensLinesTokenSource( textSourceInfo.Name, tokensIterator); // Init parser TokensLinesTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens); TracingCobolParser cobolParser = new TracingCobolParser(tokenStream); // -> activate full ambiguities detection //parser.Interpreter.PredictionMode = PredictionMode.LlExactAmbigDetection; IAntlrErrorStrategy cobolErrorStrategy = new CobolErrorStrategy(); cobolParser.ErrorHandler = cobolErrorStrategy; // Register all parse errors in a list in memory DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener(); cobolParser.RemoveErrorListeners(); cobolParser.AddErrorListener(errorListener); // Prepare to analyze the parse tree ParseTreeWalker walker = new ParseTreeWalker(); CodeElementBuilder codeElementBuilder = new CodeElementBuilder(); codeElementBuilder.Dispatcher = new CodeElementDispatcher(); codeElementBuilder.Dispatcher.CreateListeners(); // --- INCREMENTAL PARSING --- // In case of incremental parsing, parse only the code sections we need to refresh IEnumerator<ParseSection> parseSectionsEnumerator = null; ParseSection currentParseSection = null; if (refreshParseSections != null) { // Get the first code section we need to refresh parseSectionsEnumerator = refreshParseSections.GetEnumerator(); parseSectionsEnumerator.MoveNext(); currentParseSection = parseSectionsEnumerator.Current; // Seek just before the next code element starting token tokenStream.SeekToToken(currentParseSection.StartToken); tokenStream.StartLookingForStopToken(currentParseSection.StopToken); } // Parse one CodeElement at a time while advancing in the underlying token stream do { // Reset parsing error diagnostics //cobolErrorStrategy.Reset(cobolParser); errorListener.Diagnostics.Clear(); // Reset parser traces (consumed tokens) cobolParser.ResetTraces(); // Try to parse a code element starting with the current token CodeElementsParser.CodeElementContext codeElementParseTree = cobolParser.codeElement(); // If the parse tree is not empty if (codeElementParseTree.Start.Type > 0) { // Get the first line that was parsed var tokenStart = (Token)codeElementParseTree.Start; CodeElementsLine codeElementsLine; var importedToken = tokenStart as ImportedToken; if (importedToken != null) { codeElementsLine = (CodeElementsLine) importedToken.CopyDirective.TextNameSymbol.TokensLine; } else { codeElementsLine = ((CodeElementsLine) tokenStart.TokensLine); } // Register that this line was updated // COMMENTED FOR THE SAKE OF PERFORMANCE -- SEE ISSUE #160 //int updatedLineIndex = documentLines.IndexOf(codeElementsLine, codeElementsLine.InitialLineIndex); //codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, updatedLineIndex, codeElementsLine)); codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, codeElementsLine.InitialLineIndex, codeElementsLine)); // Visit the parse tree to build a first class object representing the code elements try { walker.Walk(codeElementBuilder, codeElementParseTree); } catch (Exception ex) { var code = Diagnostics.MessageCode.ImplementationError; int line = 0; int start = 0; int stop = 0; if (codeElementsLine.SourceTokens != null && codeElementsLine.SourceTokens.Count > 0){ start = codeElementsLine.SourceTokens[0].StartIndex; stop = codeElementsLine.SourceTokens[codeElementsLine.SourceTokens.Count-1].StopIndex; } codeElementsLine.AddParserDiagnostic(new ParserDiagnostic(ex.ToString(), start,stop,line, null, code)); } CodeElement codeElement = codeElementBuilder.CodeElement; if (codeElement != null) { // Attach consumed tokens and main document line numbers information to the code element codeElement.ConsumedTokens = cobolParser.ConsumedTokens; if (codeElement.ConsumedTokens == null) {// ISSUE #204: if (tokenStream.Lt(1) != null) {// if not end of file, // add next token to ConsumedTokens to know where is the CodeElement in error codeElement.ConsumedTokens.Add((Token)tokenStream.Lt(1)); // this alter CodeElements semantics: in addition to matched tokens, // it includes the first token in error if no token has been matched } } //TODO Issue #384 to discuss if this code should stay here: //This should be in a Checker, but "codeElement.ConsumedTokens" is only set after all the checkers have been called //Rule TCLIMITATION_NO_CE_ACROSS_SOURCES if (codeElement.IsAcrossSourceFile()) { DiagnosticUtils.AddError(codeElement, "A Cobol statement cannot be across 2 sources files (eg. Main program and a COPY)", MessageCode.TypeCobolParserLimitation); } // Add code element to the list codeElementsLine.AddCodeElement(codeElement); foreach (ParserDiagnostic d in errorListener.Diagnostics) { codeElement.Diagnostics.Add(d); } foreach (Diagnostic d in codeElement.Diagnostics) { codeElementsLine.AddParserDiagnostic(d); } } else { foreach (ParserDiagnostic d in errorListener.Diagnostics) { codeElementsLine.AddParserDiagnostic(d); } } } // In case of incremental parsing, directly jump to next parse section in the token stream // Else, simply start parsing the next CodeElement beginning with the next token if (currentParseSection != null) { // Check if we reached the end of the current ParseSection if (tokenStream.StreamReachedStopToken) { // Adavance to the next ParseSection if (parseSectionsEnumerator.MoveNext()) { currentParseSection = parseSectionsEnumerator.Current; tokenStream.SeekToToken(currentParseSection.StartToken); tokenStream.StartLookingForStopToken(currentParseSection.StopToken); } // No more section to parse else { break; } } } } while (tokenStream.La(1) >= 0); return codeElementsLinesChanges; }
/// <summary> /// Initial parsing of a complete document /// </summary> internal static void ParseDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<CodeElementsLine> documentLines, TypeCobolOptions compilerOptions) { ParseProcessedTokensLinesChanges(textSourceInfo, documentLines, null, null, compilerOptions); }
// --- Initialization --- /// <summary> /// Initializes a new compilation document from a list of text lines. /// This method does not scan the inserted text lines to produce tokens. /// You must explicitely call UpdateTokensLines() to start an initial scan of the document. /// </summary> public CompilationDocument(TextSourceInfo textSourceInfo, IEnumerable<ITextLine> initialTextLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider) : this(textSourceInfo, initialTextLines, compilerOptions, processedTokensDocumentProvider, null) { }