public static void ParseProgramOrClass(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<CodeElementsLine> codeElementsLines, TypeCobolOptions compilerOptions, SymbolTable customSymbols, out Program newProgram, out Class newClass, out IList<ParserDiagnostic> diagnostics) { // Create an Antlr compatible token source on top a the token iterator CodeElementsLinesTokenSource tokenSource = new CodeElementsLinesTokenSource( textSourceInfo.Name, codeElementsLines); // Init parser ITokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens); ProgramClassParser cobolParser = new ProgramClassParser(tokenStream); // -> activate full ambiguities detection //parser.Interpreter.PredictionMode = PredictionMode.LlExactAmbigDetection; // Register all parse errors in a list in memory DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener(); cobolParser.RemoveErrorListeners(); cobolParser.AddErrorListener(errorListener); // Try to parse a Cobol program or class ProgramClassParser.CobolCompilationUnitContext codeElementParseTree = cobolParser.cobolCompilationUnit(); // Visit the parse tree to build a first class object representing a Cobol program or class ParseTreeWalker walker = new ParseTreeWalker(); CobolNodeBuilder programClassBuilder = new CobolNodeBuilder(); programClassBuilder.CustomSymbols = customSymbols; programClassBuilder.Dispatcher = new NodeDispatcher(); programClassBuilder.Dispatcher.CreateListeners(); walker.Walk(programClassBuilder, codeElementParseTree); // Register compiler results newProgram = programClassBuilder.Program; newClass = programClassBuilder.Class; diagnostics = errorListener.Diagnostics; }
public void Init(string[] extensions = null, bool autoRemarks = false, bool AntlrProfiler = false) { DirectoryInfo localDirectory = new DirectoryInfo(Path.GetDirectoryName(Comparator?.paths?.SamplePath)); DocumentFormat format = Comparator?.GetSampleFormat(); TypeCobolOptions options = new TypeCobolOptions(); #if EUROINFO_RULES options.AutoRemarksEnable = autoRemarks; #endif if (extensions == null) { extensions = new[] { ".cbl", ".cpy" } } ; //comparator.paths.sextension = extensions[0].Substring(1); CompilationProject project = new CompilationProject("TEST", localDirectory.FullName, extensions, format.Encoding, format.EndOfLineDelimiter, format.FixedLineLength, format.ColumnsLayout, options); string filename = Comparator.paths.SampleName; Compiler = new FileCompiler(null, filename, project.SourceFileProvider, project, format.ColumnsLayout, options, null, false, project); if (AntlrProfiler) { Compiler.CompilationResultsForProgram.PerfStatsForCodeElementsParser.ActivateDetailedAntlrPofiling = true; Compiler.CompilationResultsForProgram.PerfStatsForTemporarySemantic.ActivateDetailedAntlrPofiling = true; } }
public CompilationDocument(TextSourceInfo textSourceInfo, IEnumerable <ITextLine> initialTextLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider, [CanBeNull] MultilineScanState scanState, List <RemarksDirective.TextNameVariation> copyTextNameVariations) { TextSourceInfo = textSourceInfo; CompilerOptions = compilerOptions; CopyTextNamesVariations = copyTextNameVariations ?? new List <RemarksDirective.TextNameVariation>(); MissingCopies = new List <CopyDirective>(); this.processedTokensDocumentProvider = processedTokensDocumentProvider; // Initialize the compilation document lines compilationDocumentLines = ImmutableList <CodeElementsLine> .Empty.ToBuilder(); // ... with the initial list of text lines received as a parameter if (initialTextLines != null) { // Insert Cobol text lines in the internal tree structure compilationDocumentLines.AddRange(initialTextLines.Select(textLine => CreateNewDocumentLine(textLine, textSourceInfo.ColumnsLayout))); } // Initialize document views versions currentTextLinesVersion = new DocumentVersion <ICobolTextLine>(this); currentTokensLinesVersion = new DocumentVersion <ITokensLine>(this); // Initialize performance stats PerfStatsForText = new PerfStatsForCompilationStep(CompilationStep.Text); PerfStatsForScanner = new PerfStatsForCompilationStep(CompilationStep.Scanner); PerfStatsForPreprocessor = new PerfStatsForParsingStep(CompilationStep.Preprocessor); initialScanStateForCopy = scanState; }
private static CodeElement[] ParseCodeElements(string cobolString, out Diagnostic[] parserDiagnostics) { // Load text document from string var textDocument = new ReadOnlyTextDocument("test string", Encoding.Default, ColumnsLayout.FreeTextFormat, ""); textDocument.LoadChars(cobolString); // Create a compilation project and a compiler for this document var typeCobolOptions = new TypeCobolOptions(); var project = new CompilationProject("test project", ".", new[] { ".cbl", ".cpy" }, DocumentFormat.FreeTextFormat.Encoding, DocumentFormat.FreeTextFormat.EndOfLineDelimiter, DocumentFormat.FreeTextFormat.FixedLineLength, DocumentFormat.FreeTextFormat.ColumnsLayout, typeCobolOptions); var compiler = new FileCompiler(textDocument, project.SourceFileProvider, project, typeCobolOptions, false, project); // Execute compilation - until the CodeElements phase ONLY compiler.CompilationResultsForProgram.UpdateTokensLines(); compiler.CompilationResultsForProgram.RefreshTokensDocumentSnapshot(); compiler.CompilationResultsForProgram.RefreshProcessedTokensDocumentSnapshot(); compiler.CompilationResultsForProgram.RefreshCodeElementsDocumentSnapshot(); // Return CodeElements and Diagnostics var codeElementsDocument = compiler.CompilationResultsForProgram.CodeElementsDocumentSnapshot; var codeElements = codeElementsDocument.CodeElements.ToArray(); parserDiagnostics = codeElementsDocument.ParserDiagnostics.ToArray(); return(codeElements); }
public Workspace(string workspaceName, string rootDirectory, string[] fileExtensions, Encoding encoding, EndOfLineDelimiter endOfLineDelimiter, int fixedLineLength, ColumnsLayout columnsLayout, TypeCobolOptions compilationOptions) { compilationProject = new CompilationProject(workspaceName, rootDirectory, fileExtensions, encoding, endOfLineDelimiter, fixedLineLength, columnsLayout, compilationOptions); OpenedFileCompilers = new Dictionary <string, FileCompiler>(3); }
/// <summary> /// Initializes a new compilation document from a list of text lines. /// This method does not scan the inserted text lines to produce tokens. /// You must explicitely call UpdateTokensLines() to start an initial scan of the document. /// </summary> public CompilationUnit(TextSourceInfo textSourceInfo, IEnumerable<ITextLine> initialTextLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider) : base(textSourceInfo, initialTextLines, compilerOptions, processedTokensDocumentProvider) { // Initialize performance stats PerfStatsForCodeElementsParser = new PerfStatsForCompilationStep(CompilationStep.CodeElementsParser); PerfStatsForProgramClassParser = new PerfStatsForCompilationStep(CompilationStep.ProgramClassParser); }
public void Init([NotNull] string path, TypeCobolOptions options, DocumentFormat format = null, IList <string> copies = null) { FileCompiler compiler; if (Compilers.TryGetValue(path, out compiler)) { return; } string filename = Path.GetFileName(path); var root = new DirectoryInfo(Directory.GetParent(path).FullName); if (format == null) { format = GetFormat(path); } CompilationProject project = new CompilationProject(path, root.FullName, Extensions, format.Encoding, format.EndOfLineDelimiter, format.FixedLineLength, format.ColumnsLayout, options); //Add copy folder into sourceFileProvider SourceFileProvider sourceFileProvider = project.SourceFileProvider; copies = copies ?? new List <string>(); foreach (var folder in copies) { sourceFileProvider.AddLocalDirectoryLibrary(folder, false, CopyExtensions, format.Encoding, format.EndOfLineDelimiter, format.FixedLineLength); } compiler = new FileCompiler(null, filename, project.SourceFileProvider, project, format.ColumnsLayout, options, CustomSymbols, false, project); Compilers.Add(path, compiler); Inits.Add(path, false); }
// --- Initialization --- /// <summary> /// Initializes a new compilation document from a list of text lines. /// This method does not scan the inserted text lines to produce tokens. /// You must explicitely call UpdateTokensLines() to start an initial scan of the document. /// </summary> public CompilationDocument(TextSourceInfo textSourceInfo, IEnumerable<ITextLine> initialTextLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider) { TextSourceInfo = textSourceInfo; CompilerOptions = compilerOptions; this.processedTokensDocumentProvider = processedTokensDocumentProvider; // Initialize the compilation document lines compilationDocumentLines = ImmutableList<CodeElementsLine>.Empty.ToBuilder(); // ... with the initial list of text lines received as a parameter if (initialTextLines != null) { // Insert Cobol text lines in the internal tree structure compilationDocumentLines.AddRange(initialTextLines.Select(textLine => CreateNewDocumentLine(textLine, textSourceInfo.ColumnsLayout))); } // Initialize document views versions currentTextLinesVersion = new DocumentVersion<ICobolTextLine>(this); currentTokensLinesVersion = new DocumentVersion<ITokensLine>(this); // Initialize performance stats PerfStatsForText = new PerfStatsForCompilationStep(CompilationStep.Text); PerfStatsForScanner = new PerfStatsForCompilationStep(CompilationStep.Scanner); PerfStatsForPreprocessor = new PerfStatsForCompilationStep(CompilationStep.Preprocessor); }
public static CompilationUnit ParseCobolFile(string textName, DocumentFormat documentFormat = null, string folder = null, ExecutionStep execToStep = ExecutionStep.SemanticCheck) { if (folder == null) { folder = "Parser" + Path.DirectorySeparatorChar + "CodeElements"; } DirectoryInfo localDirectory = new DirectoryInfo(PlatformUtils.GetPathForProjectFile(folder)); if (!localDirectory.Exists) { throw new Exception(String.Format("Directory : {0} does not exist", localDirectory.FullName)); } if (documentFormat == null) { documentFormat = DocumentFormat.RDZReferenceFormat; } TypeCobolOptions options = new TypeCobolOptions { ExecToStep = execToStep }; //Create CompilerOptions. ExecToStep / AutoRemarks / HaltOnMissingCopy have to be set here. CompilationProject project = new CompilationProject("test", //First use *.cpy as tests will use file WITH extension for program but without extension for copy inside programs => small perf gain localDirectory.FullName, new string[] { ".cpy", ".cbl" }, documentFormat.Encoding, documentFormat.EndOfLineDelimiter, documentFormat.FixedLineLength, documentFormat.ColumnsLayout, options); FileCompiler compiler = new FileCompiler(null, textName, project.SourceFileProvider, project, documentFormat.ColumnsLayout, options, null, false, project); compiler.CompileOnce(); return(compiler.CompilationResultsForProgram); }
/// <summary> /// New implementation of the Init function, to allocate a BuildProject instance rather than a /// CompilationProject. /// </summary> /// <param name="path">The path of the file to parser</param> /// <param name="format">The resulting document format</param> /// <returns>The BuildProject instance created</returns> public new BuildProject Init(string path, DocumentFormat format = null) { FileCompiler compiler; if (Compilers.TryGetValue(path, out compiler)) { return(m_Projects[path]); } string directory = Directory.GetParent(path).FullName; string filename = Path.GetFileName(path); DirectoryInfo root = new DirectoryInfo(directory); if (format == null) { format = GetFormat(path); } TypeCobolOptions options = new TypeCobolOptions(); BuildProject project = new BuildProject(BuilderEngine, path, root.FullName, new string[] { "*.cbl", "*.cpy" }, format.Encoding, format.EndOfLineDelimiter, format.FixedLineLength, format.ColumnsLayout, options); m_Projects[path] = project; compiler = new FileCompiler(null, filename, project.SourceFileProvider, project, format.ColumnsLayout, options, CustomSymbols, false); Compilers.Add(path, compiler); Inits.Add(path, false); return(project); }
public Workspace(string workspaceName, string rootDirectory, string[] fileExtensions, Encoding encoding, EndOfLineDelimiter endOfLineDelimiter, int fixedLineLength, ColumnsLayout columnsLayout, TypeCobolOptions compilationOptions) { compilationProject = new CompilationProject(workspaceName, rootDirectory, fileExtensions, encoding, endOfLineDelimiter, fixedLineLength, columnsLayout, compilationOptions); OpenedFileCompilers = new Dictionary<string, FileCompiler>(3); }
/// <summary> /// Parse an file using a NodeListener and IReport instance and compare the resulting report. /// </summary> /// <typeparam name="TCtx"></typeparam> /// <param name="fileName">The file name to parse</param> /// <param name="reportFileName">The file that contains the expected report</param> /// <param name="reportType">The Type of the IReport instance to be instantiated.</param> public static void ParseWithNodeListenerReportCompare <TCtx>(string fileName, string reportFileName, System.Type reportType) where TCtx : class { Assert.IsTrue(Tools.Reflection.IsTypeOf(reportType, typeof(IReport))); IReport report = null;//Variable to receive the created report instance. TypeCobol.Compiler.Parser.NodeListenerFactory <TCtx> factory = () => { object obj = System.Activator.CreateInstance(reportType); Assert.IsTrue(obj is NodeListener <TCtx>); TypeCobol.Compiler.Parser.NodeListener <TCtx> nodeListener = (TypeCobol.Compiler.Parser.NodeListener <TCtx>)obj; Assert.IsNotNull(nodeListener); report = (IReport)nodeListener; return(nodeListener); }; //Register the Node Listener Factory TypeCobol.Compiler.Parser.NodeDispatcher <TCtx> .RegisterStaticNodeListenerFactory(factory); try { string input = Path.Combine(ROOT_INPUT, fileName); string output = Path.Combine(ROOT_OUTPUT, reportFileName); DocumentFormat format = DocumentFormat.RDZReferenceFormat; var parser = new TypeCobol.Parser(); var typeCobolOption = new TypeCobolOptions { ExecToStep = ExecutionStep.CrossCheck }; #if EUROINFO_RULES bool autoRemarks = false; typeCobolOption.AutoRemarksEnable = autoRemarks; #endif String copyFolder = Path.Combine(Directory.GetCurrentDirectory(), ROOT_COPY); parser.Init(input, typeCobolOption, format, new List <string>() { copyFolder }); parser.Parse(input); var allDiags = parser.Results.AllDiagnostics(); if (allDiags.Count == 0) { if (report != null) { using (System.IO.StringWriter sw = new StringWriter()) { report.Report(sw); // compare with expected result string result = sw.ToString(); string expected = File.ReadAllText(output, format.Encoding); TypeCobol.Test.TestUtils.compareLines(input, result, expected); } } } } finally { TypeCobol.Compiler.Parser.NodeDispatcher <TCtx> .RemoveStaticNodeListenerFactory(factory); } }
public ImportedTokensDocument(CopyDirective copyDirective, ProcessedTokensDocument importedDocumentSource, PerfStatsForImportedDocument perfStats, TypeCobolOptions compilerOptions) { CopyDirective = copyDirective; SourceDocument = importedDocumentSource; HasReplacingDirective = copyDirective.ReplaceOperations.Count > 0; PerfStatsForImportedDocument = perfStats; CompilerOptions = compilerOptions; }
public ProcessedTokensDocument(TokensDocument previousStepSnapshot, DocumentVersion <IProcessedTokensLine> processedTokensLinesVersion, ISearchableReadOnlyList <IProcessedTokensLine> processedTokensLines, TypeCobolOptions compilerOptions) { TextSourceInfo = previousStepSnapshot.TextSourceInfo; PreviousStepSnapshot = previousStepSnapshot; CurrentVersion = processedTokensLinesVersion; Lines = processedTokensLines; CompilerOptions = compilerOptions; }
private Scanner(string line, int startIndex, int lastIndex, TokensLine tokensLine, TypeCobolOptions compilerOptions) { this.tokensLine = tokensLine; this.line = line; this.currentIndex = startIndex; this.lastIndex = lastIndex; this.compilerOptions = compilerOptions; }
public static void AttachIncrementalCompilerToTextEditorDocument(TypeCobolEditor textEditor, CompilationProject project, string textName, TypeCobolOptions compilerOptions, IObserver<IList<CompilationError>> errorObserver) { var serviceContainer = textEditor.TextArea.TextView.Services; if (serviceContainer.GetService(typeof(TypeCobolCompilerService)) != null) { serviceContainer.RemoveService(typeof(TypeCobolCompilerService)); } TypeCobolCompilerService compilerService = new TypeCobolCompilerService(textEditor, project, textName, compilerOptions, errorObserver); serviceContainer.AddService(typeof(TypeCobolCompilerService), compilerService); }
/// <summary> /// Common internal implementation for all 4 constructors above /// </summary> private FileCompiler(string libraryName, string textName, CobolFile loadedCobolFile, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ColumnsLayout columnsLayout, ITextDocument textDocument, TypeCobolOptions compilerOptions, SymbolTable customSymbols, bool isCopyFile, [CanBeNull] MultilineScanState scanState) { // 1.a Find the Cobol source file CobolFile sourceFile = null; if (textName != null) { if (sourceFileProvider.TryGetFile(libraryName, textName, out sourceFile)) { CobolFile = sourceFile; } else { throw new Exception(String.Format("Could not find a Cobol source file named {0} in {1}", textName, libraryName)); } } // 1.b Register a Cobol source file which was already loaded else if(loadedCobolFile != null) { CobolFile = loadedCobolFile; } // 2.a Load it in a new text document in memory if (textDocument == null) { TextDocument = new ReadOnlyTextDocument(sourceFile.Name, sourceFile.Encoding, columnsLayout, sourceFile.ReadChars()); } // 2.b Load it in an existing text document in memory else if (sourceFile != null) { TextDocument = textDocument; textDocument.LoadChars(sourceFile.ReadChars()); } // 2.c Use a pre-existing text document // - not yet associated with a Cobol source file // - with a Cobol source file already loaded else if (sourceFile == null || loadedCobolFile != null) { TextDocument = textDocument; } // 3. Prepare the data structures used by the different steps of the compiler if (isCopyFile) { CompilationResultsForCopy = new CompilationDocument(TextDocument.Source, TextDocument.Lines, compilerOptions, documentProvider, scanState); CompilationResultsForCopy.CustomSymbols = customSymbols; } else { CompilationResultsForProgram = new CompilationUnit(TextDocument.Source, TextDocument.Lines, compilerOptions, documentProvider); CompilationResultsForProgram.CustomSymbols = customSymbols; } CompilerOptions = compilerOptions; }
public static Parser Parse(string path, DocumentFormat format, bool autoRemarks = false) { var parser = new Parser(); var typeCobolOption = new TypeCobolOptions { ExecToStep = ExecutionStep.Generate }; #if EUROINFO_RULES typeCobolOption.AutoRemarksEnable = autoRemarks; #endif parser.Init(path, typeCobolOption, format); parser.Parse(path); return(parser); }
public TypeCobolWindow() { InitializeComponent(); // Default directory at startup currentProject = new CompilationProject("project", DEFAULT_DIRECTORY, new string[] { ".txt", "*.cbl", "*.cpy" }, DEFAULT_DOCUMENT_FORMAT.Encoding, DEFAULT_DOCUMENT_FORMAT.EndOfLineDelimiter, DEFAULT_DOCUMENT_FORMAT.FixedLineLength, DEFAULT_DOCUMENT_FORMAT.ColumnsLayout, new TypeCobolOptions()); // Default compiler options compilerOptions = new TypeCobolOptions(); // Incremental TypeCobol compilation TypeCobolCompilerService.AttachIncrementalCompilerToTextEditorDocument(textEditor, currentProject, null, compilerOptions, this); }
// -- Project creation and persistence -- /// <summary> /// Create a new Cobol compilation project in a local directory /// </summary> public CompilationProject(string projectName, string rootDirectory, string[] fileExtensions, Encoding encoding, EndOfLineDelimiter endOfLineDelimiter, int fixedLineLength, ColumnsLayout columnsLayout, TypeCobolOptions compilationOptions) { Name = projectName; RootDirectory = rootDirectory; SourceFileProvider = new SourceFileProvider(); rootDirectoryLibrary = SourceFileProvider.AddLocalDirectoryLibrary(rootDirectory, true, fileExtensions, encoding, endOfLineDelimiter, fixedLineLength); Encoding = encoding; EndOfLineDelimiter = endOfLineDelimiter; FixedLineLength = fixedLineLength; ColumnsLayout = columnsLayout; CompilationOptions = compilationOptions; CobolFiles = new Dictionary<string, CobolFile>(); CobolTextReferences = new Dictionary<string, CobolFile>(); CobolProgramCalls = new Dictionary<string, CobolFile>(); }
/// <summary> /// Create a new Cobol compilation project in a local directory /// </summary> public CompilationProject(string projectName, string rootDirectory, string[] fileExtensions, Encoding encoding, EndOfLineDelimiter endOfLineDelimiter, int fixedLineLength, ColumnsLayout columnsLayout, TypeCobolOptions compilationOptions) { Name = projectName; RootDirectory = rootDirectory; SourceFileProvider = new SourceFileProvider(); rootDirectoryLibrary = SourceFileProvider.AddLocalDirectoryLibrary(rootDirectory, false, fileExtensions, encoding, endOfLineDelimiter, fixedLineLength); Encoding = encoding; EndOfLineDelimiter = endOfLineDelimiter; FixedLineLength = fixedLineLength; ColumnsLayout = columnsLayout; CompilationOptions = compilationOptions; CobolFiles = new Dictionary <string, CobolFile>(); CobolTextReferences = new Dictionary <string, CobolFile>(); CobolProgramCalls = new Dictionary <string, CobolFile>(); }
/// <summary> /// Implement COPY REPLACING on top of an underlying tokens line iterator /// </summary> public ReplaceTokensLinesIterator(ITokensLinesIterator sourceIterator, CopyDirective copyReplacingDirective, TypeCobolOptions compilerOptions) { this.sourceIterator = sourceIterator; this.CopyReplacingDirective = copyReplacingDirective; CompilerOptions = compilerOptions; if (copyReplacingDirective.ReplaceOperations.Count > 0) { if (copyReplacingDirective.ReplaceOperations.Count == 1) { currentPosition.ReplaceOperation = copyReplacingDirective.ReplaceOperations[0]; } else { currentPosition.ReplaceOperations = copyReplacingDirective.ReplaceOperations; } } }
public static CompilationUnit ParseCobolString(string cobolString) { //Prepare var textDocument = new ReadOnlyTextDocument("Empty doc", Encoding.Default, ColumnsLayout.FreeTextFormat, ""); textDocument.LoadChars(cobolString); var typeCobolOptions = new TypeCobolOptions(); var project = new CompilationProject("Empty project", ".", new[] { ".cbl", ".cpy" }, DocumentFormat.FreeTextFormat.Encoding, DocumentFormat.FreeTextFormat.EndOfLineDelimiter, DocumentFormat.FreeTextFormat.FixedLineLength, DocumentFormat.FreeTextFormat.ColumnsLayout, typeCobolOptions); var compiler = new FileCompiler(textDocument, project.SourceFileProvider, project, typeCobolOptions, false, project); compiler.CompileOnce(); return(compiler.CompilationResultsForProgram); }
// public static void Main(string[] args) public static void main(string[] args) { // Directory where the source programs and COPY files are stored string sourcePath = @"D:\Users\Laurent\OneDrive\Dev\Visual Studio 2012\Projects\TypeCobol\TypeCobol.Test\Samples\EI Cobol samples\EI-Production"; string[] programExtensions = { ".PGM" }; string[] copyExtensions = { ".CPY" }; // List of all sample programs used to compute the statistics IList <string> textNames = new List <string>(); foreach (string programExtension in programExtensions) { foreach (string filePath in Directory.EnumerateFiles(sourcePath, "*" + programExtension)) { string textName = Path.GetFileNameWithoutExtension(filePath); textNames.Add(textName); } } // Source file format for the samples DocumentFormat docFormat = new DocumentFormat(Encoding.GetEncoding("iso8859-1"), EndOfLineDelimiter.CrLfCharacters, 80, ColumnsLayout.CobolReferenceFormat); // Initialize a compilation project TypeCobolOptions compilerOptions = new TypeCobolOptions(); CompilationProject project = new CompilationProject("samples", sourcePath, programExtensions.Concat(copyExtensions).ToArray(), docFormat.Encoding, docFormat.EndOfLineDelimiter, docFormat.FixedLineLength, docFormat.ColumnsLayout, compilerOptions); // Output files used to store the results string resultFilePath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop); string countersFile = Path.Combine(resultFilePath, "SyntaxCounters.txt"); string languageModelForProgramFile = Path.Combine(resultFilePath, "LanguageModel.Program.txt"); string languageModelForCopyFile = Path.Combine(resultFilePath, "LanguageModel.Copy.txt"); // Compute statistics Stopwatch chrono = new Stopwatch(); chrono.Start(); StatsGenerator.GenerateStatisticsForPrograms(project, textNames, Console.Out, countersFile, languageModelForProgramFile, languageModelForCopyFile); chrono.Stop(); Console.WriteLine(""); Console.WriteLine("Programs analyzed in " + Math.Round(chrono.ElapsedMilliseconds / (double)1000, 3) + " sec"); }
public void Init(string[] extensions = null, bool autoRemarks = false) { DirectoryInfo localDirectory = new DirectoryInfo(Path.GetDirectoryName(Comparator.paths.SamplePath)); DocumentFormat format = Comparator.getSampleFormat(); TypeCobolOptions options = new TypeCobolOptions(); #if EUROINFO_RULES options.AutoRemarksEnable = autoRemarks; #endif if (extensions == null) { extensions = new[] { ".cbl", ".cpy" } } ; //comparator.paths.sextension = extensions[0].Substring(1); CompilationProject project = new CompilationProject("TEST", localDirectory.FullName, extensions, format.Encoding, format.EndOfLineDelimiter, format.FixedLineLength, format.ColumnsLayout, options); string filename = Comparator.paths.SampleName; Compiler = new FileCompiler(null, filename, project.SourceFileProvider, project, format.ColumnsLayout, options, null, false, project); }
// public static void Main(string[] args) public static void main(string[] args) { // Directory where the source programs and COPY files are stored string sourcePath = @"D:\Users\Laurent\OneDrive\Dev\Visual Studio 2012\Projects\TypeCobol\TypeCobol.Test\Samples\EI Cobol samples\EI-Production"; string[] programExtensions = { "*.PGM" }; string[] copyExtensions = { "*.CPY" }; // List of all sample programs used to compute the statistics IList<string> textNames = new List<string>(); foreach (string programExtension in programExtensions) { foreach (string filePath in Directory.EnumerateFiles(sourcePath, programExtension)) { string textName = Path.GetFileNameWithoutExtension(filePath); textNames.Add(textName); } } // Source file format for the samples DocumentFormat docFormat = new DocumentFormat(Encoding.GetEncoding("iso8859-1"), EndOfLineDelimiter.CrLfCharacters, 80, ColumnsLayout.CobolReferenceFormat); // Initialize a compilation project TypeCobolOptions compilerOptions = new TypeCobolOptions(); CompilationProject project = new CompilationProject("samples", sourcePath, programExtensions.Concat(copyExtensions).ToArray(), docFormat.Encoding, docFormat.EndOfLineDelimiter, docFormat.FixedLineLength, docFormat.ColumnsLayout, compilerOptions); // Output files used to store the results string resultFilePath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop); string countersFile = Path.Combine(resultFilePath, "SyntaxCounters.txt"); string languageModelForProgramFile = Path.Combine(resultFilePath, "LanguageModel.Program.txt"); string languageModelForCopyFile = Path.Combine(resultFilePath, "LanguageModel.Copy.txt"); // Compute statistics Stopwatch chrono = new Stopwatch(); chrono.Start(); StatsGenerator.GenerateStatisticsForPrograms(project, textNames, Console.Out, countersFile, languageModelForProgramFile, languageModelForCopyFile); chrono.Stop(); Console.WriteLine(""); Console.WriteLine("Programs analyzed in " + Math.Round(chrono.ElapsedMilliseconds / (double)1000, 3) + " sec"); }
/// <summary> /// Handle the Configuration change notification. /// </summary> /// <param name="arguments">The arguments</param> public void DidChangeConfigurationParams(IEnumerable <string> arguments) { var options = TypeCobolOptionSet.GetCommonTypeCobolOptions(TypeCobolConfiguration); options.Parse(arguments); //Adding default copies folder var folder = Path.GetDirectoryName(Process.GetCurrentProcess().MainModule.FileName); TypeCobolConfiguration.CopyFolders.Add(folder + @"\DefaultCopies\"); if (TypeCobolConfiguration.Telemetry) { AnalyticsWrapper.Telemetry.DisableTelemetry = false; //If telemetry arg is passed enable telemetry } if (TypeCobolConfiguration.ExecToStep >= ExecutionStep.Generate) { TypeCobolConfiguration.ExecToStep = ExecutionStep.SemanticCheck; //Language Server does not support Cobol Generation for now } var typeCobolOptions = new TypeCobolOptions { HaltOnMissingCopy = TypeCobolConfiguration.HaltOnMissingCopyFilePath != null, ExecToStep = TypeCobolConfiguration.ExecToStep, #if EUROINFO_RULES AutoRemarksEnable = TypeCobolConfiguration.AutoRemarks #endif }; CompilationProject = new CompilationProject(WorkspaceName, RootDirectoryFullName, Extensions, TypeCobolConfiguration.Format.Encoding, TypeCobolConfiguration.Format.EndOfLineDelimiter, TypeCobolConfiguration.Format.FixedLineLength, TypeCobolConfiguration.Format.ColumnsLayout, typeCobolOptions); if (OpenedFileCompiler.Count > 0) { RefreshOpenedFiles(); } else { RefreshCustomSymbols(); } }
private TypeCobolCompilerService(TypeCobolEditor textEditor, CompilationProject project, string textName, TypeCobolOptions compilerOptions, IObserver<IList<CompilationError>> errorObserver) { TextDocument = textEditor.TextDocument; // DEMO : TaskPoolScheduler can not be used yet because of Length property access on ITextDocument by CobolCharStream BackgroundCompilationScheduler = DispatcherScheduler.Current ; // Create a new file compiler for this document FileCompiler compiler = null; if (textName != null) { compiler = new FileCompiler(null, textName, project.SourceFileProvider, project, TextDocument, compilerOptions, false); } else { compiler = new FileCompiler(TextDocument, project.SourceFileProvider, project, compilerOptions, false); } CompilationUnit = compiler.CompilationResultsForProgram; // Compile in the background compiler.StartContinuousBackgroundCompilation(400, 400, 900, 2000); // Refresh the syntax coloring for rescanned lines IObservable<DocumentChangedEvent<ITokensLine>> tokensLinesObservable = Observable.FromEvent<EventHandler<DocumentChangedEvent<ITokensLine>>,DocumentChangedEvent<ITokensLine>>( evtHandler => CompilationUnit.TokensLinesChanged += evtHandler, evtHandler => CompilationUnit.TokensLinesChanged -= evtHandler); tokensLinesObservable.Subscribe(textEditor); // Listen to all compilation errors found by the incremental compiler => on the dispatcher thread to udpate the UI // --> TO DO : there is no way yet to observe compilation errors //compilationErrorsObservable.ObserveOn(DispatcherScheduler.Current).Subscribe(errorObserver); // Trigger initial notification after document load compiler.StartDocumentProcessing(); }
/// <summary> /// Initial preprocessing of a complete document /// </summary> internal static void ProcessDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<ProcessedTokensLine> documentLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider) { ProcessTokensLinesChanges(textSourceInfo, documentLines, null, null, compilerOptions, processedTokensDocumentProvider); }
/// <summary> /// Initial preprocessing of a complete document /// </summary> internal static void ProcessDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <ProcessedTokensLine> documentLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider, List <RemarksDirective.TextNameVariation> copyTextNameVariations) { ProcessTokensLinesChanges(textSourceInfo, documentLines, null, null, compilerOptions, processedTokensDocumentProvider, copyTextNameVariations); }
/// <summary> /// Incremental parsing of a set of processed tokens lines changes /// </summary> internal static IList<DocumentChange<ICodeElementsLine>> ParseProcessedTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<CodeElementsLine> documentLines, IList<DocumentChange<IProcessedTokensLine>> processedTokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions) { // Collect all changes applied to the processed tokens lines during the incremental scan IList<DocumentChange<ICodeElementsLine>> codeElementsLinesChanges = new List<DocumentChange<ICodeElementsLine>>(); // There are 2 reasons to re-parse a tokens line after a change : // 1. The tokens line changed : these lines were already reset during the previous steps // 2. If a tokens line that changed was involved in the parsing of a multiline code element, the whole group of lines must be parsed again // --- PREPARATION PHASE : identify all parse sections where code elements need to be refreshed --- IList<ParseSection> refreshParseSections = null; // Iterate over all processed tokens changes detected by the PreprocessorStep : // - refresh all the adjacent lines participating in a CodeElement // - register the start and stop token for all sections of the document which need to be parsed again if (processedTokensLinesChanges != null) { // If the document was cleared, everything must be parsed again if (processedTokensLinesChanges[0].Type != DocumentChangeType.DocumentCleared) { refreshParseSections = new List<ParseSection>(); ParseSection lastParseSection = null; foreach (DocumentChange<IProcessedTokensLine> tokensChange in processedTokensLinesChanges) { if (lastParseSection == null || tokensChange.LineIndex > lastParseSection.StopLineIndex) { lastParseSection = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, codeElementsLinesChanges, lastParseSection); refreshParseSections.Add(lastParseSection); } } } } // --- INITIALIZE ANTLR CodeElements parser --- // Create a token iterator on top of pre-processed tokens lines ITokensLinesIterator tokensIterator = ProcessedTokensDocument.GetProcessedTokensIterator(textSourceInfo, documentLines); // Create an Antlr compatible token source on top of the token iterator TokensLinesTokenSource tokenSource = new TokensLinesTokenSource( textSourceInfo.Name, tokensIterator); // Init parser TokensLinesTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens); TracingCobolParser cobolParser = new TracingCobolParser(tokenStream); // -> activate full ambiguities detection //parser.Interpreter.PredictionMode = PredictionMode.LlExactAmbigDetection; IAntlrErrorStrategy cobolErrorStrategy = new CobolErrorStrategy(); cobolParser.ErrorHandler = cobolErrorStrategy; // Register all parse errors in a list in memory DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener(); cobolParser.RemoveErrorListeners(); cobolParser.AddErrorListener(errorListener); // Prepare to analyze the parse tree ParseTreeWalker walker = new ParseTreeWalker(); CodeElementBuilder codeElementBuilder = new CodeElementBuilder(); codeElementBuilder.Dispatcher = new CodeElementDispatcher(); codeElementBuilder.Dispatcher.CreateListeners(); // --- INCREMENTAL PARSING --- // In case of incremental parsing, parse only the code sections we need to refresh IEnumerator<ParseSection> parseSectionsEnumerator = null; ParseSection currentParseSection = null; if (refreshParseSections != null) { // Get the first code section we need to refresh parseSectionsEnumerator = refreshParseSections.GetEnumerator(); parseSectionsEnumerator.MoveNext(); currentParseSection = parseSectionsEnumerator.Current; // Seek just before the next code element starting token tokenStream.SeekToToken(currentParseSection.StartToken); tokenStream.StartLookingForStopToken(currentParseSection.StopToken); } // Parse one CodeElement at a time while advancing in the underlying token stream do { // Reset parsing error diagnostics //cobolErrorStrategy.Reset(cobolParser); errorListener.Diagnostics.Clear(); // Reset parser traces (consumed tokens) cobolParser.ResetTraces(); // Try to parse a code element starting with the current token CodeElementsParser.CodeElementContext codeElementParseTree = cobolParser.codeElement(); // If the parse tree is not empty if (codeElementParseTree.Start.Type > 0) { // Get the first line that was parsed var tokenStart = (Token)codeElementParseTree.Start; CodeElementsLine codeElementsLine; var importedToken = tokenStart as ImportedToken; if (importedToken != null) { codeElementsLine = (CodeElementsLine) importedToken.CopyDirective.TextNameSymbol.TokensLine; } else { codeElementsLine = ((CodeElementsLine) tokenStart.TokensLine); } // Register that this line was updated // COMMENTED FOR THE SAKE OF PERFORMANCE -- SEE ISSUE #160 //int updatedLineIndex = documentLines.IndexOf(codeElementsLine, codeElementsLine.InitialLineIndex); //codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, updatedLineIndex, codeElementsLine)); codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, codeElementsLine.InitialLineIndex, codeElementsLine)); // Visit the parse tree to build a first class object representing the code elements try { walker.Walk(codeElementBuilder, codeElementParseTree); } catch (Exception ex) { var code = Diagnostics.MessageCode.ImplementationError; int line = 0; int start = 0; int stop = 0; if (codeElementsLine.SourceTokens != null && codeElementsLine.SourceTokens.Count > 0){ start = codeElementsLine.SourceTokens[0].StartIndex; stop = codeElementsLine.SourceTokens[codeElementsLine.SourceTokens.Count-1].StopIndex; } codeElementsLine.AddParserDiagnostic(new ParserDiagnostic(ex.ToString(), start,stop,line, null, code)); } CodeElement codeElement = codeElementBuilder.CodeElement; if (codeElement != null) { // Attach consumed tokens and main document line numbers information to the code element codeElement.ConsumedTokens = cobolParser.ConsumedTokens; if (codeElement.ConsumedTokens == null) {// ISSUE #204: if (tokenStream.Lt(1) != null) {// if not end of file, // add next token to ConsumedTokens to know where is the CodeElement in error codeElement.ConsumedTokens.Add((Token)tokenStream.Lt(1)); // this alter CodeElements semantics: in addition to matched tokens, // it includes the first token in error if no token has been matched } } //TODO Issue #384 to discuss if this code should stay here: //This should be in a Checker, but "codeElement.ConsumedTokens" is only set after all the checkers have been called //Rule TCLIMITATION_NO_CE_ACROSS_SOURCES if (codeElement.IsAcrossSourceFile()) { DiagnosticUtils.AddError(codeElement, "A Cobol statement cannot be across 2 sources files (eg. Main program and a COPY)", MessageCode.TypeCobolParserLimitation); } // Add code element to the list codeElementsLine.AddCodeElement(codeElement); foreach (ParserDiagnostic d in errorListener.Diagnostics) { codeElement.Diagnostics.Add(d); } foreach (Diagnostic d in codeElement.Diagnostics) { codeElementsLine.AddParserDiagnostic(d); } } else { foreach (ParserDiagnostic d in errorListener.Diagnostics) { codeElementsLine.AddParserDiagnostic(d); } } } // In case of incremental parsing, directly jump to next parse section in the token stream // Else, simply start parsing the next CodeElement beginning with the next token if (currentParseSection != null) { // Check if we reached the end of the current ParseSection if (tokenStream.StreamReachedStopToken) { // Adavance to the next ParseSection if (parseSectionsEnumerator.MoveNext()) { currentParseSection = parseSectionsEnumerator.Current; tokenStream.SeekToToken(currentParseSection.StartToken); tokenStream.StartLookingForStopToken(currentParseSection.StopToken); } // No more section to parse else { break; } } } } while (tokenStream.La(1) >= 0); return codeElementsLinesChanges; }
/// <summary> /// Incremental scan of a set of text lines changes /// </summary> internal static IList<DocumentChange<ITokensLine>> ScanTextLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, IList<DocumentChange<ICobolTextLine>> textLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions) { return ScanTextLinesChanges(textSourceInfo, documentLines, textLinesChanges, prepareDocumentLineForUpdate, compilerOptions, null); }
/// <summary> /// Use a pre-existing text document, already initialized from a Cobol file /// </summary> public FileCompiler(string libraryName, string fileName, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ColumnsLayout columnsLayout, TypeCobolOptions compilerOptions, CodeModel.SymbolTable customSymbols, bool isCopyFile, MultilineScanState scanState, CompilationProject compilationProject, List <RemarksDirective.TextNameVariation> copyTextNameVariations) : this(libraryName, fileName, null, sourceFileProvider, documentProvider, columnsLayout, null, compilerOptions, customSymbols, isCopyFile, scanState, compilationProject, copyTextNameVariations) { }
/// <summary> /// Load a Cobol source file in an pre-existing text document /// </summary> public FileCompiler(string libraryName, string fileName, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ITextDocument textDocument, TypeCobolOptions compilerOptions, bool isCopyFile, CompilationProject compilationProject) : this(libraryName, fileName, null, sourceFileProvider, documentProvider, default(ColumnsLayout), textDocument, compilerOptions, null, isCopyFile, null, compilationProject, null) { }
private static MultilineScanState ScanTokensLineWithContinuations(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IList<DocumentChange<ITokensLine>> tokensLinesChanges, out int nextLineToScanIndex, out TokensLine nextLineToScan) { // Initialize out parameters if (lineToScanIndex < (documentLines.Count - 1)) { nextLineToScanIndex = lineToScanIndex + 1; nextLineToScan = documentLines[nextLineToScanIndex]; } else { nextLineToScanIndex = -1; nextLineToScan = null; } // Check if the line to scan participates in a multiline continuation // - because it is itself a continuation line bool partOfMultilineContinuation = lineToScan.Type == CobolTextLineType.Continuation; // - or because the following line is a continuation line if (!partOfMultilineContinuation && lineToScanIndex < (documentLines.Count - 1)) { nextLineToScanIndex = lineToScanIndex + 1; nextLineToScan = documentLines[nextLineToScanIndex]; partOfMultilineContinuation = nextLineToScan.Type == CobolTextLineType.Continuation; } // ^-- LIMITATION : we don't support the case where one or more comment lines follow the current line before a continuation line // Case 1 : the line is not part of a multiline continuation => we can parse it as as a standalone line if (!partOfMultilineContinuation) { // Create a new copy of the line before the update if necessary lineToScan = (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner); if (lineToScanIndex == 0) { // Scan the first line of the document Scanner.ScanFirstLine(lineToScan, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions); } else { // Get the scan state at the end of the previous line TokensLine previousLine = documentLines[lineToScanIndex - 1]; // Scan the current line with this initial scan state Scanner.ScanTokensLine(lineToScan, previousLine.ScanState, compilerOptions); } tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineUpdated, lineToScanIndex, lineToScan)); return lineToScan.ScanState; } // Case 2 : the line is part of a multiline continuation => we must parse all continuation lines as a group else { // Build a list of the lines we will have to scan as a group : IList<TokensLine> continuationLinesGroup = new List<TokensLine>(); int firstLineIndex = lineToScanIndex; continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); // Navigate backwards to the start of the multiline continuation if (lineToScan.Type == CobolTextLineType.Continuation && lineToScanIndex > 0) { int revLineToScanIndex = lineToScanIndex; IEnumerator<TokensLine> reversedEnumerator = documentLines.GetEnumerator(lineToScanIndex - 1, -1, true); while(reversedEnumerator.MoveNext()) { // Get the previous line until a non continuation and non comment line is encountered revLineToScanIndex--; lineToScan = reversedEnumerator.Current; if(lineToScan.Type != CobolTextLineType.Continuation /*&& <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line lineToScan.Type != CobolTextLineType.Comment && lineToScan.Type != CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines { firstLineIndex = revLineToScanIndex; continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); } else { break; } } } // Reuse our knowledge of the next line if it is available and if it is a continuation if (nextLineToScan != null && nextLineToScan.Type == CobolTextLineType.Continuation) { lineToScanIndex++; lineToScan = nextLineToScan; continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); nextLineToScanIndex = -1; nextLineToScan = null; } // Navigate forwards to the end of the multiline continuation if (lineToScanIndex < (documentLines.Count - 1)) { IEnumerator<TokensLine> enumerator = documentLines.GetEnumerator(lineToScanIndex + 1, -1, false); while (enumerator.MoveNext()) { // Get the next line until a non continuation and non comment line is encountered lineToScanIndex++; lineToScan = enumerator.Current; if (lineToScan.Type == CobolTextLineType.Continuation /*|| <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line lineToScan.Type == CobolTextLineType.Comment || lineToScan.Type == CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines { // Add this line at the end of the list of continuation lines continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); } else { // Save the knowledge of the next line and exit the loop nextLineToScanIndex = lineToScanIndex; nextLineToScan = lineToScan; break; } } } // Scan the group of continuation lines if (firstLineIndex == 0) { // Scan the first line group of the document Scanner.ScanFirstLineContinuationGroup(continuationLinesGroup, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions); } else { // Get the scan state at the end of the previous line TokensLine previousLine = documentLines[firstLineIndex - 1]; // Scan the current line group with this initial scan state Scanner.ScanTokensLineContinuationGroup(continuationLinesGroup, previousLine.ScanState, compilerOptions); } int updatedLineIndex = firstLineIndex; foreach(TokensLine updatedLine in continuationLinesGroup) { tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineUpdated, updatedLineIndex, updatedLine)); updatedLineIndex++; } return continuationLinesGroup[continuationLinesGroup.Count - 1].ScanState; } }
private static void ScanTokensLineWithMultilineScanState(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IList<DocumentChange<ITokensLine>> tokensLinesChanges, out int nextLineToScanIndex, out TokensLine nextLineToScan) { // Scan the current line (or continuation lines group) MultilineScanState scanState = ScanTokensLineWithContinuations(lineToScanIndex, lineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan); // Scan the following lines until we find that the scan state at the beginning of the next line has been updated while(nextLineToScan != null && nextLineToScan.InitialScanState != null && !nextLineToScan.InitialScanState.Equals(scanState)) { scanState = ScanTokensLineWithContinuations(nextLineToScanIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan); } }
/// <summary> /// Initial scan of a complete document /// </summary> public static void ScanDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, TypeCobolOptions compilerOptions) { TokensLine tokensLine = null; TokensLine nextTokensLine = null; MultilineScanState lastScanState = null; // Get the first line IEnumerator<TokensLine> documentLinesEnumerator = documentLines.GetEnumerator(); if (documentLinesEnumerator.MoveNext()) { tokensLine = documentLinesEnumerator.Current; } while (tokensLine != null) { // Peek the next line to look for continuations if (documentLinesEnumerator.MoveNext()) { nextTokensLine = documentLinesEnumerator.Current; } else { nextTokensLine = null; } // If no continuation is found, scan the current line if (nextTokensLine == null || nextTokensLine.Type != CobolTextLineType.Continuation) { if (lastScanState == null) { Scanner.ScanFirstLine(tokensLine, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions); } else { Scanner.ScanTokensLine(tokensLine, lastScanState, compilerOptions); } } // If a continuation is found on the next line, scan the continuation lines as a group else { // Build a list of the lines we will have to scan as a group : IList<TokensLine> continuationLinesGroup = new List<TokensLine>(); // Add current line continuationLinesGroup.Add(tokensLine); // Add next line tokensLine = nextTokensLine; nextTokensLine = null; continuationLinesGroup.Add(tokensLine); // Navigate forwards to the end of the multiline continuation while (documentLinesEnumerator.MoveNext()) { nextTokensLine = documentLinesEnumerator.Current; if (nextTokensLine.Type == CobolTextLineType.Continuation /*|| <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line lineToScan.Type == CobolTextLineType.Comment || lineToScan.Type == CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines { // Add this line at the end of the list of continuation lines tokensLine = nextTokensLine; nextTokensLine = null; continuationLinesGroup.Add(tokensLine); } else { // Exit the loop break; } } // Scan the whole group of continuation lines if (lastScanState == null) { Scanner.ScanFirstLineContinuationGroup(continuationLinesGroup, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions); } else { Scanner.ScanTokensLineContinuationGroup(continuationLinesGroup, lastScanState, compilerOptions); } } lastScanState = tokensLine.ScanState; tokensLine = nextTokensLine; } }
/// <summary> /// Scan a group of continuation lines /// </summary> public static void ScanTokensLineContinuationGroup(IList<TokensLine> continuationLinesGroup, MultilineScanState initialScanState, TypeCobolOptions compilerOptions) { // p54: Continuation lines // Any sentence, entry, clause, or phrase that requires more than one line can be // continued in Area B of the next line that is neither a comment line nor a blank line. // The line being continued is a continued line; the succeeding lines are continuation // lines. // Track the length of text contributed to the continuation text by each individual line TextArea[] textAreasForOriginalLinesInConcatenatedLine = new TextArea[continuationLinesGroup.Count]; int[] startIndexForTextAreasInOriginalLines = new int[continuationLinesGroup.Count]; int[] offsetForLiteralContinuationInOriginalLines = new int[continuationLinesGroup.Count]; // Initialize the continuation text with the complete source text of the first line TokensLine firstLine = continuationLinesGroup[0]; string concatenatedLine = null; if (firstLine.Type == CobolTextLineType.Source || (firstLine.Type == CobolTextLineType.Debug && initialScanState.WithDebuggingMode)) { concatenatedLine = firstLine.SourceText; } else { concatenatedLine = String.Empty; Scanner.ScanTokensLine(firstLine, initialScanState, compilerOptions); } textAreasForOriginalLinesInConcatenatedLine[0] = new TextArea(TextAreaType.Source, 0, concatenatedLine.Length -1); startIndexForTextAreasInOriginalLines[0] = firstLine.Source.StartIndex; offsetForLiteralContinuationInOriginalLines[0] = 0; // All the following lines are continuation lines // => build a character string representing the complete continuation text along the way for (int i = 1; i < continuationLinesGroup.Count; i++) { TokensLine continuationLine = continuationLinesGroup[i]; int startIndex = continuationLine.Source.StartIndex; int lastIndex = continuationLine.Source.EndIndex; string line = continuationLine.Text; // 1. Match and remove all blank characters at the beginning of the continuation line int startOfContinuationIndex = startIndex; for (; startOfContinuationIndex <= lastIndex && line[startOfContinuationIndex] == ' '; startOfContinuationIndex++) { } if (startOfContinuationIndex > startIndex) { Token whitespaceToken = new Token(TokenType.SpaceSeparator, startIndex, startOfContinuationIndex - 1, continuationLine); continuationLine.SourceTokens.Add(whitespaceToken); startIndex = startOfContinuationIndex; } if (startOfContinuationIndex <= lastIndex) { if (startOfContinuationIndex < 4) { continuationLine.AddDiagnostic(MessageCode.AreaAOfContinuationLineMustBeBlank, startOfContinuationIndex, startOfContinuationIndex); } } else { // Nothing but spaces on the continuation line continue; } // p55: Continuation of alphanumeric and national literals // Alphanumeric and national literals can be continued only when there are no DBCS // characters in the content of the literal. // The following rules apply to alphanumeric and national literals that do not contain // DBCS characters: // - If the continued line contains an alphanumeric or national literal without a // closing quotation mark, all spaces at the end of the continued line (through // column 72) are considered to be part of the literal. The continuation line must // contain a hyphen in the indicator area, and the first nonblank character must be // a quotation mark. The continuation of the literal begins with the character // immediately following the quotation mark. // - If an alphanumeric or national literal that is to be continued on the next line has // as its last character a quotation mark in column 72, the continuation line must // start with two consecutive quotation marks. This will result in a single quotation // mark as part of the value of the literal. // - If the last character on the continued line of an alphanumeric or national literal // is a single quotation mark in Area B, the continuation line can start with a single // quotation mark. This will result in two consecutive literals instead of one // continued literal. // The rules are the same when an apostrophe is used instead of a quotation mark in // delimiters. // ... p55 -> p56: examples of continuations and expected behavior ... int offsetForLiteralContinuation = 0; if (concatenatedLine.Length > 0) { // Scan the continuation text, and get its last token so far TokensLine temporaryTokensLine = TokensLine.CreateVirtualLineForInsertedToken(firstLine.InitialLineIndex, concatenatedLine); Scanner.ScanTokensLine(temporaryTokensLine, initialScanState, compilerOptions); Token lastTokenOfConcatenatedLineSoFar = temporaryTokensLine.SourceTokens[temporaryTokensLine.SourceTokens.Count - 1]; // Check if the last token so far is an alphanumeric or national literal if (lastTokenOfConcatenatedLineSoFar.TokenFamily == TokenFamily.AlphanumericLiteral) { // The continuation line must contain a hyphen in the indicator area, and the first nonblank character must be a quotation mark if (line[startOfContinuationIndex] != lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter) { continuationLine.AddDiagnostic(MessageCode.InvalidFirstCharForContinuationLine, startOfContinuationIndex, startOfContinuationIndex, lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter); } // The continuation of the literal begins with the character immediately following the quotation mark. else { offsetForLiteralContinuation = 1; // If an alphanumeric literal that is to be continued on the next line has as its last character a quotation mark in column 72, // the continuation line must start with two consecutive quotation marks. if (lastTokenOfConcatenatedLineSoFar.HasClosingDelimiter) { if ((startOfContinuationIndex + 1) > lastIndex || line[startOfContinuationIndex + 1] != lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter) { continuationLine.AddDiagnostic(MessageCode.InvalidFirstTwoCharsForContinuationLine, startOfContinuationIndex, startOfContinuationIndex + 1, lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter); // Use the first quotation mark to avoid a complete mess while scanning the rest of the line offsetForLiteralContinuation = 0; } } } } // Check if the last token so far is a floating comment else if (lastTokenOfConcatenatedLineSoFar.TokenType == TokenType.FloatingComment) { // => remove the floating comment from the text of the continuation concatenatedLine = concatenatedLine.Substring(0, concatenatedLine.Length - lastTokenOfConcatenatedLineSoFar.Length); textAreasForOriginalLinesInConcatenatedLine[i - 1] = new TextArea(TextAreaType.Source, textAreasForOriginalLinesInConcatenatedLine[i - 1].StartIndex, textAreasForOriginalLinesInConcatenatedLine[i - 1].EndIndex - lastTokenOfConcatenatedLineSoFar.Length); TokensLine lineWithFloatingComment = continuationLinesGroup[i - 1]; Token floatingCommentToken = new Token(TokenType.FloatingComment, lineWithFloatingComment.Length - lastTokenOfConcatenatedLineSoFar.Length, lineWithFloatingComment.Length - 1, lineWithFloatingComment); lineWithFloatingComment.SourceTokens.Add(floatingCommentToken); } // Check if the last token so far is a comment entry else if (lastTokenOfConcatenatedLineSoFar.TokenType == TokenType.CommentEntry) { // p105: A hyphen in the indicator area (column 7) is not permitted in comment - entries. // => impossible to ignore the continuation indicator here, it is too late // (we can not know there is a comment entry before scanning the continuation lines groups) // => register an error message continuationLine.AddDiagnostic(MessageCode.HyphenIndicatorNotPermittedInCommenEntries, continuationLine.Indicator.StartIndex + 1, continuationLine.Indicator.EndIndex + 1); } } // p54: If there is no hyphen (-) in the indicator area (column 7) of a line, the last character // of the preceding line is assumed to be followed by a space. // If there is a hyphen in the indicator area of a line, the first nonblank character of // the continuation line immediately follows the last nonblank character of the // continued line without an intervening space. // Concatenate the continuation text so far with the text of the current continuation line int startIndexOfContinuationStringInContinuationLine = startOfContinuationIndex + offsetForLiteralContinuation; int lengthOfContinuationStringInContinuationLine = lastIndex - startIndexOfContinuationStringInContinuationLine + 1; textAreasForOriginalLinesInConcatenatedLine[i] = new TextArea(TextAreaType.Source, concatenatedLine.Length, concatenatedLine.Length + lengthOfContinuationStringInContinuationLine - 1); startIndexForTextAreasInOriginalLines[i] = startIndexOfContinuationStringInContinuationLine; offsetForLiteralContinuationInOriginalLines[i] = offsetForLiteralContinuation; concatenatedLine += line.Substring(startIndexOfContinuationStringInContinuationLine, lengthOfContinuationStringInContinuationLine); } // Scan the complete continuation text as a whole TokensLine virtualContinuationTokensLine = TokensLine.CreateVirtualLineForInsertedToken(firstLine.InitialLineIndex, concatenatedLine); Scanner.ScanTokensLine(virtualContinuationTokensLine, initialScanState, compilerOptions); // Then attribute each token and diagnostic to its corresponding tokens line MultilineScanState scanState = initialScanState; for (int i = 0; i < continuationLinesGroup.Count; i++) { TokensLine originalLine = continuationLinesGroup[i]; originalLine.InitializeScanState(scanState); TextArea textAreaForOriginalLine = textAreasForOriginalLinesInConcatenatedLine[i]; int concatenatedLineToOriginalLineOffset = startIndexForTextAreasInOriginalLines[i] - textAreaForOriginalLine.StartIndex; foreach (Token token in virtualContinuationTokensLine.SourceTokens) { // Token located after the current line if(token.StartIndex > textAreaForOriginalLine.EndIndex) { break; } // Token located before the current line else if(token.StopIndex < textAreaForOriginalLine.StartIndex) { continue; } // Token completely completely included inside the current line else if(token.StartIndex >= textAreaForOriginalLine.StartIndex && token.StopIndex <= textAreaForOriginalLine.EndIndex) { int startIndexInOriginalLine = token.StartIndex + concatenatedLineToOriginalLineOffset; int stopIndexInOriginalLine = token.StopIndex + concatenatedLineToOriginalLineOffset; token.CorrectTokensLine(originalLine, startIndexInOriginalLine, stopIndexInOriginalLine); originalLine.AddToken(token); foreach(Diagnostic diag in virtualContinuationTokensLine.GetDiagnosticsForToken(token)) { originalLine.AddDiagnostic((MessageCode)diag.Info.Code, token, diag.MessageArgs); } } // Multiline continuation token only partially located on this line else { bool isContinuationFromPreviousLine = token.StartIndex < textAreaForOriginalLine.StartIndex; bool isContinuedOnNextLine = token.StopIndex > textAreaForOriginalLine.EndIndex; int startIndexInOriginalLine = 0; if (isContinuationFromPreviousLine) { startIndexInOriginalLine = startIndexForTextAreasInOriginalLines[i] - offsetForLiteralContinuationInOriginalLines[i]; } else { startIndexInOriginalLine = token.StartIndex + concatenatedLineToOriginalLineOffset; } int stopIndexInOriginalLine = 0; if (isContinuedOnNextLine) { stopIndexInOriginalLine = originalLine.Source.EndIndex; // If a continued line ends with a floating comment, the continued token ends just before the floating comment if (originalLine.SourceTokens.Count > 0 && originalLine.SourceTokens[originalLine.SourceTokens.Count - 1].TokenType == TokenType.FloatingComment) { stopIndexInOriginalLine -= originalLine.SourceTokens[originalLine.SourceTokens.Count - 1].Length; } } else { stopIndexInOriginalLine = token.StopIndex + concatenatedLineToOriginalLineOffset; } ContinuationToken continuationToken = new ContinuationToken(token, startIndexInOriginalLine, stopIndexInOriginalLine, originalLine, isContinuationFromPreviousLine, isContinuedOnNextLine); originalLine.AddToken(continuationToken); // Copy diagnostics on the first line only if(!isContinuationFromPreviousLine) { foreach (Diagnostic diag in virtualContinuationTokensLine.GetDiagnosticsForToken(token)) { originalLine.AddDiagnostic((MessageCode)diag.Info.Code, token, diag.MessageArgs); } } } } scanState = originalLine.ScanState; } }
/// <summary> /// Initial parsing of a complete document /// </summary> internal static void ParseDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <CodeElementsLine> documentLines, TypeCobolOptions compilerOptions, PerfStatsForParserInvocation perfStatsForParserInvocation) { ParseProcessedTokensLinesChanges(textSourceInfo, documentLines, null, null, compilerOptions, perfStatsForParserInvocation); }
/// <summary> /// Incremental preprocessing of a set of tokens lines changes /// </summary> internal static IList<DocumentChange<IProcessedTokensLine>> ProcessTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<ProcessedTokensLine> documentLines, IList<DocumentChange<ITokensLine>> tokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider) { // Collect all changes applied to the processed tokens lines during the incremental scan IList<DocumentChange<IProcessedTokensLine>> processedTokensLinesChanges = new List<DocumentChange<IProcessedTokensLine>>(); // There are 2 reasons to a preprocess a tokens line after a change : // 1. A tokens line changed : these lines were already reset during the previous steps // 2. If a tokens line that changed was involved in the parsing of a multiline compiler directive, the whole group of lines must be parsed again // Then, if a new COPY directive was parsed : the CompilationDocument to include must be prepared // --- PREPARATION PHASE : reset all processed tokens lines which were involved in a multiline compiler directive where at least one line changed --- // Iterate over all tokens changes detected by the ScannerStep : // refresh all the adjacent lines participating in a ContinuationTokensGroup if (tokensLinesChanges != null) { int lastLineIndexReset = -1; foreach (DocumentChange<ITokensLine> tokensChange in tokensLinesChanges) { processedTokensLinesChanges.Add(new DocumentChange<IProcessedTokensLine>(tokensChange.Type, tokensChange.LineIndex, (IProcessedTokensLine)tokensChange.NewLine)); if (tokensChange.LineIndex > lastLineIndexReset) { lastLineIndexReset = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, processedTokensLinesChanges, lastLineIndexReset); } } } // --- COMPILER DIRECTIVES PHASE : Find and parse all compiler directives --- // Init. Prepare a compiler directive parser // Create a token iterator on top of tokens lines TokensLinesIterator tokensIterator = new TokensLinesIterator( textSourceInfo.Name, documentLines, null, Token.CHANNEL_SourceTokens); // Crate an Antlr compatible token source on top a the token iterator TokensLinesTokenSource tokenSource = new TokensLinesTokenSource( textSourceInfo.Name, tokensIterator); // Init a compiler directive parser CommonTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens); CobolCompilerDirectivesParser directivesParser = new CobolCompilerDirectivesParser(tokenStream); IAntlrErrorStrategy compilerDirectiveErrorStrategy = new CompilerDirectiveErrorStrategy(); directivesParser.ErrorHandler = compilerDirectiveErrorStrategy; // Register all parse errors in a list in memory DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener(); directivesParser.RemoveErrorListeners(); directivesParser.AddErrorListener(errorListener); // Prepare to analyze the parse tree ParseTreeWalker walker = new ParseTreeWalker(); CompilerDirectiveBuilder directiveBuilder = new CompilerDirectiveBuilder(); // 1. Iterate over all compiler directive starting tokens found in the lines which were updated foreach (Token compilerDirectiveStartingToken in documentLines .Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing) .SelectMany(line => line.SourceTokens) .Where(token => token.TokenFamily == TokenFamily.CompilerDirectiveStartingKeyword)) { // 2. Reset the compiler directive parser state // Reset tokens iterator position before parsing // -> seek just before the compiler directive starting token tokensIterator.SeekToToken(compilerDirectiveStartingToken); tokensIterator.PreviousToken(); // Special case : for efficiency reasons, in EXEC SQL INCLUDE directives // only the third token INCLUDE is recognized as a compiler directive // starting keyword by the scanner. In this case, we must rewind the // iterator two tokens backwards to start parsing just before the EXEC token. if (compilerDirectiveStartingToken.TokenType == TokenType.EXEC_SQL_INCLUDE) { tokensIterator.PreviousToken(); tokensIterator.PreviousToken(); } // Reset Antlr BufferedTokenStream position tokenStream.SetTokenSource(tokenSource); // Reset parsing error diagnostics compilerDirectiveErrorStrategy.Reset(directivesParser); errorListener.Diagnostics.Clear(); // 3. Try to parse a compiler directive starting with the current token CobolCompilerDirectivesParser.CompilerDirectingStatementContext directiveParseTree = directivesParser.compilerDirectingStatement(); // 4. Visit the parse tree to build a first class object representing the compiler directive walker.Walk(directiveBuilder, directiveParseTree); CompilerDirective compilerDirective = directiveBuilder.CompilerDirective; bool errorFoundWhileParsingDirective = errorListener.Diagnostics.Count > 0 || directiveBuilder.Diagnostics.Count > 0; // 5. Get all tokens consumed while parsing the compiler directive // and partition them line by line Token startToken = (Token)directiveParseTree.Start; Token stopToken = (Token)directiveParseTree.Stop; if (stopToken == null) stopToken = startToken; MultilineTokensGroupSelection tokensSelection = tokensIterator.SelectAllTokensBetween(startToken, stopToken); if (compilerDirective != null) { // 6. Replace all matched tokens by : // - a CompilerDirectiveToken on the first line ProcessedTokensLine firstProcessedTokensLine = documentLines[tokensSelection.FirstLineIndex]; if (tokensSelection.SelectedTokensOnSeveralLines.Length == 1) { firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine( tokensSelection.TokensOnFirstLineBeforeStartToken, compilerDirective, errorFoundWhileParsingDirective, tokensSelection.SelectedTokensOnSeveralLines[0], tokensSelection.TokensOnLastLineAfterStopToken, false); } else { TokensGroup continuedTokensGroup = firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine( tokensSelection.TokensOnFirstLineBeforeStartToken, compilerDirective, errorFoundWhileParsingDirective, tokensSelection.SelectedTokensOnSeveralLines[0], null, true); // - a ContinuationTokensGroup on the following lines int selectionLineIndex = 1; int lastLineIndex = tokensSelection.FirstLineIndex + tokensSelection.SelectedTokensOnSeveralLines.Length - 1; for (int nextLineIndex = tokensSelection.FirstLineIndex + 1; nextLineIndex <= lastLineIndex; nextLineIndex++, selectionLineIndex++) { IList<Token> compilerDirectiveTokensOnNextLine = tokensSelection.SelectedTokensOnSeveralLines[selectionLineIndex]; if (compilerDirectiveTokensOnNextLine.Count > 0) { ProcessedTokensLine nextProcessedTokensLine = documentLines[nextLineIndex]; if (nextLineIndex != lastLineIndex) { continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine( continuedTokensGroup, compilerDirectiveTokensOnNextLine, null, true); } else { continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine( continuedTokensGroup, compilerDirectiveTokensOnNextLine, tokensSelection.TokensOnLastLineAfterStopToken, false); } } } } } // 7. Register compiler directive parse errors if (errorFoundWhileParsingDirective) { ProcessedTokensLine compilerDirectiveLine = documentLines[tokensSelection.FirstLineIndex]; foreach (ParserDiagnostic parserDiag in errorListener.Diagnostics) { compilerDirectiveLine.AddDiagnostic(parserDiag); } foreach (Diagnostic directiveDiag in directiveBuilder.Diagnostics) { compilerDirectiveLine.AddDiagnostic(directiveDiag); } } } // 8. Advance the state off all ProcessedTokensLines : // NeedsCompilerDirectiveParsing => NeedsCopyDirectiveProcessing if it contains a COPY directive IList<ProcessedTokensLine> parsedLinesWithCopyDirectives = null; // NeedsCompilerDirectiveParsing => Ready otherwise foreach (ProcessedTokensLine parsedLine in documentLines .Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing)) { if (parsedLine.ImportedDocuments != null) { if(parsedLinesWithCopyDirectives == null) { parsedLinesWithCopyDirectives = new List<ProcessedTokensLine>(); } parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.NeedsCopyDirectiveProcessing; parsedLinesWithCopyDirectives.Add(parsedLine); } else { parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready; } } // --- COPY IMPORT PHASE : Process COPY (REPLACING) directives --- // 1. Iterate over all updated lines containing a new COPY directive if (parsedLinesWithCopyDirectives != null) { foreach (ProcessedTokensLine tokensLineWithCopyDirective in parsedLinesWithCopyDirectives) { // Iterate over all COPY directives found on one updated line foreach (CopyDirective copyDirective in tokensLineWithCopyDirective.ImportedDocuments.Keys.ToArray<CopyDirective>()) { try { // Load (or retrieve in cache) the document referenced by the COPY directive ProcessedTokensDocument importedDocumentSource = processedTokensDocumentProvider.GetProcessedTokensDocument(copyDirective.LibraryName, copyDirective.TextName); // Store it on the current line after appying the REPLACING directive ImportedTokensDocument importedDocument = new ImportedTokensDocument(copyDirective, importedDocumentSource); tokensLineWithCopyDirective.ImportedDocuments[copyDirective] = importedDocument; } catch (Exception e) { // Text name refenced by COPY directive was not found // => register a preprocessor error on this line Token failedDirectiveToken = tokensLineWithCopyDirective.TokensWithCompilerDirectives .Where(token => token.TokenType == TokenType.CopyImportDirective && ((CompilerDirectiveToken)token).CompilerDirective == copyDirective) .First(); Diagnostic diag = new Diagnostic( MessageCode.FailedToLoadTextDocumentReferencedByCopyDirective, failedDirectiveToken.Column, failedDirectiveToken.EndColumn, e.Message); tokensLineWithCopyDirective.AddDiagnostic(diag); } } // Advance processing status of the line tokensLineWithCopyDirective.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready; } } // --- REPLACE PHASE : REPLACE directives are implemented in ReplaceTokensLinesIterator --- /* Algorithm : * * one REPLACE directive can express several replacement operations * one replacement operation can be of several types (distinguished for optimization purposes) * - SimpleTokenReplace : one source token / zero or one replacement token * - PartialWordReplace : one pure partial word / zero or one replacement token * - SimpleToMultipleTokenReplace : one source token / several replacement tokens * - MultipleTokenReplace : one first + several following source tokens / zero to many replacement tokens * * an iterator maintains a current set of replacement operations * * if nextToken is replace directive * the current set of replacement operations is updated * else * nextToken is compared to each replacement operation in turn * if single -> single source token operation : return replacement token * if single -> multiple operation : setup a secondary iterator with the list of replacement tokens * if multiple -> multiple operation * snapshot of the underlying iterator * try to match all of the following source tokens * if failure : restore snapshot and try next operation * if success : setup a secondary iterator * * token comparison sourceToken / replacementCandidate : * 1. Compare Token type * 2. If same token type and for families * AlphanumericLiteral * NumericLiteral * SyntaxLiteral * Symbol * => compare also Token text * * PartialCobolWord replacement : * p535 : The COPY statement with REPLACING phrase can be used to replace parts of words. * By inserting a dummy operand delimited by colons into the program text, * the compiler will replace the dummy operand with the required text. * Example 3 shows how this is used with the dummy operand :TAG:. * The colons serve as separators and make TAG a stand-alone operand. */ return processedTokensLinesChanges; }
/// <summary> /// Initial preprocessing of a complete document /// </summary> internal static void ProcessDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <ProcessedTokensLine> documentLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider, List <RemarksDirective.TextNameVariation> copyTextNameVariations, PerfStatsForParserInvocation perfStatsForParserInvocation, List <CopyDirective> missingCopies) { ProcessTokensLinesChanges(textSourceInfo, documentLines, null, null, compilerOptions, processedTokensDocumentProvider, copyTextNameVariations, perfStatsForParserInvocation, missingCopies); }
/// <summary> /// Load a Cobol source file in memory /// </summary> public FileCompiler(string libraryName, string fileName, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ColumnsLayout columnsLayout, TypeCobolOptions compilerOptions, CodeModel.SymbolTable customSymbols, bool isCopyFile, CompilationProject compilationProject) : this(libraryName, fileName, null, sourceFileProvider, documentProvider, columnsLayout, null, compilerOptions, customSymbols, isCopyFile, null, compilationProject, null) { }
public static void CheckTests(string rootFolder, string resultFolder, string timedResultFile, string regex, string[] include, string[] exclude, string[] copiesFolder, string skelPath, int stopAfterAsManyErrors, bool autoRemarks, string expectedResultFile) { string[] files = Directory.GetFiles(rootFolder, regex, SearchOption.AllDirectories); bool codegen = true; var format = TypeCobol.Compiler.DocumentFormat.RDZReferenceFormat; string resultFile = "GeneratedResultFile.txt"; //Initialize both files File.WriteAllText(timedResultFile, ""); if (expectedResultFile != null) { File.WriteAllText(resultFile, ""); } int tested = 0, nbFilesInError = 0, ignores = 0; TimeSpan parsingSumDuration = new TimeSpan(0); TimeSpan codeGenSumDuration = new TimeSpan(0); int parseErrors = 0; int codegenErrors = 0; int codegenDiff = 0; foreach (var file in files) { string filename = Path.GetFileName(file); AppendTextToFiles((filename + ':'), timedResultFile, resultFile); bool ignore = include.Length > 0 && !include.Contains(filename); if (!ignore) { ignore = exclude.Contains(filename); } if (ignore) { ignores++; File.AppendAllText(timedResultFile, " ignored.\n"); continue; } string path = Path.Combine(rootFolder, filename); Stopwatch watch = new Stopwatch(); watch.Start(); var document = new TypeCobol.Parser(); var options = new TypeCobolOptions { ExecToStep = ExecutionStep.CrossCheck, #if EUROINFO_RULES AutoRemarksEnable = autoRemarks #endif }; document.Init(path, options, format, copiesFolder); document.Parse(path); watch.Stop(); //TestJSONSerializer.DumpAsJSON(unit.CodeElementsDocumentSnapshot.CodeElements, filename); TimeSpan elapsed = watch.Elapsed; parsingSumDuration += elapsed; string formatted = String.Format("{0:00}m{1:00}s{2:000}ms", elapsed.Minutes, elapsed.Seconds, elapsed.Milliseconds); AppendTextToFiles(" parsed in " + formatted + "\n", timedResultFile); AppendTextToFiles("- " + document.Results.PerfStatsForText.FirstCompilationTime + " ms : text update\n", timedResultFile); AppendTextToFiles("- " + document.Results.PerfStatsForScanner.FirstCompilationTime + " ms : scanner\n", timedResultFile); AppendTextToFiles("- " + document.Results.PerfStatsForPreprocessor.FirstCompilationTime + " ms : preprocessor\n", timedResultFile); AppendTextToFiles("- " + document.Results.PerfStatsForCodeElementsParser.FirstCompilationTime + " ms : code elements parser\n", timedResultFile); AppendTextToFiles("- " + document.Results.PerfStatsForTemporarySemantic.FirstCompilationTime + " ms : temporary semantic class parser\n", timedResultFile); AppendTextToFiles("- " + document.Results.PerfStatsForProgramCrossCheck.FirstCompilationTime + " ms : cross check class parser\n", timedResultFile); tested++; Console.WriteLine(filename); bool okay = true; var diagnostics = document.Results.AllDiagnostics(); if (diagnostics.Count > 0) { okay = false; parseErrors += diagnostics.Count; } displayAndWriteErrorsToGrammarResult(diagnostics, timedResultFile, resultFile); if (!okay) { nbFilesInError++; if (nbFilesInError >= stopAfterAsManyErrors) { break; } } if (codegen && okay) { watch.Reset(); watch.Start(); var writer = new StringWriter(); //Retrieve skeletons var skeletons = !string.IsNullOrEmpty(skelPath) ? Config.Parse(skelPath) : null; var generator = new TypeCobol.Codegen.Generators.DefaultGenerator(document.Results, writer, skeletons, null); var columns = document.Results.ProgramClassDocumentSnapshot.TextSourceInfo.ColumnsLayout; generator.Generate(document.Results, columns); writer.Close(); //Write duration to GrammarResultFile watch.Stop(); elapsed = watch.Elapsed; codeGenSumDuration += elapsed; formatted = String.Format("{0:00}m{1:00}s{2:000}ms", elapsed.Minutes, elapsed.Seconds, elapsed.Milliseconds); AppendTextToFiles(" generated in " + formatted + "\n", timedResultFile); //Error during generation, no need to check the content of generated Cobol if (generator.Diagnostics != null && generator.Diagnostics.Count > 0) { codegenErrors += generator.Diagnostics.Count; displayAndWriteErrorsToGrammarResult(generator.Diagnostics, timedResultFile, resultFile); nbFilesInError++; if (nbFilesInError >= stopAfterAsManyErrors) { break; } } else { //Compare generated Cobol with expected var expected = AsLines(File.ReadAllText(path, format.Encoding)); var actual = AsLines(writer.ToString()); Directory.CreateDirectory(resultFolder); var linesKO = new List <int>(); for (int i = 0; i < Math.Min(expected.Count, actual.Count); i++) { if (!expected[i].Equals(actual[i])) { linesKO.Add(i); } } var errors = new System.Text.StringBuilder(); string fmt = Lines2FormatString(Math.Max(expected.Count, actual.Count)); if (linesKO.Count > 0 || expected.Count != actual.Count) { errors.AppendLine("--- Lines mismatch ---"); File.WriteAllLines( resultFolder + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(file) + "-Expected" + Path.GetExtension(file), expected); File.WriteAllLines( resultFolder + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(file) + "-Actual" + Path.GetExtension(file), actual); } int start = -1; for (int i = 0; i < linesKO.Count; i++) { int currentline = linesKO[i]; bool follows = i > 0 && linesKO[i - 1] == currentline - 1; if (!follows) { start = currentline; before(errors, expected, currentline, 3, fmt); } bool preceeds = i + 1 < linesKO.Count && linesKO[i + 1] == currentline + 1; if (!preceeds) { diff(errors, expected, actual, start, currentline, fmt); after(errors, expected, currentline, 3, fmt); start = -1; } } for (int i = actual.Count; i < expected.Count; i++) { errors.AppendLine(String.Format("-{0:" + fmt + "} {1}", i, expected[i])); } for (int i = expected.Count; i < actual.Count; i++) { errors.AppendLine(String.Format("+{0:" + fmt + "} {1}", i, actual[i])); } if (errors.Length > 0) { codegenDiff += linesKO.Count + Math.Abs(actual.Count - expected.Count); AppendTextToFiles(errors.ToString(), timedResultFile, resultFile); if (okay) { nbFilesInError++; } } } } else { AppendTextToFiles("\n", timedResultFile, resultFile); } } TimeSpan totalTestDuration = parsingSumDuration + codeGenSumDuration; string parsingTotalDurationFormatted = String.Format("{0:00}m{1:00}s{2:000}ms", parsingSumDuration.Minutes, parsingSumDuration.Seconds, parsingSumDuration.Milliseconds); string message = "Files tested=" + tested + "/" + files.Length + ", files in error=" + nbFilesInError + ", ignored=" + ignores + "\n"; AppendTextToFiles(message, resultFile); if (parseErrors > 0) { message += "Parsing errors: " + parseErrors + '\n'; } if (codegenErrors > 0) { message += "Codegen errors: " + codegenErrors + '\n'; } if (codegenDiff > 0) { message += "Codegen Diff: " + codegenDiff + '\n'; } message += "Parsing time: " + parsingTotalDurationFormatted; if (codegen) { string codeGenTotalDurationFormatted = string.Format("{0:00}m{1:00}s{2:000}ms", codeGenSumDuration.Minutes, codeGenSumDuration.Seconds, codeGenSumDuration.Milliseconds); string totalDurationFormatted = String.Format("{0:00}m{1:00}s{2:000}ms", totalTestDuration.Minutes, totalTestDuration.Seconds, totalTestDuration.Milliseconds); message += " + Codegen time: " + codeGenTotalDurationFormatted + " => Total time: " + totalDurationFormatted; } AppendTextToFiles(message, timedResultFile); if (nbFilesInError > 0 && expectedResultFile == null) { Assert.Fail('\n' + message); //If no expectedFile to compare throw assert if error } else if (expectedResultFile != null) //If expectedFileResult exists compare the DefaultGeneratedFile with expectedFile { StreamReader expectedResultReader = new StreamReader(new FileStream(expectedResultFile, FileMode.Open)); StreamReader actualResultReader = new StreamReader(new FileStream(resultFile, FileMode.Open)); TestUtils.compareLines("GrammarTestCompareFiles", expectedResultReader.ReadToEnd(), actualResultReader.ReadToEnd()); //The test will fail if result files are different expectedResultReader.Close(); actualResultReader.Close(); } }
/// <summary> /// Use a pre-existing text document, not yet associated with a Cobol file + Existing SymbolTable /// </summary> public FileCompiler(ITextDocument textDocument, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, TypeCobolOptions compilerOptions, SymbolTable customSymbols, bool isCopyFile, CompilationProject compilationProject) : this(null, null, null, sourceFileProvider, documentProvider, default(ColumnsLayout), textDocument, compilerOptions, customSymbols, isCopyFile, null, compilationProject, null) { }
/// <summary> /// Scan a group of continuation lines when no previous scan state object is available /// </summary> public static void ScanFirstLineContinuationGroup(IList<TokensLine> continuationLinesGroup, bool insideDataDivision, bool decimalPointIsComma, bool withDebuggingMode, Encoding encodingForAlphanumericLiterals, TypeCobolOptions compilerOptions) { MultilineScanState initialScanState = new MultilineScanState(insideDataDivision, decimalPointIsComma, withDebuggingMode, encodingForAlphanumericLiterals); ScanTokensLineContinuationGroup(continuationLinesGroup, initialScanState, compilerOptions); }
/// <summary> /// Common internal implementation for all constructors above /// </summary> private FileCompiler(string libraryName, string fileName, CobolFile loadedCobolFile, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ColumnsLayout columnsLayout, ITextDocument textDocument, TypeCobolOptions compilerOptions, SymbolTable customSymbols, bool isCopyFile, [CanBeNull] MultilineScanState scanState, CompilationProject compilationProject, List <RemarksDirective.TextNameVariation> copyTextNameVariations) { var chrono = new Stopwatch(); chrono.Start(); // 1.a Find the Cobol source file CobolFile sourceFile = null; CompilationProject = compilationProject; if (fileName != null) { if (sourceFileProvider.TryGetFile(libraryName, fileName, out sourceFile)) { CobolFile = sourceFile; } else { var message = string.IsNullOrEmpty(libraryName) ? string.Format("Cobol source file not found: {0}", fileName) : string.Format("Cobol source file not found: {0} in {1}", fileName, libraryName); throw new Exception(message); } } // 1.b Register a Cobol source file which was already loaded else if (loadedCobolFile != null) { CobolFile = loadedCobolFile; } chrono.Stop(); SourceFileSearchTime = (int)chrono.ElapsedMilliseconds; chrono.Reset(); // 2.a Load it in a new text document in memory chrono.Start(); if (textDocument == null) { TextDocument = new ReadOnlyTextDocument(sourceFile?.Name, sourceFile?.Encoding, columnsLayout, sourceFile?.ReadChars()); } // 2.b Load it in an existing text document in memory else if (sourceFile != null) { TextDocument = textDocument; textDocument.LoadChars(sourceFile.ReadChars()); } // 2.c Use a pre-existing text document // - not yet associated with a Cobol source file // - with a Cobol source file already loaded else if (sourceFile == null || loadedCobolFile != null) { TextDocument = textDocument; } chrono.Stop(); SourceFileLoadTime = (int)chrono.ElapsedMilliseconds; chrono.Reset(); // 3. Prepare the data structures used by the different steps of the compiler if (isCopyFile) { CompilationResultsForCopy = new CompilationDocument(TextDocument.Source, TextDocument.Lines, compilerOptions, documentProvider, scanState, copyTextNameVariations); CompilationResultsForCopy.CustomSymbols = customSymbols; } else { CompilationResultsForProgram = new CompilationUnit(TextDocument.Source, TextDocument.Lines, compilerOptions, documentProvider, copyTextNameVariations); CompilationResultsForProgram.CustomSymbols = customSymbols; } CompilerOptions = compilerOptions; }
/// <summary> /// Scan a line of a document /// </summary> public static void ScanTokensLine(TokensLine tokensLine, MultilineScanState initialScanState, TypeCobolOptions compilerOptions) { // Updates are forbidden after a snapshot of a specific version of a line if(!tokensLine.CanStillBeUpdatedBy(CompilationStep.Scanner)) { throw new InvalidOperationException("Can not update this TokensLine because it was already frozen by compilation step : " + tokensLine.CompilationStep.ToString()); } // Set the initial scan state for the line tokensLine.InitializeScanState(initialScanState); // Alias to refer to Cobol text line properties ICobolTextLine textLine = tokensLine; // The source section of this line of text must be split into tokens string line = tokensLine.Text; int startIndex = textLine.Source.StartIndex; int lastIndex = textLine.Source.EndIndex; #if EUROINFO_LEGACY_REPLACING_SYNTAX if (IsInsideRemarks(textLine.Type, tokensLine.SourceText)) tokensLine.ScanState.InsideRemarksDirective = true; // Try to scan REMARKS compiler directive parameters inside the comment or non-comment line if (tokensLine.ScanState.InsideRemarksDirective) { string remarksLine = textLine.SourceText; int startIndexForSignificantPart = GetStartIndexOfSignificantPart(remarksLine, tokensLine.ScanState); int firstPeriodIndex = remarksLine.IndexOf('.', startIndexForSignificantPart); int endIndexForSignificantPart = GetEndIndexOfSignificantPart(remarksLine, tokensLine.ScanState, firstPeriodIndex); string significantPart = remarksLine.Substring(startIndexForSignificantPart, endIndexForSignificantPart - startIndexForSignificantPart + 1).Trim(); if (firstPeriodIndex >= 0 || (!tokensLine.ScanState.InsideRemarksParentheses && !remarksLine.Contains("COPY"))) { tokensLine.ScanState.InsideRemarksDirective = false; // indicates the end of the REMARKS compiler directive } RemarksDirective remarksDirective = CreateRemarksDirective(significantPart, tokensLine.ScanState); if (remarksDirective != null && remarksDirective.CopyTextNamesVariations.Count > 0) { // A non empty remarks directive will replace the comment line tokensLine.AddToken(CreateCompilerDirectiveToken(remarksDirective, tokensLine, startIndex, lastIndex)); return; } } #endif // Comment line => return only one token with type CommentLine // Debug line => treated as a comment line if debugging mode was not activated if (textLine.Type == CobolTextLineType.Comment || (textLine.Type == CobolTextLineType.Debug && !tokensLine.InitialScanState.WithDebuggingMode)) { Token commentToken = new Token(TokenType.CommentLine, startIndex, lastIndex, tokensLine); tokensLine.AddToken(commentToken); return; } // Invalid indicator, the line type is unknown => the whole line text is handled as a single invalid token else if (textLine.Type == CobolTextLineType.Invalid) { // Invalid indicator => register an error tokensLine.AddDiagnostic(MessageCode.InvalidIndicatorCharacter, textLine.Indicator.StartIndex, textLine.Indicator.EndIndex, textLine.Indicator); Token invalidToken = new Token(TokenType.InvalidToken, startIndex, lastIndex, tokensLine); tokensLine.AddToken(invalidToken); return; } // Empty line => return immediately an empty list of tokens // Blank line => return only one token with type SpaceSeparator if(textLine.Type == CobolTextLineType.Blank) { if(!String.IsNullOrEmpty(line)) { Token whitespaceToken = new Token(TokenType.SpaceSeparator, startIndex, lastIndex, tokensLine); tokensLine.AddToken(whitespaceToken); } return; } // Create a stateful line scanner, and iterate over the tokens Scanner scanner = new Scanner(line, startIndex, lastIndex, tokensLine, compilerOptions); Token nextToken = null; while((nextToken = scanner.GetNextToken()) != null) { // Resolve DELETE ambiguity : DELETE + InterLiteral => DELETE_CD // Warning : DELETE and the sequence-number-field must be on the same line if(nextToken.TokenType == TokenType.IntegerLiteral && tokensLine.ScanState.KeywordsState == KeywordsSequenceState.After_DELETE) { tokensLine.ScanState.LastKeywordOrSymbolToken.CorrectType(TokenType.DELETE_CD); } tokensLine.AddToken(nextToken); } }
public static void Main(string[] args) { // Basic test program, useful to debug : compiles all sample programs located under TypeCobol.Test\Samples\EI Cobol samples\EI-Production" string currentDirectory = Directory.GetCurrentDirectory(); string projectRootPath = currentDirectory.Substring(0, currentDirectory.IndexOf(@"\TypeCobol\") + 11); string sourcePath = projectRootPath + @"TypeCobol.Test\Samples\EI Cobol samples\EI-Production"; string[] programExtensions = { "*.PGM" }; string[] copyExtensions = { "*.CPY" }; DocumentFormat docFormat = new DocumentFormat(Encoding.GetEncoding("iso8859-1"), EndOfLineDelimiter.CrLfCharacters, 80, ColumnsLayout.CobolReferenceFormat); TypeCobolOptions compilerOptions = new TypeCobolOptions(); CompilationProject project = new CompilationProject("samples", sourcePath, programExtensions.Concat(copyExtensions).ToArray(), docFormat.Encoding, docFormat.EndOfLineDelimiter, docFormat.FixedLineLength, docFormat.ColumnsLayout, compilerOptions); // Iterate over all programs in the source directory foreach (string programExtension in programExtensions) { foreach (string filePath in Directory.EnumerateFiles(sourcePath, programExtension)) { // Compile program string textName = Path.GetFileNameWithoutExtension(filePath); Console.Write(textName + " ... "); try { FileCompiler fileCompiler = new FileCompiler(null, textName, project.SourceFileProvider, project, ColumnsLayout.CobolReferenceFormat, compilerOptions.Clone(), null, false); fileCompiler.CompileOnce(); Console.WriteLine(" OK"); } catch(Exception e) { Console.WriteLine("error :"); Console.WriteLine(e.Message); } } } /* // TO DO : read compiler options on the command line // Start with the default compiler options TypeCobolOptions compilerOptions = new TypeCobolOptions(); // Simple test version // - all referenced files should be located under the current directory CompilationProject project = new CompilationProject("project", ".", new string[] { "*.cbl", "*.cpy" }, IBMCodePages.GetDotNetEncodingFromIBMCCSID(1147), EndOfLineDelimiter.FixedLengthLines, 80, ColumnsLayout.CobolReferenceFormat, compilerOptions); // - gets one file name as argument and compiles it if (args.Length == 1) { string textName = args[0]; FileCompiler fileCompiler = new FileCompiler(null, textName, project.SourceFileProvider, project, ColumnsLayout.CobolReferenceFormat, compilerOptions.Clone(), false); fileCompiler.CompileOnce(); } // - gets an optional "-continuous" flag as the first argument to activate source file monitoring and automatic recompilation else if (args.Length == 2) { if (String.Equals(args[0], "-continuous", StringComparison.InvariantCultureIgnoreCase)) { string textName = args[1]; FileCompiler fileCompiler = new FileCompiler(null, textName, project.SourceFileProvider, project, ColumnsLayout.CobolReferenceFormat, compilerOptions.Clone(), false); fileCompiler.StartContinuousBackgroundCompilation(400, 400, 900, 2000); Console.WriteLine("Processing, press enter to stop ..."); fileCompiler.StartContinuousFileProcessing(); Console.ReadLine(); } else { Console.WriteLine("ERROR : Invalid option"); } } else { Console.WriteLine("ERROR : Invalid number of arguments"); } */ }
/// <summary> /// Initial scan of a complete document /// </summary> public static void ScanDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, TypeCobolOptions compilerOptions) { ScanDocument(textSourceInfo, documentLines, compilerOptions, null); }
/// <summary> /// Incremental preprocessing of a set of tokens lines changes /// </summary> internal static IList <DocumentChange <IProcessedTokensLine> > ProcessTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <ProcessedTokensLine> documentLines, IList <DocumentChange <ITokensLine> > tokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider, List <RemarksDirective.TextNameVariation> copyTextNameVariations, PerfStatsForParserInvocation perfStatsForParserInvocation, List <CopyDirective> missingCopies) { // Collect all changes applied to the processed tokens lines during the incremental scan IList <DocumentChange <IProcessedTokensLine> > processedTokensLinesChanges = new List <DocumentChange <IProcessedTokensLine> >(); // There are 2 reasons to a preprocess a tokens line after a change : // 1. A tokens line changed : these lines were already reset during the previous steps // 2. If a tokens line that changed was involved in the parsing of a multiline compiler directive, the whole group of lines must be parsed again // Then, if a new COPY directive was parsed : the CompilationDocument to include must be prepared // --- PREPARATION PHASE : reset all processed tokens lines which were involved in a multiline compiler directive where at least one line changed --- // Iterate over all tokens changes detected by the ScannerStep : // refresh all the adjacent lines participating in a ContinuationTokensGroup if (tokensLinesChanges != null) { int lastLineIndexReset = -1; foreach (DocumentChange <ITokensLine> tokensChange in tokensLinesChanges) { processedTokensLinesChanges.Add(new DocumentChange <IProcessedTokensLine>(tokensChange.Type, tokensChange.LineIndex, (IProcessedTokensLine)tokensChange.NewLine)); if (tokensChange.LineIndex > lastLineIndexReset) { lastLineIndexReset = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, processedTokensLinesChanges, lastLineIndexReset); } } } // --- COMPILER DIRECTIVES PHASE : Find and parse all compiler directives --- // Init. Prepare a compiler directive parser // Create a token iterator on top of tokens lines TokensLinesIterator tokensIterator = new TokensLinesIterator( textSourceInfo.Name, documentLines, null, Token.CHANNEL_SourceTokens); // Crate an Antlr compatible token source on top a the token iterator TokensLinesTokenSource tokenSource = new TokensLinesTokenSource( textSourceInfo.Name, tokensIterator); // Init a compiler directive parser CommonTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens); CobolCompilerDirectivesParser directivesParser = new CobolCompilerDirectivesParser(tokenStream); // Optionnaly activate Antlr Parser performance profiling // WARNING : use this in a single-treaded context only (uses static field) if (AntlrPerformanceProfiler == null && perfStatsForParserInvocation.ActivateDetailedAntlrPofiling) { AntlrPerformanceProfiler = new AntlrPerformanceProfiler(directivesParser); } if (AntlrPerformanceProfiler != null) { // Replace the generated parser by a subclass which traces all rules invocations directivesParser = new CobolCompilerDirectivesTracingParser(tokenStream); AntlrPerformanceProfiler.BeginParsingFile(textSourceInfo, null); } IAntlrErrorStrategy compilerDirectiveErrorStrategy = new CompilerDirectiveErrorStrategy(); directivesParser.ErrorHandler = compilerDirectiveErrorStrategy; // Register all parse errors in a list in memory ParserDiagnosticErrorListener errorListener = new ParserDiagnosticErrorListener(); directivesParser.RemoveErrorListeners(); directivesParser.AddErrorListener(errorListener); // Prepare to analyze the parse tree ParseTreeWalker walker = new ParseTreeWalker(); CompilerDirectiveBuilder directiveBuilder = new CompilerDirectiveBuilder(compilerOptions, copyTextNameVariations); // 1. Iterate over all compiler directive starting tokens found in the lines which were updated foreach (Token compilerDirectiveStartingToken in documentLines .Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing) .SelectMany(line => line.SourceTokens) .Where(token => token.TokenFamily == TokenFamily.CompilerDirectiveStartingKeyword)) { // 2. Reset the compiler directive parser state // Reset tokens iterator position before parsing // -> seek just before the compiler directive starting token tokensIterator.SeekToToken(compilerDirectiveStartingToken); tokensIterator.PreviousToken(); // Special case : for efficiency reasons, in EXEC SQL INCLUDE directives // only the third token INCLUDE is recognized as a compiler directive // starting keyword by the scanner. In this case, we must rewind the // iterator two tokens backwards to start parsing just before the EXEC token. if (compilerDirectiveStartingToken.TokenType == TokenType.EXEC_SQL_INCLUDE) { tokensIterator.PreviousToken(); tokensIterator.PreviousToken(); } // Reset Antlr BufferedTokenStream position tokenStream.SetTokenSource(tokenSource); // Reset parsing error diagnostics compilerDirectiveErrorStrategy.Reset(directivesParser); // 3. Try to parse a compiler directive starting with the current token perfStatsForParserInvocation.OnStartAntlrParsing(); if (AntlrPerformanceProfiler != null) { AntlrPerformanceProfiler.BeginParsingSection(); } CobolCompilerDirectivesParser.CompilerDirectingStatementContext directiveParseTree = directivesParser.compilerDirectingStatement(); if (AntlrPerformanceProfiler != null) { AntlrPerformanceProfiler.EndParsingSection(directiveParseTree.ChildCount); } perfStatsForParserInvocation.OnStopAntlrParsing( AntlrPerformanceProfiler != null ? (int)AntlrPerformanceProfiler.CurrentFileInfo.DecisionTimeMs : 0, AntlrPerformanceProfiler != null ? AntlrPerformanceProfiler.CurrentFileInfo.RuleInvocations.Sum() : 0); perfStatsForParserInvocation.OnStartTreeBuilding(); // 4. Visit the parse tree to build a first class object representing the compiler directive walker.Walk(directiveBuilder, directiveParseTree); CompilerDirective compilerDirective = directiveBuilder.CompilerDirective; bool errorFoundWhileParsingDirective = compilerDirective == null || compilerDirective.Diagnostics != null || directiveParseTree.Diagnostics != null; // 5. Get all tokens consumed while parsing the compiler directive // and partition them line by line Token startToken = (Token)directiveParseTree.Start; Token stopToken = (Token)directiveParseTree.Stop; if (stopToken == null) { stopToken = startToken; } MultilineTokensGroupSelection tokensSelection = tokensIterator.SelectAllTokensBetween(startToken, stopToken); if (compilerDirective != null) { // 6. Replace all matched tokens by : // - a CompilerDirectiveToken on the first line ProcessedTokensLine firstProcessedTokensLine = documentLines[tokensSelection.FirstLineIndex]; if (tokensSelection.SelectedTokensOnSeveralLines.Length == 1) { firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine( tokensSelection.TokensOnFirstLineBeforeStartToken, compilerDirective, errorFoundWhileParsingDirective, tokensSelection.SelectedTokensOnSeveralLines[0], tokensSelection.TokensOnLastLineAfterStopToken, false); } else { TokensGroup continuedTokensGroup = firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine( tokensSelection.TokensOnFirstLineBeforeStartToken, compilerDirective, errorFoundWhileParsingDirective, tokensSelection.SelectedTokensOnSeveralLines[0], null, true); // - a ContinuationTokensGroup on the following lines int selectionLineIndex = 1; int lastLineIndex = tokensSelection.FirstLineIndex + tokensSelection.SelectedTokensOnSeveralLines.Length - 1; for (int nextLineIndex = tokensSelection.FirstLineIndex + 1; nextLineIndex <= lastLineIndex; nextLineIndex++, selectionLineIndex++) { IList <Token> compilerDirectiveTokensOnNextLine = tokensSelection.SelectedTokensOnSeveralLines[selectionLineIndex]; if (compilerDirectiveTokensOnNextLine.Count > 0) { ProcessedTokensLine nextProcessedTokensLine = documentLines[nextLineIndex]; if (nextLineIndex != lastLineIndex) { continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine( continuedTokensGroup, compilerDirectiveTokensOnNextLine, null, true); } else { continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine( continuedTokensGroup, compilerDirectiveTokensOnNextLine, tokensSelection.TokensOnLastLineAfterStopToken, false); } } } } } // 7. Register compiler directive parse errors if (errorFoundWhileParsingDirective) { ProcessedTokensLine compilerDirectiveLine = documentLines[tokensSelection.FirstLineIndex]; if (compilerDirective != null && compilerDirective.Diagnostics != null) { foreach (Diagnostic directiveDiag in compilerDirective.Diagnostics) { compilerDirectiveLine.AddDiagnostic(directiveDiag); } } else if (directiveParseTree.Diagnostics != null) { foreach (Diagnostic directiveDiag in directiveParseTree.Diagnostics) { if (compilerDirective != null) { compilerDirective.AddDiagnostic(directiveDiag); } compilerDirectiveLine.AddDiagnostic(directiveDiag); } } } } if (AntlrPerformanceProfiler != null) { AntlrPerformanceProfiler.EndParsingFile(directivesParser.ParseInfo.DecisionInfo, (int)(directivesParser.ParseInfo.GetTotalTimeInPrediction() / 1000000)); } // 8. Advance the state off all ProcessedTokensLines : // NeedsCompilerDirectiveParsing => NeedsCopyDirectiveProcessing if it contains a COPY directive IList <ProcessedTokensLine> parsedLinesWithCopyDirectives = null; // NeedsCompilerDirectiveParsing => Ready otherwise foreach (ProcessedTokensLine parsedLine in documentLines .Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing)) { if (parsedLine.ImportedDocuments != null) { if (parsedLinesWithCopyDirectives == null) { parsedLinesWithCopyDirectives = new List <ProcessedTokensLine>(); } parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.NeedsCopyDirectiveProcessing; parsedLinesWithCopyDirectives.Add(parsedLine); } else { parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready; } } perfStatsForParserInvocation.OnStopTreeBuilding(); // --- COPY IMPORT PHASE : Process COPY (REPLACING) directives --- foreach (var lineChange in processedTokensLinesChanges) { missingCopies.Remove(missingCopies.FirstOrDefault(c => c.COPYToken.Line == lineChange.LineIndex + 1)); } // 1. Iterate over all updated lines containing a new COPY directive if (parsedLinesWithCopyDirectives != null) { foreach (ProcessedTokensLine tokensLineWithCopyDirective in parsedLinesWithCopyDirectives) { // Iterate over all COPY directives found on one updated line foreach (CopyDirective copyDirective in tokensLineWithCopyDirective.ImportedDocuments.Keys.Where(c => c.TextName != null || c.COPYToken.TokenType == TokenType.EXEC).ToArray()) { try { PerfStatsForImportedDocument perfStats; // Load (or retrieve in cache) the document referenced by the COPY directive //Issue #315: tokensLineWithCopyDirective.ScanState must be passed because special names paragraph such as "Decimal point is comma" are declared in the enclosing program and can affect the parsing of COPY ProcessedTokensDocument importedDocumentSource = processedTokensDocumentProvider.GetProcessedTokensDocument(copyDirective.LibraryName, copyDirective.TextName, tokensLineWithCopyDirective.ScanStateBeforeCOPYToken[copyDirective.COPYToken], copyTextNameVariations, out perfStats); // Store it on the current line after applying the REPLACING directive ImportedTokensDocument importedDocument = new ImportedTokensDocument(copyDirective, importedDocumentSource, perfStats); tokensLineWithCopyDirective.ImportedDocuments[copyDirective] = importedDocument; } catch (Exception e) { if (missingCopies != null && copyDirective != null && copyDirective.COPYToken != null && !missingCopies.Contains(copyDirective)) //If list already contains the copy directive just ignore { var missingCopieToReplace = missingCopies.FirstOrDefault(c => c.COPYToken.Line == copyDirective.COPYToken.Line); missingCopies.Remove(missingCopieToReplace); missingCopies.Add(copyDirective); } // Text name refenced by COPY directive was not found // => register a preprocessor error on this line Token failedDirectiveToken = tokensLineWithCopyDirective.TokensWithCompilerDirectives .First( token => token.TokenType == TokenType.CopyImportDirective && ((CompilerDirectiveToken)token).CompilerDirective == copyDirective); Diagnostic diag = new Diagnostic( MessageCode.FailedToLoadTextDocumentReferencedByCopyDirective, failedDirectiveToken.Column, failedDirectiveToken.EndColumn, failedDirectiveToken.Line, e.Message, e); tokensLineWithCopyDirective.AddDiagnostic(diag); } } // Advance processing status of the line tokensLineWithCopyDirective.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready; } } // --- REPLACE PHASE : REPLACE directives are implemented in ReplaceTokensLinesIterator --- /* Algorithm : * * one REPLACE directive can express several replacement operations * one replacement operation can be of several types (distinguished for optimization purposes) * - SimpleTokenReplace : one source token / zero or one replacement token * - PartialWordReplace : one pure partial word / zero or one replacement token * - SimpleToMultipleTokenReplace : one source token / several replacement tokens * - MultipleTokenReplace : one first + several following source tokens / zero to many replacement tokens * * an iterator maintains a current set of replacement operations * * if nextToken is replace directive * the current set of replacement operations is updated * else * nextToken is compared to each replacement operation in turn * if single -> single source token operation : return replacement token * if single -> multiple operation : setup a secondary iterator with the list of replacement tokens * if multiple -> multiple operation * snapshot of the underlying iterator * try to match all of the following source tokens * if failure : restore snapshot and try next operation * if success : setup a secondary iterator * * token comparison sourceToken / replacementCandidate : * 1. Compare Token type * 2. If same token type and for families * AlphanumericLiteral * NumericLiteral * SyntaxLiteral * Symbol * => compare also Token text * * PartialCobolWord replacement : * p535 : The COPY statement with REPLACING phrase can be used to replace parts of words. * By inserting a dummy operand delimited by colons into the program text, * the compiler will replace the dummy operand with the required text. * Example 3 shows how this is used with the dummy operand :TAG:. * The colons serve as separators and make TAG a stand-alone operand. */ return(processedTokensLinesChanges); }
/// <summary> /// Load a Cobol source file in memory /// </summary> public FileCompiler(string libraryName, string textName, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ColumnsLayout columnsLayout, TypeCobolOptions compilerOptions, CodeModel.SymbolTable customSymbols, bool isCopyFile) : this(libraryName, textName, null, sourceFileProvider, documentProvider, columnsLayout, null, compilerOptions, customSymbols, isCopyFile) { }
public void Init(string[] extensions = null) { DirectoryInfo localDirectory = new DirectoryInfo(Path.GetDirectoryName( Comparator.paths.SamplePath)); DocumentFormat format = Comparator.getSampleFormat(); TypeCobolOptions options = new TypeCobolOptions(); if (extensions == null) extensions = new[] { "*.cbl", "*.cpy" }; //comparator.paths.sextension = extensions[0].Substring(1); CompilationProject project = new CompilationProject("TEST", localDirectory.FullName, extensions, format.Encoding, format.EndOfLineDelimiter, format.FixedLineLength, format.ColumnsLayout, options); string filename = Comparator.paths.SampleName; Compiler = new FileCompiler(null, filename, project.SourceFileProvider, project, format.ColumnsLayout, options, null, false); }
/// <summary> /// Load a Cobol source file in an pre-existing text document /// </summary> public FileCompiler(string libraryName, string textName, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ITextDocument textDocument, TypeCobolOptions compilerOptions, bool isCopyFile) : this(libraryName, textName, null, sourceFileProvider, documentProvider, default(ColumnsLayout), textDocument, compilerOptions, null, isCopyFile) { }
// --- Initialization --- /// <summary> /// Initializes a new compilation document from a list of text lines. /// This method does not scan the inserted text lines to produce tokens. /// You must explicitely call UpdateTokensLines() to start an initial scan of the document. /// </summary> public CompilationDocument(TextSourceInfo textSourceInfo, IEnumerable<ITextLine> initialTextLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider) : this(textSourceInfo, initialTextLines, compilerOptions, processedTokensDocumentProvider, null) { }
/// <summary> /// Use a pre-existing text document, already initialized from a Cobol file /// </summary> public FileCompiler(ITextDocument textDocument, CobolFile loadedCobolFile, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, TypeCobolOptions compilerOptions, bool isCopyFile) : this(null, null, null, sourceFileProvider, documentProvider, default(ColumnsLayout), textDocument, compilerOptions, null, isCopyFile) { }
/// <summary> /// Initializes a new compilation document from a list of text lines. /// This method does not scan the inserted text lines to produce tokens. /// You must explicitely call UpdateTokensLines() to start an initial scan of the document. /// </summary> public CompilationUnit(TextSourceInfo textSourceInfo, IEnumerable <ITextLine> initialTextLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider, List <RemarksDirective.TextNameVariation> copyTextNameVariations) : base(textSourceInfo, initialTextLines, compilerOptions, processedTokensDocumentProvider, copyTextNameVariations) { // Initialize performance stats PerfStatsForCodeElementsParser = new PerfStatsForParsingStep(CompilationStep.CodeElementsParser); PerfStatsForTemporarySemantic = new PerfStatsForParsingStep(CompilationStep.ProgramClassParser); PerfStatsForProgramCrossCheck = new PerfStatsForParsingStep(CompilationStep.ProgramCrossCheck); }
/// <summary> /// Initial parsing of a complete document /// </summary> internal static void ParseDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<CodeElementsLine> documentLines, TypeCobolOptions compilerOptions) { ParseProcessedTokensLinesChanges(textSourceInfo, documentLines, null, null, compilerOptions); }
/// <summary> /// Incremental parsing of a set of processed tokens lines changes /// </summary> internal static IList <DocumentChange <ICodeElementsLine> > ParseProcessedTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <CodeElementsLine> documentLines, IList <DocumentChange <IProcessedTokensLine> > processedTokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, PerfStatsForParserInvocation perfStatsForParserInvocation) { // Collect all changes applied to the processed tokens lines during the incremental scan IList <DocumentChange <ICodeElementsLine> > codeElementsLinesChanges = new List <DocumentChange <ICodeElementsLine> >(); // There are 2 reasons to re-parse a tokens line after a change : // 1. The tokens line changed : these lines were already reset during the previous steps // 2. If a tokens line that changed was involved in the parsing of a multiline code element, the whole group of lines must be parsed again // --- PREPARATION PHASE : identify all parse sections where code elements need to be refreshed --- IList <ParseSection> refreshParseSections = null; ParseSection largestRefreshParseSection = null; // Iterate over all processed tokens changes detected by the PreprocessorStep : // - refresh all the adjacent lines participating in a CodeElement // - register the start and stop token for all sections of the document which need to be parsed again if (processedTokensLinesChanges != null && processedTokensLinesChanges.Count > 0) { // If the document was cleared, everything must be parsed again if (processedTokensLinesChanges[0].Type != DocumentChangeType.DocumentCleared) { refreshParseSections = new List <ParseSection>(); ParseSection lastParseSection = null; foreach (DocumentChange <IProcessedTokensLine> tokensChange in processedTokensLinesChanges) { if (lastParseSection == null || tokensChange.LineIndex > lastParseSection.StopLineIndex) { lastParseSection = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, codeElementsLinesChanges, lastParseSection); refreshParseSections.Add(lastParseSection); } } } } if (refreshParseSections != null) { //After getting all the parts refreshed, get the largest part that has been refreshed var minParseSection = refreshParseSections.OrderBy(p => p.StartLineIndex).First(); var maxParseSection = refreshParseSections.OrderByDescending(p => p.StopLineIndex).First(); largestRefreshParseSection = new ParseSection(minParseSection.StartLineIndex, minParseSection.StartToken, maxParseSection.StopLineIndex, maxParseSection.StopToken, maxParseSection.StopTokenIsFirstTokenOfTheLine); } // --- INITIALIZE ANTLR CodeElements parser --- // Create a token iterator on top of pre-processed tokens lines ITokensLinesIterator tokensIterator = ProcessedTokensDocument.GetProcessedTokensIterator(textSourceInfo, documentLines); // Create an Antlr compatible token source on top of the token iterator TokensLinesTokenSource tokenSource = new TokensLinesTokenSource( textSourceInfo.Name, tokensIterator); // Init parser TokensLinesTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens); CodeElementsParser cobolParser = new CodeElementsParser(tokenStream); // REVERT TO STD PARSER ==> TracingCobolParser cobolParser = new TracingCobolParser(tokenStream); // Optionnaly activate Antlr Parser performance profiling // WARNING : use this in a single-treaded context only (uses static field) if (AntlrPerformanceProfiler == null && perfStatsForParserInvocation.ActivateDetailedAntlrPofiling) { AntlrPerformanceProfiler = new AntlrPerformanceProfiler(cobolParser); } if (AntlrPerformanceProfiler != null) { // Replace the generated parser by a subclass which traces all rules invocations cobolParser = new CodeElementsTracingParser(tokenStream); var tokensCountIterator = ProcessedTokensDocument.GetProcessedTokensIterator(textSourceInfo, documentLines); AntlrPerformanceProfiler.BeginParsingFile(textSourceInfo, tokensCountIterator); } // Customize error recovery strategy IAntlrErrorStrategy cobolErrorStrategy = new CodeElementErrorStrategy(); cobolParser.ErrorHandler = cobolErrorStrategy; // Register all parse errors in a list in memory ParserDiagnosticErrorListener errorListener = new ParserDiagnosticErrorListener(); cobolParser.RemoveErrorListeners(); cobolParser.AddErrorListener(errorListener); // Prepare to analyze the parse tree ParseTreeWalker walker = new ParseTreeWalker(); CodeElementBuilder codeElementBuilder = new CodeElementBuilder(); codeElementBuilder.Dispatcher = new CodeElementDispatcher(); codeElementBuilder.Dispatcher.CreateListeners(); // --- INCREMENTAL PARSING --- // In case of incremental parsing, parse only the code sections we need to refresh if (largestRefreshParseSection != null) { // Seek just before the next code element starting token tokenStream.SeekToToken(largestRefreshParseSection.StartToken); tokenStream.StartLookingForStopToken(largestRefreshParseSection.StopToken); //Remove all the code elements for the future line to parse. for (int i = largestRefreshParseSection.StartLineIndex; i < (largestRefreshParseSection.StopLineIndex == documentLines.Count - 1 && largestRefreshParseSection.StopToken == null //If the last index is equals to number of line in document, make sure to also reset the last line, otherwise, reset lines normally. ? largestRefreshParseSection.StopLineIndex + 1 : largestRefreshParseSection.StopLineIndex); i++) { if (documentLines[i].CodeElements != null) { documentLines[i].ResetCodeElements(); } } } // Reset parsing error diagnostics cobolErrorStrategy.Reset(cobolParser); // Try to parse code elements : // - starting with the current parse section Start token // - ending with the current parse section Stop token CodeElementsParser.CobolCodeElementsContext codeElementsParseTree = null; try { perfStatsForParserInvocation.OnStartAntlrParsing(); if (AntlrPerformanceProfiler != null) { AntlrPerformanceProfiler.BeginParsingSection(); } codeElementsParseTree = cobolParser.cobolCodeElements(); if (AntlrPerformanceProfiler != null) { AntlrPerformanceProfiler.EndParsingSection(codeElementsParseTree.ChildCount); } perfStatsForParserInvocation.OnStopAntlrParsing( AntlrPerformanceProfiler != null ? (int)AntlrPerformanceProfiler.CurrentFileInfo.DecisionTimeMs : 0, AntlrPerformanceProfiler != null ? AntlrPerformanceProfiler.CurrentFileInfo.RuleInvocations.Sum() : 0); } catch (Exception e) { var currentToken = (Token)cobolParser.CurrentToken; CodeElementsLine codeElementsLine = GetCodeElementsLineForToken(currentToken); if (codeElementsLine != null) { codeElementsLine.AddParserDiagnostic(new TokenDiagnostic(MessageCode.ImplementationError, currentToken, currentToken.Line, e)); } } if (codeElementsParseTree != null) { // If the parse tree is not empty if (codeElementsParseTree.codeElement() != null && codeElementsParseTree.codeElement().Length > 0) { // Analyze the parse tree for each code element foreach (var codeElementParseTree in codeElementsParseTree.codeElement()) { // Get the first line that was parsed var tokenStart = (Token)codeElementParseTree.Start; CodeElementsLine codeElementsLine = GetCodeElementsLineForToken(tokenStart); if (codeElementsLine == null) { continue; } // Register that this line was updated // COMMENTED FOR THE SAKE OF PERFORMANCE -- SEE ISSUE #160 //int updatedLineIndex = documentLines.IndexOf(codeElementsLine, codeElementsLine.LineIndex); //codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, updatedLineIndex, codeElementsLine)); codeElementsLinesChanges.Add( new DocumentChange <ICodeElementsLine>(DocumentChangeType.LineUpdated, codeElementsLine.LineIndex, codeElementsLine)); perfStatsForParserInvocation.OnStartTreeBuilding(); // Visit the parse tree to build a first class object representing the code elements try { walker.Walk(codeElementBuilder, codeElementParseTree); } catch (Exception ex) { var code = MessageCode.ImplementationError; int line = 0; int start = 0; int stop = 0; if (codeElementsLine.SourceTokens != null && codeElementsLine.SourceTokens.Count > 0) { start = codeElementsLine.SourceTokens[0].StartIndex; stop = codeElementsLine.SourceTokens[codeElementsLine.SourceTokens.Count - 1].StopIndex; } codeElementsLine.AddParserDiagnostic(new ParserDiagnostic(ex.ToString(), start, stop, line, null, code, ex)); } CodeElement codeElement = codeElementBuilder.CodeElement; if (codeElement != null) { // Attach consumed tokens and main document line numbers information to the code element if (codeElement.ConsumedTokens.Count == 0) { // ISSUE #204: var tempToken = tokenStream.Lt(1); if (tempToken != null && tempToken != Token.END_OF_FILE) { // if not end of file, // add next token to ConsumedTokens to know where is the CodeElement in error codeElement.ConsumedTokens.Add((Token)tempToken); // this alter CodeElements semantics: in addition to matched tokens, // it includes the first token in error if no token has been matched } } //TODO Issue #384 to discuss if this code should stay here: //This should be in a Checker, but "codeElement.ConsumedTokens" is only set after all the checkers have been called //Rule TCLIMITATION_NO_CE_ACROSS_SOURCES if (codeElement.IsAcrossSourceFile()) { DiagnosticUtils.AddError(codeElement, "A Cobol statement cannot be across 2 sources files (eg. Main program and a COPY)", MessageCode.TypeCobolParserLimitation); } // Add code element to the list codeElementsLine.AddCodeElement(codeElement); } } } // If the parse tree contains errors if (codeElementsParseTree.Diagnostics != null) { foreach (ParserDiagnostic d in codeElementsParseTree.Diagnostics) { if (d.OffendingSymbol != null) { CodeElementsLine codeElementsLine = GetCodeElementsLineForToken((Token)d.OffendingSymbol); if (codeElementsLine != null) { codeElementsLine.AddParserDiagnostic(d); } } } } perfStatsForParserInvocation.OnStopTreeBuilding(); } if (AntlrPerformanceProfiler != null) { AntlrPerformanceProfiler.EndParsingFile(cobolParser.ParseInfo.DecisionInfo, (int)(cobolParser.ParseInfo.GetTotalTimeInPrediction() / 1000000)); } return(codeElementsLinesChanges); }
/// <summary> /// Incremental scan of a set of text lines changes /// </summary> internal static IList<DocumentChange<ITokensLine>> ScanTextLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, IList<DocumentChange<ICobolTextLine>> textLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions) { // Collect all changes applied to the tokens lines during the incremental scan IList<DocumentChange<ITokensLine>> tokensLinesChanges = new List<DocumentChange<ITokensLine>>(); // There are 3 reasons to scan a line after a text change : // 1. New text lines which were just inserted or updated must be scanned for the first time // 2. Text lines must be scanned again if their initial scan state changed : a new scan of the previous line can alter the scan state at the beginning of the following line // 3. Continuation lines and multiline tokens : if a line participates in a continuation on several lines, scan the group of lines as a whole // IMPORTANT : the text changes are ordered in increasing order of line index for (int textChangeIndex = 0; textChangeIndex < textLinesChanges.Count; textChangeIndex++) { // Local variables used to optimize navigation in the document int nextLineToScanIndex = -1; TokensLine nextLineToScan = null; // Update tokens depending on the current text change DocumentChange<ICobolTextLine> textChange = textLinesChanges[textChangeIndex]; if (textChange.Type == DocumentChangeType.DocumentCleared) { tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.DocumentCleared, 0, null)); continue; } else if (textChange.Type == DocumentChangeType.LineInserted || textChange.Type == DocumentChangeType.LineUpdated) { // We update lines as a group below, but we remember here which lines were inserted if(textChange.Type == DocumentChangeType.LineInserted) { tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineInserted, textChange.LineIndex, (ITokensLine)textChange.NewLine)); } // Text lines which were inserted or updated must be scanned again ScanTokensLineWithMultilineScanState(textChange.LineIndex, (TokensLine)textChange.NewLine, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan); } else if (textChange.Type == DocumentChangeType.LineRemoved) { tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineRemoved, textChange.LineIndex, (ITokensLine)textChange.NewLine)); // Get the last line just before the line that was removed TokensLine previousLine = null; if (textChange.LineIndex > 0) { previousLine = documentLines[textChange.LineIndex - 1]; } // When a text line is removed : // - the previous line must be scanned again if the line which was removed was a member of a multiline continuation group if (previousLine != null && previousLine.HasTokenContinuedOnNextLine) { ScanTokensLineWithMultilineScanState(textChange.LineIndex - 1, previousLine, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan); } if (nextLineToScan == null && textChange.LineIndex < documentLines.Count) { nextLineToScanIndex = textChange.LineIndex; nextLineToScan = documentLines[nextLineToScanIndex]; } // - the next line must be scanned again if the scan state at the end of the previous line is different from the scan state at the beginning of the next line if (nextLineToScan != null && nextLineToScanIndex == textChange.LineIndex && previousLine != null && nextLineToScan.InitialScanState.Equals(previousLine.ScanState)) { ScanTokensLineWithMultilineScanState(textChange.LineIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan); } } // We can skip all text changes with an index smaller than the index of the last line which was already scanned if (nextLineToScan == null) { break; } else if (textChangeIndex < (textLinesChanges.Count - 1)) { int nextTextChangeIndex = textChangeIndex; DocumentChange<ICobolTextLine> nextTextChange = null; do { nextTextChangeIndex++; nextTextChange = textLinesChanges[nextTextChangeIndex]; } while (nextTextChangeIndex < (textLinesChanges.Count - 1) && nextTextChange.LineIndex <= nextLineToScanIndex); textChangeIndex = nextTextChangeIndex - 1; } } return tokensLinesChanges; }