public static void ParseProgramOrClass(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<CodeElementsLine> codeElementsLines, TypeCobolOptions compilerOptions, SymbolTable customSymbols, out Program newProgram, out Class newClass, out IList<ParserDiagnostic> diagnostics)
{
// Create an Antlr compatible token source on top a the token iterator
CodeElementsLinesTokenSource tokenSource = new CodeElementsLinesTokenSource(
textSourceInfo.Name,
codeElementsLines);
// Init parser
ITokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
ProgramClassParser cobolParser = new ProgramClassParser(tokenStream);
// -> activate full ambiguities detection
//parser.Interpreter.PredictionMode = PredictionMode.LlExactAmbigDetection;
// Register all parse errors in a list in memory
DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener();
cobolParser.RemoveErrorListeners();
cobolParser.AddErrorListener(errorListener);
// Try to parse a Cobol program or class
ProgramClassParser.CobolCompilationUnitContext codeElementParseTree = cobolParser.cobolCompilationUnit();
// Visit the parse tree to build a first class object representing a Cobol program or class
ParseTreeWalker walker = new ParseTreeWalker();
CobolNodeBuilder programClassBuilder = new CobolNodeBuilder();
programClassBuilder.CustomSymbols = customSymbols;
programClassBuilder.Dispatcher = new NodeDispatcher();
programClassBuilder.Dispatcher.CreateListeners();
walker.Walk(programClassBuilder, codeElementParseTree);
// Register compiler results
newProgram = programClassBuilder.Program;
newClass = programClassBuilder.Class;
diagnostics = errorListener.Diagnostics;
}
public void Init(string[] extensions = null, bool autoRemarks = false, bool AntlrProfiler = false)
{
DirectoryInfo localDirectory = new DirectoryInfo(Path.GetDirectoryName(Comparator?.paths?.SamplePath));
DocumentFormat format = Comparator?.GetSampleFormat();
TypeCobolOptions options = new TypeCobolOptions();
#if EUROINFO_RULES
options.AutoRemarksEnable = autoRemarks;
#endif
if (extensions == null)
{
extensions = new[] { ".cbl", ".cpy" }
}
;
//comparator.paths.sextension = extensions[0].Substring(1);
CompilationProject project = new CompilationProject("TEST",
localDirectory.FullName, extensions,
format.Encoding, format.EndOfLineDelimiter, format.FixedLineLength, format.ColumnsLayout, options);
string filename = Comparator.paths.SampleName;
Compiler = new FileCompiler(null, filename, project.SourceFileProvider, project, format.ColumnsLayout, options, null, false, project);
if (AntlrProfiler)
{
Compiler.CompilationResultsForProgram.PerfStatsForCodeElementsParser.ActivateDetailedAntlrPofiling = true;
Compiler.CompilationResultsForProgram.PerfStatsForTemporarySemantic.ActivateDetailedAntlrPofiling = true;
}
}
public CompilationDocument(TextSourceInfo textSourceInfo, IEnumerable <ITextLine> initialTextLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider,
[CanBeNull] MultilineScanState scanState, List <RemarksDirective.TextNameVariation> copyTextNameVariations)
{
TextSourceInfo = textSourceInfo;
CompilerOptions = compilerOptions;
CopyTextNamesVariations = copyTextNameVariations ?? new List <RemarksDirective.TextNameVariation>();
MissingCopies = new List <CopyDirective>();
this.processedTokensDocumentProvider = processedTokensDocumentProvider;
// Initialize the compilation document lines
compilationDocumentLines = ImmutableList <CodeElementsLine> .Empty.ToBuilder();
// ... with the initial list of text lines received as a parameter
if (initialTextLines != null)
{
// Insert Cobol text lines in the internal tree structure
compilationDocumentLines.AddRange(initialTextLines.Select(textLine => CreateNewDocumentLine(textLine, textSourceInfo.ColumnsLayout)));
}
// Initialize document views versions
currentTextLinesVersion = new DocumentVersion <ICobolTextLine>(this);
currentTokensLinesVersion = new DocumentVersion <ITokensLine>(this);
// Initialize performance stats
PerfStatsForText = new PerfStatsForCompilationStep(CompilationStep.Text);
PerfStatsForScanner = new PerfStatsForCompilationStep(CompilationStep.Scanner);
PerfStatsForPreprocessor = new PerfStatsForParsingStep(CompilationStep.Preprocessor);
initialScanStateForCopy = scanState;
}
private static CodeElement[] ParseCodeElements(string cobolString, out Diagnostic[] parserDiagnostics)
{
// Load text document from string
var textDocument = new ReadOnlyTextDocument("test string", Encoding.Default, ColumnsLayout.FreeTextFormat, "");
textDocument.LoadChars(cobolString);
// Create a compilation project and a compiler for this document
var typeCobolOptions = new TypeCobolOptions();
var project = new CompilationProject("test project", ".", new[] { ".cbl", ".cpy" },
DocumentFormat.FreeTextFormat.Encoding, DocumentFormat.FreeTextFormat.EndOfLineDelimiter,
DocumentFormat.FreeTextFormat.FixedLineLength, DocumentFormat.FreeTextFormat.ColumnsLayout, typeCobolOptions);
var compiler = new FileCompiler(textDocument, project.SourceFileProvider, project, typeCobolOptions, false, project);
// Execute compilation - until the CodeElements phase ONLY
compiler.CompilationResultsForProgram.UpdateTokensLines();
compiler.CompilationResultsForProgram.RefreshTokensDocumentSnapshot();
compiler.CompilationResultsForProgram.RefreshProcessedTokensDocumentSnapshot();
compiler.CompilationResultsForProgram.RefreshCodeElementsDocumentSnapshot();
// Return CodeElements and Diagnostics
var codeElementsDocument = compiler.CompilationResultsForProgram.CodeElementsDocumentSnapshot;
var codeElements = codeElementsDocument.CodeElements.ToArray();
parserDiagnostics = codeElementsDocument.ParserDiagnostics.ToArray();
return(codeElements);
}
public Workspace(string workspaceName, string rootDirectory, string[] fileExtensions,
Encoding encoding, EndOfLineDelimiter endOfLineDelimiter, int fixedLineLength, ColumnsLayout columnsLayout,
TypeCobolOptions compilationOptions)
{
compilationProject = new CompilationProject(workspaceName, rootDirectory, fileExtensions, encoding, endOfLineDelimiter, fixedLineLength, columnsLayout, compilationOptions);
OpenedFileCompilers = new Dictionary <string, FileCompiler>(3);
}
/// <summary>
/// Initializes a new compilation document from a list of text lines.
/// This method does not scan the inserted text lines to produce tokens.
/// You must explicitely call UpdateTokensLines() to start an initial scan of the document.
/// </summary>
public CompilationUnit(TextSourceInfo textSourceInfo, IEnumerable<ITextLine> initialTextLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider)
: base(textSourceInfo, initialTextLines, compilerOptions, processedTokensDocumentProvider)
{
// Initialize performance stats
PerfStatsForCodeElementsParser = new PerfStatsForCompilationStep(CompilationStep.CodeElementsParser);
PerfStatsForProgramClassParser = new PerfStatsForCompilationStep(CompilationStep.ProgramClassParser);
}
public void Init([NotNull] string path, TypeCobolOptions options, DocumentFormat format = null, IList <string> copies = null)
{
FileCompiler compiler;
if (Compilers.TryGetValue(path, out compiler))
{
return;
}
string filename = Path.GetFileName(path);
var root = new DirectoryInfo(Directory.GetParent(path).FullName);
if (format == null)
{
format = GetFormat(path);
}
CompilationProject project = new CompilationProject(path, root.FullName, Extensions,
format.Encoding, format.EndOfLineDelimiter, format.FixedLineLength, format.ColumnsLayout, options);
//Add copy folder into sourceFileProvider
SourceFileProvider sourceFileProvider = project.SourceFileProvider;
copies = copies ?? new List <string>();
foreach (var folder in copies)
{
sourceFileProvider.AddLocalDirectoryLibrary(folder, false, CopyExtensions, format.Encoding, format.EndOfLineDelimiter, format.FixedLineLength);
}
compiler = new FileCompiler(null, filename, project.SourceFileProvider, project, format.ColumnsLayout, options, CustomSymbols, false, project);
Compilers.Add(path, compiler);
Inits.Add(path, false);
}
// --- Initialization ---
/// <summary>
/// Initializes a new compilation document from a list of text lines.
/// This method does not scan the inserted text lines to produce tokens.
/// You must explicitely call UpdateTokensLines() to start an initial scan of the document.
/// </summary>
public CompilationDocument(TextSourceInfo textSourceInfo, IEnumerable<ITextLine> initialTextLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider)
{
TextSourceInfo = textSourceInfo;
CompilerOptions = compilerOptions;
this.processedTokensDocumentProvider = processedTokensDocumentProvider;
// Initialize the compilation document lines
compilationDocumentLines = ImmutableList<CodeElementsLine>.Empty.ToBuilder();
// ... with the initial list of text lines received as a parameter
if (initialTextLines != null)
{
// Insert Cobol text lines in the internal tree structure
compilationDocumentLines.AddRange(initialTextLines.Select(textLine => CreateNewDocumentLine(textLine, textSourceInfo.ColumnsLayout)));
}
// Initialize document views versions
currentTextLinesVersion = new DocumentVersion<ICobolTextLine>(this);
currentTokensLinesVersion = new DocumentVersion<ITokensLine>(this);
// Initialize performance stats
PerfStatsForText = new PerfStatsForCompilationStep(CompilationStep.Text);
PerfStatsForScanner = new PerfStatsForCompilationStep(CompilationStep.Scanner);
PerfStatsForPreprocessor = new PerfStatsForCompilationStep(CompilationStep.Preprocessor);
}
public static CompilationUnit ParseCobolFile(string textName, DocumentFormat documentFormat = null, string folder = null, ExecutionStep execToStep = ExecutionStep.SemanticCheck)
{
if (folder == null)
{
folder = "Parser" + Path.DirectorySeparatorChar + "CodeElements";
}
DirectoryInfo localDirectory = new DirectoryInfo(PlatformUtils.GetPathForProjectFile(folder));
if (!localDirectory.Exists)
{
throw new Exception(String.Format("Directory : {0} does not exist", localDirectory.FullName));
}
if (documentFormat == null)
{
documentFormat = DocumentFormat.RDZReferenceFormat;
}
TypeCobolOptions options = new TypeCobolOptions {
ExecToStep = execToStep
}; //Create CompilerOptions. ExecToStep / AutoRemarks / HaltOnMissingCopy have to be set here.
CompilationProject project = new CompilationProject("test",
//First use *.cpy as tests will use file WITH extension for program but without extension for copy inside programs => small perf gain
localDirectory.FullName, new string[] { ".cpy", ".cbl" },
documentFormat.Encoding, documentFormat.EndOfLineDelimiter, documentFormat.FixedLineLength, documentFormat.ColumnsLayout, options);
FileCompiler compiler = new FileCompiler(null, textName, project.SourceFileProvider, project, documentFormat.ColumnsLayout, options, null, false, project);
compiler.CompileOnce();
return(compiler.CompilationResultsForProgram);
}
/// <summary>
/// New implementation of the Init function, to allocate a BuildProject instance rather than a
/// CompilationProject.
/// </summary>
/// <param name="path">The path of the file to parser</param>
/// <param name="format">The resulting document format</param>
/// <returns>The BuildProject instance created</returns>
public new BuildProject Init(string path, DocumentFormat format = null)
{
FileCompiler compiler;
if (Compilers.TryGetValue(path, out compiler))
{
return(m_Projects[path]);
}
string directory = Directory.GetParent(path).FullName;
string filename = Path.GetFileName(path);
DirectoryInfo root = new DirectoryInfo(directory);
if (format == null)
{
format = GetFormat(path);
}
TypeCobolOptions options = new TypeCobolOptions();
BuildProject project = new BuildProject(BuilderEngine, path, root.FullName, new string[] { "*.cbl", "*.cpy" },
format.Encoding, format.EndOfLineDelimiter, format.FixedLineLength, format.ColumnsLayout, options);
m_Projects[path] = project;
compiler = new FileCompiler(null, filename, project.SourceFileProvider, project, format.ColumnsLayout, options, CustomSymbols, false);
Compilers.Add(path, compiler);
Inits.Add(path, false);
return(project);
}
public Workspace(string workspaceName, string rootDirectory, string[] fileExtensions,
Encoding encoding, EndOfLineDelimiter endOfLineDelimiter, int fixedLineLength, ColumnsLayout columnsLayout,
TypeCobolOptions compilationOptions)
{
compilationProject = new CompilationProject(workspaceName, rootDirectory, fileExtensions, encoding, endOfLineDelimiter, fixedLineLength, columnsLayout, compilationOptions);
OpenedFileCompilers = new Dictionary<string, FileCompiler>(3);
}
/// <summary>
/// Parse an file using a NodeListener and IReport instance and compare the resulting report.
/// </summary>
/// <typeparam name="TCtx"></typeparam>
/// <param name="fileName">The file name to parse</param>
/// <param name="reportFileName">The file that contains the expected report</param>
/// <param name="reportType">The Type of the IReport instance to be instantiated.</param>
public static void ParseWithNodeListenerReportCompare <TCtx>(string fileName, string reportFileName, System.Type reportType) where TCtx : class
{
Assert.IsTrue(Tools.Reflection.IsTypeOf(reportType, typeof(IReport)));
IReport report = null;//Variable to receive the created report instance.
TypeCobol.Compiler.Parser.NodeListenerFactory <TCtx> factory = () =>
{
object obj = System.Activator.CreateInstance(reportType);
Assert.IsTrue(obj is NodeListener <TCtx>);
TypeCobol.Compiler.Parser.NodeListener <TCtx> nodeListener = (TypeCobol.Compiler.Parser.NodeListener <TCtx>)obj;
Assert.IsNotNull(nodeListener);
report = (IReport)nodeListener;
return(nodeListener);
};
//Register the Node Listener Factory
TypeCobol.Compiler.Parser.NodeDispatcher <TCtx> .RegisterStaticNodeListenerFactory(factory);
try
{
string input = Path.Combine(ROOT_INPUT, fileName);
string output = Path.Combine(ROOT_OUTPUT, reportFileName);
DocumentFormat format = DocumentFormat.RDZReferenceFormat;
var parser = new TypeCobol.Parser();
var typeCobolOption = new TypeCobolOptions {
ExecToStep = ExecutionStep.CrossCheck
};
#if EUROINFO_RULES
bool autoRemarks = false;
typeCobolOption.AutoRemarksEnable = autoRemarks;
#endif
String copyFolder = Path.Combine(Directory.GetCurrentDirectory(), ROOT_COPY);
parser.Init(input, typeCobolOption, format, new List <string>()
{
copyFolder
});
parser.Parse(input);
var allDiags = parser.Results.AllDiagnostics();
if (allDiags.Count == 0)
{
if (report != null)
{
using (System.IO.StringWriter sw = new StringWriter())
{
report.Report(sw);
// compare with expected result
string result = sw.ToString();
string expected = File.ReadAllText(output, format.Encoding);
TypeCobol.Test.TestUtils.compareLines(input, result, expected);
}
}
}
}
finally
{
TypeCobol.Compiler.Parser.NodeDispatcher <TCtx> .RemoveStaticNodeListenerFactory(factory);
}
}
public ImportedTokensDocument(CopyDirective copyDirective, ProcessedTokensDocument importedDocumentSource, PerfStatsForImportedDocument perfStats, TypeCobolOptions compilerOptions)
{
CopyDirective = copyDirective;
SourceDocument = importedDocumentSource;
HasReplacingDirective = copyDirective.ReplaceOperations.Count > 0;
PerfStatsForImportedDocument = perfStats;
CompilerOptions = compilerOptions;
}
public ProcessedTokensDocument(TokensDocument previousStepSnapshot, DocumentVersion <IProcessedTokensLine> processedTokensLinesVersion, ISearchableReadOnlyList <IProcessedTokensLine> processedTokensLines, TypeCobolOptions compilerOptions)
{
TextSourceInfo = previousStepSnapshot.TextSourceInfo;
PreviousStepSnapshot = previousStepSnapshot;
CurrentVersion = processedTokensLinesVersion;
Lines = processedTokensLines;
CompilerOptions = compilerOptions;
}
private Scanner(string line, int startIndex, int lastIndex, TokensLine tokensLine, TypeCobolOptions compilerOptions)
{
this.tokensLine = tokensLine;
this.line = line;
this.currentIndex = startIndex;
this.lastIndex = lastIndex;
this.compilerOptions = compilerOptions;
}
public static void AttachIncrementalCompilerToTextEditorDocument(TypeCobolEditor textEditor, CompilationProject project, string textName, TypeCobolOptions compilerOptions, IObserver<IList<CompilationError>> errorObserver)
{
var serviceContainer = textEditor.TextArea.TextView.Services;
if (serviceContainer.GetService(typeof(TypeCobolCompilerService)) != null)
{
serviceContainer.RemoveService(typeof(TypeCobolCompilerService));
}
TypeCobolCompilerService compilerService = new TypeCobolCompilerService(textEditor, project, textName, compilerOptions, errorObserver);
serviceContainer.AddService(typeof(TypeCobolCompilerService), compilerService);
}
/// <summary>
/// Common internal implementation for all 4 constructors above
/// </summary>
private FileCompiler(string libraryName, string textName, CobolFile loadedCobolFile, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ColumnsLayout columnsLayout, ITextDocument textDocument, TypeCobolOptions compilerOptions, SymbolTable customSymbols, bool isCopyFile,
[CanBeNull] MultilineScanState scanState)
{
// 1.a Find the Cobol source file
CobolFile sourceFile = null;
if (textName != null)
{
if (sourceFileProvider.TryGetFile(libraryName, textName, out sourceFile))
{
CobolFile = sourceFile;
}
else
{
throw new Exception(String.Format("Could not find a Cobol source file named {0} in {1}", textName, libraryName));
}
}
// 1.b Register a Cobol source file which was already loaded
else if(loadedCobolFile != null)
{
CobolFile = loadedCobolFile;
}
// 2.a Load it in a new text document in memory
if (textDocument == null)
{
TextDocument = new ReadOnlyTextDocument(sourceFile.Name, sourceFile.Encoding, columnsLayout, sourceFile.ReadChars());
}
// 2.b Load it in an existing text document in memory
else if (sourceFile != null)
{
TextDocument = textDocument;
textDocument.LoadChars(sourceFile.ReadChars());
}
// 2.c Use a pre-existing text document
// - not yet associated with a Cobol source file
// - with a Cobol source file already loaded
else if (sourceFile == null || loadedCobolFile != null)
{
TextDocument = textDocument;
}
// 3. Prepare the data structures used by the different steps of the compiler
if (isCopyFile) {
CompilationResultsForCopy = new CompilationDocument(TextDocument.Source, TextDocument.Lines, compilerOptions, documentProvider, scanState);
CompilationResultsForCopy.CustomSymbols = customSymbols;
} else {
CompilationResultsForProgram = new CompilationUnit(TextDocument.Source, TextDocument.Lines, compilerOptions, documentProvider);
CompilationResultsForProgram.CustomSymbols = customSymbols;
}
CompilerOptions = compilerOptions;
}
public static Parser Parse(string path, DocumentFormat format, bool autoRemarks = false)
{
var parser = new Parser();
var typeCobolOption = new TypeCobolOptions {
ExecToStep = ExecutionStep.Generate
};
#if EUROINFO_RULES
typeCobolOption.AutoRemarksEnable = autoRemarks;
#endif
parser.Init(path, typeCobolOption, format);
parser.Parse(path);
return(parser);
}
public TypeCobolWindow()
{
InitializeComponent();
// Default directory at startup
currentProject = new CompilationProject("project",
DEFAULT_DIRECTORY, new string[] { ".txt", "*.cbl", "*.cpy" },
DEFAULT_DOCUMENT_FORMAT.Encoding, DEFAULT_DOCUMENT_FORMAT.EndOfLineDelimiter, DEFAULT_DOCUMENT_FORMAT.FixedLineLength, DEFAULT_DOCUMENT_FORMAT.ColumnsLayout, new TypeCobolOptions());
// Default compiler options
compilerOptions = new TypeCobolOptions();
// Incremental TypeCobol compilation
TypeCobolCompilerService.AttachIncrementalCompilerToTextEditorDocument(textEditor, currentProject, null, compilerOptions, this);
}
// -- Project creation and persistence --
/// <summary>
/// Create a new Cobol compilation project in a local directory
/// </summary>
public CompilationProject(string projectName, string rootDirectory, string[] fileExtensions, Encoding encoding, EndOfLineDelimiter endOfLineDelimiter, int fixedLineLength, ColumnsLayout columnsLayout, TypeCobolOptions compilationOptions)
{
Name = projectName;
RootDirectory = rootDirectory;
SourceFileProvider = new SourceFileProvider();
rootDirectoryLibrary = SourceFileProvider.AddLocalDirectoryLibrary(rootDirectory, true, fileExtensions, encoding, endOfLineDelimiter, fixedLineLength);
Encoding = encoding;
EndOfLineDelimiter = endOfLineDelimiter;
FixedLineLength = fixedLineLength;
ColumnsLayout = columnsLayout;
CompilationOptions = compilationOptions;
CobolFiles = new Dictionary<string, CobolFile>();
CobolTextReferences = new Dictionary<string, CobolFile>();
CobolProgramCalls = new Dictionary<string, CobolFile>();
}
/// <summary>
/// Create a new Cobol compilation project in a local directory
/// </summary>
public CompilationProject(string projectName, string rootDirectory, string[] fileExtensions, Encoding encoding, EndOfLineDelimiter endOfLineDelimiter, int fixedLineLength, ColumnsLayout columnsLayout, TypeCobolOptions compilationOptions)
{
Name = projectName;
RootDirectory = rootDirectory;
SourceFileProvider = new SourceFileProvider();
rootDirectoryLibrary = SourceFileProvider.AddLocalDirectoryLibrary(rootDirectory, false, fileExtensions, encoding, endOfLineDelimiter, fixedLineLength);
Encoding = encoding;
EndOfLineDelimiter = endOfLineDelimiter;
FixedLineLength = fixedLineLength;
ColumnsLayout = columnsLayout;
CompilationOptions = compilationOptions;
CobolFiles = new Dictionary <string, CobolFile>();
CobolTextReferences = new Dictionary <string, CobolFile>();
CobolProgramCalls = new Dictionary <string, CobolFile>();
}
/// <summary>
/// Implement COPY REPLACING on top of an underlying tokens line iterator
/// </summary>
public ReplaceTokensLinesIterator(ITokensLinesIterator sourceIterator, CopyDirective copyReplacingDirective, TypeCobolOptions compilerOptions)
{
this.sourceIterator = sourceIterator;
this.CopyReplacingDirective = copyReplacingDirective;
CompilerOptions = compilerOptions;
if (copyReplacingDirective.ReplaceOperations.Count > 0)
{
if (copyReplacingDirective.ReplaceOperations.Count == 1)
{
currentPosition.ReplaceOperation = copyReplacingDirective.ReplaceOperations[0];
}
else
{
currentPosition.ReplaceOperations = copyReplacingDirective.ReplaceOperations;
}
}
}
public static CompilationUnit ParseCobolString(string cobolString)
{
//Prepare
var textDocument = new ReadOnlyTextDocument("Empty doc", Encoding.Default, ColumnsLayout.FreeTextFormat, "");
textDocument.LoadChars(cobolString);
var typeCobolOptions = new TypeCobolOptions();
var project = new CompilationProject("Empty project", ".", new[] { ".cbl", ".cpy" },
DocumentFormat.FreeTextFormat.Encoding, DocumentFormat.FreeTextFormat.EndOfLineDelimiter,
DocumentFormat.FreeTextFormat.FixedLineLength, DocumentFormat.FreeTextFormat.ColumnsLayout, typeCobolOptions);
var compiler = new FileCompiler(textDocument, project.SourceFileProvider, project, typeCobolOptions, false, project);
compiler.CompileOnce();
return(compiler.CompilationResultsForProgram);
}
// public static void Main(string[] args)
public static void main(string[] args)
{
// Directory where the source programs and COPY files are stored
string sourcePath = @"D:\Users\Laurent\OneDrive\Dev\Visual Studio 2012\Projects\TypeCobol\TypeCobol.Test\Samples\EI Cobol samples\EI-Production";
string[] programExtensions = { ".PGM" };
string[] copyExtensions = { ".CPY" };
// List of all sample programs used to compute the statistics
IList <string> textNames = new List <string>();
foreach (string programExtension in programExtensions)
{
foreach (string filePath in Directory.EnumerateFiles(sourcePath, "*" + programExtension))
{
string textName = Path.GetFileNameWithoutExtension(filePath);
textNames.Add(textName);
}
}
// Source file format for the samples
DocumentFormat docFormat = new DocumentFormat(Encoding.GetEncoding("iso8859-1"), EndOfLineDelimiter.CrLfCharacters, 80, ColumnsLayout.CobolReferenceFormat);
// Initialize a compilation project
TypeCobolOptions compilerOptions = new TypeCobolOptions();
CompilationProject project = new CompilationProject("samples", sourcePath, programExtensions.Concat(copyExtensions).ToArray(),
docFormat.Encoding, docFormat.EndOfLineDelimiter, docFormat.FixedLineLength, docFormat.ColumnsLayout, compilerOptions);
// Output files used to store the results
string resultFilePath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop);
string countersFile = Path.Combine(resultFilePath, "SyntaxCounters.txt");
string languageModelForProgramFile = Path.Combine(resultFilePath, "LanguageModel.Program.txt");
string languageModelForCopyFile = Path.Combine(resultFilePath, "LanguageModel.Copy.txt");
// Compute statistics
Stopwatch chrono = new Stopwatch();
chrono.Start();
StatsGenerator.GenerateStatisticsForPrograms(project, textNames, Console.Out, countersFile, languageModelForProgramFile, languageModelForCopyFile);
chrono.Stop();
Console.WriteLine("");
Console.WriteLine("Programs analyzed in " + Math.Round(chrono.ElapsedMilliseconds / (double)1000, 3) + " sec");
}
public void Init(string[] extensions = null, bool autoRemarks = false)
{
DirectoryInfo localDirectory = new DirectoryInfo(Path.GetDirectoryName(Comparator.paths.SamplePath));
DocumentFormat format = Comparator.getSampleFormat();
TypeCobolOptions options = new TypeCobolOptions();
#if EUROINFO_RULES
options.AutoRemarksEnable = autoRemarks;
#endif
if (extensions == null)
{
extensions = new[] { ".cbl", ".cpy" }
}
;
//comparator.paths.sextension = extensions[0].Substring(1);
CompilationProject project = new CompilationProject("TEST",
localDirectory.FullName, extensions,
format.Encoding, format.EndOfLineDelimiter, format.FixedLineLength, format.ColumnsLayout, options);
string filename = Comparator.paths.SampleName;
Compiler = new FileCompiler(null, filename, project.SourceFileProvider, project, format.ColumnsLayout, options, null, false, project);
}
// public static void Main(string[] args)
public static void main(string[] args)
{
// Directory where the source programs and COPY files are stored
string sourcePath = @"D:\Users\Laurent\OneDrive\Dev\Visual Studio 2012\Projects\TypeCobol\TypeCobol.Test\Samples\EI Cobol samples\EI-Production";
string[] programExtensions = { "*.PGM" };
string[] copyExtensions = { "*.CPY" };
// List of all sample programs used to compute the statistics
IList<string> textNames = new List<string>();
foreach (string programExtension in programExtensions)
{
foreach (string filePath in Directory.EnumerateFiles(sourcePath, programExtension))
{
string textName = Path.GetFileNameWithoutExtension(filePath);
textNames.Add(textName);
}
}
// Source file format for the samples
DocumentFormat docFormat = new DocumentFormat(Encoding.GetEncoding("iso8859-1"), EndOfLineDelimiter.CrLfCharacters, 80, ColumnsLayout.CobolReferenceFormat);
// Initialize a compilation project
TypeCobolOptions compilerOptions = new TypeCobolOptions();
CompilationProject project = new CompilationProject("samples", sourcePath, programExtensions.Concat(copyExtensions).ToArray(),
docFormat.Encoding, docFormat.EndOfLineDelimiter, docFormat.FixedLineLength, docFormat.ColumnsLayout, compilerOptions);
// Output files used to store the results
string resultFilePath = Environment.GetFolderPath(Environment.SpecialFolder.Desktop);
string countersFile = Path.Combine(resultFilePath, "SyntaxCounters.txt");
string languageModelForProgramFile = Path.Combine(resultFilePath, "LanguageModel.Program.txt");
string languageModelForCopyFile = Path.Combine(resultFilePath, "LanguageModel.Copy.txt");
// Compute statistics
Stopwatch chrono = new Stopwatch();
chrono.Start();
StatsGenerator.GenerateStatisticsForPrograms(project, textNames, Console.Out, countersFile, languageModelForProgramFile, languageModelForCopyFile);
chrono.Stop();
Console.WriteLine("");
Console.WriteLine("Programs analyzed in " + Math.Round(chrono.ElapsedMilliseconds / (double)1000, 3) + " sec");
}
/// <summary>
/// Handle the Configuration change notification.
/// </summary>
/// <param name="arguments">The arguments</param>
public void DidChangeConfigurationParams(IEnumerable <string> arguments)
{
var options = TypeCobolOptionSet.GetCommonTypeCobolOptions(TypeCobolConfiguration);
options.Parse(arguments);
//Adding default copies folder
var folder = Path.GetDirectoryName(Process.GetCurrentProcess().MainModule.FileName);
TypeCobolConfiguration.CopyFolders.Add(folder + @"\DefaultCopies\");
if (TypeCobolConfiguration.Telemetry)
{
AnalyticsWrapper.Telemetry.DisableTelemetry = false; //If telemetry arg is passed enable telemetry
}
if (TypeCobolConfiguration.ExecToStep >= ExecutionStep.Generate)
{
TypeCobolConfiguration.ExecToStep = ExecutionStep.SemanticCheck; //Language Server does not support Cobol Generation for now
}
var typeCobolOptions = new TypeCobolOptions
{
HaltOnMissingCopy = TypeCobolConfiguration.HaltOnMissingCopyFilePath != null,
ExecToStep = TypeCobolConfiguration.ExecToStep,
#if EUROINFO_RULES
AutoRemarksEnable = TypeCobolConfiguration.AutoRemarks
#endif
};
CompilationProject = new CompilationProject(WorkspaceName, RootDirectoryFullName, Extensions, TypeCobolConfiguration.Format.Encoding, TypeCobolConfiguration.Format.EndOfLineDelimiter, TypeCobolConfiguration.Format.FixedLineLength, TypeCobolConfiguration.Format.ColumnsLayout, typeCobolOptions);
if (OpenedFileCompiler.Count > 0)
{
RefreshOpenedFiles();
}
else
{
RefreshCustomSymbols();
}
}
private TypeCobolCompilerService(TypeCobolEditor textEditor, CompilationProject project, string textName, TypeCobolOptions compilerOptions, IObserver<IList<CompilationError>> errorObserver)
{
TextDocument = textEditor.TextDocument;
// DEMO : TaskPoolScheduler can not be used yet because of Length property access on ITextDocument by CobolCharStream
BackgroundCompilationScheduler = DispatcherScheduler.Current ;
// Create a new file compiler for this document
FileCompiler compiler = null;
if (textName != null)
{
compiler = new FileCompiler(null, textName, project.SourceFileProvider, project, TextDocument, compilerOptions, false);
}
else
{
compiler = new FileCompiler(TextDocument, project.SourceFileProvider, project, compilerOptions, false);
}
CompilationUnit = compiler.CompilationResultsForProgram;
// Compile in the background
compiler.StartContinuousBackgroundCompilation(400, 400, 900, 2000);
// Refresh the syntax coloring for rescanned lines
IObservable<DocumentChangedEvent<ITokensLine>> tokensLinesObservable =
Observable.FromEvent<EventHandler<DocumentChangedEvent<ITokensLine>>,DocumentChangedEvent<ITokensLine>>(
evtHandler => CompilationUnit.TokensLinesChanged += evtHandler,
evtHandler => CompilationUnit.TokensLinesChanged -= evtHandler);
tokensLinesObservable.Subscribe(textEditor);
// Listen to all compilation errors found by the incremental compiler => on the dispatcher thread to udpate the UI
// --> TO DO : there is no way yet to observe compilation errors
//compilationErrorsObservable.ObserveOn(DispatcherScheduler.Current).Subscribe(errorObserver);
// Trigger initial notification after document load
compiler.StartDocumentProcessing();
}
/// <summary>
/// Initial preprocessing of a complete document
/// </summary>
internal static void ProcessDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<ProcessedTokensLine> documentLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider)
{
ProcessTokensLinesChanges(textSourceInfo, documentLines, null, null, compilerOptions, processedTokensDocumentProvider);
}
/// <summary>
/// Initial preprocessing of a complete document
/// </summary>
internal static void ProcessDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <ProcessedTokensLine> documentLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider, List <RemarksDirective.TextNameVariation> copyTextNameVariations)
{
ProcessTokensLinesChanges(textSourceInfo, documentLines, null, null, compilerOptions, processedTokensDocumentProvider, copyTextNameVariations);
}
/// <summary>
/// Incremental parsing of a set of processed tokens lines changes
/// </summary>
internal static IList<DocumentChange<ICodeElementsLine>> ParseProcessedTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<CodeElementsLine> documentLines, IList<DocumentChange<IProcessedTokensLine>> processedTokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions)
{
// Collect all changes applied to the processed tokens lines during the incremental scan
IList<DocumentChange<ICodeElementsLine>> codeElementsLinesChanges = new List<DocumentChange<ICodeElementsLine>>();
// There are 2 reasons to re-parse a tokens line after a change :
// 1. The tokens line changed : these lines were already reset during the previous steps
// 2. If a tokens line that changed was involved in the parsing of a multiline code element, the whole group of lines must be parsed again
// --- PREPARATION PHASE : identify all parse sections where code elements need to be refreshed ---
IList<ParseSection> refreshParseSections = null;
// Iterate over all processed tokens changes detected by the PreprocessorStep :
// - refresh all the adjacent lines participating in a CodeElement
// - register the start and stop token for all sections of the document which need to be parsed again
if (processedTokensLinesChanges != null)
{
// If the document was cleared, everything must be parsed again
if (processedTokensLinesChanges[0].Type != DocumentChangeType.DocumentCleared)
{
refreshParseSections = new List<ParseSection>();
ParseSection lastParseSection = null;
foreach (DocumentChange<IProcessedTokensLine> tokensChange in processedTokensLinesChanges)
{
if (lastParseSection == null || tokensChange.LineIndex > lastParseSection.StopLineIndex)
{
lastParseSection = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, codeElementsLinesChanges, lastParseSection);
refreshParseSections.Add(lastParseSection);
}
}
}
}
// --- INITIALIZE ANTLR CodeElements parser ---
// Create a token iterator on top of pre-processed tokens lines
ITokensLinesIterator tokensIterator = ProcessedTokensDocument.GetProcessedTokensIterator(textSourceInfo, documentLines);
// Create an Antlr compatible token source on top of the token iterator
TokensLinesTokenSource tokenSource = new TokensLinesTokenSource(
textSourceInfo.Name,
tokensIterator);
// Init parser
TokensLinesTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
TracingCobolParser cobolParser = new TracingCobolParser(tokenStream);
// -> activate full ambiguities detection
//parser.Interpreter.PredictionMode = PredictionMode.LlExactAmbigDetection;
IAntlrErrorStrategy cobolErrorStrategy = new CobolErrorStrategy();
cobolParser.ErrorHandler = cobolErrorStrategy;
// Register all parse errors in a list in memory
DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener();
cobolParser.RemoveErrorListeners();
cobolParser.AddErrorListener(errorListener);
// Prepare to analyze the parse tree
ParseTreeWalker walker = new ParseTreeWalker();
CodeElementBuilder codeElementBuilder = new CodeElementBuilder();
codeElementBuilder.Dispatcher = new CodeElementDispatcher();
codeElementBuilder.Dispatcher.CreateListeners();
// --- INCREMENTAL PARSING ---
// In case of incremental parsing, parse only the code sections we need to refresh
IEnumerator<ParseSection> parseSectionsEnumerator = null;
ParseSection currentParseSection = null;
if (refreshParseSections != null)
{
// Get the first code section we need to refresh
parseSectionsEnumerator = refreshParseSections.GetEnumerator();
parseSectionsEnumerator.MoveNext();
currentParseSection = parseSectionsEnumerator.Current;
// Seek just before the next code element starting token
tokenStream.SeekToToken(currentParseSection.StartToken);
tokenStream.StartLookingForStopToken(currentParseSection.StopToken);
}
// Parse one CodeElement at a time while advancing in the underlying token stream
do
{
// Reset parsing error diagnostics
//cobolErrorStrategy.Reset(cobolParser);
errorListener.Diagnostics.Clear();
// Reset parser traces (consumed tokens)
cobolParser.ResetTraces();
// Try to parse a code element starting with the current token
CodeElementsParser.CodeElementContext codeElementParseTree = cobolParser.codeElement();
// If the parse tree is not empty
if (codeElementParseTree.Start.Type > 0)
{
// Get the first line that was parsed
var tokenStart = (Token)codeElementParseTree.Start;
CodeElementsLine codeElementsLine;
var importedToken = tokenStart as ImportedToken;
if (importedToken != null)
{
codeElementsLine = (CodeElementsLine) importedToken.CopyDirective.TextNameSymbol.TokensLine;
}
else
{
codeElementsLine = ((CodeElementsLine) tokenStart.TokensLine);
}
// Register that this line was updated
// COMMENTED FOR THE SAKE OF PERFORMANCE -- SEE ISSUE #160
//int updatedLineIndex = documentLines.IndexOf(codeElementsLine, codeElementsLine.InitialLineIndex);
//codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, updatedLineIndex, codeElementsLine));
codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, codeElementsLine.InitialLineIndex, codeElementsLine));
// Visit the parse tree to build a first class object representing the code elements
try { walker.Walk(codeElementBuilder, codeElementParseTree); }
catch (Exception ex) {
var code = Diagnostics.MessageCode.ImplementationError;
int line = 0; int start = 0; int stop = 0;
if (codeElementsLine.SourceTokens != null && codeElementsLine.SourceTokens.Count > 0){
start = codeElementsLine.SourceTokens[0].StartIndex;
stop = codeElementsLine.SourceTokens[codeElementsLine.SourceTokens.Count-1].StopIndex;
}
codeElementsLine.AddParserDiagnostic(new ParserDiagnostic(ex.ToString(), start,stop,line, null, code));
}
CodeElement codeElement = codeElementBuilder.CodeElement;
if (codeElement != null)
{
// Attach consumed tokens and main document line numbers information to the code element
codeElement.ConsumedTokens = cobolParser.ConsumedTokens;
if (codeElement.ConsumedTokens == null) {// ISSUE #204:
if (tokenStream.Lt(1) != null) {// if not end of file,
// add next token to ConsumedTokens to know where is the CodeElement in error
codeElement.ConsumedTokens.Add((Token)tokenStream.Lt(1));
// this alter CodeElements semantics: in addition to matched tokens,
// it includes the first token in error if no token has been matched
}
}
//TODO Issue #384 to discuss if this code should stay here:
//This should be in a Checker, but "codeElement.ConsumedTokens" is only set after all the checkers have been called
//Rule TCLIMITATION_NO_CE_ACROSS_SOURCES
if (codeElement.IsAcrossSourceFile()) {
DiagnosticUtils.AddError(codeElement, "A Cobol statement cannot be across 2 sources files (eg. Main program and a COPY)", MessageCode.TypeCobolParserLimitation);
}
// Add code element to the list
codeElementsLine.AddCodeElement(codeElement);
foreach (ParserDiagnostic d in errorListener.Diagnostics)
{
codeElement.Diagnostics.Add(d);
}
foreach (Diagnostic d in codeElement.Diagnostics)
{
codeElementsLine.AddParserDiagnostic(d);
}
}
else
{
foreach (ParserDiagnostic d in errorListener.Diagnostics)
{
codeElementsLine.AddParserDiagnostic(d);
}
}
}
// In case of incremental parsing, directly jump to next parse section in the token stream
// Else, simply start parsing the next CodeElement beginning with the next token
if (currentParseSection != null)
{
// Check if we reached the end of the current ParseSection
if (tokenStream.StreamReachedStopToken)
{
// Adavance to the next ParseSection
if (parseSectionsEnumerator.MoveNext())
{
currentParseSection = parseSectionsEnumerator.Current;
tokenStream.SeekToToken(currentParseSection.StartToken);
tokenStream.StartLookingForStopToken(currentParseSection.StopToken);
}
// No more section to parse
else
{
break;
}
}
}
}
while (tokenStream.La(1) >= 0);
return codeElementsLinesChanges;
}
/// <summary>
/// Incremental scan of a set of text lines changes
/// </summary>
internal static IList<DocumentChange<ITokensLine>> ScanTextLinesChanges(TextSourceInfo textSourceInfo,
ISearchableReadOnlyList<TokensLine> documentLines, IList<DocumentChange<ICobolTextLine>> textLinesChanges,
PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions)
{
return ScanTextLinesChanges(textSourceInfo, documentLines, textLinesChanges, prepareDocumentLineForUpdate,
compilerOptions, null);
}
/// <summary>
/// Use a pre-existing text document, already initialized from a Cobol file
/// </summary>
public FileCompiler(string libraryName, string fileName, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ColumnsLayout columnsLayout, TypeCobolOptions compilerOptions, CodeModel.SymbolTable customSymbols, bool isCopyFile, MultilineScanState scanState, CompilationProject compilationProject, List <RemarksDirective.TextNameVariation> copyTextNameVariations) :
this(libraryName, fileName, null, sourceFileProvider, documentProvider, columnsLayout, null, compilerOptions, customSymbols, isCopyFile, scanState, compilationProject, copyTextNameVariations)
{
}
/// <summary>
/// Load a Cobol source file in an pre-existing text document
/// </summary>
public FileCompiler(string libraryName, string fileName, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ITextDocument textDocument, TypeCobolOptions compilerOptions, bool isCopyFile, CompilationProject compilationProject) :
this(libraryName, fileName, null, sourceFileProvider, documentProvider, default(ColumnsLayout), textDocument, compilerOptions, null, isCopyFile, null, compilationProject, null)
{
}
private static MultilineScanState ScanTokensLineWithContinuations(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IList<DocumentChange<ITokensLine>> tokensLinesChanges, out int nextLineToScanIndex, out TokensLine nextLineToScan)
{
// Initialize out parameters
if (lineToScanIndex < (documentLines.Count - 1))
{
nextLineToScanIndex = lineToScanIndex + 1;
nextLineToScan = documentLines[nextLineToScanIndex];
}
else
{
nextLineToScanIndex = -1;
nextLineToScan = null;
}
// Check if the line to scan participates in a multiline continuation
// - because it is itself a continuation line
bool partOfMultilineContinuation = lineToScan.Type == CobolTextLineType.Continuation;
// - or because the following line is a continuation line
if (!partOfMultilineContinuation && lineToScanIndex < (documentLines.Count - 1))
{
nextLineToScanIndex = lineToScanIndex + 1;
nextLineToScan = documentLines[nextLineToScanIndex];
partOfMultilineContinuation = nextLineToScan.Type == CobolTextLineType.Continuation;
}
// ^-- LIMITATION : we don't support the case where one or more comment lines follow the current line before a continuation line
// Case 1 : the line is not part of a multiline continuation => we can parse it as as a standalone line
if (!partOfMultilineContinuation)
{
// Create a new copy of the line before the update if necessary
lineToScan = (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner);
if (lineToScanIndex == 0)
{
// Scan the first line of the document
Scanner.ScanFirstLine(lineToScan, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions);
}
else
{
// Get the scan state at the end of the previous line
TokensLine previousLine = documentLines[lineToScanIndex - 1];
// Scan the current line with this initial scan state
Scanner.ScanTokensLine(lineToScan, previousLine.ScanState, compilerOptions);
}
tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineUpdated, lineToScanIndex, lineToScan));
return lineToScan.ScanState;
}
// Case 2 : the line is part of a multiline continuation => we must parse all continuation lines as a group
else
{
// Build a list of the lines we will have to scan as a group :
IList<TokensLine> continuationLinesGroup = new List<TokensLine>();
int firstLineIndex = lineToScanIndex;
continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
// Navigate backwards to the start of the multiline continuation
if (lineToScan.Type == CobolTextLineType.Continuation && lineToScanIndex > 0)
{
int revLineToScanIndex = lineToScanIndex;
IEnumerator<TokensLine> reversedEnumerator = documentLines.GetEnumerator(lineToScanIndex - 1, -1, true);
while(reversedEnumerator.MoveNext())
{
// Get the previous line until a non continuation and non comment line is encountered
revLineToScanIndex--;
lineToScan = reversedEnumerator.Current;
if(lineToScan.Type != CobolTextLineType.Continuation /*&& <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line
lineToScan.Type != CobolTextLineType.Comment && lineToScan.Type != CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines
{
firstLineIndex = revLineToScanIndex;
continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
}
else
{
break;
}
}
}
// Reuse our knowledge of the next line if it is available and if it is a continuation
if (nextLineToScan != null && nextLineToScan.Type == CobolTextLineType.Continuation)
{
lineToScanIndex++;
lineToScan = nextLineToScan;
continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
nextLineToScanIndex = -1;
nextLineToScan = null;
}
// Navigate forwards to the end of the multiline continuation
if (lineToScanIndex < (documentLines.Count - 1))
{
IEnumerator<TokensLine> enumerator = documentLines.GetEnumerator(lineToScanIndex + 1, -1, false);
while (enumerator.MoveNext())
{
// Get the next line until a non continuation and non comment line is encountered
lineToScanIndex++;
lineToScan = enumerator.Current;
if (lineToScan.Type == CobolTextLineType.Continuation /*|| <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line
lineToScan.Type == CobolTextLineType.Comment || lineToScan.Type == CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines
{
// Add this line at the end of the list of continuation lines
continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
}
else
{
// Save the knowledge of the next line and exit the loop
nextLineToScanIndex = lineToScanIndex;
nextLineToScan = lineToScan;
break;
}
}
}
// Scan the group of continuation lines
if (firstLineIndex == 0)
{
// Scan the first line group of the document
Scanner.ScanFirstLineContinuationGroup(continuationLinesGroup, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions);
}
else
{
// Get the scan state at the end of the previous line
TokensLine previousLine = documentLines[firstLineIndex - 1];
// Scan the current line group with this initial scan state
Scanner.ScanTokensLineContinuationGroup(continuationLinesGroup, previousLine.ScanState, compilerOptions);
}
int updatedLineIndex = firstLineIndex;
foreach(TokensLine updatedLine in continuationLinesGroup)
{
tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineUpdated, updatedLineIndex, updatedLine));
updatedLineIndex++;
}
return continuationLinesGroup[continuationLinesGroup.Count - 1].ScanState;
}
}
private static void ScanTokensLineWithMultilineScanState(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IList<DocumentChange<ITokensLine>> tokensLinesChanges, out int nextLineToScanIndex, out TokensLine nextLineToScan)
{
// Scan the current line (or continuation lines group)
MultilineScanState scanState = ScanTokensLineWithContinuations(lineToScanIndex, lineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan);
// Scan the following lines until we find that the scan state at the beginning of the next line has been updated
while(nextLineToScan != null && nextLineToScan.InitialScanState != null && !nextLineToScan.InitialScanState.Equals(scanState))
{
scanState = ScanTokensLineWithContinuations(nextLineToScanIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan);
}
}
/// <summary>
/// Initial scan of a complete document
/// </summary>
public static void ScanDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, TypeCobolOptions compilerOptions)
{
TokensLine tokensLine = null;
TokensLine nextTokensLine = null;
MultilineScanState lastScanState = null;
// Get the first line
IEnumerator<TokensLine> documentLinesEnumerator = documentLines.GetEnumerator();
if (documentLinesEnumerator.MoveNext())
{
tokensLine = documentLinesEnumerator.Current;
}
while (tokensLine != null)
{
// Peek the next line to look for continuations
if (documentLinesEnumerator.MoveNext())
{
nextTokensLine = documentLinesEnumerator.Current;
}
else
{
nextTokensLine = null;
}
// If no continuation is found, scan the current line
if (nextTokensLine == null || nextTokensLine.Type != CobolTextLineType.Continuation)
{
if (lastScanState == null)
{
Scanner.ScanFirstLine(tokensLine, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions);
}
else
{
Scanner.ScanTokensLine(tokensLine, lastScanState, compilerOptions);
}
}
// If a continuation is found on the next line, scan the continuation lines as a group
else
{
// Build a list of the lines we will have to scan as a group :
IList<TokensLine> continuationLinesGroup = new List<TokensLine>();
// Add current line
continuationLinesGroup.Add(tokensLine);
// Add next line
tokensLine = nextTokensLine;
nextTokensLine = null;
continuationLinesGroup.Add(tokensLine);
// Navigate forwards to the end of the multiline continuation
while (documentLinesEnumerator.MoveNext())
{
nextTokensLine = documentLinesEnumerator.Current;
if (nextTokensLine.Type == CobolTextLineType.Continuation /*|| <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line
lineToScan.Type == CobolTextLineType.Comment || lineToScan.Type == CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines
{
// Add this line at the end of the list of continuation lines
tokensLine = nextTokensLine;
nextTokensLine = null;
continuationLinesGroup.Add(tokensLine);
}
else
{
// Exit the loop
break;
}
}
// Scan the whole group of continuation lines
if (lastScanState == null)
{
Scanner.ScanFirstLineContinuationGroup(continuationLinesGroup, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions);
}
else
{
Scanner.ScanTokensLineContinuationGroup(continuationLinesGroup, lastScanState, compilerOptions);
}
}
lastScanState = tokensLine.ScanState;
tokensLine = nextTokensLine;
}
}
/// <summary>
/// Scan a group of continuation lines
/// </summary>
public static void ScanTokensLineContinuationGroup(IList<TokensLine> continuationLinesGroup, MultilineScanState initialScanState, TypeCobolOptions compilerOptions)
{
// p54: Continuation lines
// Any sentence, entry, clause, or phrase that requires more than one line can be
// continued in Area B of the next line that is neither a comment line nor a blank line.
// The line being continued is a continued line; the succeeding lines are continuation
// lines.
// Track the length of text contributed to the continuation text by each individual line
TextArea[] textAreasForOriginalLinesInConcatenatedLine = new TextArea[continuationLinesGroup.Count];
int[] startIndexForTextAreasInOriginalLines = new int[continuationLinesGroup.Count];
int[] offsetForLiteralContinuationInOriginalLines = new int[continuationLinesGroup.Count];
// Initialize the continuation text with the complete source text of the first line
TokensLine firstLine = continuationLinesGroup[0];
string concatenatedLine = null;
if (firstLine.Type == CobolTextLineType.Source || (firstLine.Type == CobolTextLineType.Debug && initialScanState.WithDebuggingMode))
{
concatenatedLine = firstLine.SourceText;
}
else
{
concatenatedLine = String.Empty;
Scanner.ScanTokensLine(firstLine, initialScanState, compilerOptions);
}
textAreasForOriginalLinesInConcatenatedLine[0] = new TextArea(TextAreaType.Source, 0, concatenatedLine.Length -1);
startIndexForTextAreasInOriginalLines[0] = firstLine.Source.StartIndex;
offsetForLiteralContinuationInOriginalLines[0] = 0;
// All the following lines are continuation lines
// => build a character string representing the complete continuation text along the way
for (int i = 1; i < continuationLinesGroup.Count; i++)
{
TokensLine continuationLine = continuationLinesGroup[i];
int startIndex = continuationLine.Source.StartIndex;
int lastIndex = continuationLine.Source.EndIndex;
string line = continuationLine.Text;
// 1. Match and remove all blank characters at the beginning of the continuation line
int startOfContinuationIndex = startIndex;
for (; startOfContinuationIndex <= lastIndex && line[startOfContinuationIndex] == ' '; startOfContinuationIndex++) { }
if (startOfContinuationIndex > startIndex)
{
Token whitespaceToken = new Token(TokenType.SpaceSeparator, startIndex, startOfContinuationIndex - 1, continuationLine);
continuationLine.SourceTokens.Add(whitespaceToken);
startIndex = startOfContinuationIndex;
}
if (startOfContinuationIndex <= lastIndex)
{
if (startOfContinuationIndex < 4)
{
continuationLine.AddDiagnostic(MessageCode.AreaAOfContinuationLineMustBeBlank, startOfContinuationIndex, startOfContinuationIndex);
}
}
else
{
// Nothing but spaces on the continuation line
continue;
}
// p55: Continuation of alphanumeric and national literals
// Alphanumeric and national literals can be continued only when there are no DBCS
// characters in the content of the literal.
// The following rules apply to alphanumeric and national literals that do not contain
// DBCS characters:
// - If the continued line contains an alphanumeric or national literal without a
// closing quotation mark, all spaces at the end of the continued line (through
// column 72) are considered to be part of the literal. The continuation line must
// contain a hyphen in the indicator area, and the first nonblank character must be
// a quotation mark. The continuation of the literal begins with the character
// immediately following the quotation mark.
// - If an alphanumeric or national literal that is to be continued on the next line has
// as its last character a quotation mark in column 72, the continuation line must
// start with two consecutive quotation marks. This will result in a single quotation
// mark as part of the value of the literal.
// - If the last character on the continued line of an alphanumeric or national literal
// is a single quotation mark in Area B, the continuation line can start with a single
// quotation mark. This will result in two consecutive literals instead of one
// continued literal.
// The rules are the same when an apostrophe is used instead of a quotation mark in
// delimiters.
// ... p55 -> p56: examples of continuations and expected behavior ...
int offsetForLiteralContinuation = 0;
if (concatenatedLine.Length > 0)
{
// Scan the continuation text, and get its last token so far
TokensLine temporaryTokensLine = TokensLine.CreateVirtualLineForInsertedToken(firstLine.InitialLineIndex, concatenatedLine);
Scanner.ScanTokensLine(temporaryTokensLine, initialScanState, compilerOptions);
Token lastTokenOfConcatenatedLineSoFar = temporaryTokensLine.SourceTokens[temporaryTokensLine.SourceTokens.Count - 1];
// Check if the last token so far is an alphanumeric or national literal
if (lastTokenOfConcatenatedLineSoFar.TokenFamily == TokenFamily.AlphanumericLiteral)
{
// The continuation line must contain a hyphen in the indicator area, and the first nonblank character must be a quotation mark
if (line[startOfContinuationIndex] != lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter)
{
continuationLine.AddDiagnostic(MessageCode.InvalidFirstCharForContinuationLine, startOfContinuationIndex, startOfContinuationIndex, lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter);
}
// The continuation of the literal begins with the character immediately following the quotation mark.
else
{
offsetForLiteralContinuation = 1;
// If an alphanumeric literal that is to be continued on the next line has as its last character a quotation mark in column 72,
// the continuation line must start with two consecutive quotation marks.
if (lastTokenOfConcatenatedLineSoFar.HasClosingDelimiter)
{
if ((startOfContinuationIndex + 1) > lastIndex || line[startOfContinuationIndex + 1] != lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter)
{
continuationLine.AddDiagnostic(MessageCode.InvalidFirstTwoCharsForContinuationLine, startOfContinuationIndex, startOfContinuationIndex + 1, lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter);
// Use the first quotation mark to avoid a complete mess while scanning the rest of the line
offsetForLiteralContinuation = 0;
}
}
}
}
// Check if the last token so far is a floating comment
else if (lastTokenOfConcatenatedLineSoFar.TokenType == TokenType.FloatingComment)
{
// => remove the floating comment from the text of the continuation
concatenatedLine = concatenatedLine.Substring(0, concatenatedLine.Length - lastTokenOfConcatenatedLineSoFar.Length);
textAreasForOriginalLinesInConcatenatedLine[i - 1] = new TextArea(TextAreaType.Source, textAreasForOriginalLinesInConcatenatedLine[i - 1].StartIndex, textAreasForOriginalLinesInConcatenatedLine[i - 1].EndIndex - lastTokenOfConcatenatedLineSoFar.Length);
TokensLine lineWithFloatingComment = continuationLinesGroup[i - 1];
Token floatingCommentToken = new Token(TokenType.FloatingComment, lineWithFloatingComment.Length - lastTokenOfConcatenatedLineSoFar.Length, lineWithFloatingComment.Length - 1, lineWithFloatingComment);
lineWithFloatingComment.SourceTokens.Add(floatingCommentToken);
}
// Check if the last token so far is a comment entry
else if (lastTokenOfConcatenatedLineSoFar.TokenType == TokenType.CommentEntry)
{
// p105: A hyphen in the indicator area (column 7) is not permitted in comment - entries.
// => impossible to ignore the continuation indicator here, it is too late
// (we can not know there is a comment entry before scanning the continuation lines groups)
// => register an error message
continuationLine.AddDiagnostic(MessageCode.HyphenIndicatorNotPermittedInCommenEntries, continuationLine.Indicator.StartIndex + 1, continuationLine.Indicator.EndIndex + 1);
}
}
// p54: If there is no hyphen (-) in the indicator area (column 7) of a line, the last character
// of the preceding line is assumed to be followed by a space.
// If there is a hyphen in the indicator area of a line, the first nonblank character of
// the continuation line immediately follows the last nonblank character of the
// continued line without an intervening space.
// Concatenate the continuation text so far with the text of the current continuation line
int startIndexOfContinuationStringInContinuationLine = startOfContinuationIndex + offsetForLiteralContinuation;
int lengthOfContinuationStringInContinuationLine = lastIndex - startIndexOfContinuationStringInContinuationLine + 1;
textAreasForOriginalLinesInConcatenatedLine[i] = new TextArea(TextAreaType.Source, concatenatedLine.Length, concatenatedLine.Length + lengthOfContinuationStringInContinuationLine - 1);
startIndexForTextAreasInOriginalLines[i] = startIndexOfContinuationStringInContinuationLine;
offsetForLiteralContinuationInOriginalLines[i] = offsetForLiteralContinuation;
concatenatedLine += line.Substring(startIndexOfContinuationStringInContinuationLine, lengthOfContinuationStringInContinuationLine);
}
// Scan the complete continuation text as a whole
TokensLine virtualContinuationTokensLine = TokensLine.CreateVirtualLineForInsertedToken(firstLine.InitialLineIndex, concatenatedLine);
Scanner.ScanTokensLine(virtualContinuationTokensLine, initialScanState, compilerOptions);
// Then attribute each token and diagnostic to its corresponding tokens line
MultilineScanState scanState = initialScanState;
for (int i = 0; i < continuationLinesGroup.Count; i++)
{
TokensLine originalLine = continuationLinesGroup[i];
originalLine.InitializeScanState(scanState);
TextArea textAreaForOriginalLine = textAreasForOriginalLinesInConcatenatedLine[i];
int concatenatedLineToOriginalLineOffset = startIndexForTextAreasInOriginalLines[i] - textAreaForOriginalLine.StartIndex;
foreach (Token token in virtualContinuationTokensLine.SourceTokens)
{
// Token located after the current line
if(token.StartIndex > textAreaForOriginalLine.EndIndex)
{
break;
}
// Token located before the current line
else if(token.StopIndex < textAreaForOriginalLine.StartIndex)
{
continue;
}
// Token completely completely included inside the current line
else if(token.StartIndex >= textAreaForOriginalLine.StartIndex && token.StopIndex <= textAreaForOriginalLine.EndIndex)
{
int startIndexInOriginalLine = token.StartIndex + concatenatedLineToOriginalLineOffset;
int stopIndexInOriginalLine = token.StopIndex + concatenatedLineToOriginalLineOffset;
token.CorrectTokensLine(originalLine, startIndexInOriginalLine, stopIndexInOriginalLine);
originalLine.AddToken(token);
foreach(Diagnostic diag in virtualContinuationTokensLine.GetDiagnosticsForToken(token))
{
originalLine.AddDiagnostic((MessageCode)diag.Info.Code, token, diag.MessageArgs);
}
}
// Multiline continuation token only partially located on this line
else
{
bool isContinuationFromPreviousLine = token.StartIndex < textAreaForOriginalLine.StartIndex;
bool isContinuedOnNextLine = token.StopIndex > textAreaForOriginalLine.EndIndex;
int startIndexInOriginalLine = 0;
if (isContinuationFromPreviousLine)
{
startIndexInOriginalLine = startIndexForTextAreasInOriginalLines[i] - offsetForLiteralContinuationInOriginalLines[i];
}
else
{
startIndexInOriginalLine = token.StartIndex + concatenatedLineToOriginalLineOffset;
}
int stopIndexInOriginalLine = 0;
if (isContinuedOnNextLine)
{
stopIndexInOriginalLine = originalLine.Source.EndIndex;
// If a continued line ends with a floating comment, the continued token ends just before the floating comment
if (originalLine.SourceTokens.Count > 0 && originalLine.SourceTokens[originalLine.SourceTokens.Count - 1].TokenType == TokenType.FloatingComment)
{
stopIndexInOriginalLine -= originalLine.SourceTokens[originalLine.SourceTokens.Count - 1].Length;
}
}
else
{
stopIndexInOriginalLine = token.StopIndex + concatenatedLineToOriginalLineOffset;
}
ContinuationToken continuationToken = new ContinuationToken(token, startIndexInOriginalLine, stopIndexInOriginalLine,
originalLine, isContinuationFromPreviousLine, isContinuedOnNextLine);
originalLine.AddToken(continuationToken);
// Copy diagnostics on the first line only
if(!isContinuationFromPreviousLine)
{
foreach (Diagnostic diag in virtualContinuationTokensLine.GetDiagnosticsForToken(token))
{
originalLine.AddDiagnostic((MessageCode)diag.Info.Code, token, diag.MessageArgs);
}
}
}
}
scanState = originalLine.ScanState;
}
}
/// <summary>
/// Initial parsing of a complete document
/// </summary>
internal static void ParseDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <CodeElementsLine> documentLines, TypeCobolOptions compilerOptions, PerfStatsForParserInvocation perfStatsForParserInvocation)
{
ParseProcessedTokensLinesChanges(textSourceInfo, documentLines, null, null, compilerOptions, perfStatsForParserInvocation);
}
/// <summary>
/// Incremental preprocessing of a set of tokens lines changes
/// </summary>
internal static IList<DocumentChange<IProcessedTokensLine>> ProcessTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<ProcessedTokensLine> documentLines, IList<DocumentChange<ITokensLine>> tokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider)
{
// Collect all changes applied to the processed tokens lines during the incremental scan
IList<DocumentChange<IProcessedTokensLine>> processedTokensLinesChanges = new List<DocumentChange<IProcessedTokensLine>>();
// There are 2 reasons to a preprocess a tokens line after a change :
// 1. A tokens line changed : these lines were already reset during the previous steps
// 2. If a tokens line that changed was involved in the parsing of a multiline compiler directive, the whole group of lines must be parsed again
// Then, if a new COPY directive was parsed : the CompilationDocument to include must be prepared
// --- PREPARATION PHASE : reset all processed tokens lines which were involved in a multiline compiler directive where at least one line changed ---
// Iterate over all tokens changes detected by the ScannerStep :
// refresh all the adjacent lines participating in a ContinuationTokensGroup
if (tokensLinesChanges != null)
{
int lastLineIndexReset = -1;
foreach (DocumentChange<ITokensLine> tokensChange in tokensLinesChanges)
{
processedTokensLinesChanges.Add(new DocumentChange<IProcessedTokensLine>(tokensChange.Type, tokensChange.LineIndex, (IProcessedTokensLine)tokensChange.NewLine));
if (tokensChange.LineIndex > lastLineIndexReset)
{
lastLineIndexReset = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, processedTokensLinesChanges, lastLineIndexReset);
}
}
}
// --- COMPILER DIRECTIVES PHASE : Find and parse all compiler directives ---
// Init. Prepare a compiler directive parser
// Create a token iterator on top of tokens lines
TokensLinesIterator tokensIterator = new TokensLinesIterator(
textSourceInfo.Name,
documentLines,
null,
Token.CHANNEL_SourceTokens);
// Crate an Antlr compatible token source on top a the token iterator
TokensLinesTokenSource tokenSource = new TokensLinesTokenSource(
textSourceInfo.Name,
tokensIterator);
// Init a compiler directive parser
CommonTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
CobolCompilerDirectivesParser directivesParser = new CobolCompilerDirectivesParser(tokenStream);
IAntlrErrorStrategy compilerDirectiveErrorStrategy = new CompilerDirectiveErrorStrategy();
directivesParser.ErrorHandler = compilerDirectiveErrorStrategy;
// Register all parse errors in a list in memory
DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener();
directivesParser.RemoveErrorListeners();
directivesParser.AddErrorListener(errorListener);
// Prepare to analyze the parse tree
ParseTreeWalker walker = new ParseTreeWalker();
CompilerDirectiveBuilder directiveBuilder = new CompilerDirectiveBuilder();
// 1. Iterate over all compiler directive starting tokens found in the lines which were updated
foreach (Token compilerDirectiveStartingToken in documentLines
.Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing)
.SelectMany(line => line.SourceTokens)
.Where(token => token.TokenFamily == TokenFamily.CompilerDirectiveStartingKeyword))
{
// 2. Reset the compiler directive parser state
// Reset tokens iterator position before parsing
// -> seek just before the compiler directive starting token
tokensIterator.SeekToToken(compilerDirectiveStartingToken);
tokensIterator.PreviousToken();
// Special case : for efficiency reasons, in EXEC SQL INCLUDE directives
// only the third token INCLUDE is recognized as a compiler directive
// starting keyword by the scanner. In this case, we must rewind the
// iterator two tokens backwards to start parsing just before the EXEC token.
if (compilerDirectiveStartingToken.TokenType == TokenType.EXEC_SQL_INCLUDE)
{
tokensIterator.PreviousToken();
tokensIterator.PreviousToken();
}
// Reset Antlr BufferedTokenStream position
tokenStream.SetTokenSource(tokenSource);
// Reset parsing error diagnostics
compilerDirectiveErrorStrategy.Reset(directivesParser);
errorListener.Diagnostics.Clear();
// 3. Try to parse a compiler directive starting with the current token
CobolCompilerDirectivesParser.CompilerDirectingStatementContext directiveParseTree = directivesParser.compilerDirectingStatement();
// 4. Visit the parse tree to build a first class object representing the compiler directive
walker.Walk(directiveBuilder, directiveParseTree);
CompilerDirective compilerDirective = directiveBuilder.CompilerDirective;
bool errorFoundWhileParsingDirective = errorListener.Diagnostics.Count > 0 || directiveBuilder.Diagnostics.Count > 0;
// 5. Get all tokens consumed while parsing the compiler directive
// and partition them line by line
Token startToken = (Token)directiveParseTree.Start;
Token stopToken = (Token)directiveParseTree.Stop;
if (stopToken == null) stopToken = startToken;
MultilineTokensGroupSelection tokensSelection = tokensIterator.SelectAllTokensBetween(startToken, stopToken);
if (compilerDirective != null)
{
// 6. Replace all matched tokens by :
// - a CompilerDirectiveToken on the first line
ProcessedTokensLine firstProcessedTokensLine = documentLines[tokensSelection.FirstLineIndex];
if (tokensSelection.SelectedTokensOnSeveralLines.Length == 1)
{
firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine(
tokensSelection.TokensOnFirstLineBeforeStartToken,
compilerDirective, errorFoundWhileParsingDirective,
tokensSelection.SelectedTokensOnSeveralLines[0],
tokensSelection.TokensOnLastLineAfterStopToken, false);
}
else
{
TokensGroup continuedTokensGroup = firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine(
tokensSelection.TokensOnFirstLineBeforeStartToken,
compilerDirective, errorFoundWhileParsingDirective,
tokensSelection.SelectedTokensOnSeveralLines[0],
null, true);
// - a ContinuationTokensGroup on the following lines
int selectionLineIndex = 1;
int lastLineIndex = tokensSelection.FirstLineIndex + tokensSelection.SelectedTokensOnSeveralLines.Length - 1;
for (int nextLineIndex = tokensSelection.FirstLineIndex + 1; nextLineIndex <= lastLineIndex; nextLineIndex++, selectionLineIndex++)
{
IList<Token> compilerDirectiveTokensOnNextLine = tokensSelection.SelectedTokensOnSeveralLines[selectionLineIndex];
if (compilerDirectiveTokensOnNextLine.Count > 0)
{
ProcessedTokensLine nextProcessedTokensLine = documentLines[nextLineIndex];
if (nextLineIndex != lastLineIndex)
{
continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine(
continuedTokensGroup,
compilerDirectiveTokensOnNextLine,
null, true);
}
else
{
continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine(
continuedTokensGroup,
compilerDirectiveTokensOnNextLine,
tokensSelection.TokensOnLastLineAfterStopToken, false);
}
}
}
}
}
// 7. Register compiler directive parse errors
if (errorFoundWhileParsingDirective)
{
ProcessedTokensLine compilerDirectiveLine = documentLines[tokensSelection.FirstLineIndex];
foreach (ParserDiagnostic parserDiag in errorListener.Diagnostics)
{
compilerDirectiveLine.AddDiagnostic(parserDiag);
}
foreach (Diagnostic directiveDiag in directiveBuilder.Diagnostics)
{
compilerDirectiveLine.AddDiagnostic(directiveDiag);
}
}
}
// 8. Advance the state off all ProcessedTokensLines :
// NeedsCompilerDirectiveParsing => NeedsCopyDirectiveProcessing if it contains a COPY directive
IList<ProcessedTokensLine> parsedLinesWithCopyDirectives = null;
// NeedsCompilerDirectiveParsing => Ready otherwise
foreach (ProcessedTokensLine parsedLine in documentLines
.Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing))
{
if (parsedLine.ImportedDocuments != null)
{
if(parsedLinesWithCopyDirectives == null)
{
parsedLinesWithCopyDirectives = new List<ProcessedTokensLine>();
}
parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.NeedsCopyDirectiveProcessing;
parsedLinesWithCopyDirectives.Add(parsedLine);
}
else
{
parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready;
}
}
// --- COPY IMPORT PHASE : Process COPY (REPLACING) directives ---
// 1. Iterate over all updated lines containing a new COPY directive
if (parsedLinesWithCopyDirectives != null)
{
foreach (ProcessedTokensLine tokensLineWithCopyDirective in parsedLinesWithCopyDirectives)
{
// Iterate over all COPY directives found on one updated line
foreach (CopyDirective copyDirective in tokensLineWithCopyDirective.ImportedDocuments.Keys.ToArray<CopyDirective>())
{
try
{
// Load (or retrieve in cache) the document referenced by the COPY directive
ProcessedTokensDocument importedDocumentSource = processedTokensDocumentProvider.GetProcessedTokensDocument(copyDirective.LibraryName, copyDirective.TextName);
// Store it on the current line after appying the REPLACING directive
ImportedTokensDocument importedDocument = new ImportedTokensDocument(copyDirective, importedDocumentSource);
tokensLineWithCopyDirective.ImportedDocuments[copyDirective] = importedDocument;
}
catch (Exception e)
{
// Text name refenced by COPY directive was not found
// => register a preprocessor error on this line
Token failedDirectiveToken = tokensLineWithCopyDirective.TokensWithCompilerDirectives
.Where(token => token.TokenType == TokenType.CopyImportDirective && ((CompilerDirectiveToken)token).CompilerDirective == copyDirective)
.First();
Diagnostic diag = new Diagnostic(
MessageCode.FailedToLoadTextDocumentReferencedByCopyDirective,
failedDirectiveToken.Column, failedDirectiveToken.EndColumn,
e.Message);
tokensLineWithCopyDirective.AddDiagnostic(diag);
}
}
// Advance processing status of the line
tokensLineWithCopyDirective.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready;
}
}
// --- REPLACE PHASE : REPLACE directives are implemented in ReplaceTokensLinesIterator ---
/* Algorithm :
*
* one REPLACE directive can express several replacement operations
* one replacement operation can be of several types (distinguished for optimization purposes)
* - SimpleTokenReplace : one source token / zero or one replacement token
* - PartialWordReplace : one pure partial word / zero or one replacement token
* - SimpleToMultipleTokenReplace : one source token / several replacement tokens
* - MultipleTokenReplace : one first + several following source tokens / zero to many replacement tokens
*
* an iterator maintains a current set of replacement operations
*
* if nextToken is replace directive
* the current set of replacement operations is updated
* else
* nextToken is compared to each replacement operation in turn
* if single -> single source token operation : return replacement token
* if single -> multiple operation : setup a secondary iterator with the list of replacement tokens
* if multiple -> multiple operation
* snapshot of the underlying iterator
* try to match all of the following source tokens
* if failure : restore snapshot and try next operation
* if success : setup a secondary iterator
*
* token comparison sourceToken / replacementCandidate :
* 1. Compare Token type
* 2. If same token type and for families
* AlphanumericLiteral
* NumericLiteral
* SyntaxLiteral
* Symbol
* => compare also Token text
*
* PartialCobolWord replacement :
* p535 : The COPY statement with REPLACING phrase can be used to replace parts of words.
* By inserting a dummy operand delimited by colons into the program text,
* the compiler will replace the dummy operand with the required text.
* Example 3 shows how this is used with the dummy operand :TAG:.
* The colons serve as separators and make TAG a stand-alone operand.
*/
return processedTokensLinesChanges;
}
/// <summary>
/// Initial preprocessing of a complete document
/// </summary>
internal static void ProcessDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <ProcessedTokensLine> documentLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider, List <RemarksDirective.TextNameVariation> copyTextNameVariations, PerfStatsForParserInvocation perfStatsForParserInvocation, List <CopyDirective> missingCopies)
{
ProcessTokensLinesChanges(textSourceInfo, documentLines, null, null, compilerOptions, processedTokensDocumentProvider, copyTextNameVariations, perfStatsForParserInvocation, missingCopies);
}
/// <summary>
/// Load a Cobol source file in memory
/// </summary>
public FileCompiler(string libraryName, string fileName, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ColumnsLayout columnsLayout, TypeCobolOptions compilerOptions, CodeModel.SymbolTable customSymbols, bool isCopyFile, CompilationProject compilationProject) :
this(libraryName, fileName, null, sourceFileProvider, documentProvider, columnsLayout, null, compilerOptions, customSymbols, isCopyFile, null, compilationProject, null)
{
}
public static void CheckTests(string rootFolder, string resultFolder, string timedResultFile, string regex, string[] include, string[] exclude, string[] copiesFolder, string skelPath, int stopAfterAsManyErrors, bool autoRemarks, string expectedResultFile)
{
string[] files = Directory.GetFiles(rootFolder, regex, SearchOption.AllDirectories);
bool codegen = true;
var format = TypeCobol.Compiler.DocumentFormat.RDZReferenceFormat;
string resultFile = "GeneratedResultFile.txt";
//Initialize both files
File.WriteAllText(timedResultFile, "");
if (expectedResultFile != null)
{
File.WriteAllText(resultFile, "");
}
int tested = 0, nbFilesInError = 0, ignores = 0;
TimeSpan parsingSumDuration = new TimeSpan(0);
TimeSpan codeGenSumDuration = new TimeSpan(0);
int parseErrors = 0;
int codegenErrors = 0;
int codegenDiff = 0;
foreach (var file in files)
{
string filename = Path.GetFileName(file);
AppendTextToFiles((filename + ':'), timedResultFile, resultFile);
bool ignore = include.Length > 0 && !include.Contains(filename);
if (!ignore)
{
ignore = exclude.Contains(filename);
}
if (ignore)
{
ignores++;
File.AppendAllText(timedResultFile, " ignored.\n");
continue;
}
string path = Path.Combine(rootFolder, filename);
Stopwatch watch = new Stopwatch();
watch.Start();
var document = new TypeCobol.Parser();
var options = new TypeCobolOptions
{
ExecToStep = ExecutionStep.CrossCheck,
#if EUROINFO_RULES
AutoRemarksEnable = autoRemarks
#endif
};
document.Init(path, options, format, copiesFolder);
document.Parse(path);
watch.Stop();
//TestJSONSerializer.DumpAsJSON(unit.CodeElementsDocumentSnapshot.CodeElements, filename);
TimeSpan elapsed = watch.Elapsed;
parsingSumDuration += elapsed;
string formatted = String.Format("{0:00}m{1:00}s{2:000}ms", elapsed.Minutes, elapsed.Seconds, elapsed.Milliseconds);
AppendTextToFiles(" parsed in " + formatted + "\n", timedResultFile);
AppendTextToFiles("- " + document.Results.PerfStatsForText.FirstCompilationTime + " ms : text update\n", timedResultFile);
AppendTextToFiles("- " + document.Results.PerfStatsForScanner.FirstCompilationTime + " ms : scanner\n", timedResultFile);
AppendTextToFiles("- " + document.Results.PerfStatsForPreprocessor.FirstCompilationTime + " ms : preprocessor\n", timedResultFile);
AppendTextToFiles("- " + document.Results.PerfStatsForCodeElementsParser.FirstCompilationTime + " ms : code elements parser\n", timedResultFile);
AppendTextToFiles("- " + document.Results.PerfStatsForTemporarySemantic.FirstCompilationTime + " ms : temporary semantic class parser\n", timedResultFile);
AppendTextToFiles("- " + document.Results.PerfStatsForProgramCrossCheck.FirstCompilationTime + " ms : cross check class parser\n", timedResultFile);
tested++;
Console.WriteLine(filename);
bool okay = true;
var diagnostics = document.Results.AllDiagnostics();
if (diagnostics.Count > 0)
{
okay = false;
parseErrors += diagnostics.Count;
}
displayAndWriteErrorsToGrammarResult(diagnostics, timedResultFile, resultFile);
if (!okay)
{
nbFilesInError++;
if (nbFilesInError >= stopAfterAsManyErrors)
{
break;
}
}
if (codegen && okay)
{
watch.Reset();
watch.Start();
var writer = new StringWriter();
//Retrieve skeletons
var skeletons = !string.IsNullOrEmpty(skelPath) ? Config.Parse(skelPath) : null;
var generator = new TypeCobol.Codegen.Generators.DefaultGenerator(document.Results, writer, skeletons, null);
var columns = document.Results.ProgramClassDocumentSnapshot.TextSourceInfo.ColumnsLayout;
generator.Generate(document.Results, columns);
writer.Close();
//Write duration to GrammarResultFile
watch.Stop();
elapsed = watch.Elapsed;
codeGenSumDuration += elapsed;
formatted = String.Format("{0:00}m{1:00}s{2:000}ms", elapsed.Minutes, elapsed.Seconds, elapsed.Milliseconds);
AppendTextToFiles(" generated in " + formatted + "\n", timedResultFile);
//Error during generation, no need to check the content of generated Cobol
if (generator.Diagnostics != null && generator.Diagnostics.Count > 0)
{
codegenErrors += generator.Diagnostics.Count;
displayAndWriteErrorsToGrammarResult(generator.Diagnostics, timedResultFile, resultFile);
nbFilesInError++;
if (nbFilesInError >= stopAfterAsManyErrors)
{
break;
}
}
else
{
//Compare generated Cobol with expected
var expected = AsLines(File.ReadAllText(path, format.Encoding));
var actual = AsLines(writer.ToString());
Directory.CreateDirectory(resultFolder);
var linesKO = new List <int>();
for (int i = 0; i < Math.Min(expected.Count, actual.Count); i++)
{
if (!expected[i].Equals(actual[i]))
{
linesKO.Add(i);
}
}
var errors = new System.Text.StringBuilder();
string fmt = Lines2FormatString(Math.Max(expected.Count, actual.Count));
if (linesKO.Count > 0 || expected.Count != actual.Count)
{
errors.AppendLine("--- Lines mismatch ---");
File.WriteAllLines(
resultFolder + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(file) + "-Expected" +
Path.GetExtension(file), expected);
File.WriteAllLines(
resultFolder + Path.DirectorySeparatorChar + Path.GetFileNameWithoutExtension(file) + "-Actual" +
Path.GetExtension(file), actual);
}
int start = -1;
for (int i = 0; i < linesKO.Count; i++)
{
int currentline = linesKO[i];
bool follows = i > 0 && linesKO[i - 1] == currentline - 1;
if (!follows)
{
start = currentline;
before(errors, expected, currentline, 3, fmt);
}
bool preceeds = i + 1 < linesKO.Count && linesKO[i + 1] == currentline + 1;
if (!preceeds)
{
diff(errors, expected, actual, start, currentline, fmt);
after(errors, expected, currentline, 3, fmt);
start = -1;
}
}
for (int i = actual.Count; i < expected.Count; i++)
{
errors.AppendLine(String.Format("-{0:" + fmt + "} {1}", i, expected[i]));
}
for (int i = expected.Count; i < actual.Count; i++)
{
errors.AppendLine(String.Format("+{0:" + fmt + "} {1}", i, actual[i]));
}
if (errors.Length > 0)
{
codegenDiff += linesKO.Count + Math.Abs(actual.Count - expected.Count);
AppendTextToFiles(errors.ToString(), timedResultFile, resultFile);
if (okay)
{
nbFilesInError++;
}
}
}
}
else
{
AppendTextToFiles("\n", timedResultFile, resultFile);
}
}
TimeSpan totalTestDuration = parsingSumDuration + codeGenSumDuration;
string parsingTotalDurationFormatted = String.Format("{0:00}m{1:00}s{2:000}ms", parsingSumDuration.Minutes, parsingSumDuration.Seconds, parsingSumDuration.Milliseconds);
string message = "Files tested=" + tested + "/" + files.Length + ", files in error=" + nbFilesInError + ", ignored=" + ignores + "\n";
AppendTextToFiles(message, resultFile);
if (parseErrors > 0)
{
message += "Parsing errors: " + parseErrors + '\n';
}
if (codegenErrors > 0)
{
message += "Codegen errors: " + codegenErrors + '\n';
}
if (codegenDiff > 0)
{
message += "Codegen Diff: " + codegenDiff + '\n';
}
message += "Parsing time: " + parsingTotalDurationFormatted;
if (codegen)
{
string codeGenTotalDurationFormatted = string.Format("{0:00}m{1:00}s{2:000}ms", codeGenSumDuration.Minutes, codeGenSumDuration.Seconds, codeGenSumDuration.Milliseconds);
string totalDurationFormatted = String.Format("{0:00}m{1:00}s{2:000}ms", totalTestDuration.Minutes, totalTestDuration.Seconds, totalTestDuration.Milliseconds);
message += " + Codegen time: " + codeGenTotalDurationFormatted + " => Total time: " + totalDurationFormatted;
}
AppendTextToFiles(message, timedResultFile);
if (nbFilesInError > 0 && expectedResultFile == null)
{
Assert.Fail('\n' + message); //If no expectedFile to compare throw assert if error
}
else if (expectedResultFile != null) //If expectedFileResult exists compare the DefaultGeneratedFile with expectedFile
{
StreamReader expectedResultReader = new StreamReader(new FileStream(expectedResultFile, FileMode.Open));
StreamReader actualResultReader = new StreamReader(new FileStream(resultFile, FileMode.Open));
TestUtils.compareLines("GrammarTestCompareFiles", expectedResultReader.ReadToEnd(), actualResultReader.ReadToEnd()); //The test will fail if result files are different
expectedResultReader.Close();
actualResultReader.Close();
}
}
/// <summary>
/// Use a pre-existing text document, not yet associated with a Cobol file + Existing SymbolTable
/// </summary>
public FileCompiler(ITextDocument textDocument, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, TypeCobolOptions compilerOptions, SymbolTable customSymbols, bool isCopyFile, CompilationProject compilationProject) :
this(null, null, null, sourceFileProvider, documentProvider, default(ColumnsLayout), textDocument, compilerOptions, customSymbols, isCopyFile, null, compilationProject, null)
{
}
/// <summary>
/// Scan a group of continuation lines when no previous scan state object is available
/// </summary>
public static void ScanFirstLineContinuationGroup(IList<TokensLine> continuationLinesGroup, bool insideDataDivision, bool decimalPointIsComma, bool withDebuggingMode, Encoding encodingForAlphanumericLiterals, TypeCobolOptions compilerOptions)
{
MultilineScanState initialScanState = new MultilineScanState(insideDataDivision, decimalPointIsComma, withDebuggingMode, encodingForAlphanumericLiterals);
ScanTokensLineContinuationGroup(continuationLinesGroup, initialScanState, compilerOptions);
}
/// <summary>
/// Common internal implementation for all constructors above
/// </summary>
private FileCompiler(string libraryName, string fileName, CobolFile loadedCobolFile, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ColumnsLayout columnsLayout, ITextDocument textDocument, TypeCobolOptions compilerOptions, SymbolTable customSymbols, bool isCopyFile,
[CanBeNull] MultilineScanState scanState, CompilationProject compilationProject, List <RemarksDirective.TextNameVariation> copyTextNameVariations)
{
var chrono = new Stopwatch();
chrono.Start();
// 1.a Find the Cobol source file
CobolFile sourceFile = null;
CompilationProject = compilationProject;
if (fileName != null)
{
if (sourceFileProvider.TryGetFile(libraryName, fileName, out sourceFile))
{
CobolFile = sourceFile;
}
else
{
var message = string.IsNullOrEmpty(libraryName) ? string.Format("Cobol source file not found: {0}", fileName)
: string.Format("Cobol source file not found: {0} in {1}", fileName, libraryName);
throw new Exception(message);
}
}
// 1.b Register a Cobol source file which was already loaded
else if (loadedCobolFile != null)
{
CobolFile = loadedCobolFile;
}
chrono.Stop();
SourceFileSearchTime = (int)chrono.ElapsedMilliseconds;
chrono.Reset();
// 2.a Load it in a new text document in memory
chrono.Start();
if (textDocument == null)
{
TextDocument = new ReadOnlyTextDocument(sourceFile?.Name, sourceFile?.Encoding, columnsLayout, sourceFile?.ReadChars());
}
// 2.b Load it in an existing text document in memory
else if (sourceFile != null)
{
TextDocument = textDocument;
textDocument.LoadChars(sourceFile.ReadChars());
}
// 2.c Use a pre-existing text document
// - not yet associated with a Cobol source file
// - with a Cobol source file already loaded
else if (sourceFile == null || loadedCobolFile != null)
{
TextDocument = textDocument;
}
chrono.Stop();
SourceFileLoadTime = (int)chrono.ElapsedMilliseconds;
chrono.Reset();
// 3. Prepare the data structures used by the different steps of the compiler
if (isCopyFile)
{
CompilationResultsForCopy = new CompilationDocument(TextDocument.Source, TextDocument.Lines, compilerOptions, documentProvider, scanState, copyTextNameVariations);
CompilationResultsForCopy.CustomSymbols = customSymbols;
}
else
{
CompilationResultsForProgram = new CompilationUnit(TextDocument.Source, TextDocument.Lines, compilerOptions, documentProvider, copyTextNameVariations);
CompilationResultsForProgram.CustomSymbols = customSymbols;
}
CompilerOptions = compilerOptions;
}
/// <summary>
/// Scan a line of a document
/// </summary>
public static void ScanTokensLine(TokensLine tokensLine, MultilineScanState initialScanState, TypeCobolOptions compilerOptions)
{
// Updates are forbidden after a snapshot of a specific version of a line
if(!tokensLine.CanStillBeUpdatedBy(CompilationStep.Scanner))
{
throw new InvalidOperationException("Can not update this TokensLine because it was already frozen by compilation step : " + tokensLine.CompilationStep.ToString());
}
// Set the initial scan state for the line
tokensLine.InitializeScanState(initialScanState);
// Alias to refer to Cobol text line properties
ICobolTextLine textLine = tokensLine;
// The source section of this line of text must be split into tokens
string line = tokensLine.Text;
int startIndex = textLine.Source.StartIndex;
int lastIndex = textLine.Source.EndIndex;
#if EUROINFO_LEGACY_REPLACING_SYNTAX
if (IsInsideRemarks(textLine.Type, tokensLine.SourceText)) tokensLine.ScanState.InsideRemarksDirective = true;
// Try to scan REMARKS compiler directive parameters inside the comment or non-comment line
if (tokensLine.ScanState.InsideRemarksDirective) {
string remarksLine = textLine.SourceText;
int startIndexForSignificantPart = GetStartIndexOfSignificantPart(remarksLine, tokensLine.ScanState);
int firstPeriodIndex = remarksLine.IndexOf('.', startIndexForSignificantPart);
int endIndexForSignificantPart = GetEndIndexOfSignificantPart(remarksLine, tokensLine.ScanState, firstPeriodIndex);
string significantPart = remarksLine.Substring(startIndexForSignificantPart, endIndexForSignificantPart - startIndexForSignificantPart + 1).Trim();
if (firstPeriodIndex >= 0 || (!tokensLine.ScanState.InsideRemarksParentheses && !remarksLine.Contains("COPY"))) {
tokensLine.ScanState.InsideRemarksDirective = false; // indicates the end of the REMARKS compiler directive
}
RemarksDirective remarksDirective = CreateRemarksDirective(significantPart, tokensLine.ScanState);
if (remarksDirective != null && remarksDirective.CopyTextNamesVariations.Count > 0) {
// A non empty remarks directive will replace the comment line
tokensLine.AddToken(CreateCompilerDirectiveToken(remarksDirective, tokensLine, startIndex, lastIndex));
return;
}
}
#endif
// Comment line => return only one token with type CommentLine
// Debug line => treated as a comment line if debugging mode was not activated
if (textLine.Type == CobolTextLineType.Comment ||
(textLine.Type == CobolTextLineType.Debug && !tokensLine.InitialScanState.WithDebuggingMode))
{
Token commentToken = new Token(TokenType.CommentLine, startIndex, lastIndex, tokensLine);
tokensLine.AddToken(commentToken);
return;
}
// Invalid indicator, the line type is unknown => the whole line text is handled as a single invalid token
else if (textLine.Type == CobolTextLineType.Invalid)
{
// Invalid indicator => register an error
tokensLine.AddDiagnostic(MessageCode.InvalidIndicatorCharacter, textLine.Indicator.StartIndex, textLine.Indicator.EndIndex, textLine.Indicator);
Token invalidToken = new Token(TokenType.InvalidToken, startIndex, lastIndex, tokensLine);
tokensLine.AddToken(invalidToken);
return;
}
// Empty line => return immediately an empty list of tokens
// Blank line => return only one token with type SpaceSeparator
if(textLine.Type == CobolTextLineType.Blank)
{
if(!String.IsNullOrEmpty(line))
{
Token whitespaceToken = new Token(TokenType.SpaceSeparator, startIndex, lastIndex, tokensLine);
tokensLine.AddToken(whitespaceToken);
}
return;
}
// Create a stateful line scanner, and iterate over the tokens
Scanner scanner = new Scanner(line, startIndex, lastIndex, tokensLine, compilerOptions);
Token nextToken = null;
while((nextToken = scanner.GetNextToken()) != null)
{
// Resolve DELETE ambiguity : DELETE + InterLiteral => DELETE_CD
// Warning : DELETE and the sequence-number-field must be on the same line
if(nextToken.TokenType == TokenType.IntegerLiteral && tokensLine.ScanState.KeywordsState == KeywordsSequenceState.After_DELETE)
{
tokensLine.ScanState.LastKeywordOrSymbolToken.CorrectType(TokenType.DELETE_CD);
}
tokensLine.AddToken(nextToken);
}
}
public static void Main(string[] args)
{
// Basic test program, useful to debug : compiles all sample programs located under TypeCobol.Test\Samples\EI Cobol samples\EI-Production"
string currentDirectory = Directory.GetCurrentDirectory();
string projectRootPath = currentDirectory.Substring(0, currentDirectory.IndexOf(@"\TypeCobol\") + 11);
string sourcePath = projectRootPath + @"TypeCobol.Test\Samples\EI Cobol samples\EI-Production";
string[] programExtensions = { "*.PGM" };
string[] copyExtensions = { "*.CPY" };
DocumentFormat docFormat = new DocumentFormat(Encoding.GetEncoding("iso8859-1"), EndOfLineDelimiter.CrLfCharacters, 80, ColumnsLayout.CobolReferenceFormat);
TypeCobolOptions compilerOptions = new TypeCobolOptions();
CompilationProject project = new CompilationProject("samples", sourcePath, programExtensions.Concat(copyExtensions).ToArray(),
docFormat.Encoding, docFormat.EndOfLineDelimiter, docFormat.FixedLineLength, docFormat.ColumnsLayout, compilerOptions);
// Iterate over all programs in the source directory
foreach (string programExtension in programExtensions)
{
foreach (string filePath in Directory.EnumerateFiles(sourcePath, programExtension))
{
// Compile program
string textName = Path.GetFileNameWithoutExtension(filePath);
Console.Write(textName + " ... ");
try
{
FileCompiler fileCompiler = new FileCompiler(null, textName, project.SourceFileProvider, project, ColumnsLayout.CobolReferenceFormat, compilerOptions.Clone(), null, false);
fileCompiler.CompileOnce();
Console.WriteLine(" OK");
}
catch(Exception e)
{
Console.WriteLine("error :");
Console.WriteLine(e.Message);
}
}
}
/*
// TO DO : read compiler options on the command line
// Start with the default compiler options
TypeCobolOptions compilerOptions = new TypeCobolOptions();
// Simple test version
// - all referenced files should be located under the current directory
CompilationProject project = new CompilationProject("project", ".", new string[] { "*.cbl", "*.cpy" },
IBMCodePages.GetDotNetEncodingFromIBMCCSID(1147), EndOfLineDelimiter.FixedLengthLines, 80, ColumnsLayout.CobolReferenceFormat, compilerOptions);
// - gets one file name as argument and compiles it
if (args.Length == 1)
{
string textName = args[0];
FileCompiler fileCompiler = new FileCompiler(null, textName, project.SourceFileProvider, project, ColumnsLayout.CobolReferenceFormat, compilerOptions.Clone(), false);
fileCompiler.CompileOnce();
}
// - gets an optional "-continuous" flag as the first argument to activate source file monitoring and automatic recompilation
else if (args.Length == 2)
{
if (String.Equals(args[0], "-continuous", StringComparison.InvariantCultureIgnoreCase))
{
string textName = args[1];
FileCompiler fileCompiler = new FileCompiler(null, textName, project.SourceFileProvider, project, ColumnsLayout.CobolReferenceFormat, compilerOptions.Clone(), false);
fileCompiler.StartContinuousBackgroundCompilation(400, 400, 900, 2000);
Console.WriteLine("Processing, press enter to stop ...");
fileCompiler.StartContinuousFileProcessing();
Console.ReadLine();
}
else
{
Console.WriteLine("ERROR : Invalid option");
}
}
else
{
Console.WriteLine("ERROR : Invalid number of arguments");
}
*/
}
/// <summary>
/// Initial scan of a complete document
/// </summary>
public static void ScanDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines,
TypeCobolOptions compilerOptions)
{
ScanDocument(textSourceInfo, documentLines, compilerOptions, null);
}
/// <summary>
/// Incremental preprocessing of a set of tokens lines changes
/// </summary>
internal static IList <DocumentChange <IProcessedTokensLine> > ProcessTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <ProcessedTokensLine> documentLines, IList <DocumentChange <ITokensLine> > tokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider, List <RemarksDirective.TextNameVariation> copyTextNameVariations, PerfStatsForParserInvocation perfStatsForParserInvocation, List <CopyDirective> missingCopies)
{
// Collect all changes applied to the processed tokens lines during the incremental scan
IList <DocumentChange <IProcessedTokensLine> > processedTokensLinesChanges = new List <DocumentChange <IProcessedTokensLine> >();
// There are 2 reasons to a preprocess a tokens line after a change :
// 1. A tokens line changed : these lines were already reset during the previous steps
// 2. If a tokens line that changed was involved in the parsing of a multiline compiler directive, the whole group of lines must be parsed again
// Then, if a new COPY directive was parsed : the CompilationDocument to include must be prepared
// --- PREPARATION PHASE : reset all processed tokens lines which were involved in a multiline compiler directive where at least one line changed ---
// Iterate over all tokens changes detected by the ScannerStep :
// refresh all the adjacent lines participating in a ContinuationTokensGroup
if (tokensLinesChanges != null)
{
int lastLineIndexReset = -1;
foreach (DocumentChange <ITokensLine> tokensChange in tokensLinesChanges)
{
processedTokensLinesChanges.Add(new DocumentChange <IProcessedTokensLine>(tokensChange.Type, tokensChange.LineIndex, (IProcessedTokensLine)tokensChange.NewLine));
if (tokensChange.LineIndex > lastLineIndexReset)
{
lastLineIndexReset = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, processedTokensLinesChanges, lastLineIndexReset);
}
}
}
// --- COMPILER DIRECTIVES PHASE : Find and parse all compiler directives ---
// Init. Prepare a compiler directive parser
// Create a token iterator on top of tokens lines
TokensLinesIterator tokensIterator = new TokensLinesIterator(
textSourceInfo.Name,
documentLines,
null,
Token.CHANNEL_SourceTokens);
// Crate an Antlr compatible token source on top a the token iterator
TokensLinesTokenSource tokenSource = new TokensLinesTokenSource(
textSourceInfo.Name,
tokensIterator);
// Init a compiler directive parser
CommonTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
CobolCompilerDirectivesParser directivesParser = new CobolCompilerDirectivesParser(tokenStream);
// Optionnaly activate Antlr Parser performance profiling
// WARNING : use this in a single-treaded context only (uses static field)
if (AntlrPerformanceProfiler == null && perfStatsForParserInvocation.ActivateDetailedAntlrPofiling)
{
AntlrPerformanceProfiler = new AntlrPerformanceProfiler(directivesParser);
}
if (AntlrPerformanceProfiler != null)
{
// Replace the generated parser by a subclass which traces all rules invocations
directivesParser = new CobolCompilerDirectivesTracingParser(tokenStream);
AntlrPerformanceProfiler.BeginParsingFile(textSourceInfo, null);
}
IAntlrErrorStrategy compilerDirectiveErrorStrategy = new CompilerDirectiveErrorStrategy();
directivesParser.ErrorHandler = compilerDirectiveErrorStrategy;
// Register all parse errors in a list in memory
ParserDiagnosticErrorListener errorListener = new ParserDiagnosticErrorListener();
directivesParser.RemoveErrorListeners();
directivesParser.AddErrorListener(errorListener);
// Prepare to analyze the parse tree
ParseTreeWalker walker = new ParseTreeWalker();
CompilerDirectiveBuilder directiveBuilder = new CompilerDirectiveBuilder(compilerOptions, copyTextNameVariations);
// 1. Iterate over all compiler directive starting tokens found in the lines which were updated
foreach (Token compilerDirectiveStartingToken in documentLines
.Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing)
.SelectMany(line => line.SourceTokens)
.Where(token => token.TokenFamily == TokenFamily.CompilerDirectiveStartingKeyword))
{
// 2. Reset the compiler directive parser state
// Reset tokens iterator position before parsing
// -> seek just before the compiler directive starting token
tokensIterator.SeekToToken(compilerDirectiveStartingToken);
tokensIterator.PreviousToken();
// Special case : for efficiency reasons, in EXEC SQL INCLUDE directives
// only the third token INCLUDE is recognized as a compiler directive
// starting keyword by the scanner. In this case, we must rewind the
// iterator two tokens backwards to start parsing just before the EXEC token.
if (compilerDirectiveStartingToken.TokenType == TokenType.EXEC_SQL_INCLUDE)
{
tokensIterator.PreviousToken();
tokensIterator.PreviousToken();
}
// Reset Antlr BufferedTokenStream position
tokenStream.SetTokenSource(tokenSource);
// Reset parsing error diagnostics
compilerDirectiveErrorStrategy.Reset(directivesParser);
// 3. Try to parse a compiler directive starting with the current token
perfStatsForParserInvocation.OnStartAntlrParsing();
if (AntlrPerformanceProfiler != null)
{
AntlrPerformanceProfiler.BeginParsingSection();
}
CobolCompilerDirectivesParser.CompilerDirectingStatementContext directiveParseTree = directivesParser.compilerDirectingStatement();
if (AntlrPerformanceProfiler != null)
{
AntlrPerformanceProfiler.EndParsingSection(directiveParseTree.ChildCount);
}
perfStatsForParserInvocation.OnStopAntlrParsing(
AntlrPerformanceProfiler != null ? (int)AntlrPerformanceProfiler.CurrentFileInfo.DecisionTimeMs : 0,
AntlrPerformanceProfiler != null ? AntlrPerformanceProfiler.CurrentFileInfo.RuleInvocations.Sum() : 0);
perfStatsForParserInvocation.OnStartTreeBuilding();
// 4. Visit the parse tree to build a first class object representing the compiler directive
walker.Walk(directiveBuilder, directiveParseTree);
CompilerDirective compilerDirective = directiveBuilder.CompilerDirective;
bool errorFoundWhileParsingDirective = compilerDirective == null || compilerDirective.Diagnostics != null || directiveParseTree.Diagnostics != null;
// 5. Get all tokens consumed while parsing the compiler directive
// and partition them line by line
Token startToken = (Token)directiveParseTree.Start;
Token stopToken = (Token)directiveParseTree.Stop;
if (stopToken == null)
{
stopToken = startToken;
}
MultilineTokensGroupSelection tokensSelection = tokensIterator.SelectAllTokensBetween(startToken, stopToken);
if (compilerDirective != null)
{
// 6. Replace all matched tokens by :
// - a CompilerDirectiveToken on the first line
ProcessedTokensLine firstProcessedTokensLine = documentLines[tokensSelection.FirstLineIndex];
if (tokensSelection.SelectedTokensOnSeveralLines.Length == 1)
{
firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine(
tokensSelection.TokensOnFirstLineBeforeStartToken,
compilerDirective, errorFoundWhileParsingDirective,
tokensSelection.SelectedTokensOnSeveralLines[0],
tokensSelection.TokensOnLastLineAfterStopToken, false);
}
else
{
TokensGroup continuedTokensGroup = firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine(
tokensSelection.TokensOnFirstLineBeforeStartToken,
compilerDirective, errorFoundWhileParsingDirective,
tokensSelection.SelectedTokensOnSeveralLines[0],
null, true);
// - a ContinuationTokensGroup on the following lines
int selectionLineIndex = 1;
int lastLineIndex = tokensSelection.FirstLineIndex + tokensSelection.SelectedTokensOnSeveralLines.Length - 1;
for (int nextLineIndex = tokensSelection.FirstLineIndex + 1; nextLineIndex <= lastLineIndex; nextLineIndex++, selectionLineIndex++)
{
IList <Token> compilerDirectiveTokensOnNextLine = tokensSelection.SelectedTokensOnSeveralLines[selectionLineIndex];
if (compilerDirectiveTokensOnNextLine.Count > 0)
{
ProcessedTokensLine nextProcessedTokensLine = documentLines[nextLineIndex];
if (nextLineIndex != lastLineIndex)
{
continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine(
continuedTokensGroup,
compilerDirectiveTokensOnNextLine,
null, true);
}
else
{
continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine(
continuedTokensGroup,
compilerDirectiveTokensOnNextLine,
tokensSelection.TokensOnLastLineAfterStopToken, false);
}
}
}
}
}
// 7. Register compiler directive parse errors
if (errorFoundWhileParsingDirective)
{
ProcessedTokensLine compilerDirectiveLine = documentLines[tokensSelection.FirstLineIndex];
if (compilerDirective != null && compilerDirective.Diagnostics != null)
{
foreach (Diagnostic directiveDiag in compilerDirective.Diagnostics)
{
compilerDirectiveLine.AddDiagnostic(directiveDiag);
}
}
else if (directiveParseTree.Diagnostics != null)
{
foreach (Diagnostic directiveDiag in directiveParseTree.Diagnostics)
{
if (compilerDirective != null)
{
compilerDirective.AddDiagnostic(directiveDiag);
}
compilerDirectiveLine.AddDiagnostic(directiveDiag);
}
}
}
}
if (AntlrPerformanceProfiler != null)
{
AntlrPerformanceProfiler.EndParsingFile(directivesParser.ParseInfo.DecisionInfo, (int)(directivesParser.ParseInfo.GetTotalTimeInPrediction() / 1000000));
}
// 8. Advance the state off all ProcessedTokensLines :
// NeedsCompilerDirectiveParsing => NeedsCopyDirectiveProcessing if it contains a COPY directive
IList <ProcessedTokensLine> parsedLinesWithCopyDirectives = null;
// NeedsCompilerDirectiveParsing => Ready otherwise
foreach (ProcessedTokensLine parsedLine in documentLines
.Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing))
{
if (parsedLine.ImportedDocuments != null)
{
if (parsedLinesWithCopyDirectives == null)
{
parsedLinesWithCopyDirectives = new List <ProcessedTokensLine>();
}
parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.NeedsCopyDirectiveProcessing;
parsedLinesWithCopyDirectives.Add(parsedLine);
}
else
{
parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready;
}
}
perfStatsForParserInvocation.OnStopTreeBuilding();
// --- COPY IMPORT PHASE : Process COPY (REPLACING) directives ---
foreach (var lineChange in processedTokensLinesChanges)
{
missingCopies.Remove(missingCopies.FirstOrDefault(c => c.COPYToken.Line == lineChange.LineIndex + 1));
}
// 1. Iterate over all updated lines containing a new COPY directive
if (parsedLinesWithCopyDirectives != null)
{
foreach (ProcessedTokensLine tokensLineWithCopyDirective in parsedLinesWithCopyDirectives)
{
// Iterate over all COPY directives found on one updated line
foreach (CopyDirective copyDirective in tokensLineWithCopyDirective.ImportedDocuments.Keys.Where(c => c.TextName != null || c.COPYToken.TokenType == TokenType.EXEC).ToArray())
{
try
{
PerfStatsForImportedDocument perfStats;
// Load (or retrieve in cache) the document referenced by the COPY directive
//Issue #315: tokensLineWithCopyDirective.ScanState must be passed because special names paragraph such as "Decimal point is comma" are declared in the enclosing program and can affect the parsing of COPY
ProcessedTokensDocument importedDocumentSource =
processedTokensDocumentProvider.GetProcessedTokensDocument(copyDirective.LibraryName,
copyDirective.TextName,
tokensLineWithCopyDirective.ScanStateBeforeCOPYToken[copyDirective.COPYToken], copyTextNameVariations, out perfStats);
// Store it on the current line after applying the REPLACING directive
ImportedTokensDocument importedDocument = new ImportedTokensDocument(copyDirective,
importedDocumentSource, perfStats);
tokensLineWithCopyDirective.ImportedDocuments[copyDirective] = importedDocument;
}
catch (Exception e)
{
if (missingCopies != null && copyDirective != null && copyDirective.COPYToken != null && !missingCopies.Contains(copyDirective)) //If list already contains the copy directive just ignore
{
var missingCopieToReplace = missingCopies.FirstOrDefault(c => c.COPYToken.Line == copyDirective.COPYToken.Line);
missingCopies.Remove(missingCopieToReplace);
missingCopies.Add(copyDirective);
}
// Text name refenced by COPY directive was not found
// => register a preprocessor error on this line
Token failedDirectiveToken = tokensLineWithCopyDirective.TokensWithCompilerDirectives
.First(
token =>
token.TokenType == TokenType.CopyImportDirective &&
((CompilerDirectiveToken)token).CompilerDirective == copyDirective);
Diagnostic diag = new Diagnostic(
MessageCode.FailedToLoadTextDocumentReferencedByCopyDirective,
failedDirectiveToken.Column, failedDirectiveToken.EndColumn,
failedDirectiveToken.Line, e.Message, e);
tokensLineWithCopyDirective.AddDiagnostic(diag);
}
}
// Advance processing status of the line
tokensLineWithCopyDirective.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready;
}
}
// --- REPLACE PHASE : REPLACE directives are implemented in ReplaceTokensLinesIterator ---
/* Algorithm :
*
* one REPLACE directive can express several replacement operations
* one replacement operation can be of several types (distinguished for optimization purposes)
* - SimpleTokenReplace : one source token / zero or one replacement token
* - PartialWordReplace : one pure partial word / zero or one replacement token
* - SimpleToMultipleTokenReplace : one source token / several replacement tokens
* - MultipleTokenReplace : one first + several following source tokens / zero to many replacement tokens
*
* an iterator maintains a current set of replacement operations
*
* if nextToken is replace directive
* the current set of replacement operations is updated
* else
* nextToken is compared to each replacement operation in turn
* if single -> single source token operation : return replacement token
* if single -> multiple operation : setup a secondary iterator with the list of replacement tokens
* if multiple -> multiple operation
* snapshot of the underlying iterator
* try to match all of the following source tokens
* if failure : restore snapshot and try next operation
* if success : setup a secondary iterator
*
* token comparison sourceToken / replacementCandidate :
* 1. Compare Token type
* 2. If same token type and for families
* AlphanumericLiteral
* NumericLiteral
* SyntaxLiteral
* Symbol
* => compare also Token text
*
* PartialCobolWord replacement :
* p535 : The COPY statement with REPLACING phrase can be used to replace parts of words.
* By inserting a dummy operand delimited by colons into the program text,
* the compiler will replace the dummy operand with the required text.
* Example 3 shows how this is used with the dummy operand :TAG:.
* The colons serve as separators and make TAG a stand-alone operand.
*/
return(processedTokensLinesChanges);
}
/// <summary>
/// Load a Cobol source file in memory
/// </summary>
public FileCompiler(string libraryName, string textName, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ColumnsLayout columnsLayout, TypeCobolOptions compilerOptions, CodeModel.SymbolTable customSymbols, bool isCopyFile)
: this(libraryName, textName, null, sourceFileProvider, documentProvider, columnsLayout, null, compilerOptions, customSymbols, isCopyFile)
{
}
public void Init(string[] extensions = null)
{
DirectoryInfo localDirectory = new DirectoryInfo(Path.GetDirectoryName( Comparator.paths.SamplePath));
DocumentFormat format = Comparator.getSampleFormat();
TypeCobolOptions options = new TypeCobolOptions();
if (extensions == null) extensions = new[] { "*.cbl", "*.cpy" };
//comparator.paths.sextension = extensions[0].Substring(1);
CompilationProject project = new CompilationProject("TEST",
localDirectory.FullName, extensions,
format.Encoding, format.EndOfLineDelimiter, format.FixedLineLength, format.ColumnsLayout, options);
string filename = Comparator.paths.SampleName;
Compiler = new FileCompiler(null, filename, project.SourceFileProvider, project, format.ColumnsLayout, options, null, false);
}
/// <summary>
/// Load a Cobol source file in an pre-existing text document
/// </summary>
public FileCompiler(string libraryName, string textName, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ITextDocument textDocument, TypeCobolOptions compilerOptions, bool isCopyFile)
: this(libraryName, textName, null, sourceFileProvider, documentProvider, default(ColumnsLayout), textDocument, compilerOptions, null, isCopyFile)
{
}
// --- Initialization ---
/// <summary>
/// Initializes a new compilation document from a list of text lines.
/// This method does not scan the inserted text lines to produce tokens.
/// You must explicitely call UpdateTokensLines() to start an initial scan of the document.
/// </summary>
public CompilationDocument(TextSourceInfo textSourceInfo, IEnumerable<ITextLine> initialTextLines,
TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider)
: this(textSourceInfo, initialTextLines, compilerOptions, processedTokensDocumentProvider, null)
{
}
/// <summary>
/// Use a pre-existing text document, already initialized from a Cobol file
/// </summary>
public FileCompiler(ITextDocument textDocument, CobolFile loadedCobolFile, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, TypeCobolOptions compilerOptions, bool isCopyFile)
: this(null, null, null, sourceFileProvider, documentProvider, default(ColumnsLayout), textDocument, compilerOptions, null, isCopyFile)
{
}
/// <summary>
/// Initializes a new compilation document from a list of text lines.
/// This method does not scan the inserted text lines to produce tokens.
/// You must explicitely call UpdateTokensLines() to start an initial scan of the document.
/// </summary>
public CompilationUnit(TextSourceInfo textSourceInfo, IEnumerable <ITextLine> initialTextLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider, List <RemarksDirective.TextNameVariation> copyTextNameVariations) :
base(textSourceInfo, initialTextLines, compilerOptions, processedTokensDocumentProvider, copyTextNameVariations)
{
// Initialize performance stats
PerfStatsForCodeElementsParser = new PerfStatsForParsingStep(CompilationStep.CodeElementsParser);
PerfStatsForTemporarySemantic = new PerfStatsForParsingStep(CompilationStep.ProgramClassParser);
PerfStatsForProgramCrossCheck = new PerfStatsForParsingStep(CompilationStep.ProgramCrossCheck);
}
/// <summary>
/// Initial parsing of a complete document
/// </summary>
internal static void ParseDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<CodeElementsLine> documentLines, TypeCobolOptions compilerOptions)
{
ParseProcessedTokensLinesChanges(textSourceInfo, documentLines, null, null, compilerOptions);
}
/// <summary>
/// Incremental parsing of a set of processed tokens lines changes
/// </summary>
internal static IList <DocumentChange <ICodeElementsLine> > ParseProcessedTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <CodeElementsLine> documentLines, IList <DocumentChange <IProcessedTokensLine> > processedTokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, PerfStatsForParserInvocation perfStatsForParserInvocation)
{
// Collect all changes applied to the processed tokens lines during the incremental scan
IList <DocumentChange <ICodeElementsLine> > codeElementsLinesChanges = new List <DocumentChange <ICodeElementsLine> >();
// There are 2 reasons to re-parse a tokens line after a change :
// 1. The tokens line changed : these lines were already reset during the previous steps
// 2. If a tokens line that changed was involved in the parsing of a multiline code element, the whole group of lines must be parsed again
// --- PREPARATION PHASE : identify all parse sections where code elements need to be refreshed ---
IList <ParseSection> refreshParseSections = null;
ParseSection largestRefreshParseSection = null;
// Iterate over all processed tokens changes detected by the PreprocessorStep :
// - refresh all the adjacent lines participating in a CodeElement
// - register the start and stop token for all sections of the document which need to be parsed again
if (processedTokensLinesChanges != null && processedTokensLinesChanges.Count > 0)
{
// If the document was cleared, everything must be parsed again
if (processedTokensLinesChanges[0].Type != DocumentChangeType.DocumentCleared)
{
refreshParseSections = new List <ParseSection>();
ParseSection lastParseSection = null;
foreach (DocumentChange <IProcessedTokensLine> tokensChange in processedTokensLinesChanges)
{
if (lastParseSection == null || tokensChange.LineIndex > lastParseSection.StopLineIndex)
{
lastParseSection = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, codeElementsLinesChanges, lastParseSection);
refreshParseSections.Add(lastParseSection);
}
}
}
}
if (refreshParseSections != null)
{
//After getting all the parts refreshed, get the largest part that has been refreshed
var minParseSection = refreshParseSections.OrderBy(p => p.StartLineIndex).First();
var maxParseSection = refreshParseSections.OrderByDescending(p => p.StopLineIndex).First();
largestRefreshParseSection = new ParseSection(minParseSection.StartLineIndex,
minParseSection.StartToken, maxParseSection.StopLineIndex, maxParseSection.StopToken,
maxParseSection.StopTokenIsFirstTokenOfTheLine);
}
// --- INITIALIZE ANTLR CodeElements parser ---
// Create a token iterator on top of pre-processed tokens lines
ITokensLinesIterator tokensIterator = ProcessedTokensDocument.GetProcessedTokensIterator(textSourceInfo, documentLines);
// Create an Antlr compatible token source on top of the token iterator
TokensLinesTokenSource tokenSource = new TokensLinesTokenSource(
textSourceInfo.Name,
tokensIterator);
// Init parser
TokensLinesTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
CodeElementsParser cobolParser = new CodeElementsParser(tokenStream);
// REVERT TO STD PARSER ==> TracingCobolParser cobolParser = new TracingCobolParser(tokenStream);
// Optionnaly activate Antlr Parser performance profiling
// WARNING : use this in a single-treaded context only (uses static field)
if (AntlrPerformanceProfiler == null && perfStatsForParserInvocation.ActivateDetailedAntlrPofiling)
{
AntlrPerformanceProfiler = new AntlrPerformanceProfiler(cobolParser);
}
if (AntlrPerformanceProfiler != null)
{
// Replace the generated parser by a subclass which traces all rules invocations
cobolParser = new CodeElementsTracingParser(tokenStream);
var tokensCountIterator = ProcessedTokensDocument.GetProcessedTokensIterator(textSourceInfo, documentLines);
AntlrPerformanceProfiler.BeginParsingFile(textSourceInfo, tokensCountIterator);
}
// Customize error recovery strategy
IAntlrErrorStrategy cobolErrorStrategy = new CodeElementErrorStrategy();
cobolParser.ErrorHandler = cobolErrorStrategy;
// Register all parse errors in a list in memory
ParserDiagnosticErrorListener errorListener = new ParserDiagnosticErrorListener();
cobolParser.RemoveErrorListeners();
cobolParser.AddErrorListener(errorListener);
// Prepare to analyze the parse tree
ParseTreeWalker walker = new ParseTreeWalker();
CodeElementBuilder codeElementBuilder = new CodeElementBuilder();
codeElementBuilder.Dispatcher = new CodeElementDispatcher();
codeElementBuilder.Dispatcher.CreateListeners();
// --- INCREMENTAL PARSING ---
// In case of incremental parsing, parse only the code sections we need to refresh
if (largestRefreshParseSection != null)
{
// Seek just before the next code element starting token
tokenStream.SeekToToken(largestRefreshParseSection.StartToken);
tokenStream.StartLookingForStopToken(largestRefreshParseSection.StopToken);
//Remove all the code elements for the future line to parse.
for (int i = largestRefreshParseSection.StartLineIndex;
i < (largestRefreshParseSection.StopLineIndex == documentLines.Count - 1 && largestRefreshParseSection.StopToken == null //If the last index is equals to number of line in document, make sure to also reset the last line, otherwise, reset lines normally.
? largestRefreshParseSection.StopLineIndex + 1
: largestRefreshParseSection.StopLineIndex);
i++)
{
if (documentLines[i].CodeElements != null)
{
documentLines[i].ResetCodeElements();
}
}
}
// Reset parsing error diagnostics
cobolErrorStrategy.Reset(cobolParser);
// Try to parse code elements :
// - starting with the current parse section Start token
// - ending with the current parse section Stop token
CodeElementsParser.CobolCodeElementsContext codeElementsParseTree = null;
try
{
perfStatsForParserInvocation.OnStartAntlrParsing();
if (AntlrPerformanceProfiler != null)
{
AntlrPerformanceProfiler.BeginParsingSection();
}
codeElementsParseTree = cobolParser.cobolCodeElements();
if (AntlrPerformanceProfiler != null)
{
AntlrPerformanceProfiler.EndParsingSection(codeElementsParseTree.ChildCount);
}
perfStatsForParserInvocation.OnStopAntlrParsing(
AntlrPerformanceProfiler != null
? (int)AntlrPerformanceProfiler.CurrentFileInfo.DecisionTimeMs
: 0,
AntlrPerformanceProfiler != null
? AntlrPerformanceProfiler.CurrentFileInfo.RuleInvocations.Sum()
: 0);
}
catch (Exception e)
{
var currentToken = (Token)cobolParser.CurrentToken;
CodeElementsLine codeElementsLine = GetCodeElementsLineForToken(currentToken);
if (codeElementsLine != null)
{
codeElementsLine.AddParserDiagnostic(new TokenDiagnostic(MessageCode.ImplementationError,
currentToken, currentToken.Line, e));
}
}
if (codeElementsParseTree != null)
{
// If the parse tree is not empty
if (codeElementsParseTree.codeElement() != null && codeElementsParseTree.codeElement().Length > 0)
{
// Analyze the parse tree for each code element
foreach (var codeElementParseTree in codeElementsParseTree.codeElement())
{
// Get the first line that was parsed
var tokenStart = (Token)codeElementParseTree.Start;
CodeElementsLine codeElementsLine = GetCodeElementsLineForToken(tokenStart);
if (codeElementsLine == null)
{
continue;
}
// Register that this line was updated
// COMMENTED FOR THE SAKE OF PERFORMANCE -- SEE ISSUE #160
//int updatedLineIndex = documentLines.IndexOf(codeElementsLine, codeElementsLine.LineIndex);
//codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, updatedLineIndex, codeElementsLine));
codeElementsLinesChanges.Add(
new DocumentChange <ICodeElementsLine>(DocumentChangeType.LineUpdated,
codeElementsLine.LineIndex, codeElementsLine));
perfStatsForParserInvocation.OnStartTreeBuilding();
// Visit the parse tree to build a first class object representing the code elements
try
{
walker.Walk(codeElementBuilder, codeElementParseTree);
}
catch (Exception ex)
{
var code = MessageCode.ImplementationError;
int line = 0;
int start = 0;
int stop = 0;
if (codeElementsLine.SourceTokens != null && codeElementsLine.SourceTokens.Count > 0)
{
start = codeElementsLine.SourceTokens[0].StartIndex;
stop =
codeElementsLine.SourceTokens[codeElementsLine.SourceTokens.Count - 1].StopIndex;
}
codeElementsLine.AddParserDiagnostic(new ParserDiagnostic(ex.ToString(), start, stop,
line, null, code, ex));
}
CodeElement codeElement = codeElementBuilder.CodeElement;
if (codeElement != null)
{
// Attach consumed tokens and main document line numbers information to the code element
if (codeElement.ConsumedTokens.Count == 0)
{
// ISSUE #204:
var tempToken = tokenStream.Lt(1);
if (tempToken != null && tempToken != Token.END_OF_FILE)
{
// if not end of file,
// add next token to ConsumedTokens to know where is the CodeElement in error
codeElement.ConsumedTokens.Add((Token)tempToken);
// this alter CodeElements semantics: in addition to matched tokens,
// it includes the first token in error if no token has been matched
}
}
//TODO Issue #384 to discuss if this code should stay here:
//This should be in a Checker, but "codeElement.ConsumedTokens" is only set after all the checkers have been called
//Rule TCLIMITATION_NO_CE_ACROSS_SOURCES
if (codeElement.IsAcrossSourceFile())
{
DiagnosticUtils.AddError(codeElement,
"A Cobol statement cannot be across 2 sources files (eg. Main program and a COPY)",
MessageCode.TypeCobolParserLimitation);
}
// Add code element to the list
codeElementsLine.AddCodeElement(codeElement);
}
}
}
// If the parse tree contains errors
if (codeElementsParseTree.Diagnostics != null)
{
foreach (ParserDiagnostic d in codeElementsParseTree.Diagnostics)
{
if (d.OffendingSymbol != null)
{
CodeElementsLine codeElementsLine =
GetCodeElementsLineForToken((Token)d.OffendingSymbol);
if (codeElementsLine != null)
{
codeElementsLine.AddParserDiagnostic(d);
}
}
}
}
perfStatsForParserInvocation.OnStopTreeBuilding();
}
if (AntlrPerformanceProfiler != null)
{
AntlrPerformanceProfiler.EndParsingFile(cobolParser.ParseInfo.DecisionInfo, (int)(cobolParser.ParseInfo.GetTotalTimeInPrediction() / 1000000));
}
return(codeElementsLinesChanges);
}
/// <summary>
/// Incremental scan of a set of text lines changes
/// </summary>
internal static IList<DocumentChange<ITokensLine>> ScanTextLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, IList<DocumentChange<ICobolTextLine>> textLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions)
{
// Collect all changes applied to the tokens lines during the incremental scan
IList<DocumentChange<ITokensLine>> tokensLinesChanges = new List<DocumentChange<ITokensLine>>();
// There are 3 reasons to scan a line after a text change :
// 1. New text lines which were just inserted or updated must be scanned for the first time
// 2. Text lines must be scanned again if their initial scan state changed : a new scan of the previous line can alter the scan state at the beginning of the following line
// 3. Continuation lines and multiline tokens : if a line participates in a continuation on several lines, scan the group of lines as a whole
// IMPORTANT : the text changes are ordered in increasing order of line index
for (int textChangeIndex = 0; textChangeIndex < textLinesChanges.Count; textChangeIndex++)
{
// Local variables used to optimize navigation in the document
int nextLineToScanIndex = -1;
TokensLine nextLineToScan = null;
// Update tokens depending on the current text change
DocumentChange<ICobolTextLine> textChange = textLinesChanges[textChangeIndex];
if (textChange.Type == DocumentChangeType.DocumentCleared)
{
tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.DocumentCleared, 0, null));
continue;
}
else if (textChange.Type == DocumentChangeType.LineInserted || textChange.Type == DocumentChangeType.LineUpdated)
{
// We update lines as a group below, but we remember here which lines were inserted
if(textChange.Type == DocumentChangeType.LineInserted)
{
tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineInserted, textChange.LineIndex, (ITokensLine)textChange.NewLine));
}
// Text lines which were inserted or updated must be scanned again
ScanTokensLineWithMultilineScanState(textChange.LineIndex, (TokensLine)textChange.NewLine, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan);
}
else if (textChange.Type == DocumentChangeType.LineRemoved)
{
tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineRemoved, textChange.LineIndex, (ITokensLine)textChange.NewLine));
// Get the last line just before the line that was removed
TokensLine previousLine = null;
if (textChange.LineIndex > 0)
{
previousLine = documentLines[textChange.LineIndex - 1];
}
// When a text line is removed :
// - the previous line must be scanned again if the line which was removed was a member of a multiline continuation group
if (previousLine != null && previousLine.HasTokenContinuedOnNextLine)
{
ScanTokensLineWithMultilineScanState(textChange.LineIndex - 1, previousLine, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan);
}
if (nextLineToScan == null && textChange.LineIndex < documentLines.Count)
{
nextLineToScanIndex = textChange.LineIndex;
nextLineToScan = documentLines[nextLineToScanIndex];
}
// - the next line must be scanned again if the scan state at the end of the previous line is different from the scan state at the beginning of the next line
if (nextLineToScan != null && nextLineToScanIndex == textChange.LineIndex && previousLine != null &&
nextLineToScan.InitialScanState.Equals(previousLine.ScanState))
{
ScanTokensLineWithMultilineScanState(textChange.LineIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan);
}
}
// We can skip all text changes with an index smaller than the index of the last line which was already scanned
if (nextLineToScan == null)
{
break;
}
else if (textChangeIndex < (textLinesChanges.Count - 1))
{
int nextTextChangeIndex = textChangeIndex;
DocumentChange<ICobolTextLine> nextTextChange = null;
do
{
nextTextChangeIndex++;
nextTextChange = textLinesChanges[nextTextChangeIndex];
}
while (nextTextChangeIndex < (textLinesChanges.Count - 1) &&
nextTextChange.LineIndex <= nextLineToScanIndex);
textChangeIndex = nextTextChangeIndex - 1;
}
}
return tokensLinesChanges;
}
