public void CheckPerformance() { // Sample program properties string folder = "Parser" + Path.DirectorySeparatorChar + "Samples"; string textName = "BigBatch"; DocumentFormat documentFormat = DocumentFormat.RDZReferenceFormat; // Create a FileCompiler for this program DirectoryInfo localDirectory = new DirectoryInfo(PlatformUtils.GetPathForProjectFile(folder)); if (!localDirectory.Exists) { throw new Exception(String.Format("Directory : {0} does not exist", localDirectory.FullName)); } CompilationProject project = new CompilationProject("test", localDirectory.FullName, new string[] { ".cbl", ".cpy" }, documentFormat.Encoding, documentFormat.EndOfLineDelimiter, documentFormat.FixedLineLength, documentFormat.ColumnsLayout, new TypeCobolOptions()); FileCompiler compiler = new FileCompiler(null, textName, project.SourceFileProvider, project, documentFormat.ColumnsLayout, new TypeCobolOptions(), null, false, project); // Execute a first (complete) compilation compiler.CompileOnce(); // Append one line in the middle of the program ITextLine newLine = new TextLineSnapshot(9211, "094215D DISPLAY '-ICLAUA = ' ICLAUA. 0000000", null); TextChangedEvent textChangedEvent = new TextChangedEvent(); textChangedEvent.TextChanges.Add(new TextChange(TextChangeType.LineInserted, 9211, newLine)); compiler.CompilationResultsForProgram.UpdateTextLines(textChangedEvent); // Execute a second (incremental) compilation compiler.CompileOnce(); // Display a performance report StringBuilder report = new StringBuilder(); report.AppendLine("Program properties :"); report.AppendLine("- " + compiler.CompilationResultsForProgram.CobolTextLines.Count + " lines"); report.AppendLine("- " + compiler.CompilationResultsForProgram.CodeElementsDocumentSnapshot.CodeElements.Count() + " code elements"); report.AppendLine("First compilation performance"); report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForText.FirstCompilationTime + " ms : text update"); report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForScanner.FirstCompilationTime + " ms : scanner"); report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForPreprocessor.FirstCompilationTime + " ms : preprocessor"); report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForCodeElementsParser.FirstCompilationTime + " ms : code elements parser"); report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForTemporarySemantic.FirstCompilationTime + " ms : temporary semantic class parser"); report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForProgramCrossCheck.FirstCompilationTime + " ms : cross check class parser"); report.AppendLine("Incremental compilation performance"); report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForText.LastRefreshTime + " ms : text update"); report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForScanner.LastRefreshTime + " ms : scanner"); report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForPreprocessor.LastRefreshTime + " ms : preprocessor"); report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForCodeElementsParser.LastRefreshTime + " ms : code elements parser"); report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForTemporarySemantic.LastRefreshTime + " ms : temporary semantic class parser"); report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForProgramCrossCheck.LastRefreshTime + " ms : cross check class parser"); Console.WriteLine(report.ToString()); }
public static CompilationUnit ParseCobolFile(string textName, DocumentFormat documentFormat = null, string folder = null, ExecutionStep execToStep = ExecutionStep.SemanticCheck) { if (folder == null) { folder = "Parser" + Path.DirectorySeparatorChar + "CodeElements"; } DirectoryInfo localDirectory = new DirectoryInfo(PlatformUtils.GetPathForProjectFile(folder)); if (!localDirectory.Exists) { throw new Exception(String.Format("Directory : {0} does not exist", localDirectory.FullName)); } if (documentFormat == null) { documentFormat = DocumentFormat.RDZReferenceFormat; } TypeCobolOptions options = new TypeCobolOptions { ExecToStep = execToStep }; //Create CompilerOptions. ExecToStep / AutoRemarks / HaltOnMissingCopy have to be set here. CompilationProject project = new CompilationProject("test", //First use *.cpy as tests will use file WITH extension for program but without extension for copy inside programs => small perf gain localDirectory.FullName, new string[] { ".cpy", ".cbl" }, documentFormat.Encoding, documentFormat.EndOfLineDelimiter, documentFormat.FixedLineLength, documentFormat.ColumnsLayout, options); FileCompiler compiler = new FileCompiler(null, textName, project.SourceFileProvider, project, documentFormat.ColumnsLayout, options, null, false, project); compiler.CompileOnce(); return(compiler.CompilationResultsForProgram); }
public void Parse(string path, TextChangedEvent e = null) { //if the server is restarted during Eclipse lifetime, then we need to init the parser //This is useful when debugging. Perhaps it'll be deleted at the end if (!Compilers.ContainsKey(path)) { Init(path, new TypeCobolOptions { ExecToStep = ExecutionStep.Generate }); } Compiler = Compilers[path]; Compiler.CompilationResultsForProgram.TextLinesChanged += OnTextLine; Compiler.CompilationResultsForProgram.CodeElementsLinesChanged += OnCodeElementLine; if (!Inits[path]) { Inits[path] = true; // no need to update with the same content as at compiler creation } else if (e != null) { Compiler.CompilationResultsForProgram.UpdateTextLines(e); } try { Compiler.CompileOnce(); } catch (Exception ex) { throw new ParsingException(MessageCode.SyntaxErrorInParser, ex.Message, path, ex, true, true); } MissingCopys = Compiler.CompilationResultsForProgram.MissingCopies.Select(c => c.TextName).Distinct().ToList(); Compiler.CompilationResultsForProgram.TextLinesChanged -= OnTextLine; Compiler.CompilationResultsForProgram.CodeElementsLinesChanged -= OnCodeElementLine; }
public void Parse(string path, TextChangedEvent e = null) { //if the server is restarted during Eclipse lifetime, then we need to init the parser //This is useful when debugging. Perhaps it'll be deleted at the end if (!Compilers.ContainsKey(path)) { Init(path, new TypeCobolOptions { ExecToStep = ExecutionStep.Generate }); } Compiler = Compilers[path]; Compiler.CompilationResultsForProgram.TextLinesChanged += OnTextLine; Compiler.CompilationResultsForProgram.CodeElementsLinesChanged += OnCodeElementLine; if (!Inits[path]) { Inits[path] = true; // no need to update with the same content as at compiler creation } else { Compiler.CompilationResultsForProgram.UpdateTextLines(e); } try { Compiler.CompileOnce(); } catch (Exception ex) { Observer.OnError(ex); System.Console.WriteLine(ex.ToString()); } MissingCopys = Compiler.CompilationProject.MissingCopys; Compiler.CompilationResultsForProgram.TextLinesChanged -= OnTextLine; Compiler.CompilationResultsForProgram.CodeElementsLinesChanged -= OnCodeElementLine; }
public static void CheckAllFilesForExceptions() { CompilationProject project = new CompilationProject("test", PlatformUtils.GetPathForProjectFile(@"Compiler\Scanner\Samples"), new string[] { ".txt" }, IBMCodePages.GetDotNetEncodingFromIBMCCSID(1147), EndOfLineDelimiter.FixedLengthLines, 80, ColumnsLayout.CobolReferenceFormat, new TypeCobolOptions()); //int filesCount = 0; //int linesCount = 0; //Stopwatch chrono = new Stopwatch(); foreach (string fileName in PlatformUtils.ListFilesInSubdirectory(@"Compiler\Scanner\Samples")) { string textName = Path.GetFileNameWithoutExtension(fileName); // Initialize a CompilationDocument FileCompiler compiler = new FileCompiler(null, textName, project.SourceFileProvider, project, ColumnsLayout.CobolReferenceFormat, new TypeCobolOptions(), null, true, project); // Start compilation try { //chrono.Start(); compiler.CompileOnce(); //chrono.Stop(); } catch (Exception e) { throw new Exception("Error while scanning file " + fileName, e); } // Stats //filesCount++; //linesCount += compiler.CompilationResultsForCopy.TokensDocumentSnapshot.Lines.Count; //string result = compiler.CompilationResultsForCopy.TokensDocumentSnapshot.GetDebugString(); } // throw new Exception("Test OK for " + filesCount + " files and " + linesCount + " lines : " + chrono.ElapsedMilliseconds + " ms"); }
/// <summary> /// Update the text contents of the file /// </summary> public void UpdateSourceFile(Uri fileUri, TextChangedEvent textChangedEvent) { FileCompiler fileCompilerToUpdate = null; if (OpenedFileCompiler.TryGetValue(fileUri, out fileCompilerToUpdate)) { _semanticUpdaterTimer?.Stop(); fileCompilerToUpdate.CompilationResultsForProgram.UpdateTextLines(textChangedEvent); if (IsLsrSourceTesting) { //Log text lines string var sb = new StringBuilder(); foreach (var cobolTextLine in fileCompilerToUpdate.CompilationResultsForProgram.CobolTextLines) { sb.AppendLine(cobolTextLine.SourceText); } _Logger(sb.ToString(), fileUri); } var handler = new Action <object, ExecutionStepEventArgs>((sender, args) => { ExecutionStepEventHandler(sender, args, fileUri); }); //Subscribe to FileCompilerEvent fileCompilerToUpdate.ExecutionStepEventHandler += handler.Invoke; var execStep = LsrTestOptions.ExecutionStep(fileCompilerToUpdate.CompilerOptions.ExecToStep); if (execStep > ExecutionStep.SyntaxCheck) { execStep = ExecutionStep.SyntaxCheck; //The maximum execstep authorize for incremental parsing is SyntaxCheck, } //further it's for semantic, which is handle by NodeRefresh method fileCompilerToUpdate.CompileOnce(execStep, fileCompilerToUpdate.CompilerOptions.HaltOnMissingCopy, fileCompilerToUpdate.CompilerOptions.UseAntlrProgramParsing); fileCompilerToUpdate.ExecutionStepEventHandler -= handler.Invoke; if (LsrTestOptions == LsrTestingOptions.NoLsrTesting || LsrTestOptions == LsrTestingOptions.LsrSemanticPhaseTesting) { if (!_timerDisabled) //If TimerDisabled is false, create a timer to automatically launch Node phase { lock (_fileCompilerWaittingForNodePhase) { if (!_fileCompilerWaittingForNodePhase.Contains(fileCompilerToUpdate)) { _fileCompilerWaittingForNodePhase.Add(fileCompilerToUpdate); //Store that this fileCompiler will soon need a Node Phase } } _semanticUpdaterTimer = new System.Timers.Timer(750); _semanticUpdaterTimer.Elapsed += (sender, e) => TimerEvent(sender, e, fileCompilerToUpdate); _semanticUpdaterTimer.Start(); } } } }
public static void ExecuteInceremental(FileCompiler compiler, TestUtils.CompilationStats stats) { // Execute a first (complete) compilation compiler.CompileOnce(); //Iterate multiple times over an incremental change stats.IterationNumber = 20; for (int i = 0; i < stats.IterationNumber; i++) { // Append one line in the middle of the program ITextLine newLine = new TextLineSnapshot(9211, "094215D DISPLAY '-ICLAUA = ' ICLAUA. 0000000", null); TextChangedEvent textChangedEvent = new TextChangedEvent(); textChangedEvent.TextChanges.Add(new TextChange(TextChangeType.LineInserted, 9211, newLine)); compiler.CompilationResultsForProgram.UpdateTextLines(textChangedEvent); // Execute a second (incremental) compilation compiler.CompileOnce(); //Accumulate results stats.AverageTextUpdateTime += compiler.CompilationResultsForProgram.PerfStatsForText.LastRefreshTime; stats.AverageScannerTime += compiler.CompilationResultsForProgram.PerfStatsForScanner.LastRefreshTime; stats.AveragePreprocessorTime += compiler.CompilationResultsForProgram.PerfStatsForPreprocessor.LastRefreshTime; stats.AverageCodeElementParserTime += compiler.CompilationResultsForProgram.PerfStatsForCodeElementsParser.LastRefreshTime; stats.AverateTemporarySemanticsParserTime += compiler.CompilationResultsForProgram.PerfStatsForTemporarySemantic.LastRefreshTime; stats.AverageCrossCheckerParserTime += compiler.CompilationResultsForProgram.PerfStatsForProgramCrossCheck.LastRefreshTime; } //Compute average time needed for each phase stats.AverageTextUpdateTime = (int)stats.AverageTextUpdateTime / stats.IterationNumber; stats.AverageScannerTime = (int)stats.AverageScannerTime / stats.IterationNumber; stats.AveragePreprocessorTime = (int)stats.AveragePreprocessorTime / stats.IterationNumber; stats.AverageCodeElementParserTime = (int)stats.AverageCodeElementParserTime / stats.IterationNumber; stats.AverateTemporarySemanticsParserTime = (int)stats.AverateTemporarySemanticsParserTime / stats.IterationNumber; stats.AverageCrossCheckerParserTime = (int)stats.AverageCrossCheckerParserTime / stats.IterationNumber; stats.AverageTotalProcessingTime = stats.AverageCodeElementParserTime + stats.AverageCrossCheckerParserTime + stats.AveragePreprocessorTime + stats.AverageScannerTime + stats.AverageTextUpdateTime + stats.AverateTemporarySemanticsParserTime; }
/// <summary> /// Start continuous background compilation on a newly opened file /// </summary> public void OpenSourceFile(Uri fileUri, string sourceText, LsrTestingOptions lsrOptions) { string fileName = Path.GetFileName(fileUri.LocalPath); ITextDocument initialTextDocumentLines = new ReadOnlyTextDocument(fileName, TypeCobolConfiguration.Format.Encoding, TypeCobolConfiguration.Format.ColumnsLayout, sourceText); FileCompiler fileCompiler = null; #if EUROINFO_RULES //Issue #583 SymbolTable arrangedCustomSymbol = null; var inputFileName = fileName.Substring(0, 8); var matchingPgm = _customSymbols.Programs.Keys.FirstOrDefault( k => k.Equals(inputFileName, StringComparison.InvariantCultureIgnoreCase)); if (matchingPgm != null) { arrangedCustomSymbol = new SymbolTable(_customSymbols, SymbolTable.Scope.Namespace); var prog = _customSymbols.Programs.Values.SelectMany(p => p).Where(p => p.Name != matchingPgm); arrangedCustomSymbol.CopyAllPrograms(new List <List <Program> >() { prog.ToList() }); arrangedCustomSymbol.Programs.Remove(matchingPgm); } fileCompiler = new FileCompiler(initialTextDocumentLines, CompilationProject.SourceFileProvider, CompilationProject, CompilationProject.CompilationOptions, arrangedCustomSymbol ?? _customSymbols, false, CompilationProject); #else fileCompiler = new FileCompiler(initialTextDocumentLines, CompilationProject.SourceFileProvider, CompilationProject, CompilationProject.CompilationOptions, _customSymbols, false, CompilationProject); #endif fileCompiler.CompilationResultsForProgram.UpdateTokensLines(); lock (OpenedFileCompiler) { if (OpenedFileCompiler.ContainsKey(fileUri)) { CloseSourceFile(fileUri); //Close and remove the previous opened file. } OpenedFileCompiler.Add(fileUri, fileCompiler); fileCompiler.CompilationResultsForProgram.ProgramClassChanged += ProgramClassChanged; } fileCompiler.CompilationResultsForProgram.SetOwnerThread(Thread.CurrentThread); if (lsrOptions != LsrTestingOptions.LsrSourceDocumentTesting) { fileCompiler.CompileOnce(lsrOptions.ExecutionStep(fileCompiler.CompilerOptions.ExecToStep.Value), fileCompiler.CompilerOptions.HaltOnMissingCopy); //Let's parse file for the first time after opening. } }
public static CompilationUnit ParseCobolString(string cobolString) { //Prepare var textDocument = new ReadOnlyTextDocument("Empty doc", Encoding.Default, ColumnsLayout.FreeTextFormat, ""); textDocument.LoadChars(cobolString); var typeCobolOptions = new TypeCobolOptions(); var project = new CompilationProject("Empty project", ".", new[] { ".cbl", ".cpy" }, DocumentFormat.FreeTextFormat.Encoding, DocumentFormat.FreeTextFormat.EndOfLineDelimiter, DocumentFormat.FreeTextFormat.FixedLineLength, DocumentFormat.FreeTextFormat.ColumnsLayout, typeCobolOptions); var compiler = new FileCompiler(textDocument, project.SourceFileProvider, project, typeCobolOptions, false, project); compiler.CompileOnce(); return(compiler.CompilationResultsForProgram); }
public static CompilationDocument ScanCobolFile(string relativePath, string textName, DocumentFormat documentFormat) { DirectoryInfo localDirectory = new DirectoryInfo(PlatformUtils.GetPathForProjectFile(relativePath)); if (!localDirectory.Exists) { throw new Exception(String.Format("Directory : {0} does not exist", relativePath)); } CompilationProject project = new CompilationProject("test", localDirectory.FullName, new string[] { ".cbl", ".cpy" }, documentFormat.Encoding, documentFormat.EndOfLineDelimiter, documentFormat.FixedLineLength, documentFormat.ColumnsLayout, new TypeCobolOptions()); FileCompiler compiler = new FileCompiler(null, textName, project.SourceFileProvider, project, documentFormat.ColumnsLayout, new TypeCobolOptions(), null, true, project); compiler.CompileOnce(); return(compiler.CompilationResultsForCopy); }
public void Parse() { try { Compiler.CompileOnce(); } catch (Exception e) { Observer.OnError(e); } }
public static void GenerateStatisticsForPrograms(CompilationProject project, IEnumerable <string> textNames, TextWriter console, string countersFile, string languageModelForProgramFile, string languageModelForCopyFile) { // Initialize statistics vars // 1. Program analysis after preprocessing // - total number of lines per program (including expanded COPY directives) long[] linesCountDistributionCategories = { 500, 1000, 1500, 2000, 3000, 5000, 7500, 10000, 15000, 20000, 30000, 50000, int.MaxValue }; StatsCounter <CobolTextLineType> linesCounter = new StatsCounter <CobolTextLineType>(linesCountDistributionCategories); // - total number of tokens per program (after preprocessing) long[] tokensCountDistributionCategories = { 1500, 3000, 4500, 6000, 9000, 15000, 22500, 30000, 45000, 60000, 90000, 150000, int.MaxValue }; StatsCounter <TokenType> tokensCounter = new StatsCounter <TokenType>(tokensCountDistributionCategories); // - number of copies per program long[] copiesCountDistributionCategories = { 0, 5, 10, 15, 20, 30, 50, int.MaxValue }; StatsCounter <CopyDirectiveType> copiesCounter = new StatsCounter <CopyDirectiveType>(copiesCountDistributionCategories); // - number of replaced tokens per program long[] replacedTokensCountDistributionCategories = { 50, 100, 150, 200, 300, 500, 1000, 2000, 5000, 10000, 20000, int.MaxValue }; StatsCounter <TokenType> replacedTokensCounter = new StatsCounter <TokenType>(tokensCountDistributionCategories); // - number of code elements per program long[] codeElementsCountDistributionCategories = { 100, 200, 300, 400, 500, 750, 1000, 1500, 2000, 5000, 10000, int.MaxValue }; StatsCounter <CodeElementType> codeElementCounter = new StatsCounter <CodeElementType>(codeElementsCountDistributionCategories); // 2. Program files before preprocessing // - number of lines per program file (before preprocessing) long[] linesCountPerProgramFileDistributionCategories = { 25, 50, 100, 150, 200, 300, 500, 1000, 1500, 2000, 3000, int.MaxValue }; StatsCounter <CobolTextLineType> linesPerProgramFileCounter = new StatsCounter <CobolTextLineType>(linesCountPerProgramFileDistributionCategories); // - number of tokens per program file (before preprocessing) long[] tokensCountPerProgramFileDistributionCategories = { 1500, 3000, 4500, 6000, 9000, 15000, 22500, 30000, 45000, 60000, 90000, 150000, int.MaxValue }; StatsCounter <TokenType> tokensPerProgramFileCounter = new StatsCounter <TokenType>(tokensCountPerProgramFileDistributionCategories); // - number of compiler directives per program file long[] compilerDirectivesPerProgramFileCountDistributionCategories = { 0, 5, 10, 15, 20, 30, 50, 75, 100, 200, int.MaxValue }; StatsCounter <CompilerDirectiveType> compilerDirectivesPerProgramFileCounter = new StatsCounter <CompilerDirectiveType>(compilerDirectivesPerProgramFileCountDistributionCategories); // 3. Copy files before preprocessing // - number of references to each copy file IDictionary <string, int> copyFileReferenceCount = new Dictionary <string, int>(); // - number of lines per copy file (before preprocessing) long[] linesCountPerCopyFileDistributionCategories = { 25, 50, 100, 150, 200, 300, 500, 1000, 1500, 2000, 3000, int.MaxValue }; StatsCounter <CobolTextLineType> linesPerCopyFileCounter = new StatsCounter <CobolTextLineType>(linesCountPerCopyFileDistributionCategories); // - number of tokens per copy file (before preprocessing) long[] tokensCountPerCopyFileDistributionCategories = { 1500, 3000, 4500, 6000, 9000, 15000, 22500, 30000, 45000, 60000, 90000, 150000, int.MaxValue }; StatsCounter <TokenType> tokensPerCopyFileCounter = new StatsCounter <TokenType>(tokensCountPerCopyFileDistributionCategories); // - number of compiler directives per copy file long[] compilerDirectivesPerCopyFileCountDistributionCategories = { 0, 5, 10, 15, 20, 30, 50, 75, 100, 200, int.MaxValue }; StatsCounter <CompilerDirectiveType> compilerDirectivesPerCopyFileCounter = new StatsCounter <CompilerDirectiveType>(compilerDirectivesPerCopyFileCountDistributionCategories); // 4. Language models // - language model to predict the next word in a program LanguageModelGenerator languageModelForProgram = new LanguageModelGenerator(); // - language model to predict the next word in a copy LanguageModelGenerator languageModelForCopy = new LanguageModelGenerator(); // -- Compile and compute stats -- foreach (string textName in textNames) { console.Write(textName + " : compilation ... "); int programCopiesNotFound = 0; try { // Compile program FileCompiler fileCompiler = new FileCompiler(null, textName, project.SourceFileProvider, project, project.ColumnsLayout, project.CompilationOptions.Clone(), null, false, project); fileCompiler.CompileOnce(); CompilationUnit compilationResult = fileCompiler.CompilationResultsForProgram; programCopiesNotFound = 0; // Compute stats console.Write(" OK, compute stats ... "); // STATS for PROGRAM linesCounter.OnBeginProgram(); tokensCounter.OnBeginProgram(); copiesCounter.OnBeginProgram(); replacedTokensCounter.OnBeginProgram(); codeElementCounter.OnBeginProgram(); linesPerProgramFileCounter.OnBeginProgram(); tokensPerProgramFileCounter.OnBeginProgram(); compilerDirectivesPerProgramFileCounter.OnBeginProgram(); languageModelForProgram.OnBeginProgram(); // Iterate over program file lines foreach (var line in compilationResult.CodeElementsDocumentSnapshot.Lines) { // + count lines linesCounter.OnElement((int)line.Type); linesPerProgramFileCounter.OnElement((int)line.Type); // Use symbol information known at parsing time for the tokens to build a language model if (line.CodeElements != null) { foreach (var codeElement in line.CodeElements) { if (codeElement.SymbolInformationForTokens.Count > 0) { languageModelForProgram.AddSymbolInformationForTokens(codeElement.SymbolInformationForTokens); } } } if (line.ImportedDocuments != null) { var symbolInformationForTokens = new Dictionary <Token, SymbolInformation>(); foreach (var copyDirective in line.ImportedDocuments.Keys) { if (copyDirective.TextNameSymbol != null) { symbolInformationForTokens.Add(copyDirective.TextNameSymbol, new ExternalName(new AlphanumericValue(copyDirective.TextNameSymbol), SymbolType.TextName)); } if (copyDirective.LibraryNameSymbol != null) { symbolInformationForTokens.Add(copyDirective.TextNameSymbol, new ExternalName(new AlphanumericValue(copyDirective.LibraryNameSymbol), SymbolType.LibraryName)); } } languageModelForProgram.AddSymbolInformationForTokens(symbolInformationForTokens); } // Iterate over tokens on this line foreach (var token in line.SourceTokens) { // + count tokens and build language model tokensPerProgramFileCounter.OnElement((int)token.TokenType); languageModelForProgram.OnToken(token); } // Iterate over compiler directives on this line if (line.HasCompilerDirectives) { foreach (var token in line.TokensWithCompilerDirectives) { CompilerDirectiveToken compilerDirectiveToken = token as CompilerDirectiveToken; if (compilerDirectiveToken != null) { compilerDirectivesPerProgramFileCounter.OnElement((int)compilerDirectiveToken.CompilerDirective.Type); } } } // Iterate over COPY directives on this line if (line.ImportedDocuments != null) { foreach (CopyDirective copyDirective in line.ImportedDocuments.Keys) { // + count COPY directives CopyDirectiveType copyDirectiveType = CopyDirectiveType.Copy; #if EUROINFO_RULES if (copyDirective.InsertSuffixChar) { copyDirectiveType = CopyDirectiveType.CopyReplacingRemarks; } else if (copyDirective.RemoveFirst01Level) { copyDirectiveType = CopyDirectiveType.CopyRemarks; } #endif if (copyDirective.ReplaceOperations != null && copyDirective.ReplaceOperations.Count > 0) { copyDirectiveType = CopyDirectiveType.CopyReplacing; } copiesCounter.OnElement((int)copyDirectiveType); var importedDocument = line.ImportedDocuments[copyDirective]; if (importedDocument == null) { // + count missing COPY files for this program programCopiesNotFound++; } else { // + count references to copy files // AND check if copy file has already been analyzed string copyFileReference = copyDirective.LibraryName + ":" + copyDirective.TextName; if (copyFileReferenceCount.ContainsKey(copyFileReference)) { copyFileReferenceCount[copyFileReference] = copyFileReferenceCount[copyFileReference] + 1; // Iterate over copy file lines foreach (var copyLine in importedDocument.SourceDocument.Lines) { // + count lines inside COPY file linesCounter.OnElement((int)copyLine.Type); linesPerCopyFileCounter.OnElement((int)copyLine.Type); } } else { copyFileReferenceCount.Add(copyFileReference, 1); // STATS FOR COPY linesPerCopyFileCounter.OnBeginProgram(); tokensPerCopyFileCounter.OnBeginProgram(); compilerDirectivesPerCopyFileCounter.OnBeginProgram(); languageModelForCopy.OnBeginProgram(); // Iterate over copy file lines foreach (var copyLine in importedDocument.SourceDocument.Lines) { // + count lines inside COPY file linesCounter.OnElement((int)copyLine.Type); linesPerCopyFileCounter.OnElement((int)copyLine.Type); // Use symbol information known at parsing time for the tokens to build a language model if (copyLine.ImportedDocuments != null) { var symbolInformationForTokens = new Dictionary <Token, SymbolInformation>(); foreach (var copyDirective2 in line.ImportedDocuments.Keys) { if (copyDirective2.TextNameSymbol != null) { symbolInformationForTokens.Add(copyDirective2.TextNameSymbol, new ExternalName(new AlphanumericValue(copyDirective2.TextNameSymbol), SymbolType.TextName)); } if (copyDirective2.LibraryNameSymbol != null) { symbolInformationForTokens.Add(copyDirective2.TextNameSymbol, new ExternalName(new AlphanumericValue(copyDirective2.LibraryNameSymbol), SymbolType.LibraryName)); } } languageModelForCopy.AddSymbolInformationForTokens(symbolInformationForTokens); } // Iterate over tokens on this line foreach (var token in copyLine.SourceTokens) { // + count tokens and build language model tokensPerCopyFileCounter.OnElement((int)token.TokenType); languageModelForCopy.OnToken(token); } // Iterate over compiler directives on this line if (copyLine.HasCompilerDirectives) { foreach (var token in copyLine.TokensWithCompilerDirectives) { CompilerDirectiveToken compilerDirectiveToken = token as CompilerDirectiveToken; if (compilerDirectiveToken != null) { compilerDirectivesPerCopyFileCounter.OnElement((int)compilerDirectiveToken.CompilerDirective.Type); } } } } linesPerCopyFileCounter.OnEndProgram(); tokensPerCopyFileCounter.OnEndProgram(); compilerDirectivesPerCopyFileCounter.OnEndProgram(); } } } } // Iterate over code elements on this line if (line.CodeElements != null) { foreach (var codeElement in line.CodeElements) { codeElementCounter.OnElement((int)codeElement.Type); } } } // Iterate over tokens AFTER preprocessing ITokensLinesIterator processedTokensIterator = compilationResult.ProcessedTokensDocumentSnapshot.ProcessedTokens; Token processedToken = null; while ((processedToken = processedTokensIterator.NextToken()) != Token.END_OF_FILE) { tokensCounter.OnElement((int)processedToken.TokenType); ReplacedToken replacedToken = processedToken as ReplacedToken; if (replacedToken != null) { replacedTokensCounter.OnElement((int)replacedToken.OriginalToken.TokenType); } else if (processedToken is ReplacedTokenGroup) { replacedTokensCounter.OnElement((int)TokenType.ContinuationTokenGroup); } } linesCounter.OnEndProgram(); tokensCounter.OnEndProgram(); copiesCounter.OnEndProgram(); replacedTokensCounter.OnEndProgram(); codeElementCounter.OnEndProgram(); linesPerProgramFileCounter.OnEndProgram(); tokensPerProgramFileCounter.OnEndProgram(); compilerDirectivesPerProgramFileCounter.OnEndProgram(); } catch (Exception e) { console.WriteLine("ERROR :"); console.WriteLine(e.Message); } finally { console.Write("FINISHED"); if (programCopiesNotFound == 0) { console.WriteLine(); } else { console.WriteLine(" (" + programCopiesNotFound + " missing COPY)"); } } } // Compute language models languageModelForProgram.ComputeProbabilities(); languageModelForCopy.ComputeProbabilities(); // Write results files console.WriteLine(""); console.WriteLine("Writing statistics results to " + countersFile); using (StreamWriter writer = new StreamWriter(countersFile)) { writer.WriteLine("1. Program analysis after preprocessing"); writer.WriteLine(); WriteTitle(writer, "Total number of lines per program (including expanded COPY directives)"); linesCounter.DisplayResults(writer); WriteTitle(writer, "Total number of tokens per program (after preprocessing)"); tokensCounter.DisplayResults(writer); WriteTitle(writer, "Number of copies per program"); copiesCounter.DisplayResults(writer); WriteTitle(writer, "Number of replaced tokens per program"); replacedTokensCounter.DisplayResults(writer); WriteTitle(writer, "Number of code elements per program"); codeElementCounter.DisplayResults(writer); writer.WriteLine("2. Program files before preprocessing"); writer.WriteLine(); WriteTitle(writer, "Number of lines per program file (before preprocessing)"); linesPerProgramFileCounter.DisplayResults(writer); WriteTitle(writer, "Number of tokens per program file (before preprocessing)"); tokensPerProgramFileCounter.DisplayResults(writer); WriteTitle(writer, "Number of compiler directives per program file"); compilerDirectivesPerProgramFileCounter.DisplayResults(writer); writer.WriteLine("3. Copy files before preprocessing"); writer.WriteLine(); WriteTitle(writer, "Number of references to each copy file"); // copyFileReferenceCount = new Dictionary<string, int>() WriteTitle(writer, "Number of lines per copy file (before preprocessing)"); linesPerCopyFileCounter.DisplayResults(writer); WriteTitle(writer, "Number of tokens per copy file (before preprocessing)"); tokensPerCopyFileCounter.DisplayResults(writer); WriteTitle(writer, "Number of compiler directives per copy file"); compilerDirectivesPerCopyFileCounter.DisplayResults(writer); } console.WriteLine("Done"); console.WriteLine("Writing language model for program to " + languageModelForProgramFile); using (StreamWriter writer = new StreamWriter(languageModelForProgramFile)) { languageModelForProgram.WriteModelFile(writer, console); } console.WriteLine("Writing language model for copy to " + languageModelForCopyFile); using (StreamWriter writer = new StreamWriter(languageModelForCopyFile)) { languageModelForCopy.WriteModelFile(writer, console); } }
public static void Main(string[] args) { // Basic test program, useful to debug : compiles all sample programs located under TypeCobol.Test\Samples\EI Cobol samples\EI-Production" string currentDirectory = Directory.GetCurrentDirectory(); string projectRootPath = currentDirectory.Substring(0, currentDirectory.IndexOf(@"\TypeCobol\") + 11); string sourcePath = projectRootPath + @"TypeCobol.Test\Samples\EI Cobol samples\EI-Production"; string[] programExtensions = { ".PGM" }; string[] copyExtensions = { ".CPY" }; DocumentFormat docFormat = new DocumentFormat(Encoding.GetEncoding("iso8859-1"), EndOfLineDelimiter.CrLfCharacters, 80, ColumnsLayout.CobolReferenceFormat); TypeCobolOptions compilerOptions = new TypeCobolOptions(); CompilationProject project = new CompilationProject("samples", sourcePath, programExtensions.Concat(copyExtensions).ToArray(), docFormat.Encoding, docFormat.EndOfLineDelimiter, docFormat.FixedLineLength, docFormat.ColumnsLayout, compilerOptions); // Iterate over all programs in the source directory foreach (string programExtension in programExtensions) { foreach (string filePath in Directory.EnumerateFiles(sourcePath, "*" + programExtension)) { // Compile program string textName = Path.GetFileNameWithoutExtension(filePath); Console.Write(textName + " ... "); try { FileCompiler fileCompiler = new FileCompiler(null, textName, project.SourceFileProvider, project, ColumnsLayout.CobolReferenceFormat, compilerOptions.Clone(), null, false, project); fileCompiler.CompileOnce(); Console.WriteLine(" OK"); } catch (Exception e) { Console.WriteLine("error :"); Console.WriteLine(e.Message); } } } /* * // TO DO : read compiler options on the command line * // Start with the default compiler options * TypeCobolOptions compilerOptions = new TypeCobolOptions(); * * // Simple test version * // - all referenced files should be located under the current directory * * CompilationProject project = new CompilationProject("project", ".", new string[] { ".cbl", ".cpy" }, * IBMCodePages.GetDotNetEncodingFromIBMCCSID(1147), EndOfLineDelimiter.FixedLengthLines, 80, ColumnsLayout.CobolReferenceFormat, compilerOptions); * * // - gets one file name as argument and compiles it * if (args.Length == 1) * { * string textName = args[0]; * * FileCompiler fileCompiler = new FileCompiler(null, textName, project.SourceFileProvider, project, ColumnsLayout.CobolReferenceFormat, compilerOptions.Clone(), false); * fileCompiler.CompileOnce(); * } * // - gets an optional "-continuous" flag as the first argument to activate source file monitoring and automatic recompilation * else if (args.Length == 2) * { * if (String.Equals(args[0], "-continuous", StringComparison.InvariantCultureIgnoreCase)) * { * string textName = args[1]; * * FileCompiler fileCompiler = new FileCompiler(null, textName, project.SourceFileProvider, project, ColumnsLayout.CobolReferenceFormat, compilerOptions.Clone(), false); * fileCompiler.StartContinuousBackgroundCompilation(400, 400, 900, 2000); * * Console.WriteLine("Processing, press enter to stop ..."); * fileCompiler.StartContinuousFileProcessing(); * * Console.ReadLine(); * } * else * { * Console.WriteLine("ERROR : Invalid option"); * } * } * else * { * Console.WriteLine("ERROR : Invalid number of arguments"); * } */ }