public void CheckPerformance()
{
// Sample program properties
string folder = "Parser" + Path.DirectorySeparatorChar + "Samples";
string textName = "BigBatch";
DocumentFormat documentFormat = DocumentFormat.RDZReferenceFormat;
// Create a FileCompiler for this program
DirectoryInfo localDirectory = new DirectoryInfo(PlatformUtils.GetPathForProjectFile(folder));
if (!localDirectory.Exists)
{
throw new Exception(String.Format("Directory : {0} does not exist", localDirectory.FullName));
}
CompilationProject project = new CompilationProject("test",
localDirectory.FullName, new string[] { ".cbl", ".cpy" },
documentFormat.Encoding, documentFormat.EndOfLineDelimiter, documentFormat.FixedLineLength, documentFormat.ColumnsLayout, new TypeCobolOptions());
FileCompiler compiler = new FileCompiler(null, textName, project.SourceFileProvider, project, documentFormat.ColumnsLayout, new TypeCobolOptions(), null, false, project);
// Execute a first (complete) compilation
compiler.CompileOnce();
// Append one line in the middle of the program
ITextLine newLine = new TextLineSnapshot(9211, "094215D DISPLAY '-ICLAUA = ' ICLAUA. 0000000", null);
TextChangedEvent textChangedEvent = new TextChangedEvent();
textChangedEvent.TextChanges.Add(new TextChange(TextChangeType.LineInserted, 9211, newLine));
compiler.CompilationResultsForProgram.UpdateTextLines(textChangedEvent);
// Execute a second (incremental) compilation
compiler.CompileOnce();
// Display a performance report
StringBuilder report = new StringBuilder();
report.AppendLine("Program properties :");
report.AppendLine("- " + compiler.CompilationResultsForProgram.CobolTextLines.Count + " lines");
report.AppendLine("- " + compiler.CompilationResultsForProgram.CodeElementsDocumentSnapshot.CodeElements.Count() + " code elements");
report.AppendLine("First compilation performance");
report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForText.FirstCompilationTime + " ms : text update");
report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForScanner.FirstCompilationTime + " ms : scanner");
report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForPreprocessor.FirstCompilationTime + " ms : preprocessor");
report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForCodeElementsParser.FirstCompilationTime + " ms : code elements parser");
report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForTemporarySemantic.FirstCompilationTime + " ms : temporary semantic class parser");
report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForProgramCrossCheck.FirstCompilationTime + " ms : cross check class parser");
report.AppendLine("Incremental compilation performance");
report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForText.LastRefreshTime + " ms : text update");
report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForScanner.LastRefreshTime + " ms : scanner");
report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForPreprocessor.LastRefreshTime + " ms : preprocessor");
report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForCodeElementsParser.LastRefreshTime + " ms : code elements parser");
report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForTemporarySemantic.LastRefreshTime + " ms : temporary semantic class parser");
report.AppendLine("- " + compiler.CompilationResultsForProgram.PerfStatsForProgramCrossCheck.LastRefreshTime + " ms : cross check class parser");
Console.WriteLine(report.ToString());
}
public static CompilationUnit ParseCobolFile(string textName, DocumentFormat documentFormat = null, string folder = null, ExecutionStep execToStep = ExecutionStep.SemanticCheck)
{
if (folder == null)
{
folder = "Parser" + Path.DirectorySeparatorChar + "CodeElements";
}
DirectoryInfo localDirectory = new DirectoryInfo(PlatformUtils.GetPathForProjectFile(folder));
if (!localDirectory.Exists)
{
throw new Exception(String.Format("Directory : {0} does not exist", localDirectory.FullName));
}
if (documentFormat == null)
{
documentFormat = DocumentFormat.RDZReferenceFormat;
}
TypeCobolOptions options = new TypeCobolOptions {
ExecToStep = execToStep
}; //Create CompilerOptions. ExecToStep / AutoRemarks / HaltOnMissingCopy have to be set here.
CompilationProject project = new CompilationProject("test",
//First use *.cpy as tests will use file WITH extension for program but without extension for copy inside programs => small perf gain
localDirectory.FullName, new string[] { ".cpy", ".cbl" },
documentFormat.Encoding, documentFormat.EndOfLineDelimiter, documentFormat.FixedLineLength, documentFormat.ColumnsLayout, options);
FileCompiler compiler = new FileCompiler(null, textName, project.SourceFileProvider, project, documentFormat.ColumnsLayout, options, null, false, project);
compiler.CompileOnce();
return(compiler.CompilationResultsForProgram);
}
public void Parse(string path, TextChangedEvent e = null)
{
//if the server is restarted during Eclipse lifetime, then we need to init the parser
//This is useful when debugging. Perhaps it'll be deleted at the end
if (!Compilers.ContainsKey(path))
{
Init(path, new TypeCobolOptions {
ExecToStep = ExecutionStep.Generate
});
}
Compiler = Compilers[path];
Compiler.CompilationResultsForProgram.TextLinesChanged += OnTextLine;
Compiler.CompilationResultsForProgram.CodeElementsLinesChanged += OnCodeElementLine;
if (!Inits[path])
{
Inits[path] = true; // no need to update with the same content as at compiler creation
}
else if (e != null)
{
Compiler.CompilationResultsForProgram.UpdateTextLines(e);
}
try { Compiler.CompileOnce(); }
catch (Exception ex) {
throw new ParsingException(MessageCode.SyntaxErrorInParser, ex.Message, path, ex, true, true);
}
MissingCopys = Compiler.CompilationResultsForProgram.MissingCopies.Select(c => c.TextName).Distinct().ToList();
Compiler.CompilationResultsForProgram.TextLinesChanged -= OnTextLine;
Compiler.CompilationResultsForProgram.CodeElementsLinesChanged -= OnCodeElementLine;
}
public void Parse(string path, TextChangedEvent e = null)
{
//if the server is restarted during Eclipse lifetime, then we need to init the parser
//This is useful when debugging. Perhaps it'll be deleted at the end
if (!Compilers.ContainsKey(path))
{
Init(path, new TypeCobolOptions {
ExecToStep = ExecutionStep.Generate
});
}
Compiler = Compilers[path];
Compiler.CompilationResultsForProgram.TextLinesChanged += OnTextLine;
Compiler.CompilationResultsForProgram.CodeElementsLinesChanged += OnCodeElementLine;
if (!Inits[path])
{
Inits[path] = true; // no need to update with the same content as at compiler creation
}
else
{
Compiler.CompilationResultsForProgram.UpdateTextLines(e);
}
try { Compiler.CompileOnce(); }
catch (Exception ex) {
Observer.OnError(ex);
System.Console.WriteLine(ex.ToString());
}
MissingCopys = Compiler.CompilationProject.MissingCopys;
Compiler.CompilationResultsForProgram.TextLinesChanged -= OnTextLine;
Compiler.CompilationResultsForProgram.CodeElementsLinesChanged -= OnCodeElementLine;
}
public static void CheckAllFilesForExceptions()
{
CompilationProject project = new CompilationProject("test",
PlatformUtils.GetPathForProjectFile(@"Compiler\Scanner\Samples"), new string[] { ".txt" },
IBMCodePages.GetDotNetEncodingFromIBMCCSID(1147), EndOfLineDelimiter.FixedLengthLines, 80, ColumnsLayout.CobolReferenceFormat, new TypeCobolOptions());
//int filesCount = 0;
//int linesCount = 0;
//Stopwatch chrono = new Stopwatch();
foreach (string fileName in PlatformUtils.ListFilesInSubdirectory(@"Compiler\Scanner\Samples"))
{
string textName = Path.GetFileNameWithoutExtension(fileName);
// Initialize a CompilationDocument
FileCompiler compiler = new FileCompiler(null, textName, project.SourceFileProvider, project, ColumnsLayout.CobolReferenceFormat, new TypeCobolOptions(), null, true, project);
// Start compilation
try
{
//chrono.Start();
compiler.CompileOnce();
//chrono.Stop();
}
catch (Exception e)
{
throw new Exception("Error while scanning file " + fileName, e);
}
// Stats
//filesCount++;
//linesCount += compiler.CompilationResultsForCopy.TokensDocumentSnapshot.Lines.Count;
//string result = compiler.CompilationResultsForCopy.TokensDocumentSnapshot.GetDebugString();
}
// throw new Exception("Test OK for " + filesCount + " files and " + linesCount + " lines : " + chrono.ElapsedMilliseconds + " ms");
}
/// <summary>
/// Update the text contents of the file
/// </summary>
public void UpdateSourceFile(Uri fileUri, TextChangedEvent textChangedEvent)
{
FileCompiler fileCompilerToUpdate = null;
if (OpenedFileCompiler.TryGetValue(fileUri, out fileCompilerToUpdate))
{
_semanticUpdaterTimer?.Stop();
fileCompilerToUpdate.CompilationResultsForProgram.UpdateTextLines(textChangedEvent);
if (IsLsrSourceTesting)
{
//Log text lines string
var sb = new StringBuilder();
foreach (var cobolTextLine in fileCompilerToUpdate.CompilationResultsForProgram.CobolTextLines)
{
sb.AppendLine(cobolTextLine.SourceText);
}
_Logger(sb.ToString(), fileUri);
}
var handler = new Action <object, ExecutionStepEventArgs>((sender, args) => { ExecutionStepEventHandler(sender, args, fileUri); });
//Subscribe to FileCompilerEvent
fileCompilerToUpdate.ExecutionStepEventHandler += handler.Invoke;
var execStep = LsrTestOptions.ExecutionStep(fileCompilerToUpdate.CompilerOptions.ExecToStep);
if (execStep > ExecutionStep.SyntaxCheck)
{
execStep = ExecutionStep.SyntaxCheck; //The maximum execstep authorize for incremental parsing is SyntaxCheck,
}
//further it's for semantic, which is handle by NodeRefresh method
fileCompilerToUpdate.CompileOnce(execStep, fileCompilerToUpdate.CompilerOptions.HaltOnMissingCopy, fileCompilerToUpdate.CompilerOptions.UseAntlrProgramParsing);
fileCompilerToUpdate.ExecutionStepEventHandler -= handler.Invoke;
if (LsrTestOptions == LsrTestingOptions.NoLsrTesting || LsrTestOptions == LsrTestingOptions.LsrSemanticPhaseTesting)
{
if (!_timerDisabled) //If TimerDisabled is false, create a timer to automatically launch Node phase
{
lock (_fileCompilerWaittingForNodePhase)
{
if (!_fileCompilerWaittingForNodePhase.Contains(fileCompilerToUpdate))
{
_fileCompilerWaittingForNodePhase.Add(fileCompilerToUpdate); //Store that this fileCompiler will soon need a Node Phase
}
}
_semanticUpdaterTimer = new System.Timers.Timer(750);
_semanticUpdaterTimer.Elapsed += (sender, e) => TimerEvent(sender, e, fileCompilerToUpdate);
_semanticUpdaterTimer.Start();
}
}
}
}
public static void ExecuteInceremental(FileCompiler compiler, TestUtils.CompilationStats stats)
{
// Execute a first (complete) compilation
compiler.CompileOnce();
//Iterate multiple times over an incremental change
stats.IterationNumber = 20;
for (int i = 0; i < stats.IterationNumber; i++)
{
// Append one line in the middle of the program
ITextLine newLine = new TextLineSnapshot(9211, "094215D DISPLAY '-ICLAUA = ' ICLAUA. 0000000", null);
TextChangedEvent textChangedEvent = new TextChangedEvent();
textChangedEvent.TextChanges.Add(new TextChange(TextChangeType.LineInserted, 9211, newLine));
compiler.CompilationResultsForProgram.UpdateTextLines(textChangedEvent);
// Execute a second (incremental) compilation
compiler.CompileOnce();
//Accumulate results
stats.AverageTextUpdateTime += compiler.CompilationResultsForProgram.PerfStatsForText.LastRefreshTime;
stats.AverageScannerTime += compiler.CompilationResultsForProgram.PerfStatsForScanner.LastRefreshTime;
stats.AveragePreprocessorTime += compiler.CompilationResultsForProgram.PerfStatsForPreprocessor.LastRefreshTime;
stats.AverageCodeElementParserTime += compiler.CompilationResultsForProgram.PerfStatsForCodeElementsParser.LastRefreshTime;
stats.AverateTemporarySemanticsParserTime += compiler.CompilationResultsForProgram.PerfStatsForTemporarySemantic.LastRefreshTime;
stats.AverageCrossCheckerParserTime += compiler.CompilationResultsForProgram.PerfStatsForProgramCrossCheck.LastRefreshTime;
}
//Compute average time needed for each phase
stats.AverageTextUpdateTime = (int)stats.AverageTextUpdateTime / stats.IterationNumber;
stats.AverageScannerTime = (int)stats.AverageScannerTime / stats.IterationNumber;
stats.AveragePreprocessorTime = (int)stats.AveragePreprocessorTime / stats.IterationNumber;
stats.AverageCodeElementParserTime = (int)stats.AverageCodeElementParserTime / stats.IterationNumber;
stats.AverateTemporarySemanticsParserTime = (int)stats.AverateTemporarySemanticsParserTime / stats.IterationNumber;
stats.AverageCrossCheckerParserTime = (int)stats.AverageCrossCheckerParserTime / stats.IterationNumber;
stats.AverageTotalProcessingTime = stats.AverageCodeElementParserTime +
stats.AverageCrossCheckerParserTime +
stats.AveragePreprocessorTime +
stats.AverageScannerTime +
stats.AverageTextUpdateTime +
stats.AverateTemporarySemanticsParserTime;
}
/// <summary>
/// Start continuous background compilation on a newly opened file
/// </summary>
public void OpenSourceFile(Uri fileUri, string sourceText, LsrTestingOptions lsrOptions)
{
string fileName = Path.GetFileName(fileUri.LocalPath);
ITextDocument initialTextDocumentLines = new ReadOnlyTextDocument(fileName, TypeCobolConfiguration.Format.Encoding, TypeCobolConfiguration.Format.ColumnsLayout, sourceText);
FileCompiler fileCompiler = null;
#if EUROINFO_RULES //Issue #583
SymbolTable arrangedCustomSymbol = null;
var inputFileName = fileName.Substring(0, 8);
var matchingPgm =
_customSymbols.Programs.Keys.FirstOrDefault(
k => k.Equals(inputFileName, StringComparison.InvariantCultureIgnoreCase));
if (matchingPgm != null)
{
arrangedCustomSymbol = new SymbolTable(_customSymbols, SymbolTable.Scope.Namespace);
var prog = _customSymbols.Programs.Values.SelectMany(p => p).Where(p => p.Name != matchingPgm);
arrangedCustomSymbol.CopyAllPrograms(new List <List <Program> >()
{
prog.ToList()
});
arrangedCustomSymbol.Programs.Remove(matchingPgm);
}
fileCompiler = new FileCompiler(initialTextDocumentLines, CompilationProject.SourceFileProvider,
CompilationProject, CompilationProject.CompilationOptions, arrangedCustomSymbol ?? _customSymbols,
false, CompilationProject);
#else
fileCompiler = new FileCompiler(initialTextDocumentLines, CompilationProject.SourceFileProvider, CompilationProject, CompilationProject.CompilationOptions, _customSymbols, false, CompilationProject);
#endif
fileCompiler.CompilationResultsForProgram.UpdateTokensLines();
lock (OpenedFileCompiler)
{
if (OpenedFileCompiler.ContainsKey(fileUri))
{
CloseSourceFile(fileUri); //Close and remove the previous opened file.
}
OpenedFileCompiler.Add(fileUri, fileCompiler);
fileCompiler.CompilationResultsForProgram.ProgramClassChanged += ProgramClassChanged;
}
fileCompiler.CompilationResultsForProgram.SetOwnerThread(Thread.CurrentThread);
if (lsrOptions != LsrTestingOptions.LsrSourceDocumentTesting)
{
fileCompiler.CompileOnce(lsrOptions.ExecutionStep(fileCompiler.CompilerOptions.ExecToStep.Value), fileCompiler.CompilerOptions.HaltOnMissingCopy); //Let's parse file for the first time after opening.
}
}
public static CompilationUnit ParseCobolString(string cobolString)
{
//Prepare
var textDocument = new ReadOnlyTextDocument("Empty doc", Encoding.Default, ColumnsLayout.FreeTextFormat, "");
textDocument.LoadChars(cobolString);
var typeCobolOptions = new TypeCobolOptions();
var project = new CompilationProject("Empty project", ".", new[] { ".cbl", ".cpy" },
DocumentFormat.FreeTextFormat.Encoding, DocumentFormat.FreeTextFormat.EndOfLineDelimiter,
DocumentFormat.FreeTextFormat.FixedLineLength, DocumentFormat.FreeTextFormat.ColumnsLayout, typeCobolOptions);
var compiler = new FileCompiler(textDocument, project.SourceFileProvider, project, typeCobolOptions, false, project);
compiler.CompileOnce();
return(compiler.CompilationResultsForProgram);
}
public static CompilationDocument ScanCobolFile(string relativePath, string textName, DocumentFormat documentFormat)
{
DirectoryInfo localDirectory = new DirectoryInfo(PlatformUtils.GetPathForProjectFile(relativePath));
if (!localDirectory.Exists)
{
throw new Exception(String.Format("Directory : {0} does not exist", relativePath));
}
CompilationProject project = new CompilationProject("test",
localDirectory.FullName, new string[] { ".cbl", ".cpy" },
documentFormat.Encoding, documentFormat.EndOfLineDelimiter, documentFormat.FixedLineLength, documentFormat.ColumnsLayout, new TypeCobolOptions());
FileCompiler compiler = new FileCompiler(null, textName, project.SourceFileProvider, project, documentFormat.ColumnsLayout, new TypeCobolOptions(), null, true, project);
compiler.CompileOnce();
return(compiler.CompilationResultsForCopy);
}
public void Parse()
{
try { Compiler.CompileOnce(); }
catch (Exception e) { Observer.OnError(e); }
}
public static void GenerateStatisticsForPrograms(CompilationProject project, IEnumerable <string> textNames, TextWriter console, string countersFile, string languageModelForProgramFile, string languageModelForCopyFile)
{
// Initialize statistics vars
// 1. Program analysis after preprocessing
// - total number of lines per program (including expanded COPY directives)
long[] linesCountDistributionCategories = { 500, 1000, 1500, 2000, 3000, 5000, 7500, 10000, 15000, 20000, 30000, 50000, int.MaxValue };
StatsCounter <CobolTextLineType> linesCounter = new StatsCounter <CobolTextLineType>(linesCountDistributionCategories);
// - total number of tokens per program (after preprocessing)
long[] tokensCountDistributionCategories = { 1500, 3000, 4500, 6000, 9000, 15000, 22500, 30000, 45000, 60000, 90000, 150000, int.MaxValue };
StatsCounter <TokenType> tokensCounter = new StatsCounter <TokenType>(tokensCountDistributionCategories);
// - number of copies per program
long[] copiesCountDistributionCategories = { 0, 5, 10, 15, 20, 30, 50, int.MaxValue };
StatsCounter <CopyDirectiveType> copiesCounter = new StatsCounter <CopyDirectiveType>(copiesCountDistributionCategories);
// - number of replaced tokens per program
long[] replacedTokensCountDistributionCategories = { 50, 100, 150, 200, 300, 500, 1000, 2000, 5000, 10000, 20000, int.MaxValue };
StatsCounter <TokenType> replacedTokensCounter = new StatsCounter <TokenType>(tokensCountDistributionCategories);
// - number of code elements per program
long[] codeElementsCountDistributionCategories = { 100, 200, 300, 400, 500, 750, 1000, 1500, 2000, 5000, 10000, int.MaxValue };
StatsCounter <CodeElementType> codeElementCounter = new StatsCounter <CodeElementType>(codeElementsCountDistributionCategories);
// 2. Program files before preprocessing
// - number of lines per program file (before preprocessing)
long[] linesCountPerProgramFileDistributionCategories = { 25, 50, 100, 150, 200, 300, 500, 1000, 1500, 2000, 3000, int.MaxValue };
StatsCounter <CobolTextLineType> linesPerProgramFileCounter = new StatsCounter <CobolTextLineType>(linesCountPerProgramFileDistributionCategories);
// - number of tokens per program file (before preprocessing)
long[] tokensCountPerProgramFileDistributionCategories = { 1500, 3000, 4500, 6000, 9000, 15000, 22500, 30000, 45000, 60000, 90000, 150000, int.MaxValue };
StatsCounter <TokenType> tokensPerProgramFileCounter = new StatsCounter <TokenType>(tokensCountPerProgramFileDistributionCategories);
// - number of compiler directives per program file
long[] compilerDirectivesPerProgramFileCountDistributionCategories = { 0, 5, 10, 15, 20, 30, 50, 75, 100, 200, int.MaxValue };
StatsCounter <CompilerDirectiveType> compilerDirectivesPerProgramFileCounter = new StatsCounter <CompilerDirectiveType>(compilerDirectivesPerProgramFileCountDistributionCategories);
// 3. Copy files before preprocessing
// - number of references to each copy file
IDictionary <string, int> copyFileReferenceCount = new Dictionary <string, int>();
// - number of lines per copy file (before preprocessing)
long[] linesCountPerCopyFileDistributionCategories = { 25, 50, 100, 150, 200, 300, 500, 1000, 1500, 2000, 3000, int.MaxValue };
StatsCounter <CobolTextLineType> linesPerCopyFileCounter = new StatsCounter <CobolTextLineType>(linesCountPerCopyFileDistributionCategories);
// - number of tokens per copy file (before preprocessing)
long[] tokensCountPerCopyFileDistributionCategories = { 1500, 3000, 4500, 6000, 9000, 15000, 22500, 30000, 45000, 60000, 90000, 150000, int.MaxValue };
StatsCounter <TokenType> tokensPerCopyFileCounter = new StatsCounter <TokenType>(tokensCountPerCopyFileDistributionCategories);
// - number of compiler directives per copy file
long[] compilerDirectivesPerCopyFileCountDistributionCategories = { 0, 5, 10, 15, 20, 30, 50, 75, 100, 200, int.MaxValue };
StatsCounter <CompilerDirectiveType> compilerDirectivesPerCopyFileCounter = new StatsCounter <CompilerDirectiveType>(compilerDirectivesPerCopyFileCountDistributionCategories);
// 4. Language models
// - language model to predict the next word in a program
LanguageModelGenerator languageModelForProgram = new LanguageModelGenerator();
// - language model to predict the next word in a copy
LanguageModelGenerator languageModelForCopy = new LanguageModelGenerator();
// -- Compile and compute stats --
foreach (string textName in textNames)
{
console.Write(textName + " : compilation ... ");
int programCopiesNotFound = 0;
try
{
// Compile program
FileCompiler fileCompiler = new FileCompiler(null, textName, project.SourceFileProvider, project, project.ColumnsLayout, project.CompilationOptions.Clone(), null, false, project);
fileCompiler.CompileOnce();
CompilationUnit compilationResult = fileCompiler.CompilationResultsForProgram;
programCopiesNotFound = 0;
// Compute stats
console.Write(" OK, compute stats ... ");
// STATS for PROGRAM
linesCounter.OnBeginProgram();
tokensCounter.OnBeginProgram();
copiesCounter.OnBeginProgram();
replacedTokensCounter.OnBeginProgram();
codeElementCounter.OnBeginProgram();
linesPerProgramFileCounter.OnBeginProgram();
tokensPerProgramFileCounter.OnBeginProgram();
compilerDirectivesPerProgramFileCounter.OnBeginProgram();
languageModelForProgram.OnBeginProgram();
// Iterate over program file lines
foreach (var line in compilationResult.CodeElementsDocumentSnapshot.Lines)
{
// + count lines
linesCounter.OnElement((int)line.Type);
linesPerProgramFileCounter.OnElement((int)line.Type);
// Use symbol information known at parsing time for the tokens to build a language model
if (line.CodeElements != null)
{
foreach (var codeElement in line.CodeElements)
{
if (codeElement.SymbolInformationForTokens.Count > 0)
{
languageModelForProgram.AddSymbolInformationForTokens(codeElement.SymbolInformationForTokens);
}
}
}
if (line.ImportedDocuments != null)
{
var symbolInformationForTokens = new Dictionary <Token, SymbolInformation>();
foreach (var copyDirective in line.ImportedDocuments.Keys)
{
if (copyDirective.TextNameSymbol != null)
{
symbolInformationForTokens.Add(copyDirective.TextNameSymbol, new ExternalName(new AlphanumericValue(copyDirective.TextNameSymbol), SymbolType.TextName));
}
if (copyDirective.LibraryNameSymbol != null)
{
symbolInformationForTokens.Add(copyDirective.TextNameSymbol, new ExternalName(new AlphanumericValue(copyDirective.LibraryNameSymbol), SymbolType.LibraryName));
}
}
languageModelForProgram.AddSymbolInformationForTokens(symbolInformationForTokens);
}
// Iterate over tokens on this line
foreach (var token in line.SourceTokens)
{
// + count tokens and build language model
tokensPerProgramFileCounter.OnElement((int)token.TokenType);
languageModelForProgram.OnToken(token);
}
// Iterate over compiler directives on this line
if (line.HasCompilerDirectives)
{
foreach (var token in line.TokensWithCompilerDirectives)
{
CompilerDirectiveToken compilerDirectiveToken = token as CompilerDirectiveToken;
if (compilerDirectiveToken != null)
{
compilerDirectivesPerProgramFileCounter.OnElement((int)compilerDirectiveToken.CompilerDirective.Type);
}
}
}
// Iterate over COPY directives on this line
if (line.ImportedDocuments != null)
{
foreach (CopyDirective copyDirective in line.ImportedDocuments.Keys)
{
// + count COPY directives
CopyDirectiveType copyDirectiveType = CopyDirectiveType.Copy;
#if EUROINFO_RULES
if (copyDirective.InsertSuffixChar)
{
copyDirectiveType = CopyDirectiveType.CopyReplacingRemarks;
}
else if (copyDirective.RemoveFirst01Level)
{
copyDirectiveType = CopyDirectiveType.CopyRemarks;
}
#endif
if (copyDirective.ReplaceOperations != null && copyDirective.ReplaceOperations.Count > 0)
{
copyDirectiveType = CopyDirectiveType.CopyReplacing;
}
copiesCounter.OnElement((int)copyDirectiveType);
var importedDocument = line.ImportedDocuments[copyDirective];
if (importedDocument == null)
{
// + count missing COPY files for this program
programCopiesNotFound++;
}
else
{
// + count references to copy files
// AND check if copy file has already been analyzed
string copyFileReference = copyDirective.LibraryName + ":" + copyDirective.TextName;
if (copyFileReferenceCount.ContainsKey(copyFileReference))
{
copyFileReferenceCount[copyFileReference] = copyFileReferenceCount[copyFileReference] + 1;
// Iterate over copy file lines
foreach (var copyLine in importedDocument.SourceDocument.Lines)
{
// + count lines inside COPY file
linesCounter.OnElement((int)copyLine.Type);
linesPerCopyFileCounter.OnElement((int)copyLine.Type);
}
}
else
{
copyFileReferenceCount.Add(copyFileReference, 1);
// STATS FOR COPY
linesPerCopyFileCounter.OnBeginProgram();
tokensPerCopyFileCounter.OnBeginProgram();
compilerDirectivesPerCopyFileCounter.OnBeginProgram();
languageModelForCopy.OnBeginProgram();
// Iterate over copy file lines
foreach (var copyLine in importedDocument.SourceDocument.Lines)
{
// + count lines inside COPY file
linesCounter.OnElement((int)copyLine.Type);
linesPerCopyFileCounter.OnElement((int)copyLine.Type);
// Use symbol information known at parsing time for the tokens to build a language model
if (copyLine.ImportedDocuments != null)
{
var symbolInformationForTokens = new Dictionary <Token, SymbolInformation>();
foreach (var copyDirective2 in line.ImportedDocuments.Keys)
{
if (copyDirective2.TextNameSymbol != null)
{
symbolInformationForTokens.Add(copyDirective2.TextNameSymbol, new ExternalName(new AlphanumericValue(copyDirective2.TextNameSymbol), SymbolType.TextName));
}
if (copyDirective2.LibraryNameSymbol != null)
{
symbolInformationForTokens.Add(copyDirective2.TextNameSymbol, new ExternalName(new AlphanumericValue(copyDirective2.LibraryNameSymbol), SymbolType.LibraryName));
}
}
languageModelForCopy.AddSymbolInformationForTokens(symbolInformationForTokens);
}
// Iterate over tokens on this line
foreach (var token in copyLine.SourceTokens)
{
// + count tokens and build language model
tokensPerCopyFileCounter.OnElement((int)token.TokenType);
languageModelForCopy.OnToken(token);
}
// Iterate over compiler directives on this line
if (copyLine.HasCompilerDirectives)
{
foreach (var token in copyLine.TokensWithCompilerDirectives)
{
CompilerDirectiveToken compilerDirectiveToken = token as CompilerDirectiveToken;
if (compilerDirectiveToken != null)
{
compilerDirectivesPerCopyFileCounter.OnElement((int)compilerDirectiveToken.CompilerDirective.Type);
}
}
}
}
linesPerCopyFileCounter.OnEndProgram();
tokensPerCopyFileCounter.OnEndProgram();
compilerDirectivesPerCopyFileCounter.OnEndProgram();
}
}
}
}
// Iterate over code elements on this line
if (line.CodeElements != null)
{
foreach (var codeElement in line.CodeElements)
{
codeElementCounter.OnElement((int)codeElement.Type);
}
}
}
// Iterate over tokens AFTER preprocessing
ITokensLinesIterator processedTokensIterator = compilationResult.ProcessedTokensDocumentSnapshot.ProcessedTokens;
Token processedToken = null;
while ((processedToken = processedTokensIterator.NextToken()) != Token.END_OF_FILE)
{
tokensCounter.OnElement((int)processedToken.TokenType);
ReplacedToken replacedToken = processedToken as ReplacedToken;
if (replacedToken != null)
{
replacedTokensCounter.OnElement((int)replacedToken.OriginalToken.TokenType);
}
else if (processedToken is ReplacedTokenGroup)
{
replacedTokensCounter.OnElement((int)TokenType.ContinuationTokenGroup);
}
}
linesCounter.OnEndProgram();
tokensCounter.OnEndProgram();
copiesCounter.OnEndProgram();
replacedTokensCounter.OnEndProgram();
codeElementCounter.OnEndProgram();
linesPerProgramFileCounter.OnEndProgram();
tokensPerProgramFileCounter.OnEndProgram();
compilerDirectivesPerProgramFileCounter.OnEndProgram();
}
catch (Exception e)
{
console.WriteLine("ERROR :");
console.WriteLine(e.Message);
}
finally
{
console.Write("FINISHED");
if (programCopiesNotFound == 0)
{
console.WriteLine();
}
else
{
console.WriteLine(" (" + programCopiesNotFound + " missing COPY)");
}
}
}
// Compute language models
languageModelForProgram.ComputeProbabilities();
languageModelForCopy.ComputeProbabilities();
// Write results files
console.WriteLine("");
console.WriteLine("Writing statistics results to " + countersFile);
using (StreamWriter writer = new StreamWriter(countersFile))
{
writer.WriteLine("1. Program analysis after preprocessing");
writer.WriteLine();
WriteTitle(writer, "Total number of lines per program (including expanded COPY directives)");
linesCounter.DisplayResults(writer);
WriteTitle(writer, "Total number of tokens per program (after preprocessing)");
tokensCounter.DisplayResults(writer);
WriteTitle(writer, "Number of copies per program");
copiesCounter.DisplayResults(writer);
WriteTitle(writer, "Number of replaced tokens per program");
replacedTokensCounter.DisplayResults(writer);
WriteTitle(writer, "Number of code elements per program");
codeElementCounter.DisplayResults(writer);
writer.WriteLine("2. Program files before preprocessing");
writer.WriteLine();
WriteTitle(writer, "Number of lines per program file (before preprocessing)");
linesPerProgramFileCounter.DisplayResults(writer);
WriteTitle(writer, "Number of tokens per program file (before preprocessing)");
tokensPerProgramFileCounter.DisplayResults(writer);
WriteTitle(writer, "Number of compiler directives per program file");
compilerDirectivesPerProgramFileCounter.DisplayResults(writer);
writer.WriteLine("3. Copy files before preprocessing");
writer.WriteLine();
WriteTitle(writer, "Number of references to each copy file");
// copyFileReferenceCount = new Dictionary<string, int>()
WriteTitle(writer, "Number of lines per copy file (before preprocessing)");
linesPerCopyFileCounter.DisplayResults(writer);
WriteTitle(writer, "Number of tokens per copy file (before preprocessing)");
tokensPerCopyFileCounter.DisplayResults(writer);
WriteTitle(writer, "Number of compiler directives per copy file");
compilerDirectivesPerCopyFileCounter.DisplayResults(writer);
}
console.WriteLine("Done");
console.WriteLine("Writing language model for program to " + languageModelForProgramFile);
using (StreamWriter writer = new StreamWriter(languageModelForProgramFile))
{
languageModelForProgram.WriteModelFile(writer, console);
}
console.WriteLine("Writing language model for copy to " + languageModelForCopyFile);
using (StreamWriter writer = new StreamWriter(languageModelForCopyFile))
{
languageModelForCopy.WriteModelFile(writer, console);
}
}
public static void Main(string[] args)
{
// Basic test program, useful to debug : compiles all sample programs located under TypeCobol.Test\Samples\EI Cobol samples\EI-Production"
string currentDirectory = Directory.GetCurrentDirectory();
string projectRootPath = currentDirectory.Substring(0, currentDirectory.IndexOf(@"\TypeCobol\") + 11);
string sourcePath = projectRootPath + @"TypeCobol.Test\Samples\EI Cobol samples\EI-Production";
string[] programExtensions = { ".PGM" };
string[] copyExtensions = { ".CPY" };
DocumentFormat docFormat = new DocumentFormat(Encoding.GetEncoding("iso8859-1"), EndOfLineDelimiter.CrLfCharacters, 80, ColumnsLayout.CobolReferenceFormat);
TypeCobolOptions compilerOptions = new TypeCobolOptions();
CompilationProject project = new CompilationProject("samples", sourcePath, programExtensions.Concat(copyExtensions).ToArray(),
docFormat.Encoding, docFormat.EndOfLineDelimiter, docFormat.FixedLineLength, docFormat.ColumnsLayout, compilerOptions);
// Iterate over all programs in the source directory
foreach (string programExtension in programExtensions)
{
foreach (string filePath in Directory.EnumerateFiles(sourcePath, "*" + programExtension))
{
// Compile program
string textName = Path.GetFileNameWithoutExtension(filePath);
Console.Write(textName + " ... ");
try
{
FileCompiler fileCompiler = new FileCompiler(null, textName, project.SourceFileProvider, project, ColumnsLayout.CobolReferenceFormat, compilerOptions.Clone(), null, false, project);
fileCompiler.CompileOnce();
Console.WriteLine(" OK");
}
catch (Exception e)
{
Console.WriteLine("error :");
Console.WriteLine(e.Message);
}
}
}
/*
* // TO DO : read compiler options on the command line
* // Start with the default compiler options
* TypeCobolOptions compilerOptions = new TypeCobolOptions();
*
* // Simple test version
* // - all referenced files should be located under the current directory
*
* CompilationProject project = new CompilationProject("project", ".", new string[] { ".cbl", ".cpy" },
* IBMCodePages.GetDotNetEncodingFromIBMCCSID(1147), EndOfLineDelimiter.FixedLengthLines, 80, ColumnsLayout.CobolReferenceFormat, compilerOptions);
*
* // - gets one file name as argument and compiles it
* if (args.Length == 1)
* {
* string textName = args[0];
*
* FileCompiler fileCompiler = new FileCompiler(null, textName, project.SourceFileProvider, project, ColumnsLayout.CobolReferenceFormat, compilerOptions.Clone(), false);
* fileCompiler.CompileOnce();
* }
* // - gets an optional "-continuous" flag as the first argument to activate source file monitoring and automatic recompilation
* else if (args.Length == 2)
* {
* if (String.Equals(args[0], "-continuous", StringComparison.InvariantCultureIgnoreCase))
* {
* string textName = args[1];
*
* FileCompiler fileCompiler = new FileCompiler(null, textName, project.SourceFileProvider, project, ColumnsLayout.CobolReferenceFormat, compilerOptions.Clone(), false);
* fileCompiler.StartContinuousBackgroundCompilation(400, 400, 900, 2000);
*
* Console.WriteLine("Processing, press enter to stop ...");
* fileCompiler.StartContinuousFileProcessing();
*
* Console.ReadLine();
* }
* else
* {
* Console.WriteLine("ERROR : Invalid option");
* }
* }
* else
* {
* Console.WriteLine("ERROR : Invalid number of arguments");
* }
*/
}
