public static void CupParseProgramOrClass(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <CodeElementsLine> codeElementsLines, TypeCobolOptions compilerOptions, SymbolTable customSymbols, PerfStatsForParserInvocation perfStatsForParserInvocation, out SourceFile root, out List <Diagnostic> diagnostics, out Dictionary <CodeElement, Node> nodeCodeElementLinkers)
{
PrepareCupParser();
#if DEBUG_ANTRL_CUP_TIME
var t1 = DateTime.UtcNow;
#endif
CodeElementTokenizer scanner = new CodeElementTokenizer(codeElementsLines);
CupParser.TypeCobolProgramParser parser = new CupParser.TypeCobolProgramParser(scanner);
CupParserTypeCobolProgramDiagnosticErrorReporter diagReporter = new CupParserTypeCobolProgramDiagnosticErrorReporter();
parser.ErrorReporter = diagReporter;
ProgramClassBuilder builder = new ProgramClassBuilder();
parser.Builder = builder;
ParserDiagnostic programClassBuilderError = null;
builder.SyntaxTree = new SyntaxTree <CodeElement>(); //Initializie SyntaxTree for the current source file
builder.CustomSymbols = customSymbols;
builder.Dispatcher = new NodeDispatcher <CodeElement>();
builder.Dispatcher.CreateListeners();
// Try to parse a Cobol program or class, with cup w are also building the The Syntax Tree Node
perfStatsForParserInvocation.OnStartParsing();
try
{
TUVienna.CS_CUP.Runtime.Symbol symbol = parser.parse();
}
catch (Exception ex)
{
var code = Diagnostics.MessageCode.ImplementationError;
programClassBuilderError = new ParserDiagnostic(ex.ToString(), null, null, code, ex);
}
perfStatsForParserInvocation.OnStopParsing(0, 0);
#if DEBUG_ANTRL_CUP_TIME
var t2 = DateTime.UtcNow;
var t = t2 - t1;
System.Diagnostics.Debug.WriteLine("Time[" + textSourceInfo.Name + "];" + t.Milliseconds);
#endif
root = builder.SyntaxTree.Root; //Set output root node
perfStatsForParserInvocation.OnStartTreeBuilding();
//Create link between data definition an Types, will be stored in SymbolTable
root.AcceptASTVisitor(new TypeCobolLinker());
//Stop measuring tree building performance
perfStatsForParserInvocation.OnStopTreeBuilding();
// Register compiler results
diagnostics = diagReporter.Diagnostics ?? new List <Diagnostic>();
nodeCodeElementLinkers = builder.NodeCodeElementLinkers;
if (programClassBuilderError != null)
{
diagnostics.Add(programClassBuilderError);
}
}
/// <summary>
/// Incremental preprocessing of a set of tokens lines changes
/// </summary>
internal static IList <DocumentChange <IProcessedTokensLine> > CupProcessTokensLinesChanges(
TextSourceInfo textSourceInfo, ISearchableReadOnlyList <ProcessedTokensLine> documentLines,
IList <DocumentChange <ITokensLine> > tokensLinesChanges,
PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions,
IProcessedTokensDocumentProvider processedTokensDocumentProvider,
List <RemarksDirective.TextNameVariation> copyTextNameVariations,
PerfStatsForParserInvocation perfStatsForParserInvocation, List <CopyDirective> missingCopies)
{
// Collect all changes applied to the processed tokens lines during the incremental scan
IList <DocumentChange <IProcessedTokensLine> > processedTokensLinesChanges = new List <DocumentChange <IProcessedTokensLine> >();
// There are 2 reasons to a preprocess a tokens line after a change :
// 1. A tokens line changed : these lines were already reset during the previous steps
// 2. If a tokens line that changed was involved in the parsing of a multiline compiler directive, the whole group of lines must be parsed again
// Then, if a new COPY directive was parsed : the CompilationDocument to include must be prepared
// --- PREPARATION PHASE : reset all processed tokens lines which were involved in a multiline compiler directive where at least one line changed ---
// Iterate over all tokens changes detected by the ScannerStep :
// refresh all the adjacent lines participating in a ContinuationTokensGroup
if (tokensLinesChanges != null)
{
int lastLineIndexReset = -1;
foreach (DocumentChange <ITokensLine> tokensChange in tokensLinesChanges)
{
processedTokensLinesChanges.Add(new DocumentChange <IProcessedTokensLine>(tokensChange.Type, tokensChange.LineIndex, (IProcessedTokensLine)tokensChange.NewLine));
if (tokensChange.LineIndex > lastLineIndexReset)
{
lastLineIndexReset = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, processedTokensLinesChanges, lastLineIndexReset);
}
}
}
// --- COMPILER DIRECTIVES PHASE : Find and parse all compiler directives ---
// Init. Prepare a compiler directive parser
// Create a Cup Cobol Words token iterator on top of tokens lines
CobolWordsTokenizer tokensIterator = new CobolWordsTokenizer(
textSourceInfo.Name,
documentLines,
null,
Token.CHANNEL_SourceTokens);
// Init a CUP compiler directive parser
CupCobolErrorStrategy cupCobolErrorStrategy = new CupPreprocessor.CompilerDirectiveErrorStrategy();
// Prepare to analyze the parse tree
TypeCobol.Compiler.CupPreprocessor.CompilerDirectiveBuilder directiveBuilder = new TypeCobol.Compiler.CupPreprocessor.CompilerDirectiveBuilder(compilerOptions, copyTextNameVariations);
// 1. Iterate over all compiler directive starting tokens found in the lines which were updated
foreach (Token compilerDirectiveStartingToken in documentLines
.Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing)
.SelectMany(line => line.SourceTokens)
.Where(token => token.TokenFamily == TokenFamily.CompilerDirectiveStartingKeyword))
{
// 2. Reset the compiler directive parser state
// Reset tokens iterator position before parsing
// -> seek just before the compiler directive starting token
tokensIterator.SeekToToken(compilerDirectiveStartingToken);
tokensIterator.PreviousToken();
// Special case : for efficiency reasons, in EXEC SQL INCLUDE directives
// only the third token INCLUDE is recognized as a compiler directive
// starting keyword by the scanner. In this case, we must rewind the
// iterator two tokens backwards to start parsing just before the EXEC token.
if (compilerDirectiveStartingToken.TokenType == TokenType.EXEC_SQL)
{
tokensIterator.PreviousToken();
tokensIterator.PreviousToken();
}
// Reset Cup Tokenizer
tokensIterator.Reset(false);
// Reset parsing error diagnostics
cupCobolErrorStrategy.Diagnostics = null;
// 3. Try to parse a compiler directive starting with the current token
perfStatsForParserInvocation.OnStartParsing();
TypeCobol.Compiler.CupPreprocessor.CobolCompilerDirectivesParser directivesParser =
new TypeCobol.Compiler.CupPreprocessor.CobolCompilerDirectivesParser(tokensIterator);
directivesParser.ErrorReporter = cupCobolErrorStrategy;
directivesParser.Builder = directiveBuilder;
TUVienna.CS_CUP.Runtime.Symbol ppSymbol = directivesParser.parse();
perfStatsForParserInvocation.OnStartTreeBuilding();
// 4. Visit the parse tree to build a first class object representing the compiler directive
CompilerDirective compilerDirective = directiveBuilder.CompilerDirective;
bool errorFoundWhileParsingDirective = compilerDirective == null || compilerDirective.Diagnostics != null || cupCobolErrorStrategy.Diagnostics != null;
// 5. Get all tokens consumed while parsing the compiler directive
// and partition them line by line
Token startToken = tokensIterator.FirstToken;
Token stopToken = tokensIterator.LastToken;
if (stopToken == null)
{
stopToken = startToken;
}
MultilineTokensGroupSelection tokensSelection = tokensIterator.SelectAllTokensBetween(startToken, stopToken);
if (compilerDirective != null)
{
// 6. Replace all matched tokens by :
// - a CompilerDirectiveToken on the first line
ProcessedTokensLine firstProcessedTokensLine = documentLines[tokensSelection.FirstLineIndex];
if (tokensSelection.SelectedTokensOnSeveralLines.Length == 1)
{
firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine(
tokensSelection.TokensOnFirstLineBeforeStartToken,
compilerDirective, errorFoundWhileParsingDirective,
tokensSelection.SelectedTokensOnSeveralLines[0],
tokensSelection.TokensOnLastLineAfterStopToken, false);
}
else
{
TokensGroup continuedTokensGroup = firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine(
tokensSelection.TokensOnFirstLineBeforeStartToken,
compilerDirective, errorFoundWhileParsingDirective,
tokensSelection.SelectedTokensOnSeveralLines[0],
null, true);
// - a ContinuationTokensGroup on the following lines
int selectionLineIndex = 1;
int lastLineIndex = tokensSelection.FirstLineIndex + tokensSelection.SelectedTokensOnSeveralLines.Length - 1;
for (int nextLineIndex = tokensSelection.FirstLineIndex + 1; nextLineIndex <= lastLineIndex; nextLineIndex++, selectionLineIndex++)
{
IList <Token> compilerDirectiveTokensOnNextLine = tokensSelection.SelectedTokensOnSeveralLines[selectionLineIndex];
if (compilerDirectiveTokensOnNextLine.Count > 0)
{
ProcessedTokensLine nextProcessedTokensLine = documentLines[nextLineIndex];
if (nextLineIndex != lastLineIndex)
{
continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine(
continuedTokensGroup,
compilerDirectiveTokensOnNextLine,
null, true);
}
else
{
continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine(
continuedTokensGroup,
compilerDirectiveTokensOnNextLine,
tokensSelection.TokensOnLastLineAfterStopToken, false);
}
}
}
}
}
// 7. Register compiler directive parse errors
if (errorFoundWhileParsingDirective)
{
ProcessedTokensLine compilerDirectiveLine = documentLines[tokensSelection.FirstLineIndex];
if (compilerDirective != null && compilerDirective.Diagnostics != null)
{
foreach (Diagnostic directiveDiag in compilerDirective.Diagnostics)
{
compilerDirectiveLine.AddDiagnostic(directiveDiag);
}
}
else if (cupCobolErrorStrategy.Diagnostics != null)
{
foreach (Diagnostic directiveDiag in cupCobolErrorStrategy.Diagnostics)
{
if (compilerDirective != null)
{
compilerDirective.AddDiagnostic(directiveDiag);
}
compilerDirectiveLine.AddDiagnostic(directiveDiag);
}
}
}
}
// 8. Advance the state off all ProcessedTokensLines :
// NeedsCompilerDirectiveParsing => NeedsCopyDirectiveProcessing if it contains a COPY directive
IList <ProcessedTokensLine> parsedLinesWithCopyDirectives = null;
// NeedsCompilerDirectiveParsing => Ready otherwise
foreach (ProcessedTokensLine parsedLine in documentLines
.Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing))
{
if (parsedLine.ImportedDocuments != null)
{
if (parsedLinesWithCopyDirectives == null)
{
parsedLinesWithCopyDirectives = new List <ProcessedTokensLine>();
}
parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.NeedsCopyDirectiveProcessing;
parsedLinesWithCopyDirectives.Add(parsedLine);
}
else
{
parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready;
}
}
perfStatsForParserInvocation.OnStopTreeBuilding();
// --- COPY IMPORT PHASE : Process COPY (REPLACING) directives ---
foreach (var lineChange in processedTokensLinesChanges)
{
missingCopies.Remove(missingCopies.FirstOrDefault(c => c.COPYToken.Line == lineChange.LineIndex + 1));
}
// 1. Iterate over all updated lines containing a new COPY directive
if (parsedLinesWithCopyDirectives != null)
{
foreach (ProcessedTokensLine tokensLineWithCopyDirective in parsedLinesWithCopyDirectives)
{
// Iterate over all COPY directives found on one updated line
foreach (
CopyDirective copyDirective in
tokensLineWithCopyDirective.ImportedDocuments.Keys.Where(
c => c.TextName != null || c.COPYToken.TokenType == TokenType.EXEC).ToArray())
{
try
{
PerfStatsForImportedDocument perfStats;
// Load (or retrieve in cache) the document referenced by the COPY directive
//Issue #315: tokensLineWithCopyDirective.ScanState must be passed because special names paragraph such as "Decimal point is comma" are declared in the enclosing program and can affect the parsing of COPY
ProcessedTokensDocument importedDocumentSource =
processedTokensDocumentProvider.GetProcessedTokensDocument(copyDirective.LibraryName,
copyDirective.TextName,
tokensLineWithCopyDirective.ScanStateBeforeCOPYToken[copyDirective.COPYToken],
copyTextNameVariations, out perfStats);
// Store it on the current line after applying the REPLACING directive
ImportedTokensDocument importedDocument = new ImportedTokensDocument(copyDirective,
importedDocumentSource, perfStats, compilerOptions);
tokensLineWithCopyDirective.ImportedDocuments[copyDirective] = importedDocument;
}
catch (Exception e)
{
if (missingCopies != null && copyDirective != null && copyDirective.COPYToken != null &&
!missingCopies.Contains(copyDirective))
//If list already contains the copy directive just ignore
{
var missingCopieToReplace =
missingCopies.FirstOrDefault(c => c.COPYToken.Line == copyDirective.COPYToken.Line);
missingCopies.Remove(missingCopieToReplace);
missingCopies.Add(copyDirective);
}
// Text name refenced by COPY directive was not found
// => register a preprocessor error on this line
Token failedDirectiveToken = tokensLineWithCopyDirective.TokensWithCompilerDirectives
.First(
token =>
token.TokenType == TokenType.CopyImportDirective &&
((CompilerDirectiveToken)token).CompilerDirective == copyDirective);
Diagnostic diag = new Diagnostic(
MessageCode.FailedToLoadTextDocumentReferencedByCopyDirective,
failedDirectiveToken.Column, failedDirectiveToken.EndColumn,
failedDirectiveToken.Line, e.Message, e);
tokensLineWithCopyDirective.AddDiagnostic(diag);
}
}
// Advance processing status of the line
tokensLineWithCopyDirective.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready;
}
}
// --- REPLACE PHASE : REPLACE directives are implemented in ReplaceTokensLinesIterator ---
/* Algorithm :
*
* one REPLACE directive can express several replacement operations
* one replacement operation can be of several types (distinguished for optimization purposes)
* - SimpleTokenReplace : one source token / zero or one replacement token
* - PartialWordReplace : one pure partial word / zero or one replacement token
* - SimpleToMultipleTokenReplace : one source token / several replacement tokens
* - MultipleTokenReplace : one first + several following source tokens / zero to many replacement tokens
*
* an iterator maintains a current set of replacement operations
*
* if nextToken is replace directive
* the current set of replacement operations is updated
* else
* nextToken is compared to each replacement operation in turn
* if single -> single source token operation : return replacement token
* if single -> multiple operation : setup a secondary iterator with the list of replacement tokens
* if multiple -> multiple operation
* snapshot of the underlying iterator
* try to match all of the following source tokens
* if failure : restore snapshot and try next operation
* if success : setup a secondary iterator
*
* token comparison sourceToken / replacementCandidate :
* 1. Compare Token type
* 2. If same token type and for families
* AlphanumericLiteral
* NumericLiteral
* SyntaxLiteral
* Symbol
* => compare also Token text
*
* PartialCobolWord replacement :
* p535 : The COPY statement with REPLACING phrase can be used to replace parts of words.
* By inserting a dummy operand delimited by colons into the program text,
* the compiler will replace the dummy operand with the required text.
* Example 3 shows how this is used with the dummy operand :TAG:.
* The colons serve as separators and make TAG a stand-alone operand.
*/
return(processedTokensLinesChanges);
}
