public override void EnterBasisCompilerStatement(CobolCompilerDirectivesParser.BasisCompilerStatementContext context)
{
var basisDirective = new BasisDirective();
CompilerDirective = basisDirective;
if (context.textName() != null)
{
basisDirective.BasisName = GetTextName(context.textName());
basisDirective.TextNameSymbol = ParseTreeUtils.GetFirstToken(context.textName());
}
}
public override void EnterControlCblOption(CobolCompilerDirectivesParser.ControlCblOptionContext context)
{
string option = null;
TryGetUserDefinedWord(context.enumeratedValue1().UserDefinedWord(), ref option);
if (option != null)
{
ControlCblDirective.ControlCblOption optionValue;
if(Enum.TryParse<ControlCblDirective.ControlCblOption>(option, out optionValue))
{
((ControlCblDirective)CompilerDirective).OptionsList.Add(optionValue);
}
else
{
Token errorToken = ParseTreeUtils.GetTokenFromTerminalNode(context.enumeratedValue1().UserDefinedWord());
Diagnostic diag = new Diagnostic(
MessageCode.InvalidControlCblCompilerStatementOption,
errorToken.Column, errorToken.EndColumn,
option);
Diagnostics.Add(diag);
}
}
}
private string GetTextName(CobolCompilerDirectivesParser.TextNameContext context)
{
if (context == null) return null;
return GetName(context.externalName5().alphanumericValue5());
}
private string GetName(CobolCompilerDirectivesParser.AlphanumericValue5Context context)
{
if (context == null) return null;
string result = null;
TryGetAlphanumericLiteralValue(context.alphanumericLiteralToken(), ref result);
TryGetUserDefinedWord(context.UserDefinedWord(), ref result);
if (result != null) result = result.Trim('\'').Trim('\"');
return result;
}
public override void EnterTitleCompilerStatement(CobolCompilerDirectivesParser.TitleCompilerStatementContext context)
{
TitleDirective titleDirective = new TitleDirective();
CompilerDirective = titleDirective;
string title = ParseTreeUtils.GetAlphanumericLiteral(context.alphanumericValue2());
titleDirective.Title = title;
}
public override void EnterDeleteCompilerStatement(CobolCompilerDirectivesParser.DeleteCompilerStatementContext context)
{
CompilerDirective = new DeleteDirective();
}
/// <summary>
/// Incremental preprocessing of a set of tokens lines changes
/// </summary>
internal static IList<DocumentChange<IProcessedTokensLine>> ProcessTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList<ProcessedTokensLine> documentLines, IList<DocumentChange<ITokensLine>> tokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider)
{
// Collect all changes applied to the processed tokens lines during the incremental scan
IList<DocumentChange<IProcessedTokensLine>> processedTokensLinesChanges = new List<DocumentChange<IProcessedTokensLine>>();
// There are 2 reasons to a preprocess a tokens line after a change :
// 1. A tokens line changed : these lines were already reset during the previous steps
// 2. If a tokens line that changed was involved in the parsing of a multiline compiler directive, the whole group of lines must be parsed again
// Then, if a new COPY directive was parsed : the CompilationDocument to include must be prepared
// --- PREPARATION PHASE : reset all processed tokens lines which were involved in a multiline compiler directive where at least one line changed ---
// Iterate over all tokens changes detected by the ScannerStep :
// refresh all the adjacent lines participating in a ContinuationTokensGroup
if (tokensLinesChanges != null)
{
int lastLineIndexReset = -1;
foreach (DocumentChange<ITokensLine> tokensChange in tokensLinesChanges)
{
processedTokensLinesChanges.Add(new DocumentChange<IProcessedTokensLine>(tokensChange.Type, tokensChange.LineIndex, (IProcessedTokensLine)tokensChange.NewLine));
if (tokensChange.LineIndex > lastLineIndexReset)
{
lastLineIndexReset = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, processedTokensLinesChanges, lastLineIndexReset);
}
}
}
// --- COMPILER DIRECTIVES PHASE : Find and parse all compiler directives ---
// Init. Prepare a compiler directive parser
// Create a token iterator on top of tokens lines
TokensLinesIterator tokensIterator = new TokensLinesIterator(
textSourceInfo.Name,
documentLines,
null,
Token.CHANNEL_SourceTokens);
// Crate an Antlr compatible token source on top a the token iterator
TokensLinesTokenSource tokenSource = new TokensLinesTokenSource(
textSourceInfo.Name,
tokensIterator);
// Init a compiler directive parser
CommonTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
CobolCompilerDirectivesParser directivesParser = new CobolCompilerDirectivesParser(tokenStream);
IAntlrErrorStrategy compilerDirectiveErrorStrategy = new CompilerDirectiveErrorStrategy();
directivesParser.ErrorHandler = compilerDirectiveErrorStrategy;
// Register all parse errors in a list in memory
DiagnosticSyntaxErrorListener errorListener = new DiagnosticSyntaxErrorListener();
directivesParser.RemoveErrorListeners();
directivesParser.AddErrorListener(errorListener);
// Prepare to analyze the parse tree
ParseTreeWalker walker = new ParseTreeWalker();
CompilerDirectiveBuilder directiveBuilder = new CompilerDirectiveBuilder();
// 1. Iterate over all compiler directive starting tokens found in the lines which were updated
foreach (Token compilerDirectiveStartingToken in documentLines
.Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing)
.SelectMany(line => line.SourceTokens)
.Where(token => token.TokenFamily == TokenFamily.CompilerDirectiveStartingKeyword))
{
// 2. Reset the compiler directive parser state
// Reset tokens iterator position before parsing
// -> seek just before the compiler directive starting token
tokensIterator.SeekToToken(compilerDirectiveStartingToken);
tokensIterator.PreviousToken();
// Special case : for efficiency reasons, in EXEC SQL INCLUDE directives
// only the third token INCLUDE is recognized as a compiler directive
// starting keyword by the scanner. In this case, we must rewind the
// iterator two tokens backwards to start parsing just before the EXEC token.
if (compilerDirectiveStartingToken.TokenType == TokenType.EXEC_SQL_INCLUDE)
{
tokensIterator.PreviousToken();
tokensIterator.PreviousToken();
}
// Reset Antlr BufferedTokenStream position
tokenStream.SetTokenSource(tokenSource);
// Reset parsing error diagnostics
compilerDirectiveErrorStrategy.Reset(directivesParser);
errorListener.Diagnostics.Clear();
// 3. Try to parse a compiler directive starting with the current token
CobolCompilerDirectivesParser.CompilerDirectingStatementContext directiveParseTree = directivesParser.compilerDirectingStatement();
// 4. Visit the parse tree to build a first class object representing the compiler directive
walker.Walk(directiveBuilder, directiveParseTree);
CompilerDirective compilerDirective = directiveBuilder.CompilerDirective;
bool errorFoundWhileParsingDirective = errorListener.Diagnostics.Count > 0 || directiveBuilder.Diagnostics.Count > 0;
// 5. Get all tokens consumed while parsing the compiler directive
// and partition them line by line
Token startToken = (Token)directiveParseTree.Start;
Token stopToken = (Token)directiveParseTree.Stop;
if (stopToken == null) stopToken = startToken;
MultilineTokensGroupSelection tokensSelection = tokensIterator.SelectAllTokensBetween(startToken, stopToken);
if (compilerDirective != null)
{
// 6. Replace all matched tokens by :
// - a CompilerDirectiveToken on the first line
ProcessedTokensLine firstProcessedTokensLine = documentLines[tokensSelection.FirstLineIndex];
if (tokensSelection.SelectedTokensOnSeveralLines.Length == 1)
{
firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine(
tokensSelection.TokensOnFirstLineBeforeStartToken,
compilerDirective, errorFoundWhileParsingDirective,
tokensSelection.SelectedTokensOnSeveralLines[0],
tokensSelection.TokensOnLastLineAfterStopToken, false);
}
else
{
TokensGroup continuedTokensGroup = firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine(
tokensSelection.TokensOnFirstLineBeforeStartToken,
compilerDirective, errorFoundWhileParsingDirective,
tokensSelection.SelectedTokensOnSeveralLines[0],
null, true);
// - a ContinuationTokensGroup on the following lines
int selectionLineIndex = 1;
int lastLineIndex = tokensSelection.FirstLineIndex + tokensSelection.SelectedTokensOnSeveralLines.Length - 1;
for (int nextLineIndex = tokensSelection.FirstLineIndex + 1; nextLineIndex <= lastLineIndex; nextLineIndex++, selectionLineIndex++)
{
IList<Token> compilerDirectiveTokensOnNextLine = tokensSelection.SelectedTokensOnSeveralLines[selectionLineIndex];
if (compilerDirectiveTokensOnNextLine.Count > 0)
{
ProcessedTokensLine nextProcessedTokensLine = documentLines[nextLineIndex];
if (nextLineIndex != lastLineIndex)
{
continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine(
continuedTokensGroup,
compilerDirectiveTokensOnNextLine,
null, true);
}
else
{
continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine(
continuedTokensGroup,
compilerDirectiveTokensOnNextLine,
tokensSelection.TokensOnLastLineAfterStopToken, false);
}
}
}
}
}
// 7. Register compiler directive parse errors
if (errorFoundWhileParsingDirective)
{
ProcessedTokensLine compilerDirectiveLine = documentLines[tokensSelection.FirstLineIndex];
foreach (ParserDiagnostic parserDiag in errorListener.Diagnostics)
{
compilerDirectiveLine.AddDiagnostic(parserDiag);
}
foreach (Diagnostic directiveDiag in directiveBuilder.Diagnostics)
{
compilerDirectiveLine.AddDiagnostic(directiveDiag);
}
}
}
// 8. Advance the state off all ProcessedTokensLines :
// NeedsCompilerDirectiveParsing => NeedsCopyDirectiveProcessing if it contains a COPY directive
IList<ProcessedTokensLine> parsedLinesWithCopyDirectives = null;
// NeedsCompilerDirectiveParsing => Ready otherwise
foreach (ProcessedTokensLine parsedLine in documentLines
.Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing))
{
if (parsedLine.ImportedDocuments != null)
{
if(parsedLinesWithCopyDirectives == null)
{
parsedLinesWithCopyDirectives = new List<ProcessedTokensLine>();
}
parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.NeedsCopyDirectiveProcessing;
parsedLinesWithCopyDirectives.Add(parsedLine);
}
else
{
parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready;
}
}
// --- COPY IMPORT PHASE : Process COPY (REPLACING) directives ---
// 1. Iterate over all updated lines containing a new COPY directive
if (parsedLinesWithCopyDirectives != null)
{
foreach (ProcessedTokensLine tokensLineWithCopyDirective in parsedLinesWithCopyDirectives)
{
// Iterate over all COPY directives found on one updated line
foreach (CopyDirective copyDirective in tokensLineWithCopyDirective.ImportedDocuments.Keys.ToArray<CopyDirective>())
{
try
{
// Load (or retrieve in cache) the document referenced by the COPY directive
ProcessedTokensDocument importedDocumentSource = processedTokensDocumentProvider.GetProcessedTokensDocument(copyDirective.LibraryName, copyDirective.TextName);
// Store it on the current line after appying the REPLACING directive
ImportedTokensDocument importedDocument = new ImportedTokensDocument(copyDirective, importedDocumentSource);
tokensLineWithCopyDirective.ImportedDocuments[copyDirective] = importedDocument;
}
catch (Exception e)
{
// Text name refenced by COPY directive was not found
// => register a preprocessor error on this line
Token failedDirectiveToken = tokensLineWithCopyDirective.TokensWithCompilerDirectives
.Where(token => token.TokenType == TokenType.CopyImportDirective && ((CompilerDirectiveToken)token).CompilerDirective == copyDirective)
.First();
Diagnostic diag = new Diagnostic(
MessageCode.FailedToLoadTextDocumentReferencedByCopyDirective,
failedDirectiveToken.Column, failedDirectiveToken.EndColumn,
e.Message);
tokensLineWithCopyDirective.AddDiagnostic(diag);
}
}
// Advance processing status of the line
tokensLineWithCopyDirective.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready;
}
}
// --- REPLACE PHASE : REPLACE directives are implemented in ReplaceTokensLinesIterator ---
/* Algorithm :
*
* one REPLACE directive can express several replacement operations
* one replacement operation can be of several types (distinguished for optimization purposes)
* - SimpleTokenReplace : one source token / zero or one replacement token
* - PartialWordReplace : one pure partial word / zero or one replacement token
* - SimpleToMultipleTokenReplace : one source token / several replacement tokens
* - MultipleTokenReplace : one first + several following source tokens / zero to many replacement tokens
*
* an iterator maintains a current set of replacement operations
*
* if nextToken is replace directive
* the current set of replacement operations is updated
* else
* nextToken is compared to each replacement operation in turn
* if single -> single source token operation : return replacement token
* if single -> multiple operation : setup a secondary iterator with the list of replacement tokens
* if multiple -> multiple operation
* snapshot of the underlying iterator
* try to match all of the following source tokens
* if failure : restore snapshot and try next operation
* if success : setup a secondary iterator
*
* token comparison sourceToken / replacementCandidate :
* 1. Compare Token type
* 2. If same token type and for families
* AlphanumericLiteral
* NumericLiteral
* SyntaxLiteral
* Symbol
* => compare also Token text
*
* PartialCobolWord replacement :
* p535 : The COPY statement with REPLACING phrase can be used to replace parts of words.
* By inserting a dummy operand delimited by colons into the program text,
* the compiler will replace the dummy operand with the required text.
* Example 3 shows how this is used with the dummy operand :TAG:.
* The colons serve as separators and make TAG a stand-alone operand.
*/
return processedTokensLinesChanges;
}
public override void EnterReplaceCompilerStatement(CobolCompilerDirectivesParser.ReplaceCompilerStatementContext context)
{
ReplaceDirective replaceDirective = new ReplaceDirective(context.OFF() == null ? CompilerDirectiveType.REPLACE : CompilerDirectiveType.REPLACE_OFF);
CompilerDirective = replaceDirective;
if(context.pseudoText() !=null)
{
// Data used to build the current replace operation
Token comparisonToken = null;
Token[] followingComparisonTokens = null;
Token replacementToken = null;
Token[] replacementTokens = null;
// Used to distinguish pseudo-text1 and pseudo-text2
int pseudoTextIndex = 0;
foreach(CobolCompilerDirectivesParser.PseudoTextContext pseudoTextContext in context.pseudoText())
{
/*// Comparison tokens
if (pseudoTextIndex % 2 == 0)
{
if (pseudoTextContext._pseudoTextTokens != null && pseudoTextContext._pseudoTextTokens.Count > 0)
{
comparisonToken = (Token)pseudoTextContext._pseudoTextTokens[0];
if(pseudoTextContext._pseudoTextTokens.Count > 1)
{
followingComparisonTokens = new Token[pseudoTextContext._pseudoTextTokens.Count - 1];
for(int i = 1 ; i < pseudoTextContext._pseudoTextTokens.Count ; i++)
{
followingComparisonTokens[i - 1] = (Token)pseudoTextContext._pseudoTextTokens[i];
}
}
}
pseudoTextIndex++;
}
// Replacement tokens
else
{
if (pseudoTextContext._pseudoTextTokens != null && pseudoTextContext._pseudoTextTokens.Count > 0)
{
if (followingComparisonTokens == null && pseudoTextContext._pseudoTextTokens.Count == 1)
{
replacementToken = (Token)pseudoTextContext._pseudoTextTokens[0];
}
else
{
replacementTokens = new Token[pseudoTextContext._pseudoTextTokens.Count];
for (int i = 0; i < pseudoTextContext._pseudoTextTokens.Count; i++)
{
replacementTokens[i] = (Token)pseudoTextContext._pseudoTextTokens[i];
}
}
}
// Build replace operation
ReplaceOperation replaceOperation = null;
if(followingComparisonTokens == null)
{
if(replacementTokens == null)
{
if (comparisonToken == null || comparisonToken.TokenType != TokenType.PartialCobolWord)
{
replaceOperation = new SingleTokenReplaceOperation(comparisonToken, replacementToken);
}
else
{
replaceOperation = new PartialWordReplaceOperation(comparisonToken, replacementToken);
}
}
else
{
replaceOperation = new SingleToMultipleTokensReplaceOperation(comparisonToken, replacementTokens);
}
}
else
{
replaceOperation = new MultipleTokensReplaceOperation(comparisonToken, followingComparisonTokens, replacementTokens);
}
replaceDirective.ReplaceOperations.Add(replaceOperation);
// Reset everything for the next replace operation
pseudoTextIndex = 0;
comparisonToken = null;
followingComparisonTokens = null;
replacementToken = null;
replacementTokens = null;
}*/
BuildReplaceOperation(replaceDirective.ReplaceOperations, ref comparisonToken, ref followingComparisonTokens, ref replacementToken, ref replacementTokens, ref pseudoTextIndex, pseudoTextContext._pseudoTextTokens);
}
}
}
public override void EnterSequenceNumberField(CobolCompilerDirectivesParser.SequenceNumberFieldContext context)
{
DeleteDirective deleteDirective = (DeleteDirective)CompilerDirective;
bool isFirst = true;
int previous = -42;
foreach(ITerminalNode node in context.IntegerLiteral()) {
int current = (int)ParseTreeUtils.GetIntegerLiteral(node);
if (isFirst) {
previous = current;
isFirst = false;
} else {
DeleteDirective.SequenceNumberRange range = new DeleteDirective.SequenceNumberRange();
range.From = previous;
if(current<0) {
range.To = -current;
isFirst = true;
} else {
range.To = previous;
previous = current;
}
deleteDirective.SequenceNumberRangesList.Add(range);
}
}
if (!isFirst && previous >= 0) {
DeleteDirective.SequenceNumberRange range = new DeleteDirective.SequenceNumberRange();
range.From = previous;
range.To = previous;
deleteDirective.SequenceNumberRangesList.Add(range);
}
}
public override void EnterInsertCompilerStatement(CobolCompilerDirectivesParser.InsertCompilerStatementContext context)
{
InsertDirective insertDirective = new InsertDirective();
CompilerDirective = insertDirective;
if (context.sequenceNumber() != null && context.sequenceNumber().IntegerLiteral() != null)
{
insertDirective.SequenceNumber = (int)ParseTreeUtils.GetIntegerLiteral(context.sequenceNumber().IntegerLiteral());
if (insertDirective.SequenceNumber < 0) {
Token errorToken = ParseTreeUtils.GetTokenFromTerminalNode(context.sequenceNumber().IntegerLiteral());
Diagnostic error = new Diagnostic(
MessageCode.InvalidNumericLiteralFormat,
errorToken.Column, errorToken.EndColumn,
"TODO");
Diagnostics.Add(error);//TODO proper diagnostic error
}
}
}
public override void EnterReadyOrResetTraceCompilerStatement(CobolCompilerDirectivesParser.ReadyOrResetTraceCompilerStatementContext context)
{
CompilerDirective = new ReadyOrResetTraceDirective(context.READY() != null ? CompilerDirectiveType.READY_TRACE : CompilerDirectiveType.RESET_TRACE);
}
public override void EnterExecSqlIncludeStatement(CobolCompilerDirectivesParser.ExecSqlIncludeStatementContext context)
{
var copyDirective = new CopyDirective(CompilerDirectiveType.EXEC_SQL_INCLUDE);
CompilerDirective = copyDirective;
if (context.copyCompilerStatementBody() != null)
{
var textNameContext = context.copyCompilerStatementBody().qualifiedTextName().textName();
if (textNameContext != null)
{
string textName = GetTextName(textNameContext);
copyDirective.TextName = textName;
copyDirective.TextNameSymbol = ParseTreeUtils.GetFirstToken(textNameContext);
}
var libraryNameContext = context.copyCompilerStatementBody().qualifiedTextName().libraryName();
if (libraryNameContext != null)
{
copyDirective.LibraryName = GetLibraryName(libraryNameContext);
copyDirective.LibraryNameSymbol = ParseTreeUtils.GetFirstToken(libraryNameContext);
}
}
}
public override void EnterEnterCompilerStatement(CobolCompilerDirectivesParser.EnterCompilerStatementContext context)
{
EnterDirective enterDirective = new EnterDirective();
CompilerDirective = enterDirective;
if(context.languageName() != null)
{
string languageName = null;
TryGetUserDefinedWord(context.languageName().UserDefinedWord(), ref languageName);
enterDirective.LanguageName = languageName;
}
if(context.routineName() != null)
{
string routineName = null;
TryGetUserDefinedWord(context.routineName().UserDefinedWord(), ref routineName);
enterDirective.RoutineName = routineName;
}
}
public override void EnterEjectCompilerStatement(CobolCompilerDirectivesParser.EjectCompilerStatementContext context)
{
CompilerDirective = new EjectDirective();
}
// --- Visiting the tree ---
public override void EnterCompilerDirectingStatement(CobolCompilerDirectivesParser.CompilerDirectingStatementContext context)
{
CompilerDirective = null;
Diagnostics = new List<Diagnostic>();
}
public override void EnterServiceLabelCompilerStatement(CobolCompilerDirectivesParser.ServiceLabelCompilerStatementContext context)
{
CompilerDirective = new ServiceLabelDirective();
}
public override void EnterControlCblCompilerStatement(CobolCompilerDirectivesParser.ControlCblCompilerStatementContext context)
{
CompilerDirective = new ControlCblDirective(context.ASTERISK_CONTROL() != null ? CompilerDirectiveType.ASTERISK_CONTROL : CompilerDirectiveType.ASTERISK_CBL);
}
public override void EnterServiceReloadCompilerStatement(CobolCompilerDirectivesParser.ServiceReloadCompilerStatementContext context)
{
ServiceReloadDirective serviceReloadDirective = new ServiceReloadDirective();
CompilerDirective = serviceReloadDirective;
string userDefinedWord = null;
TryGetUserDefinedWord(context.UserDefinedWord(), ref userDefinedWord);
serviceReloadDirective.UserDefinedWord = userDefinedWord;
}
public override void EnterCopyCompilerStatement(CobolCompilerDirectivesParser.CopyCompilerStatementContext context)
{
CompilerDirective = new CopyDirective(CompilerDirectiveType.COPY);
}
public override void EnterSkipCompilerStatement(CobolCompilerDirectivesParser.SkipCompilerStatementContext context)
{
if(context.SKIP1() != null)
{
CompilerDirective = new SkipDirective(CompilerDirectiveType.SKIP1);
}
else if (context.SKIP2() != null)
{
CompilerDirective = new SkipDirective(CompilerDirectiveType.SKIP2);
}
else if (context.SKIP3() != null)
{
CompilerDirective = new SkipDirective(CompilerDirectiveType.SKIP3);
}
}
/// <summary>
/// Incremental preprocessing of a set of tokens lines changes
/// </summary>
internal static IList <DocumentChange <IProcessedTokensLine> > ProcessTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <ProcessedTokensLine> documentLines, IList <DocumentChange <ITokensLine> > tokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider, List <RemarksDirective.TextNameVariation> copyTextNameVariations, PerfStatsForParserInvocation perfStatsForParserInvocation, List <CopyDirective> missingCopies)
{
// Collect all changes applied to the processed tokens lines during the incremental scan
IList <DocumentChange <IProcessedTokensLine> > processedTokensLinesChanges = new List <DocumentChange <IProcessedTokensLine> >();
// There are 2 reasons to a preprocess a tokens line after a change :
// 1. A tokens line changed : these lines were already reset during the previous steps
// 2. If a tokens line that changed was involved in the parsing of a multiline compiler directive, the whole group of lines must be parsed again
// Then, if a new COPY directive was parsed : the CompilationDocument to include must be prepared
// --- PREPARATION PHASE : reset all processed tokens lines which were involved in a multiline compiler directive where at least one line changed ---
// Iterate over all tokens changes detected by the ScannerStep :
// refresh all the adjacent lines participating in a ContinuationTokensGroup
if (tokensLinesChanges != null)
{
int lastLineIndexReset = -1;
foreach (DocumentChange <ITokensLine> tokensChange in tokensLinesChanges)
{
processedTokensLinesChanges.Add(new DocumentChange <IProcessedTokensLine>(tokensChange.Type, tokensChange.LineIndex, (IProcessedTokensLine)tokensChange.NewLine));
if (tokensChange.LineIndex > lastLineIndexReset)
{
lastLineIndexReset = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, processedTokensLinesChanges, lastLineIndexReset);
}
}
}
// --- COMPILER DIRECTIVES PHASE : Find and parse all compiler directives ---
// Init. Prepare a compiler directive parser
// Create a token iterator on top of tokens lines
TokensLinesIterator tokensIterator = new TokensLinesIterator(
textSourceInfo.Name,
documentLines,
null,
Token.CHANNEL_SourceTokens);
// Crate an Antlr compatible token source on top a the token iterator
TokensLinesTokenSource tokenSource = new TokensLinesTokenSource(
textSourceInfo.Name,
tokensIterator);
// Init a compiler directive parser
CommonTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
CobolCompilerDirectivesParser directivesParser = new CobolCompilerDirectivesParser(tokenStream);
// Optionnaly activate Antlr Parser performance profiling
// WARNING : use this in a single-treaded context only (uses static field)
if (AntlrPerformanceProfiler == null && perfStatsForParserInvocation.ActivateDetailedAntlrPofiling)
{
AntlrPerformanceProfiler = new AntlrPerformanceProfiler(directivesParser);
}
if (AntlrPerformanceProfiler != null)
{
// Replace the generated parser by a subclass which traces all rules invocations
directivesParser = new CobolCompilerDirectivesTracingParser(tokenStream);
AntlrPerformanceProfiler.BeginParsingFile(textSourceInfo, null);
}
IAntlrErrorStrategy compilerDirectiveErrorStrategy = new CompilerDirectiveErrorStrategy();
directivesParser.ErrorHandler = compilerDirectiveErrorStrategy;
// Register all parse errors in a list in memory
ParserDiagnosticErrorListener errorListener = new ParserDiagnosticErrorListener();
directivesParser.RemoveErrorListeners();
directivesParser.AddErrorListener(errorListener);
// Prepare to analyze the parse tree
ParseTreeWalker walker = new ParseTreeWalker();
CompilerDirectiveBuilder directiveBuilder = new CompilerDirectiveBuilder(compilerOptions, copyTextNameVariations);
// 1. Iterate over all compiler directive starting tokens found in the lines which were updated
foreach (Token compilerDirectiveStartingToken in documentLines
.Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing)
.SelectMany(line => line.SourceTokens)
.Where(token => token.TokenFamily == TokenFamily.CompilerDirectiveStartingKeyword))
{
// 2. Reset the compiler directive parser state
// Reset tokens iterator position before parsing
// -> seek just before the compiler directive starting token
tokensIterator.SeekToToken(compilerDirectiveStartingToken);
tokensIterator.PreviousToken();
// Special case : for efficiency reasons, in EXEC SQL INCLUDE directives
// only the third token INCLUDE is recognized as a compiler directive
// starting keyword by the scanner. In this case, we must rewind the
// iterator two tokens backwards to start parsing just before the EXEC token.
if (compilerDirectiveStartingToken.TokenType == TokenType.EXEC_SQL_INCLUDE)
{
tokensIterator.PreviousToken();
tokensIterator.PreviousToken();
}
// Reset Antlr BufferedTokenStream position
tokenStream.SetTokenSource(tokenSource);
// Reset parsing error diagnostics
compilerDirectiveErrorStrategy.Reset(directivesParser);
// 3. Try to parse a compiler directive starting with the current token
perfStatsForParserInvocation.OnStartAntlrParsing();
if (AntlrPerformanceProfiler != null)
{
AntlrPerformanceProfiler.BeginParsingSection();
}
CobolCompilerDirectivesParser.CompilerDirectingStatementContext directiveParseTree = directivesParser.compilerDirectingStatement();
if (AntlrPerformanceProfiler != null)
{
AntlrPerformanceProfiler.EndParsingSection(directiveParseTree.ChildCount);
}
perfStatsForParserInvocation.OnStopAntlrParsing(
AntlrPerformanceProfiler != null ? (int)AntlrPerformanceProfiler.CurrentFileInfo.DecisionTimeMs : 0,
AntlrPerformanceProfiler != null ? AntlrPerformanceProfiler.CurrentFileInfo.RuleInvocations.Sum() : 0);
perfStatsForParserInvocation.OnStartTreeBuilding();
// 4. Visit the parse tree to build a first class object representing the compiler directive
walker.Walk(directiveBuilder, directiveParseTree);
CompilerDirective compilerDirective = directiveBuilder.CompilerDirective;
bool errorFoundWhileParsingDirective = compilerDirective == null || compilerDirective.Diagnostics != null || directiveParseTree.Diagnostics != null;
// 5. Get all tokens consumed while parsing the compiler directive
// and partition them line by line
Token startToken = (Token)directiveParseTree.Start;
Token stopToken = (Token)directiveParseTree.Stop;
if (stopToken == null)
{
stopToken = startToken;
}
MultilineTokensGroupSelection tokensSelection = tokensIterator.SelectAllTokensBetween(startToken, stopToken);
if (compilerDirective != null)
{
// 6. Replace all matched tokens by :
// - a CompilerDirectiveToken on the first line
ProcessedTokensLine firstProcessedTokensLine = documentLines[tokensSelection.FirstLineIndex];
if (tokensSelection.SelectedTokensOnSeveralLines.Length == 1)
{
firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine(
tokensSelection.TokensOnFirstLineBeforeStartToken,
compilerDirective, errorFoundWhileParsingDirective,
tokensSelection.SelectedTokensOnSeveralLines[0],
tokensSelection.TokensOnLastLineAfterStopToken, false);
}
else
{
TokensGroup continuedTokensGroup = firstProcessedTokensLine.InsertCompilerDirectiveTokenOnFirstLine(
tokensSelection.TokensOnFirstLineBeforeStartToken,
compilerDirective, errorFoundWhileParsingDirective,
tokensSelection.SelectedTokensOnSeveralLines[0],
null, true);
// - a ContinuationTokensGroup on the following lines
int selectionLineIndex = 1;
int lastLineIndex = tokensSelection.FirstLineIndex + tokensSelection.SelectedTokensOnSeveralLines.Length - 1;
for (int nextLineIndex = tokensSelection.FirstLineIndex + 1; nextLineIndex <= lastLineIndex; nextLineIndex++, selectionLineIndex++)
{
IList <Token> compilerDirectiveTokensOnNextLine = tokensSelection.SelectedTokensOnSeveralLines[selectionLineIndex];
if (compilerDirectiveTokensOnNextLine.Count > 0)
{
ProcessedTokensLine nextProcessedTokensLine = documentLines[nextLineIndex];
if (nextLineIndex != lastLineIndex)
{
continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine(
continuedTokensGroup,
compilerDirectiveTokensOnNextLine,
null, true);
}
else
{
continuedTokensGroup = nextProcessedTokensLine.InsertCompilerDirectiveTokenOnNextLine(
continuedTokensGroup,
compilerDirectiveTokensOnNextLine,
tokensSelection.TokensOnLastLineAfterStopToken, false);
}
}
}
}
}
// 7. Register compiler directive parse errors
if (errorFoundWhileParsingDirective)
{
ProcessedTokensLine compilerDirectiveLine = documentLines[tokensSelection.FirstLineIndex];
if (compilerDirective != null && compilerDirective.Diagnostics != null)
{
foreach (Diagnostic directiveDiag in compilerDirective.Diagnostics)
{
compilerDirectiveLine.AddDiagnostic(directiveDiag);
}
}
else if (directiveParseTree.Diagnostics != null)
{
foreach (Diagnostic directiveDiag in directiveParseTree.Diagnostics)
{
if (compilerDirective != null)
{
compilerDirective.AddDiagnostic(directiveDiag);
}
compilerDirectiveLine.AddDiagnostic(directiveDiag);
}
}
}
}
if (AntlrPerformanceProfiler != null)
{
AntlrPerformanceProfiler.EndParsingFile(directivesParser.ParseInfo.DecisionInfo, (int)(directivesParser.ParseInfo.GetTotalTimeInPrediction() / 1000000));
}
// 8. Advance the state off all ProcessedTokensLines :
// NeedsCompilerDirectiveParsing => NeedsCopyDirectiveProcessing if it contains a COPY directive
IList <ProcessedTokensLine> parsedLinesWithCopyDirectives = null;
// NeedsCompilerDirectiveParsing => Ready otherwise
foreach (ProcessedTokensLine parsedLine in documentLines
.Where(line => line.PreprocessingState == ProcessedTokensLine.PreprocessorState.NeedsCompilerDirectiveParsing))
{
if (parsedLine.ImportedDocuments != null)
{
if (parsedLinesWithCopyDirectives == null)
{
parsedLinesWithCopyDirectives = new List <ProcessedTokensLine>();
}
parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.NeedsCopyDirectiveProcessing;
parsedLinesWithCopyDirectives.Add(parsedLine);
}
else
{
parsedLine.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready;
}
}
perfStatsForParserInvocation.OnStopTreeBuilding();
// --- COPY IMPORT PHASE : Process COPY (REPLACING) directives ---
foreach (var lineChange in processedTokensLinesChanges)
{
missingCopies.Remove(missingCopies.FirstOrDefault(c => c.COPYToken.Line == lineChange.LineIndex + 1));
}
// 1. Iterate over all updated lines containing a new COPY directive
if (parsedLinesWithCopyDirectives != null)
{
foreach (ProcessedTokensLine tokensLineWithCopyDirective in parsedLinesWithCopyDirectives)
{
// Iterate over all COPY directives found on one updated line
foreach (CopyDirective copyDirective in tokensLineWithCopyDirective.ImportedDocuments.Keys.Where(c => c.TextName != null || c.COPYToken.TokenType == TokenType.EXEC).ToArray())
{
try
{
PerfStatsForImportedDocument perfStats;
// Load (or retrieve in cache) the document referenced by the COPY directive
//Issue #315: tokensLineWithCopyDirective.ScanState must be passed because special names paragraph such as "Decimal point is comma" are declared in the enclosing program and can affect the parsing of COPY
ProcessedTokensDocument importedDocumentSource =
processedTokensDocumentProvider.GetProcessedTokensDocument(copyDirective.LibraryName,
copyDirective.TextName,
tokensLineWithCopyDirective.ScanStateBeforeCOPYToken[copyDirective.COPYToken], copyTextNameVariations, out perfStats);
// Store it on the current line after applying the REPLACING directive
ImportedTokensDocument importedDocument = new ImportedTokensDocument(copyDirective,
importedDocumentSource, perfStats);
tokensLineWithCopyDirective.ImportedDocuments[copyDirective] = importedDocument;
}
catch (Exception e)
{
if (missingCopies != null && copyDirective != null && copyDirective.COPYToken != null && !missingCopies.Contains(copyDirective)) //If list already contains the copy directive just ignore
{
var missingCopieToReplace = missingCopies.FirstOrDefault(c => c.COPYToken.Line == copyDirective.COPYToken.Line);
missingCopies.Remove(missingCopieToReplace);
missingCopies.Add(copyDirective);
}
// Text name refenced by COPY directive was not found
// => register a preprocessor error on this line
Token failedDirectiveToken = tokensLineWithCopyDirective.TokensWithCompilerDirectives
.First(
token =>
token.TokenType == TokenType.CopyImportDirective &&
((CompilerDirectiveToken)token).CompilerDirective == copyDirective);
Diagnostic diag = new Diagnostic(
MessageCode.FailedToLoadTextDocumentReferencedByCopyDirective,
failedDirectiveToken.Column, failedDirectiveToken.EndColumn,
failedDirectiveToken.Line, e.Message, e);
tokensLineWithCopyDirective.AddDiagnostic(diag);
}
}
// Advance processing status of the line
tokensLineWithCopyDirective.PreprocessingState = ProcessedTokensLine.PreprocessorState.Ready;
}
}
// --- REPLACE PHASE : REPLACE directives are implemented in ReplaceTokensLinesIterator ---
/* Algorithm :
*
* one REPLACE directive can express several replacement operations
* one replacement operation can be of several types (distinguished for optimization purposes)
* - SimpleTokenReplace : one source token / zero or one replacement token
* - PartialWordReplace : one pure partial word / zero or one replacement token
* - SimpleToMultipleTokenReplace : one source token / several replacement tokens
* - MultipleTokenReplace : one first + several following source tokens / zero to many replacement tokens
*
* an iterator maintains a current set of replacement operations
*
* if nextToken is replace directive
* the current set of replacement operations is updated
* else
* nextToken is compared to each replacement operation in turn
* if single -> single source token operation : return replacement token
* if single -> multiple operation : setup a secondary iterator with the list of replacement tokens
* if multiple -> multiple operation
* snapshot of the underlying iterator
* try to match all of the following source tokens
* if failure : restore snapshot and try next operation
* if success : setup a secondary iterator
*
* token comparison sourceToken / replacementCandidate :
* 1. Compare Token type
* 2. If same token type and for families
* AlphanumericLiteral
* NumericLiteral
* SyntaxLiteral
* Symbol
* => compare also Token text
*
* PartialCobolWord replacement :
* p535 : The COPY statement with REPLACING phrase can be used to replace parts of words.
* By inserting a dummy operand delimited by colons into the program text,
* the compiler will replace the dummy operand with the required text.
* Example 3 shows how this is used with the dummy operand :TAG:.
* The colons serve as separators and make TAG a stand-alone operand.
*/
return(processedTokensLinesChanges);
}
public override void EnterCopyCompilerStatementBody(CobolCompilerDirectivesParser.CopyCompilerStatementBodyContext context)
{
var copy = (CopyDirective)CompilerDirective;
if (context.qualifiedTextName() != null) {
var ctxt = context.qualifiedTextName();
copy.TextName = GetTextName(ctxt.textName());
copy.TextNameSymbol = ParseTreeUtils.GetFirstToken(ctxt.textName());
#if EUROINFO_LEGACY_REPLACING_SYNTAX
if (copy.TextName != null) {
// Find the list of copy text names variations declared by previous REMARKS compiler directives
var variations = ((TokensLine)copy.TextNameSymbol.TokensLine).InitialScanState.CopyTextNamesVariations;
if (variations != null && variations.Count > 0) {
// Check if the current text name was mentioned in a REMARKS compiler directive
var declaration = variations.Find(d => String.Equals(d.TextNameWithSuffix, copy.TextName, StringComparison.InvariantCultureIgnoreCase));
if (declaration != null) {
// Declaration found => apply the legacy REPLACING semantics to the copy directive
copy.RemoveFirst01Level = true;
if (declaration.HasSuffix) {
copy.TextName = declaration.TextName;
copy.InsertSuffixChar = true;
copy.SuffixChar = declaration.SuffixChar;
}
}
}
}
#endif
copy.LibraryName = GetLibraryName(ctxt.libraryName());
copy.LibraryNameSymbol = ParseTreeUtils.GetFirstToken(ctxt.libraryName());
}
copy.Suppress = (context.SUPPRESS() != null);
// REPLACING
if(context.copyReplacingOperand() != null)
{
// Data used to build the current replace operation
Token comparisonToken = null;
Token[] followingComparisonTokens = null;
Token replacementToken = null;
Token[] replacementTokens = null;
// Used to distinguish pseudo-text1 and pseudo-text2
int pseudoTextIndex = 0;
foreach (CobolCompilerDirectivesParser.CopyReplacingOperandContext replacingOperandContext in context.copyReplacingOperand())
{
// Get relevant tokens
IList<IToken> operandTokens = null;
if (replacingOperandContext.pseudoText() != null) { // Pseudo-text => List of tokens
if (replacingOperandContext.pseudoText()._pseudoTextTokens != null)
operandTokens = replacingOperandContext.pseudoText()._pseudoTextTokens;
} else { // Single token
if (replacingOperandContext.literalOrUserDefinedWordOReservedWordExceptCopy() != null) {
var terminalNode = ParseTreeUtils.GetFirstTerminalNode(replacingOperandContext.literalOrUserDefinedWordOReservedWordExceptCopy());
operandTokens = new List<IToken>(1);
operandTokens.Add(terminalNode.Symbol);
}
}
BuildReplaceOperation(copy.ReplaceOperations, ref comparisonToken, ref followingComparisonTokens, ref replacementToken, ref replacementTokens, ref pseudoTextIndex, operandTokens);
}
}
}
