/// <summary>
/// Incremental parsing of a set of processed tokens lines changes
/// </summary>
internal static IList <DocumentChange <ICodeElementsLine> > ParseProcessedTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <CodeElementsLine> documentLines, IList <DocumentChange <IProcessedTokensLine> > processedTokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, PerfStatsForParserInvocation perfStatsForParserInvocation)
{
// Collect all changes applied to the processed tokens lines during the incremental scan
IList <DocumentChange <ICodeElementsLine> > codeElementsLinesChanges = new List <DocumentChange <ICodeElementsLine> >();
// There are 2 reasons to re-parse a tokens line after a change :
// 1. The tokens line changed : these lines were already reset during the previous steps
// 2. If a tokens line that changed was involved in the parsing of a multiline code element, the whole group of lines must be parsed again
// --- PREPARATION PHASE : identify all parse sections where code elements need to be refreshed ---
IList <ParseSection> refreshParseSections = null;
ParseSection largestRefreshParseSection = null;
// Iterate over all processed tokens changes detected by the PreprocessorStep :
// - refresh all the adjacent lines participating in a CodeElement
// - register the start and stop token for all sections of the document which need to be parsed again
if (processedTokensLinesChanges != null && processedTokensLinesChanges.Count > 0)
{
// If the document was cleared, everything must be parsed again
if (processedTokensLinesChanges[0].Type != DocumentChangeType.DocumentCleared)
{
refreshParseSections = new List <ParseSection>();
ParseSection lastParseSection = null;
foreach (DocumentChange <IProcessedTokensLine> tokensChange in processedTokensLinesChanges)
{
if (lastParseSection == null || tokensChange.LineIndex > lastParseSection.StopLineIndex)
{
lastParseSection = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, codeElementsLinesChanges, lastParseSection);
refreshParseSections.Add(lastParseSection);
}
}
}
}
if (refreshParseSections != null)
{
//After getting all the parts refreshed, get the largest part that has been refreshed
var minParseSection = refreshParseSections.OrderBy(p => p.StartLineIndex).First();
var maxParseSection = refreshParseSections.OrderByDescending(p => p.StopLineIndex).First();
largestRefreshParseSection = new ParseSection(minParseSection.StartLineIndex,
minParseSection.StartToken, maxParseSection.StopLineIndex, maxParseSection.StopToken,
maxParseSection.StopTokenIsFirstTokenOfTheLine);
}
// --- INITIALIZE ANTLR CodeElements parser ---
// Create a token iterator on top of pre-processed tokens lines
ITokensLinesIterator tokensIterator = ProcessedTokensDocument.GetProcessedTokensIterator(textSourceInfo, documentLines);
// Create an Antlr compatible token source on top of the token iterator
TokensLinesTokenSource tokenSource = new TokensLinesTokenSource(
textSourceInfo.Name,
tokensIterator);
// Init parser
TokensLinesTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
CodeElementsParser cobolParser = new CodeElementsParser(tokenStream);
// REVERT TO STD PARSER ==> TracingCobolParser cobolParser = new TracingCobolParser(tokenStream);
// Optionnaly activate Antlr Parser performance profiling
// WARNING : use this in a single-treaded context only (uses static field)
if (AntlrPerformanceProfiler == null && perfStatsForParserInvocation.ActivateDetailedAntlrPofiling)
{
AntlrPerformanceProfiler = new AntlrPerformanceProfiler(cobolParser);
}
if (AntlrPerformanceProfiler != null)
{
// Replace the generated parser by a subclass which traces all rules invocations
cobolParser = new CodeElementsTracingParser(tokenStream);
var tokensCountIterator = ProcessedTokensDocument.GetProcessedTokensIterator(textSourceInfo, documentLines);
AntlrPerformanceProfiler.BeginParsingFile(textSourceInfo, tokensCountIterator);
}
// Customize error recovery strategy
IAntlrErrorStrategy cobolErrorStrategy = new CodeElementErrorStrategy();
cobolParser.ErrorHandler = cobolErrorStrategy;
// Register all parse errors in a list in memory
ParserDiagnosticErrorListener errorListener = new ParserDiagnosticErrorListener();
cobolParser.RemoveErrorListeners();
cobolParser.AddErrorListener(errorListener);
// Prepare to analyze the parse tree
ParseTreeWalker walker = new ParseTreeWalker();
CodeElementBuilder codeElementBuilder = new CodeElementBuilder();
codeElementBuilder.Dispatcher = new CodeElementDispatcher();
codeElementBuilder.Dispatcher.CreateListeners();
// --- INCREMENTAL PARSING ---
// In case of incremental parsing, parse only the code sections we need to refresh
if (largestRefreshParseSection != null)
{
// Seek just before the next code element starting token
tokenStream.SeekToToken(largestRefreshParseSection.StartToken);
tokenStream.StartLookingForStopToken(largestRefreshParseSection.StopToken);
//Remove all the code elements for the future line to parse.
for (int i = largestRefreshParseSection.StartLineIndex;
i < (largestRefreshParseSection.StopLineIndex == documentLines.Count - 1 && largestRefreshParseSection.StopToken == null //If the last index is equals to number of line in document, make sure to also reset the last line, otherwise, reset lines normally.
? largestRefreshParseSection.StopLineIndex + 1
: largestRefreshParseSection.StopLineIndex);
i++)
{
if (documentLines[i].CodeElements != null)
{
documentLines[i].ResetCodeElements();
}
}
}
// Reset parsing error diagnostics
cobolErrorStrategy.Reset(cobolParser);
// Try to parse code elements :
// - starting with the current parse section Start token
// - ending with the current parse section Stop token
CodeElementsParser.CobolCodeElementsContext codeElementsParseTree = null;
try
{
perfStatsForParserInvocation.OnStartAntlrParsing();
if (AntlrPerformanceProfiler != null)
{
AntlrPerformanceProfiler.BeginParsingSection();
}
codeElementsParseTree = cobolParser.cobolCodeElements();
if (AntlrPerformanceProfiler != null)
{
AntlrPerformanceProfiler.EndParsingSection(codeElementsParseTree.ChildCount);
}
perfStatsForParserInvocation.OnStopAntlrParsing(
AntlrPerformanceProfiler != null
? (int)AntlrPerformanceProfiler.CurrentFileInfo.DecisionTimeMs
: 0,
AntlrPerformanceProfiler != null
? AntlrPerformanceProfiler.CurrentFileInfo.RuleInvocations.Sum()
: 0);
}
catch (Exception e)
{
var currentToken = (Token)cobolParser.CurrentToken;
CodeElementsLine codeElementsLine = GetCodeElementsLineForToken(currentToken);
if (codeElementsLine != null)
{
codeElementsLine.AddParserDiagnostic(new TokenDiagnostic(MessageCode.ImplementationError,
currentToken, currentToken.Line, e));
}
}
if (codeElementsParseTree != null)
{
// If the parse tree is not empty
if (codeElementsParseTree.codeElement() != null && codeElementsParseTree.codeElement().Length > 0)
{
// Analyze the parse tree for each code element
foreach (var codeElementParseTree in codeElementsParseTree.codeElement())
{
// Get the first line that was parsed
var tokenStart = (Token)codeElementParseTree.Start;
CodeElementsLine codeElementsLine = GetCodeElementsLineForToken(tokenStart);
if (codeElementsLine == null)
{
continue;
}
// Register that this line was updated
// COMMENTED FOR THE SAKE OF PERFORMANCE -- SEE ISSUE #160
//int updatedLineIndex = documentLines.IndexOf(codeElementsLine, codeElementsLine.LineIndex);
//codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, updatedLineIndex, codeElementsLine));
codeElementsLinesChanges.Add(
new DocumentChange <ICodeElementsLine>(DocumentChangeType.LineUpdated,
codeElementsLine.LineIndex, codeElementsLine));
perfStatsForParserInvocation.OnStartTreeBuilding();
// Visit the parse tree to build a first class object representing the code elements
try
{
walker.Walk(codeElementBuilder, codeElementParseTree);
}
catch (Exception ex)
{
var code = MessageCode.ImplementationError;
int line = 0;
int start = 0;
int stop = 0;
if (codeElementsLine.SourceTokens != null && codeElementsLine.SourceTokens.Count > 0)
{
start = codeElementsLine.SourceTokens[0].StartIndex;
stop =
codeElementsLine.SourceTokens[codeElementsLine.SourceTokens.Count - 1].StopIndex;
}
codeElementsLine.AddParserDiagnostic(new ParserDiagnostic(ex.ToString(), start, stop,
line, null, code, ex));
}
CodeElement codeElement = codeElementBuilder.CodeElement;
if (codeElement != null)
{
// Attach consumed tokens and main document line numbers information to the code element
if (codeElement.ConsumedTokens.Count == 0)
{
// ISSUE #204:
var tempToken = tokenStream.Lt(1);
if (tempToken != null && tempToken != Token.END_OF_FILE)
{
// if not end of file,
// add next token to ConsumedTokens to know where is the CodeElement in error
codeElement.ConsumedTokens.Add((Token)tempToken);
// this alter CodeElements semantics: in addition to matched tokens,
// it includes the first token in error if no token has been matched
}
}
//TODO Issue #384 to discuss if this code should stay here:
//This should be in a Checker, but "codeElement.ConsumedTokens" is only set after all the checkers have been called
//Rule TCLIMITATION_NO_CE_ACROSS_SOURCES
if (codeElement.IsAcrossSourceFile())
{
DiagnosticUtils.AddError(codeElement,
"A Cobol statement cannot be across 2 sources files (eg. Main program and a COPY)",
MessageCode.TypeCobolParserLimitation);
}
// Add code element to the list
codeElementsLine.AddCodeElement(codeElement);
}
}
}
// If the parse tree contains errors
if (codeElementsParseTree.Diagnostics != null)
{
foreach (ParserDiagnostic d in codeElementsParseTree.Diagnostics)
{
if (d.OffendingSymbol != null)
{
CodeElementsLine codeElementsLine =
GetCodeElementsLineForToken((Token)d.OffendingSymbol);
if (codeElementsLine != null)
{
codeElementsLine.AddParserDiagnostic(d);
}
}
}
}
perfStatsForParserInvocation.OnStopTreeBuilding();
}
if (AntlrPerformanceProfiler != null)
{
AntlrPerformanceProfiler.EndParsingFile(cobolParser.ParseInfo.DecisionInfo, (int)(cobolParser.ParseInfo.GetTotalTimeInPrediction() / 1000000));
}
return(codeElementsLinesChanges);
}
/// <summary>
/// Illustration : lines with code elements continued from previous line or with a continuation on next line (before update)
/// SW represents a code element starting word
/// [ SW x]
/// [ x]
/// [SW x]
/// [ SW ]
/// [ x SW]
/// A DocumentChange intersects with a previously parsed multiline code element if :
/// * LineInserted :
/// - all previous lines until the last starting word
/// (get previous lines until the previous code element is found)
/// - all the next lines until the next starting word
/// (get next lines until the next code element is found,
/// do not reset the last line if the next code element starts at the beginning of the line)
/// * LineUpdated / LineRemoved :
/// - all previous lines until the last starting word
/// (get previous lines until the previous code element is found)
/// - all the next lines until the next starting word
/// (get next lines until the next code element is found,
/// do not reset the last line if the next code element starts at the beginning of the line)
/// When navigating to previous or next line searching for a code element, we can stop when a fresh insertion / update is encountered.
/// When we reset a line which was not directly updated, and where the code element started in the middle of the line,
/// we must "remember" at which token we must start parsing.
/// In conclusion, the incremental parsing step for code elements is divided in two steps :
/// 1. List all the starting and stop tokens of sections to parse with the rules above
/// 2. Parse code elements beginning with starting token and until we reach stop token
/// </summary>
private static ParseSection CheckIfAdjacentLinesNeedRefresh(DocumentChangeType changeType, int lineIndex, ISearchableReadOnlyList <CodeElementsLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, IList <DocumentChange <ICodeElementsLine> > codeElementsLinesChanges, ParseSection lastParseSection)
{
ParseSection currentParseSection = new ParseSection();
// Navigate backwards to the start of the multiline code element
if (lineIndex > 0)
{
int previousLineIndex = lineIndex;
int lastLineIndexReset = lastParseSection != null ? lastParseSection.StopLineIndex : -1;
IEnumerator <CodeElementsLine> reversedEnumerator = documentLines.GetEnumerator(previousLineIndex - 1, -1, true);
bool previousLineHasCodeElements = false;
while (reversedEnumerator.MoveNext() && (--previousLineIndex > lastLineIndexReset))
{
// Get the previous line until the first code element is encountered
CodeElementsLine previousLine = reversedEnumerator.Current;
// The start of the parse section is delimited by the previous CodeElement
if (previousLine != null)
{
previousLineHasCodeElements = previousLine.HasCodeElements;
if (previousLineHasCodeElements)
{
currentParseSection.StartLineIndex = previousLineIndex;
currentParseSection.StartToken = previousLine.CodeElements.First().ConsumedTokens.FirstOrDefault();
}
// All lines contained in the parse section could be modified, and should be reset
previousLine = (CodeElementsLine)prepareDocumentLineForUpdate(previousLineIndex, previousLine, CompilationStep.CodeElementsParser);
previousLine.ResetCodeElements();
codeElementsLinesChanges.Add(new DocumentChange <ICodeElementsLine>(DocumentChangeType.LineUpdated, previousLineIndex, previousLine));
}
// Stop iterating backwards as soon as the start of an old CodeElement is found
if (previousLineHasCodeElements)
{
break;
}
}
// If no CodeElement was found on the previous lines, current parse section starts at the beggining of the file
// (because last parseSection could only stop at a CodeElement or at the end of the file)
if (!previousLineHasCodeElements)
{
currentParseSection.StartLineIndex = 0;
currentParseSection.StartToken = null;
}
}
// If line 0 was updated, current parse section starts at the beggining of the file
else
{
currentParseSection.StartLineIndex = 0;
currentParseSection.StartToken = null;
}
// Navigate forwards to the end of the multiline code element
if (lineIndex < (documentLines.Count - 1))
{
int nextLineIndex = lineIndex;
IEnumerator <CodeElementsLine> enumerator = documentLines.GetEnumerator(nextLineIndex + 1, -1, false);
bool nextLineHasCodeElements = false;
while (enumerator.MoveNext())
{
// Get the next line until non continuation line is encountered
nextLineIndex++;
CodeElementsLine nextLine = enumerator.Current;
// Check if the next CodeElement found starts at the beginning of the line
if (nextLine != null)
{
nextLineHasCodeElements = nextLine.HasCodeElements;
bool nextCodeElementStartsAtTheBeginningOfTheLine = false;
if (nextLineHasCodeElements)
{
try
{
Token startTokenForNextParseSection = nextLine.CodeElements.First().ConsumedTokens.FirstOrDefault();
Token firstSourceTokenOfThisLine = nextLine.TokensWithCompilerDirectives.First(token => token.Channel == Token.CHANNEL_SourceTokens);
nextCodeElementStartsAtTheBeginningOfTheLine = startTokenForNextParseSection == firstSourceTokenOfThisLine;
}
catch (System.InvalidOperationException /*e*/)
{//JCM: 28/08/2017: I noticed that this Exception can occur if: it doesn't exists a token which verifies the predicate: token.Channel == Token.CHANNEL_SourceToken
nextCodeElementStartsAtTheBeginningOfTheLine = false;
}
}
// All lines contained in the parse section could be modified
if (!nextCodeElementStartsAtTheBeginningOfTheLine)
{
// TO DO : ERROR below, will not work if we have source tokens from previous CodeElement + one other CodeElement on the same line
// => the other CodeElement will be deleted by prepareDocumentLineForUpdate and not parsed again
nextLine = (CodeElementsLine)prepareDocumentLineForUpdate(nextLineIndex, nextLine, CompilationStep.CodeElementsParser);
codeElementsLinesChanges.Add(new DocumentChange <ICodeElementsLine>(DocumentChangeType.LineUpdated, nextLineIndex, nextLine));
}
// Stop iterating forwards as soon as the start of an old CodeElement is found
if (nextLineHasCodeElements)
{
currentParseSection.StopLineIndex = nextLineIndex;
currentParseSection.StopToken = nextLine.CodeElements.First().ConsumedTokens.FirstOrDefault();
currentParseSection.StopTokenIsFirstTokenOfTheLine = nextCodeElementStartsAtTheBeginningOfTheLine;
break;
}
}
}
// If no CodeElement was found on the next lines, current parse section current parse section ends at the end of the file
if (!nextLineHasCodeElements)
{
currentParseSection.StopLineIndex = nextLineIndex;
currentParseSection.StopToken = null;
currentParseSection.StopTokenIsFirstTokenOfTheLine = false;
}
}
// If last line was updated, or if no CodeElement was found after the updated line, current parse section ends at the end of the file
else
{
currentParseSection.StopLineIndex = documentLines.Count - 1;
currentParseSection.StopToken = null;
currentParseSection.StopTokenIsFirstTokenOfTheLine = false;
}
return(currentParseSection);
}
private void parsePosViewINIFiles(string fname)
{
if (!File.Exists(fname))
{
return;
}
string headingRegex = "^\\[(\\S+)\\]$";
string keyRegex = "(?<key>[\\w]+)=(?<value>.*)$";
Regex heading = new Regex(headingRegex);
Regex key = new Regex(keyRegex);
ParseSection section = ParseSection.none;
System.Collections.ArrayList labels = new System.Collections.ArrayList();
try
{
using (StreamReader reader = new StreamReader(fname))
{
while (!reader.EndOfStream)
{
string line = reader.ReadLine();
Match m = heading.Match(line);
if (m.Success) //it was a heading!
{
section = parseSectionHeading(m.Groups[1].Value);
continue;
}
if (section == ParseSection.none)
{
continue; //if we are out of a section, move on
}
//not a header check for a body match
m = key.Match(line);
if (!m.Success)
{
continue;
}
string k = m.Groups["key"].Value;
string v = m.Groups["value"].Value;
switch (section)
{
case ParseSection.global:
parseGlobalKeyValuePair(k, v);
break;
case ParseSection.label:
labels.Add(v);
break;
}
}
_labels = (string[])labels.ToArray(typeof(string));
}
}
catch (Exception ex)
{
throw new InvalidDataException(String.Format("Pos ini file ({0}) is in an invalid format!", fname), ex);
}
}
/// <summary>
/// Incremental parsing of a set of processed tokens lines changes
/// </summary>
internal static IList <DocumentChange <ICodeElementsLine> > ParseProcessedTokensLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <CodeElementsLine> documentLines, IList <DocumentChange <IProcessedTokensLine> > processedTokensLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions)
{
// Collect all changes applied to the processed tokens lines during the incremental scan
IList <DocumentChange <ICodeElementsLine> > codeElementsLinesChanges = new List <DocumentChange <ICodeElementsLine> >();
// There are 2 reasons to re-parse a tokens line after a change :
// 1. The tokens line changed : these lines were already reset during the previous steps
// 2. If a tokens line that changed was involved in the parsing of a multiline code element, the whole group of lines must be parsed again
// --- PREPARATION PHASE : identify all parse sections where code elements need to be refreshed ---
IList <ParseSection> refreshParseSections = null;
// Iterate over all processed tokens changes detected by the PreprocessorStep :
// - refresh all the adjacent lines participating in a CodeElement
// - register the start and stop token for all sections of the document which need to be parsed again
if (processedTokensLinesChanges != null)
{
// If the document was cleared, everything must be parsed again
if (processedTokensLinesChanges[0].Type != DocumentChangeType.DocumentCleared)
{
refreshParseSections = new List <ParseSection>();
ParseSection lastParseSection = null;
foreach (DocumentChange <IProcessedTokensLine> tokensChange in processedTokensLinesChanges)
{
if (lastParseSection == null || tokensChange.LineIndex > lastParseSection.StopLineIndex)
{
lastParseSection = CheckIfAdjacentLinesNeedRefresh(tokensChange.Type, tokensChange.LineIndex, documentLines, prepareDocumentLineForUpdate, codeElementsLinesChanges, lastParseSection);
refreshParseSections.Add(lastParseSection);
}
}
}
}
// --- INITIALIZE ANTLR CodeElements parser ---
// Create a token iterator on top of pre-processed tokens lines
ITokensLinesIterator tokensIterator = ProcessedTokensDocument.GetProcessedTokensIterator(textSourceInfo, documentLines);
// Create an Antlr compatible token source on top of the token iterator
TokensLinesTokenSource tokenSource = new TokensLinesTokenSource(
textSourceInfo.Name,
tokensIterator);
// Init parser
TokensLinesTokenStream tokenStream = new TokensLinesTokenStream(tokenSource, Token.CHANNEL_SourceTokens);
CodeElementsParser cobolParser = new CodeElementsParser(tokenStream);
// REVERT TO STD PARSER ==> TracingCobolParser cobolParser = new TracingCobolParser(tokenStream);
// Customize error recovery strategy
IAntlrErrorStrategy cobolErrorStrategy = new CodeElementErrorStrategy();
cobolParser.ErrorHandler = cobolErrorStrategy;
// Register all parse errors in a list in memory
ParserDiagnosticErrorListener errorListener = new ParserDiagnosticErrorListener();
cobolParser.RemoveErrorListeners();
cobolParser.AddErrorListener(errorListener);
// Prepare to analyze the parse tree
ParseTreeWalker walker = new ParseTreeWalker();
CodeElementBuilder codeElementBuilder = new CodeElementBuilder();
codeElementBuilder.Dispatcher = new CodeElementDispatcher();
codeElementBuilder.Dispatcher.CreateListeners();
// --- INCREMENTAL PARSING ---
// In case of incremental parsing, parse only the code sections we need to refresh
IEnumerator <ParseSection> parseSectionsEnumerator = null;
ParseSection currentParseSection = null;
if (refreshParseSections != null)
{
// Get the first code section we need to refresh
parseSectionsEnumerator = refreshParseSections.GetEnumerator();
parseSectionsEnumerator.MoveNext();
currentParseSection = parseSectionsEnumerator.Current;
// Seek just before the next code element starting token
tokenStream.SeekToToken(currentParseSection.StartToken);
tokenStream.StartLookingForStopToken(currentParseSection.StopToken);
}
// Parse a list of code elements for each parse section while advancing in the underlying token stream
do
{
// Reset parsing error diagnostics
cobolErrorStrategy.Reset(cobolParser);
// Try to parse code elements :
// - starting with the current parse section Start token
// - ending with the current parse section Stop token
CodeElementsParser.CobolCodeElementsContext codeElementsParseTree = null;
try {
codeElementsParseTree = cobolParser.cobolCodeElements();
} catch (Exception e) {
var currentToken = (Token)cobolParser.CurrentToken;
CodeElementsLine codeElementsLine = GetCodeElementsLineForToken(currentToken);
codeElementsLine.AddParserDiagnostic(new TokenDiagnostic(MessageCode.ImplementationError, currentToken, currentToken.Line, e));
}
if (codeElementsParseTree != null)
{
// If the parse tree is not empty
if (codeElementsParseTree.codeElement() != null && codeElementsParseTree.codeElement().Length > 0)
{
// Analyze the parse tree for each code element
foreach (var codeElementParseTree in codeElementsParseTree.codeElement())
{
// Get the first line that was parsed
var tokenStart = (Token)codeElementParseTree.Start;
CodeElementsLine codeElementsLine = GetCodeElementsLineForToken(tokenStart);
// Register that this line was updated
// COMMENTED FOR THE SAKE OF PERFORMANCE -- SEE ISSUE #160
//int updatedLineIndex = documentLines.IndexOf(codeElementsLine, codeElementsLine.InitialLineIndex);
//codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, updatedLineIndex, codeElementsLine));
codeElementsLinesChanges.Add(new DocumentChange <ICodeElementsLine>(DocumentChangeType.LineUpdated, codeElementsLine.InitialLineIndex, codeElementsLine));
// Visit the parse tree to build a first class object representing the code elements
try { walker.Walk(codeElementBuilder, codeElementParseTree); }
catch (Exception ex)
{
var code = MessageCode.ImplementationError;
int line = 0; int start = 0; int stop = 0;
if (codeElementsLine.SourceTokens != null && codeElementsLine.SourceTokens.Count > 0)
{
start = codeElementsLine.SourceTokens[0].StartIndex;
stop = codeElementsLine.SourceTokens[codeElementsLine.SourceTokens.Count - 1].StopIndex;
}
codeElementsLine.AddParserDiagnostic(new ParserDiagnostic(ex.ToString(), start, stop, line, null, code));
}
CodeElement codeElement = codeElementBuilder.CodeElement;
if (codeElement != null)
{
// Attach consumed tokens and main document line numbers information to the code element
if (codeElement.ConsumedTokens.Count == 0)
{ // ISSUE #204:
if (tokenStream.Lt(1) != null)
{ // if not end of file,
// add next token to ConsumedTokens to know where is the CodeElement in error
codeElement.ConsumedTokens.Add((Token)tokenStream.Lt(1));
// this alter CodeElements semantics: in addition to matched tokens,
// it includes the first token in error if no token has been matched
}
}
//TODO Issue #384 to discuss if this code should stay here:
//This should be in a Checker, but "codeElement.ConsumedTokens" is only set after all the checkers have been called
//Rule TCLIMITATION_NO_CE_ACROSS_SOURCES
if (codeElement.IsAcrossSourceFile())
{
DiagnosticUtils.AddError(codeElement, "A Cobol statement cannot be across 2 sources files (eg. Main program and a COPY)", MessageCode.TypeCobolParserLimitation);
}
// Add code element to the list
codeElementsLine.AddCodeElement(codeElement);
if (codeElement.Diagnostics != null)
{
foreach (Diagnostic d in codeElement.Diagnostics)
{
codeElementsLine.AddParserDiagnostic(d);
}
}
}
}
}
// If the parse tree contains errors
if (codeElementsParseTree.Diagnostics != null)
{
foreach (ParserDiagnostic d in codeElementsParseTree.Diagnostics)
{
if (d.OffendingSymbol != null)
{
CodeElementsLine codeElementsLine = GetCodeElementsLineForToken((Token)d.OffendingSymbol);
codeElementsLine.AddParserDiagnostic(d);
}
}
}
}
// In case of incremental parsing, directly jump to next parse section in the token stream
// Else, simply start parsing the next CodeElement beginning with the next token
if (currentParseSection != null)
{
// Adavance to the next ParseSection
if (parseSectionsEnumerator.MoveNext())
{
currentParseSection = parseSectionsEnumerator.Current;
tokenStream.SeekToToken(currentParseSection.StartToken);
tokenStream.StartLookingForStopToken(currentParseSection.StopToken);
}
// No more section to parse
else
{
break;
}
}
}while (tokenStream.La(1) >= 0);
return(codeElementsLinesChanges);
}
/// <summary>
/// Illustration : lines with code elements continued from previous line or with a continuation on next line (before update)
/// SW represents a code element starting word
/// [ SW x]
/// [ x]
/// [SW x]
/// [ SW ]
/// [ x SW]
/// A DocumentChange intersects with a previously parsed multiline code element if :
/// * LineInserted :
/// - all previous lines until the last starting word
/// (get previous lines until the previous code element is found)
/// - all the next lines until the next starting word
/// (get next lines until the next code element is found,
/// do not reset the last line if the next code element starts at the beginning of the line)
/// * LineUpdated / LineRemoved :
/// - all previous lines until the last starting word
/// (get previous lines until the previous code element is found)
/// - all the next lines until the next starting word
/// (get next lines until the next code element is found,
/// do not reset the last line if the next code element starts at the beginning of the line)
/// When navigating to previous or next line searching for a code element, we can stop when a fresh insertion / update is encountered.
/// When we reset a line which was not directly updated, and where the code element started in the middle of the line,
/// we must "remember" at which token we must start parsing.
/// In conclusion, the incremental parsing step for code elements is divided in two steps :
/// 1. List all the starting and stop tokens of sections to parse with the rules above
/// 2. Parse code elements beginning with starting token and until we reach stop token
/// </summary>
private static ParseSection CheckIfAdjacentLinesNeedRefresh(DocumentChangeType changeType, int lineIndex, ISearchableReadOnlyList<CodeElementsLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, IList<DocumentChange<ICodeElementsLine>> codeElementsLinesChanges, ParseSection lastParseSection)
{
ParseSection currentParseSection = new ParseSection();
// Navigate backwards to the start of the multiline code element
if (lineIndex > 0)
{
int previousLineIndex = lineIndex;
int lastLineIndexReset = lastParseSection != null ? lastParseSection.StopLineIndex : - 1;
IEnumerator<CodeElementsLine> reversedEnumerator = documentLines.GetEnumerator(previousLineIndex - 1, -1, true);
while (reversedEnumerator.MoveNext() && (--previousLineIndex > lastLineIndexReset))
{
// Get the previous line until the first code element is encountered
CodeElementsLine previousLine = reversedEnumerator.Current;
// The start of the parse section is delimited by the previous CodeElement
bool previousLineHasCodeElements = previousLine.HasCodeElements;
if (previousLineHasCodeElements)
{
currentParseSection.StartLineIndex = previousLineIndex;
currentParseSection.StartToken = previousLine.CodeElements[0].ConsumedTokens[0];
}
// All lines contained in the parse section could be modified, and should be reset
previousLine = (CodeElementsLine)prepareDocumentLineForUpdate(previousLineIndex, previousLine, CompilationStep.CodeElementsParser);
previousLine.ResetCodeElements();
codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, previousLineIndex, previousLine));
// Stop iterating backwards as soon as the start of an old CodeElement is found
if (previousLineHasCodeElements)
{
break;
}
}
}
// Navigate forwards to the end of the multiline code element
if (lineIndex < (documentLines.Count - 1))
{
int nextLineIndex = lineIndex;
IEnumerator<CodeElementsLine> enumerator = documentLines.GetEnumerator(nextLineIndex + 1, -1, true);
while (enumerator.MoveNext())
{
// Get the next line until non continuation line is encountered
nextLineIndex++;
CodeElementsLine nextLine = enumerator.Current;
// Check if the next CodeElement found starts at the beginning of the line
bool nextCodeElementStartsAtTheBeginningOfTheLine = false;
if (nextLine.HasCodeElements)
{
Token startTokenForNextParseSection = nextLine.CodeElements[0].ConsumedTokens[0];
Token firstSourceTokenOfThisLine = nextLine.TokensWithCompilerDirectives.First(token => token.Channel == Token.CHANNEL_SourceTokens);
nextCodeElementStartsAtTheBeginningOfTheLine = startTokenForNextParseSection == firstSourceTokenOfThisLine;
}
// All lines contained in the parse section could be modified
if (!nextCodeElementStartsAtTheBeginningOfTheLine)
{
nextLine = (CodeElementsLine)prepareDocumentLineForUpdate(nextLineIndex, nextLine, CompilationStep.CodeElementsParser);
codeElementsLinesChanges.Add(new DocumentChange<ICodeElementsLine>(DocumentChangeType.LineUpdated, nextLineIndex, nextLine));
}
// Stop iterating forwards as soon as the start of an old CodeElement is found
if (nextLine.HasCodeElements)
{
if (!nextCodeElementStartsAtTheBeginningOfTheLine)
{
currentParseSection.StopLineIndex = nextLineIndex;
currentParseSection.StopToken = nextLine.CodeElements[0].ConsumedTokens[0];
}
else
{
currentParseSection.StopLineIndex = nextLineIndex;
while (currentParseSection.StopToken == null && currentParseSection.StopLineIndex >= 1)
{
currentParseSection.StopLineIndex = currentParseSection.StopLineIndex - 1;
currentParseSection.StopToken = documentLines[currentParseSection.StopLineIndex].TokensWithCompilerDirectives.Last(token => token.Channel == Token.CHANNEL_SourceTokens);
}
}
currentParseSection.StopTokenIsLastTokenOfTheLine = nextCodeElementStartsAtTheBeginningOfTheLine;
break;
}
}
}
// Current parse section ends with the updated line
else
{
currentParseSection.StopLineIndex = lineIndex + 1;
while (currentParseSection.StopToken == null && currentParseSection.StopLineIndex >= 1)
{
currentParseSection.StopLineIndex = currentParseSection.StopLineIndex - 1;
currentParseSection.StopToken = documentLines[currentParseSection.StopLineIndex].TokensWithCompilerDirectives.Last(token => token.Channel == Token.CHANNEL_SourceTokens);
}
currentParseSection.StopTokenIsLastTokenOfTheLine = true;
}
return currentParseSection;
}
private static ParseSection ParseLine(
string current,
ParseSection section,
Dictionary <string, string> properties,
List <Affix> affixes,
List <Pattern> patterns)
{
if (current.StartsWith("~"))
{
var index = 1;
while (index > current.Length)
{
index++;
if (current[index] != ' ')
{
break;
}
}
var sectionName = current.Substring(index);
switch (sectionName.Trim().ToLowerInvariant())
{
case Information:
section = ParseSection.Information;
break;
case Affixes:
section = ParseSection.Affixes;
break;
case Patterns:
section = ParseSection.Patterns;
break;
}
}
else
{
switch (section)
{
case ParseSection.Information:
{
// Sample Property:
// Name : Value
var parts = current.Split(':');
if (parts.Length == 2)
{
properties.Add(parts[0].Trim(), parts[1].Trim());
}
}
break;
case ParseSection.Affixes:
{
var parts = current.Split(':');
if (parts.Length == 2)
{
var name = parts[0].Trim().ToLowerInvariant(); // Affix names are case-insensitive
var variants = parts[1].Split(',').Select(v => v.Trim()).ToArray();
affixes.Add(new Affix(name, variants));
}
}
break;
case ParseSection.Patterns:
// TODO: Make sure all of the affixes are loaded before the patterns
patterns.Add(new Pattern(current));
break;
}
}
return(section);
}
