// --- Incremental compilation process ---
protected void CopyTokensLineProperties(TokensLine previousLineVersion)
{
this.lastSourceIndex = previousLineVersion.lastSourceIndex;
this.InitialScanState = previousLineVersion.InitialScanState;
this.ScanStateBeforeCOPYToken = previousLineVersion.ScanStateBeforeCOPYToken;
this.ScanState = previousLineVersion.ScanState;
this.SourceTokens = previousLineVersion.SourceTokens;
this._ScannerDiagnostics = previousLineVersion.ScannerDiagnostics;
this.HasTokenContinuationFromPreviousLine = previousLineVersion.HasTokenContinuationFromPreviousLine;
this.HasTokenContinuedOnNextLine = previousLineVersion.HasTokenContinuedOnNextLine;
CompilationStep = Concurrency.CompilationStep.Scanner;
}
public static void ScanDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <TokensLine> documentLines, TypeCobolOptions compilerOptions, List <RemarksDirective.TextNameVariation> copyTextNameVariations,
[CanBeNull] MultilineScanState initialScanState)
{
TokensLine tokensLine = null;
TokensLine nextTokensLine = null;
MultilineScanState lastScanState = initialScanState;
// Get the first line
IEnumerator <TokensLine> documentLinesEnumerator = documentLines.GetEnumerator();
if (documentLinesEnumerator.MoveNext())
{
tokensLine = documentLinesEnumerator.Current;
}
while (tokensLine != null)
{
// Peek the next line to look for continuations
if (documentLinesEnumerator.MoveNext())
{
nextTokensLine = documentLinesEnumerator.Current;
}
else
{
nextTokensLine = null;
}
// If no continuation is found, scan the current line
if (nextTokensLine == null || nextTokensLine.Type != CobolTextLineType.Continuation)
{
if (lastScanState == null)
{
Scanner.ScanFirstLine(tokensLine, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations);
}
else
{
Scanner.ScanTokensLine(tokensLine, lastScanState, compilerOptions, copyTextNameVariations);
}
}
// If a continuation is found on the next line, scan the continuation lines as a group
else
{
// Build a list of the lines we will have to scan as a group :
IList <TokensLine> continuationLinesGroup = new List <TokensLine>();
// Add current line
continuationLinesGroup.Add(tokensLine);
// Add next line
tokensLine = nextTokensLine;
nextTokensLine = null;
continuationLinesGroup.Add(tokensLine);
// Navigate forwards to the end of the multiline continuation
while (documentLinesEnumerator.MoveNext())
{
nextTokensLine = documentLinesEnumerator.Current;
if (nextTokensLine.Type == CobolTextLineType.Continuation /*|| <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line
* lineToScan.Type == CobolTextLineType.Comment || lineToScan.Type == CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines
{
// Add this line at the end of the list of continuation lines
tokensLine = nextTokensLine;
nextTokensLine = null;
continuationLinesGroup.Add(tokensLine);
}
else
{
// Exit the loop
break;
}
}
// Scan the whole group of continuation lines
if (lastScanState == null)
{
Scanner.ScanFirstLineContinuationGroup(continuationLinesGroup, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations);
}
else
{
Scanner.ScanTokensLineContinuationGroup(continuationLinesGroup, lastScanState, compilerOptions, copyTextNameVariations);
}
}
lastScanState = tokensLine.ScanState;
tokensLine = nextTokensLine;
}
}
private static void ScanTokensLineWithMultilineScanState(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList <TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, List <RemarksDirective.TextNameVariation> copyTextNameVariations, IList <DocumentChange <ITokensLine> > tokensLinesChanges, MultilineScanState initialScanState, out int nextLineToScanIndex, out TokensLine nextLineToScan)
{
// Scan the current line (or continuation lines group)
MultilineScanState scanState = ScanTokensLineWithContinuations(lineToScanIndex, lineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, initialScanState, out nextLineToScanIndex, out nextLineToScan);
// Scan the following lines until we find that the scan state at the beginning of the next line has been updated
while (nextLineToScan != null && nextLineToScan.InitialScanState != null && !nextLineToScan.InitialScanState.Equals(scanState))
{
scanState = ScanTokensLineWithContinuations(nextLineToScanIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, scanState, out nextLineToScanIndex, out nextLineToScan);
}
}
private static MultilineScanState ScanTokensLineWithContinuations(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList <TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, List <RemarksDirective.TextNameVariation> copyTextNameVariations, IList <DocumentChange <ITokensLine> > tokensLinesChanges,
[CanBeNull] MultilineScanState scanState, out int nextLineToScanIndex, out TokensLine nextLineToScan)
{
// Initialize out parameters
if (lineToScanIndex < (documentLines.Count - 1))
{
nextLineToScanIndex = lineToScanIndex + 1;
nextLineToScan = documentLines[nextLineToScanIndex];
}
else
{
nextLineToScanIndex = -1;
nextLineToScan = null;
}
// Check if the line to scan participates in a multiline continuation
// - because it is itself a continuation line
bool partOfMultilineContinuation = lineToScan.Type == CobolTextLineType.Continuation;
// - or because the following line is a continuation line
if (!partOfMultilineContinuation && lineToScanIndex < (documentLines.Count - 1))
{
nextLineToScanIndex = lineToScanIndex + 1;
nextLineToScan = documentLines[nextLineToScanIndex];
partOfMultilineContinuation = nextLineToScan.Type == CobolTextLineType.Continuation;
}
// ^-- LIMITATION : we don't support the case where one or more comment lines follow the current line before a continuation line
// Case 1 : the line is not part of a multiline continuation => we can parse it as as a standalone line
if (!partOfMultilineContinuation)
{
// Create a new copy of the line before the update if necessary
lineToScan = (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner);
if (lineToScanIndex == 0)
{
if (scanState != null)
{
// Scan the first line of the document
Scanner.ScanFirstLine(lineToScan, scanState.InsideDataDivision, scanState.DecimalPointIsComma,
scanState.WithDebuggingMode, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations);
}
else
{
Scanner.ScanFirstLine(lineToScan, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations);
}
}
else
{
// Get the scan state at the end of the previous line
TokensLine previousLine = documentLines[lineToScanIndex - 1];
// Scan the current line with this initial scan state
Scanner.ScanTokensLine(lineToScan, previousLine.ScanState, compilerOptions, copyTextNameVariations);
}
tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.LineUpdated, lineToScanIndex, lineToScan));
return(lineToScan.ScanState);
}
// Case 2 : the line is part of a multiline continuation => we must parse all continuation lines as a group
else
{
// Build a list of the lines we will have to scan as a group :
IList <TokensLine> continuationLinesGroup = new List <TokensLine>();
int firstLineIndex = lineToScanIndex;
continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
// Navigate backwards to the start of the multiline continuation
if (lineToScan.Type == CobolTextLineType.Continuation && lineToScanIndex > 0)
{
int revLineToScanIndex = lineToScanIndex;
IEnumerator <TokensLine> reversedEnumerator = documentLines.GetEnumerator(lineToScanIndex - 1, -1, true);
while (reversedEnumerator.MoveNext())
{
// Get the previous line until a non continuation and non comment line is encountered
revLineToScanIndex--;
lineToScan = reversedEnumerator.Current;
if (lineToScan.Type != CobolTextLineType.Continuation /*&& <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line
* lineToScan.Type != CobolTextLineType.Comment && lineToScan.Type != CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines
{
firstLineIndex = revLineToScanIndex;
continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
}
else
{
break;
}
}
}
// Reuse our knowledge of the next line if it is available and if it is a continuation
if (nextLineToScan != null && nextLineToScan.Type == CobolTextLineType.Continuation)
{
lineToScanIndex++;
lineToScan = nextLineToScan;
continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
nextLineToScanIndex = -1;
nextLineToScan = null;
}
// Navigate forwards to the end of the multiline continuation
if (lineToScanIndex < (documentLines.Count - 1))
{
IEnumerator <TokensLine> enumerator = documentLines.GetEnumerator(lineToScanIndex + 1, -1, false);
while (enumerator.MoveNext())
{
// Get the next line until a non continuation and non comment line is encountered
lineToScanIndex++;
lineToScan = enumerator.Current;
if (lineToScan.Type == CobolTextLineType.Continuation /*|| <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line
* lineToScan.Type == CobolTextLineType.Comment || lineToScan.Type == CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines
{
// Add this line at the end of the list of continuation lines
continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
}
else
{
// Save the knowledge of the next line and exit the loop
nextLineToScanIndex = lineToScanIndex;
nextLineToScan = lineToScan;
break;
}
}
}
// Scan the group of continuation lines
if (firstLineIndex == 0)
{
// Scan the first line group of the document
Scanner.ScanFirstLineContinuationGroup(continuationLinesGroup, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations);
}
else
{
// Get the scan state at the end of the previous line
TokensLine previousLine = documentLines[firstLineIndex - 1];
// Scan the current line group with this initial scan state
Scanner.ScanTokensLineContinuationGroup(continuationLinesGroup, previousLine.ScanState, compilerOptions, copyTextNameVariations);
}
int updatedLineIndex = firstLineIndex;
foreach (TokensLine updatedLine in continuationLinesGroup)
{
tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.LineUpdated, updatedLineIndex, updatedLine));
updatedLineIndex++;
}
return(continuationLinesGroup[continuationLinesGroup.Count - 1].ScanState);
}
}
internal static IList <DocumentChange <ITokensLine> > ScanTextLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <TokensLine> documentLines, IList <DocumentChange <ICobolTextLine> > textLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, List <RemarksDirective.TextNameVariation> copyTextNameVariations,
[CanBeNull] MultilineScanState scanState)
{
// Collect all changes applied to the tokens lines during the incremental scan
IList <DocumentChange <ITokensLine> > tokensLinesChanges = new List <DocumentChange <ITokensLine> >();
// There are 3 reasons to scan a line after a text change :
// 1. New text lines which were just inserted or updated must be scanned for the first time
// 2. Text lines must be scanned again if their initial scan state changed : a new scan of the previous line can alter the scan state at the beginning of the following line
// 3. Continuation lines and multiline tokens : if a line participates in a continuation on several lines, scan the group of lines as a whole
// IMPORTANT : the text changes are ordered in increasing order of line index
for (int textChangeIndex = 0; textChangeIndex < textLinesChanges.Count; textChangeIndex++)
{
// Local variables used to optimize navigation in the document
int nextLineToScanIndex = -1;
TokensLine nextLineToScan = null;
// Update tokens depending on the current text change
DocumentChange <ICobolTextLine> textChange = textLinesChanges[textChangeIndex];
if (textChange.Type == DocumentChangeType.DocumentCleared)
{
tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.DocumentCleared, 0, null));
continue;
}
else if (textChange.Type == DocumentChangeType.LineInserted || textChange.Type == DocumentChangeType.LineUpdated)
{
// We update lines as a group below, but we remember here which lines were inserted
if (textChange.Type == DocumentChangeType.LineInserted)
{
tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.LineInserted, textChange.LineIndex, (ITokensLine)textChange.NewLine));
}
// Text lines which were inserted or updated must be scanned again
ScanTokensLineWithMultilineScanState(textChange.LineIndex, (TokensLine)textChange.NewLine, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, scanState, out nextLineToScanIndex, out nextLineToScan);
}
else if (textChange.Type == DocumentChangeType.LineRemoved)
{
tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.LineRemoved, textChange.LineIndex, (ITokensLine)textChange.NewLine));
// Get the last line just before the line that was removed
TokensLine previousLine = null;
if (textChange.LineIndex > 0)
{
previousLine = documentLines[textChange.LineIndex - 1];
}
// When a text line is removed :
// - the previous line must be scanned again if the line which was removed was a member of a multiline continuation group
if (previousLine != null && previousLine.HasTokenContinuedOnNextLine)
{
ScanTokensLineWithMultilineScanState(textChange.LineIndex - 1, previousLine, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, scanState, out nextLineToScanIndex, out nextLineToScan);
}
if (nextLineToScan == null && textChange.LineIndex < documentLines.Count)
{
nextLineToScanIndex = textChange.LineIndex;
nextLineToScan = documentLines[nextLineToScanIndex];
}
// - the next line must be scanned again if the scan state at the end of the previous line is different from the scan state at the beginning of the next line
if (nextLineToScan != null && nextLineToScanIndex == textChange.LineIndex && previousLine != null &&
nextLineToScan.InitialScanState.Equals(previousLine.ScanState))
{
ScanTokensLineWithMultilineScanState(textChange.LineIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, scanState, out nextLineToScanIndex, out nextLineToScan);
}
}
// We can skip all text changes with an index smaller than the index of the last line which was already scanned
if (nextLineToScan == null)
{
break;
}
else if (textChangeIndex < (textLinesChanges.Count - 1))
{
int nextTextChangeIndex = textChangeIndex;
DocumentChange <ICobolTextLine> nextTextChange = null;
do
{
nextTextChangeIndex++;
nextTextChange = textLinesChanges[nextTextChangeIndex];
}while (nextTextChangeIndex < (textLinesChanges.Count - 1) &&
nextTextChange.LineIndex <= nextLineToScanIndex);
textChangeIndex = nextTextChangeIndex - 1;
}
}
return(tokensLinesChanges);
}
