/// <summary>
/// Clone the current scanner state
/// </summary>
public MultilineScanState Clone()
{
MultilineScanState clone = new MultilineScanState(InsideDataDivision, DecimalPointIsComma, WithDebuggingMode, EncodingForAlphanumericLiterals);
#if EUROINFO_LEGACY_REPLACING_SYNTAX
clone.InsideRemarksDirective = InsideRemarksDirective;
clone.InsideRemarksParentheses = InsideRemarksParentheses;
if(CopyTextNamesVariations != null)
{
clone.CopyTextNamesVariations = CopyTextNamesVariations;
}
#endif
if (SymbolicCharacters != null)
{
clone.SymbolicCharacters = new List<string>(SymbolicCharacters);
}
clone.KeywordsState = KeywordsState;
if (LastToken != null) clone.LastToken = LastToken;
if (LastKeywordOrSymbolToken != null) clone.LastKeywordOrSymbolToken = LastKeywordOrSymbolToken;
return clone;
}
private static void ScanTokensLineWithMultilineScanState(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IList<DocumentChange<ITokensLine>> tokensLinesChanges, MultilineScanState initialScanState, out int nextLineToScanIndex, out TokensLine nextLineToScan)
{
// Scan the current line (or continuation lines group)
MultilineScanState scanState = ScanTokensLineWithContinuations(lineToScanIndex, lineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, initialScanState, out nextLineToScanIndex, out nextLineToScan);
// Scan the following lines until we find that the scan state at the beginning of the next line has been updated
while(nextLineToScan != null && nextLineToScan.InitialScanState != null && !nextLineToScan.InitialScanState.Equals(scanState))
{
scanState = ScanTokensLineWithContinuations(nextLineToScanIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, scanState, out nextLineToScanIndex, out nextLineToScan);
}
}
internal void InitializeScanState(MultilineScanState initialScanState)
{
InitialScanState = initialScanState;
ScanState = initialScanState != null?initialScanState.Clone() : null;
}
private static MultilineScanState ScanTokensLineWithContinuations(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList <TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, List <RemarksDirective.TextNameVariation> copyTextNameVariations, IList <DocumentChange <ITokensLine> > tokensLinesChanges,
[CanBeNull] MultilineScanState scanState, out int nextLineToScanIndex, out TokensLine nextLineToScan)
{
// Initialize out parameters
if (lineToScanIndex < (documentLines.Count - 1))
{
nextLineToScanIndex = lineToScanIndex + 1;
nextLineToScan = documentLines[nextLineToScanIndex];
}
else
{
nextLineToScanIndex = -1;
nextLineToScan = null;
}
// Check if the line to scan participates in a multiline continuation
// - because it is itself a continuation line
bool partOfMultilineContinuation = lineToScan.Type == CobolTextLineType.Continuation;
// - or because the following line is a continuation line
if (!partOfMultilineContinuation && lineToScanIndex < (documentLines.Count - 1))
{
nextLineToScanIndex = lineToScanIndex + 1;
nextLineToScan = documentLines[nextLineToScanIndex];
partOfMultilineContinuation = nextLineToScan.Type == CobolTextLineType.Continuation;
}
// ^-- LIMITATION : we don't support the case where one or more comment lines follow the current line before a continuation line
// Case 1 : the line is not part of a multiline continuation => we can parse it as as a standalone line
if (!partOfMultilineContinuation)
{
// Create a new copy of the line before the update if necessary
lineToScan = (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner);
if (lineToScanIndex == 0)
{
if (scanState != null)
{
// Scan the first line of the document
Scanner.ScanFirstLine(lineToScan, scanState.InsideDataDivision, scanState.DecimalPointIsComma,
scanState.WithDebuggingMode, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations);
}
else
{
Scanner.ScanFirstLine(lineToScan, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations);
}
}
else
{
// Get the scan state at the end of the previous line
TokensLine previousLine = documentLines[lineToScanIndex - 1];
// Scan the current line with this initial scan state
Scanner.ScanTokensLine(lineToScan, previousLine.ScanState, compilerOptions, copyTextNameVariations);
}
tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.LineUpdated, lineToScanIndex, lineToScan));
return(lineToScan.ScanState);
}
// Case 2 : the line is part of a multiline continuation => we must parse all continuation lines as a group
else
{
// Build a list of the lines we will have to scan as a group :
IList <TokensLine> continuationLinesGroup = new List <TokensLine>();
int firstLineIndex = lineToScanIndex;
continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
// Navigate backwards to the start of the multiline continuation
if (lineToScan.Type == CobolTextLineType.Continuation && lineToScanIndex > 0)
{
int revLineToScanIndex = lineToScanIndex;
IEnumerator <TokensLine> reversedEnumerator = documentLines.GetEnumerator(lineToScanIndex - 1, -1, true);
while (reversedEnumerator.MoveNext())
{
// Get the previous line until a non continuation and non comment line is encountered
revLineToScanIndex--;
lineToScan = reversedEnumerator.Current;
if (lineToScan.Type != CobolTextLineType.Continuation /*&& <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line
* lineToScan.Type != CobolTextLineType.Comment && lineToScan.Type != CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines
{
firstLineIndex = revLineToScanIndex;
continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
}
else
{
break;
}
}
}
// Reuse our knowledge of the next line if it is available and if it is a continuation
if (nextLineToScan != null && nextLineToScan.Type == CobolTextLineType.Continuation)
{
lineToScanIndex++;
lineToScan = nextLineToScan;
continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
nextLineToScanIndex = -1;
nextLineToScan = null;
}
// Navigate forwards to the end of the multiline continuation
if (lineToScanIndex < (documentLines.Count - 1))
{
IEnumerator <TokensLine> enumerator = documentLines.GetEnumerator(lineToScanIndex + 1, -1, false);
while (enumerator.MoveNext())
{
// Get the next line until a non continuation and non comment line is encountered
lineToScanIndex++;
lineToScan = enumerator.Current;
if (lineToScan.Type == CobolTextLineType.Continuation /*|| <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line
* lineToScan.Type == CobolTextLineType.Comment || lineToScan.Type == CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines
{
// Add this line at the end of the list of continuation lines
continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner));
}
else
{
// Save the knowledge of the next line and exit the loop
nextLineToScanIndex = lineToScanIndex;
nextLineToScan = lineToScan;
break;
}
}
}
// Scan the group of continuation lines
if (firstLineIndex == 0)
{
// Scan the first line group of the document
Scanner.ScanFirstLineContinuationGroup(continuationLinesGroup, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations);
}
else
{
// Get the scan state at the end of the previous line
TokensLine previousLine = documentLines[firstLineIndex - 1];
// Scan the current line group with this initial scan state
Scanner.ScanTokensLineContinuationGroup(continuationLinesGroup, previousLine.ScanState, compilerOptions, copyTextNameVariations);
}
int updatedLineIndex = firstLineIndex;
foreach (TokensLine updatedLine in continuationLinesGroup)
{
tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.LineUpdated, updatedLineIndex, updatedLine));
updatedLineIndex++;
}
return(continuationLinesGroup[continuationLinesGroup.Count - 1].ScanState);
}
}
public static void ScanDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <TokensLine> documentLines, TypeCobolOptions compilerOptions, List <RemarksDirective.TextNameVariation> copyTextNameVariations,
[CanBeNull] MultilineScanState initialScanState)
{
TokensLine tokensLine = null;
TokensLine nextTokensLine = null;
MultilineScanState lastScanState = initialScanState;
// Get the first line
IEnumerator <TokensLine> documentLinesEnumerator = documentLines.GetEnumerator();
if (documentLinesEnumerator.MoveNext())
{
tokensLine = documentLinesEnumerator.Current;
}
while (tokensLine != null)
{
// Peek the next line to look for continuations
if (documentLinesEnumerator.MoveNext())
{
nextTokensLine = documentLinesEnumerator.Current;
}
else
{
nextTokensLine = null;
}
// If no continuation is found, scan the current line
if (nextTokensLine == null || nextTokensLine.Type != CobolTextLineType.Continuation)
{
if (lastScanState == null)
{
Scanner.ScanFirstLine(tokensLine, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations);
}
else
{
Scanner.ScanTokensLine(tokensLine, lastScanState, compilerOptions, copyTextNameVariations);
}
}
// If a continuation is found on the next line, scan the continuation lines as a group
else
{
// Build a list of the lines we will have to scan as a group :
IList <TokensLine> continuationLinesGroup = new List <TokensLine>();
// Add current line
continuationLinesGroup.Add(tokensLine);
// Add next line
tokensLine = nextTokensLine;
nextTokensLine = null;
continuationLinesGroup.Add(tokensLine);
// Navigate forwards to the end of the multiline continuation
while (documentLinesEnumerator.MoveNext())
{
nextTokensLine = documentLinesEnumerator.Current;
if (nextTokensLine.Type == CobolTextLineType.Continuation /*|| <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line
* lineToScan.Type == CobolTextLineType.Comment || lineToScan.Type == CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines
{
// Add this line at the end of the list of continuation lines
tokensLine = nextTokensLine;
nextTokensLine = null;
continuationLinesGroup.Add(tokensLine);
}
else
{
// Exit the loop
break;
}
}
// Scan the whole group of continuation lines
if (lastScanState == null)
{
Scanner.ScanFirstLineContinuationGroup(continuationLinesGroup, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations);
}
else
{
Scanner.ScanTokensLineContinuationGroup(continuationLinesGroup, lastScanState, compilerOptions, copyTextNameVariations);
}
}
lastScanState = tokensLine.ScanState;
tokensLine = nextTokensLine;
}
}
internal static IList <DocumentChange <ITokensLine> > ScanTextLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <TokensLine> documentLines, IList <DocumentChange <ICobolTextLine> > textLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, List <RemarksDirective.TextNameVariation> copyTextNameVariations,
[CanBeNull] MultilineScanState scanState)
{
// Collect all changes applied to the tokens lines during the incremental scan
IList <DocumentChange <ITokensLine> > tokensLinesChanges = new List <DocumentChange <ITokensLine> >();
// There are 3 reasons to scan a line after a text change :
// 1. New text lines which were just inserted or updated must be scanned for the first time
// 2. Text lines must be scanned again if their initial scan state changed : a new scan of the previous line can alter the scan state at the beginning of the following line
// 3. Continuation lines and multiline tokens : if a line participates in a continuation on several lines, scan the group of lines as a whole
// IMPORTANT : the text changes are ordered in increasing order of line index
for (int textChangeIndex = 0; textChangeIndex < textLinesChanges.Count; textChangeIndex++)
{
// Local variables used to optimize navigation in the document
int nextLineToScanIndex = -1;
TokensLine nextLineToScan = null;
// Update tokens depending on the current text change
DocumentChange <ICobolTextLine> textChange = textLinesChanges[textChangeIndex];
if (textChange.Type == DocumentChangeType.DocumentCleared)
{
tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.DocumentCleared, 0, null));
continue;
}
else if (textChange.Type == DocumentChangeType.LineInserted || textChange.Type == DocumentChangeType.LineUpdated)
{
// We update lines as a group below, but we remember here which lines were inserted
if (textChange.Type == DocumentChangeType.LineInserted)
{
tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.LineInserted, textChange.LineIndex, (ITokensLine)textChange.NewLine));
}
// Text lines which were inserted or updated must be scanned again
ScanTokensLineWithMultilineScanState(textChange.LineIndex, (TokensLine)textChange.NewLine, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, scanState, out nextLineToScanIndex, out nextLineToScan);
}
else if (textChange.Type == DocumentChangeType.LineRemoved)
{
tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.LineRemoved, textChange.LineIndex, (ITokensLine)textChange.NewLine));
// Get the last line just before the line that was removed
TokensLine previousLine = null;
if (textChange.LineIndex > 0)
{
previousLine = documentLines[textChange.LineIndex - 1];
}
// When a text line is removed :
// - the previous line must be scanned again if the line which was removed was a member of a multiline continuation group
if (previousLine != null && previousLine.HasTokenContinuedOnNextLine)
{
ScanTokensLineWithMultilineScanState(textChange.LineIndex - 1, previousLine, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, scanState, out nextLineToScanIndex, out nextLineToScan);
}
if (nextLineToScan == null && textChange.LineIndex < documentLines.Count)
{
nextLineToScanIndex = textChange.LineIndex;
nextLineToScan = documentLines[nextLineToScanIndex];
}
// - the next line must be scanned again if the scan state at the end of the previous line is different from the scan state at the beginning of the next line
if (nextLineToScan != null && nextLineToScanIndex == textChange.LineIndex && previousLine != null &&
nextLineToScan.InitialScanState.Equals(previousLine.ScanState))
{
ScanTokensLineWithMultilineScanState(textChange.LineIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, scanState, out nextLineToScanIndex, out nextLineToScan);
}
}
// We can skip all text changes with an index smaller than the index of the last line which was already scanned
if (nextLineToScan == null)
{
break;
}
else if (textChangeIndex < (textLinesChanges.Count - 1))
{
int nextTextChangeIndex = textChangeIndex;
DocumentChange <ICobolTextLine> nextTextChange = null;
do
{
nextTextChangeIndex++;
nextTextChange = textLinesChanges[nextTextChangeIndex];
}while (nextTextChangeIndex < (textLinesChanges.Count - 1) &&
nextTextChange.LineIndex <= nextLineToScanIndex);
textChangeIndex = nextTextChangeIndex - 1;
}
}
return(tokensLinesChanges);
}
private static void ScanTokensLineWithMultilineScanState(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList <TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, List <RemarksDirective.TextNameVariation> copyTextNameVariations, IList <DocumentChange <ITokensLine> > tokensLinesChanges, MultilineScanState initialScanState, out int nextLineToScanIndex, out TokensLine nextLineToScan)
{
// Scan the current line (or continuation lines group)
MultilineScanState scanState = ScanTokensLineWithContinuations(lineToScanIndex, lineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, initialScanState, out nextLineToScanIndex, out nextLineToScan);
// Scan the following lines until we find that the scan state at the beginning of the next line has been updated
while (nextLineToScan != null && nextLineToScan.InitialScanState != null && !nextLineToScan.InitialScanState.Equals(scanState))
{
scanState = ScanTokensLineWithContinuations(nextLineToScanIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, scanState, out nextLineToScanIndex, out nextLineToScan);
}
}
internal void InitializeScanState(MultilineScanState initialScanState)
{
InitialScanState = initialScanState;
ScanState = initialScanState.Clone();
}
/// <summary>
/// Use a pre-existing text document, already initialized from a Cobol file
/// </summary>
public FileCompiler(string libraryName, string textName, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ColumnsLayout columnsLayout, TypeCobolOptions compilerOptions, CodeModel.SymbolTable customSymbols, bool isCopyFile, MultilineScanState scanState)
: this(libraryName, textName, null, sourceFileProvider, documentProvider, columnsLayout, null, compilerOptions, customSymbols, isCopyFile, scanState)
{
}
/// <summary>
/// Scan a group of continuation lines when no previous scan state object is available
/// </summary>
public static void ScanFirstLineContinuationGroup(IList<TokensLine> continuationLinesGroup, bool insideDataDivision, bool decimalPointIsComma, bool withDebuggingMode, Encoding encodingForAlphanumericLiterals, TypeCobolOptions compilerOptions)
{
MultilineScanState initialScanState = new MultilineScanState(insideDataDivision, decimalPointIsComma, withDebuggingMode, encodingForAlphanumericLiterals);
ScanTokensLineContinuationGroup(continuationLinesGroup, initialScanState, compilerOptions);
}
/// <summary>
/// Scan a line of a document
/// </summary>
public static void ScanTokensLine(TokensLine tokensLine, MultilineScanState initialScanState, TypeCobolOptions compilerOptions)
{
// Updates are forbidden after a snapshot of a specific version of a line
if(!tokensLine.CanStillBeUpdatedBy(CompilationStep.Scanner))
{
throw new InvalidOperationException("Can not update this TokensLine because it was already frozen by compilation step : " + tokensLine.CompilationStep.ToString());
}
// Set the initial scan state for the line
tokensLine.InitializeScanState(initialScanState);
// Alias to refer to Cobol text line properties
ICobolTextLine textLine = tokensLine;
// The source section of this line of text must be split into tokens
string line = tokensLine.Text;
int startIndex = textLine.Source.StartIndex;
int lastIndex = textLine.Source.EndIndex;
#if EUROINFO_LEGACY_REPLACING_SYNTAX
if (IsInsideRemarks(textLine.Type, tokensLine.SourceText)) tokensLine.ScanState.InsideRemarksDirective = true;
// Try to scan REMARKS compiler directive parameters inside the comment or non-comment line
if (tokensLine.ScanState.InsideRemarksDirective) {
string remarksLine = textLine.SourceText;
int startIndexForSignificantPart = GetStartIndexOfSignificantPart(remarksLine, tokensLine.ScanState);
int firstPeriodIndex = remarksLine.IndexOf('.', startIndexForSignificantPart);
int endIndexForSignificantPart = GetEndIndexOfSignificantPart(remarksLine, tokensLine.ScanState, firstPeriodIndex);
string significantPart = remarksLine.Substring(startIndexForSignificantPart, endIndexForSignificantPart - startIndexForSignificantPart + 1).Trim();
if (firstPeriodIndex >= 0 || (!tokensLine.ScanState.InsideRemarksParentheses && !remarksLine.Contains("COPY"))) {
tokensLine.ScanState.InsideRemarksDirective = false; // indicates the end of the REMARKS compiler directive
}
RemarksDirective remarksDirective = CreateRemarksDirective(significantPart, tokensLine.ScanState);
if (remarksDirective != null && remarksDirective.CopyTextNamesVariations.Count > 0) {
// A non empty remarks directive will replace the comment line
tokensLine.AddToken(CreateCompilerDirectiveToken(remarksDirective, tokensLine, startIndex, lastIndex));
return;
}
}
#endif
// Comment line => return only one token with type CommentLine
// Debug line => treated as a comment line if debugging mode was not activated
if (textLine.Type == CobolTextLineType.Comment ||
(textLine.Type == CobolTextLineType.Debug && !tokensLine.InitialScanState.WithDebuggingMode))
{
Token commentToken = new Token(TokenType.CommentLine, startIndex, lastIndex, tokensLine);
tokensLine.AddToken(commentToken);
return;
}
// Invalid indicator, the line type is unknown => the whole line text is handled as a single invalid token
else if (textLine.Type == CobolTextLineType.Invalid)
{
// Invalid indicator => register an error
tokensLine.AddDiagnostic(MessageCode.InvalidIndicatorCharacter, textLine.Indicator.StartIndex, textLine.Indicator.EndIndex, textLine.Indicator);
Token invalidToken = new Token(TokenType.InvalidToken, startIndex, lastIndex, tokensLine);
tokensLine.AddToken(invalidToken);
return;
}
// Empty line => return immediately an empty list of tokens
// Blank line => return only one token with type SpaceSeparator
if(textLine.Type == CobolTextLineType.Blank)
{
if(!String.IsNullOrEmpty(line))
{
Token whitespaceToken = new Token(TokenType.SpaceSeparator, startIndex, lastIndex, tokensLine);
tokensLine.AddToken(whitespaceToken);
}
return;
}
// Create a stateful line scanner, and iterate over the tokens
Scanner scanner = new Scanner(line, startIndex, lastIndex, tokensLine, compilerOptions);
Token nextToken = null;
while((nextToken = scanner.GetNextToken()) != null)
{
// Resolve DELETE ambiguity : DELETE + InterLiteral => DELETE_CD
// Warning : DELETE and the sequence-number-field must be on the same line
if(nextToken.TokenType == TokenType.IntegerLiteral && tokensLine.ScanState.KeywordsState == KeywordsSequenceState.After_DELETE)
{
tokensLine.ScanState.LastKeywordOrSymbolToken.CorrectType(TokenType.DELETE_CD);
}
tokensLine.AddToken(nextToken);
}
}
private static int GetStartIndexOfSignificantPart(string line, MultilineScanState state)
{
int start = Math.Max(line.IndexOf(' ') +1, line.IndexOf('=') +1);
if (!state.InsideRemarksParentheses) {
int firstLParenIndex = line.IndexOf('(');
state.InsideRemarksParentheses = (firstLParenIndex >= 0);
start = Math.Max(start, firstLParenIndex +1);
}
return start;
}
private static int GetEndIndexOfSignificantPart(string line, MultilineScanState state, int firstPeriodIndex)
{
int end = line.Length -1;
if (state.InsideRemarksParentheses) {
int firstRParenIndex = line.IndexOf(')');
if (firstRParenIndex >= 0) {
end = firstRParenIndex -1;
state.InsideRemarksParentheses = false;
}
if (firstPeriodIndex >= 0 && firstPeriodIndex < firstRParenIndex)
end = firstPeriodIndex - 1;
}
return end;
}
private static RemarksDirective CreateRemarksDirective(string significantPart, MultilineScanState state)
{
if (significantPart.Length < 1) return null;
var remarksDirective = new RemarksDirective();
foreach (string candidateName in significantPart.Split(' ')) {
if (candidateName.Length == 7 || candidateName.Length == 8) {
RemarksDirective.TextNameVariation textName = new RemarksDirective.TextNameVariation(candidateName);
remarksDirective.CopyTextNamesVariations.Add(textName);
}
else if (!String.IsNullOrWhiteSpace(candidateName)) {
// A string which is not a text name is an error : stop scanning here
remarksDirective = null;
state.InsideRemarksDirective = false;
break;
}
}
return remarksDirective;
}
/// <summary>
/// Scan a group of continuation lines
/// </summary>
public static void ScanTokensLineContinuationGroup(IList<TokensLine> continuationLinesGroup, MultilineScanState initialScanState, TypeCobolOptions compilerOptions)
{
// p54: Continuation lines
// Any sentence, entry, clause, or phrase that requires more than one line can be
// continued in Area B of the next line that is neither a comment line nor a blank line.
// The line being continued is a continued line; the succeeding lines are continuation
// lines.
// Track the length of text contributed to the continuation text by each individual line
TextArea[] textAreasForOriginalLinesInConcatenatedLine = new TextArea[continuationLinesGroup.Count];
int[] startIndexForTextAreasInOriginalLines = new int[continuationLinesGroup.Count];
int[] offsetForLiteralContinuationInOriginalLines = new int[continuationLinesGroup.Count];
// Initialize the continuation text with the complete source text of the first line
TokensLine firstLine = continuationLinesGroup[0];
string concatenatedLine = null;
if (firstLine.Type == CobolTextLineType.Source || (firstLine.Type == CobolTextLineType.Debug && initialScanState.WithDebuggingMode))
{
concatenatedLine = firstLine.SourceText;
}
else
{
concatenatedLine = String.Empty;
Scanner.ScanTokensLine(firstLine, initialScanState, compilerOptions);
}
textAreasForOriginalLinesInConcatenatedLine[0] = new TextArea(TextAreaType.Source, 0, concatenatedLine.Length -1);
startIndexForTextAreasInOriginalLines[0] = firstLine.Source.StartIndex;
offsetForLiteralContinuationInOriginalLines[0] = 0;
// All the following lines are continuation lines
// => build a character string representing the complete continuation text along the way
for (int i = 1; i < continuationLinesGroup.Count; i++)
{
TokensLine continuationLine = continuationLinesGroup[i];
int startIndex = continuationLine.Source.StartIndex;
int lastIndex = continuationLine.Source.EndIndex;
string line = continuationLine.Text;
// 1. Match and remove all blank characters at the beginning of the continuation line
int startOfContinuationIndex = startIndex;
for (; startOfContinuationIndex <= lastIndex && line[startOfContinuationIndex] == ' '; startOfContinuationIndex++) { }
if (startOfContinuationIndex > startIndex)
{
Token whitespaceToken = new Token(TokenType.SpaceSeparator, startIndex, startOfContinuationIndex - 1, continuationLine);
continuationLine.SourceTokens.Add(whitespaceToken);
startIndex = startOfContinuationIndex;
}
if (startOfContinuationIndex <= lastIndex)
{
if (startOfContinuationIndex < 4)
{
continuationLine.AddDiagnostic(MessageCode.AreaAOfContinuationLineMustBeBlank, startOfContinuationIndex, startOfContinuationIndex);
}
}
else
{
// Nothing but spaces on the continuation line
continue;
}
// p55: Continuation of alphanumeric and national literals
// Alphanumeric and national literals can be continued only when there are no DBCS
// characters in the content of the literal.
// The following rules apply to alphanumeric and national literals that do not contain
// DBCS characters:
// - If the continued line contains an alphanumeric or national literal without a
// closing quotation mark, all spaces at the end of the continued line (through
// column 72) are considered to be part of the literal. The continuation line must
// contain a hyphen in the indicator area, and the first nonblank character must be
// a quotation mark. The continuation of the literal begins with the character
// immediately following the quotation mark.
// - If an alphanumeric or national literal that is to be continued on the next line has
// as its last character a quotation mark in column 72, the continuation line must
// start with two consecutive quotation marks. This will result in a single quotation
// mark as part of the value of the literal.
// - If the last character on the continued line of an alphanumeric or national literal
// is a single quotation mark in Area B, the continuation line can start with a single
// quotation mark. This will result in two consecutive literals instead of one
// continued literal.
// The rules are the same when an apostrophe is used instead of a quotation mark in
// delimiters.
// ... p55 -> p56: examples of continuations and expected behavior ...
int offsetForLiteralContinuation = 0;
if (concatenatedLine.Length > 0)
{
// Scan the continuation text, and get its last token so far
TokensLine temporaryTokensLine = TokensLine.CreateVirtualLineForInsertedToken(firstLine.InitialLineIndex, concatenatedLine);
Scanner.ScanTokensLine(temporaryTokensLine, initialScanState, compilerOptions);
Token lastTokenOfConcatenatedLineSoFar = temporaryTokensLine.SourceTokens[temporaryTokensLine.SourceTokens.Count - 1];
// Check if the last token so far is an alphanumeric or national literal
if (lastTokenOfConcatenatedLineSoFar.TokenFamily == TokenFamily.AlphanumericLiteral)
{
// The continuation line must contain a hyphen in the indicator area, and the first nonblank character must be a quotation mark
if (line[startOfContinuationIndex] != lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter)
{
continuationLine.AddDiagnostic(MessageCode.InvalidFirstCharForContinuationLine, startOfContinuationIndex, startOfContinuationIndex, lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter);
}
// The continuation of the literal begins with the character immediately following the quotation mark.
else
{
offsetForLiteralContinuation = 1;
// If an alphanumeric literal that is to be continued on the next line has as its last character a quotation mark in column 72,
// the continuation line must start with two consecutive quotation marks.
if (lastTokenOfConcatenatedLineSoFar.HasClosingDelimiter)
{
if ((startOfContinuationIndex + 1) > lastIndex || line[startOfContinuationIndex + 1] != lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter)
{
continuationLine.AddDiagnostic(MessageCode.InvalidFirstTwoCharsForContinuationLine, startOfContinuationIndex, startOfContinuationIndex + 1, lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter);
// Use the first quotation mark to avoid a complete mess while scanning the rest of the line
offsetForLiteralContinuation = 0;
}
}
}
}
// Check if the last token so far is a floating comment
else if (lastTokenOfConcatenatedLineSoFar.TokenType == TokenType.FloatingComment)
{
// => remove the floating comment from the text of the continuation
concatenatedLine = concatenatedLine.Substring(0, concatenatedLine.Length - lastTokenOfConcatenatedLineSoFar.Length);
textAreasForOriginalLinesInConcatenatedLine[i - 1] = new TextArea(TextAreaType.Source, textAreasForOriginalLinesInConcatenatedLine[i - 1].StartIndex, textAreasForOriginalLinesInConcatenatedLine[i - 1].EndIndex - lastTokenOfConcatenatedLineSoFar.Length);
TokensLine lineWithFloatingComment = continuationLinesGroup[i - 1];
Token floatingCommentToken = new Token(TokenType.FloatingComment, lineWithFloatingComment.Length - lastTokenOfConcatenatedLineSoFar.Length, lineWithFloatingComment.Length - 1, lineWithFloatingComment);
lineWithFloatingComment.SourceTokens.Add(floatingCommentToken);
}
// Check if the last token so far is a comment entry
else if (lastTokenOfConcatenatedLineSoFar.TokenType == TokenType.CommentEntry)
{
// p105: A hyphen in the indicator area (column 7) is not permitted in comment - entries.
// => impossible to ignore the continuation indicator here, it is too late
// (we can not know there is a comment entry before scanning the continuation lines groups)
// => register an error message
continuationLine.AddDiagnostic(MessageCode.HyphenIndicatorNotPermittedInCommenEntries, continuationLine.Indicator.StartIndex + 1, continuationLine.Indicator.EndIndex + 1);
}
}
// p54: If there is no hyphen (-) in the indicator area (column 7) of a line, the last character
// of the preceding line is assumed to be followed by a space.
// If there is a hyphen in the indicator area of a line, the first nonblank character of
// the continuation line immediately follows the last nonblank character of the
// continued line without an intervening space.
// Concatenate the continuation text so far with the text of the current continuation line
int startIndexOfContinuationStringInContinuationLine = startOfContinuationIndex + offsetForLiteralContinuation;
int lengthOfContinuationStringInContinuationLine = lastIndex - startIndexOfContinuationStringInContinuationLine + 1;
textAreasForOriginalLinesInConcatenatedLine[i] = new TextArea(TextAreaType.Source, concatenatedLine.Length, concatenatedLine.Length + lengthOfContinuationStringInContinuationLine - 1);
startIndexForTextAreasInOriginalLines[i] = startIndexOfContinuationStringInContinuationLine;
offsetForLiteralContinuationInOriginalLines[i] = offsetForLiteralContinuation;
concatenatedLine += line.Substring(startIndexOfContinuationStringInContinuationLine, lengthOfContinuationStringInContinuationLine);
}
// Scan the complete continuation text as a whole
TokensLine virtualContinuationTokensLine = TokensLine.CreateVirtualLineForInsertedToken(firstLine.InitialLineIndex, concatenatedLine);
Scanner.ScanTokensLine(virtualContinuationTokensLine, initialScanState, compilerOptions);
// Then attribute each token and diagnostic to its corresponding tokens line
MultilineScanState scanState = initialScanState;
for (int i = 0; i < continuationLinesGroup.Count; i++)
{
TokensLine originalLine = continuationLinesGroup[i];
originalLine.InitializeScanState(scanState);
TextArea textAreaForOriginalLine = textAreasForOriginalLinesInConcatenatedLine[i];
int concatenatedLineToOriginalLineOffset = startIndexForTextAreasInOriginalLines[i] - textAreaForOriginalLine.StartIndex;
foreach (Token token in virtualContinuationTokensLine.SourceTokens)
{
// Token located after the current line
if(token.StartIndex > textAreaForOriginalLine.EndIndex)
{
break;
}
// Token located before the current line
else if(token.StopIndex < textAreaForOriginalLine.StartIndex)
{
continue;
}
// Token completely completely included inside the current line
else if(token.StartIndex >= textAreaForOriginalLine.StartIndex && token.StopIndex <= textAreaForOriginalLine.EndIndex)
{
int startIndexInOriginalLine = token.StartIndex + concatenatedLineToOriginalLineOffset;
int stopIndexInOriginalLine = token.StopIndex + concatenatedLineToOriginalLineOffset;
token.CorrectTokensLine(originalLine, startIndexInOriginalLine, stopIndexInOriginalLine);
originalLine.AddToken(token);
foreach(Diagnostic diag in virtualContinuationTokensLine.GetDiagnosticsForToken(token))
{
originalLine.AddDiagnostic((MessageCode)diag.Info.Code, token, diag.MessageArgs);
}
}
// Multiline continuation token only partially located on this line
else
{
bool isContinuationFromPreviousLine = token.StartIndex < textAreaForOriginalLine.StartIndex;
bool isContinuedOnNextLine = token.StopIndex > textAreaForOriginalLine.EndIndex;
int startIndexInOriginalLine = 0;
if (isContinuationFromPreviousLine)
{
startIndexInOriginalLine = startIndexForTextAreasInOriginalLines[i] - offsetForLiteralContinuationInOriginalLines[i];
}
else
{
startIndexInOriginalLine = token.StartIndex + concatenatedLineToOriginalLineOffset;
}
int stopIndexInOriginalLine = 0;
if (isContinuedOnNextLine)
{
stopIndexInOriginalLine = originalLine.Source.EndIndex;
// If a continued line ends with a floating comment, the continued token ends just before the floating comment
if (originalLine.SourceTokens.Count > 0 && originalLine.SourceTokens[originalLine.SourceTokens.Count - 1].TokenType == TokenType.FloatingComment)
{
stopIndexInOriginalLine -= originalLine.SourceTokens[originalLine.SourceTokens.Count - 1].Length;
}
}
else
{
stopIndexInOriginalLine = token.StopIndex + concatenatedLineToOriginalLineOffset;
}
ContinuationToken continuationToken = new ContinuationToken(token, startIndexInOriginalLine, stopIndexInOriginalLine,
originalLine, isContinuationFromPreviousLine, isContinuedOnNextLine);
originalLine.AddToken(continuationToken);
// Copy diagnostics on the first line only
if(!isContinuationFromPreviousLine)
{
foreach (Diagnostic diag in virtualContinuationTokensLine.GetDiagnosticsForToken(token))
{
originalLine.AddDiagnostic((MessageCode)diag.Info.Code, token, diag.MessageArgs);
}
}
}
}
scanState = originalLine.ScanState;
}
}
public CompilationDocument(TextSourceInfo textSourceInfo, IEnumerable<ITextLine> initialTextLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider,
[CanBeNull] MultilineScanState scanState)
{
TextSourceInfo = textSourceInfo;
CompilerOptions = compilerOptions;
this.processedTokensDocumentProvider = processedTokensDocumentProvider;
// Initialize the compilation document lines
compilationDocumentLines = ImmutableList<CodeElementsLine>.Empty.ToBuilder();
// ... with the initial list of text lines received as a parameter
if (initialTextLines != null)
{
// Insert Cobol text lines in the internal tree structure
compilationDocumentLines.AddRange(initialTextLines.Select(textLine => CreateNewDocumentLine(textLine, textSourceInfo.ColumnsLayout)));
}
// Initialize document views versions
currentTextLinesVersion = new DocumentVersion<ICobolTextLine>(this);
currentTokensLinesVersion = new DocumentVersion<ITokensLine>(this);
// Initialize performance stats
PerfStatsForText = new PerfStatsForCompilationStep(CompilationStep.Text);
PerfStatsForScanner = new PerfStatsForCompilationStep(CompilationStep.Scanner);
PerfStatsForPreprocessor = new PerfStatsForCompilationStep(CompilationStep.Preprocessor);
initialScanStateForCopy = scanState;
}
