/// <summary> /// Clone the current scanner state /// </summary> public MultilineScanState Clone() { MultilineScanState clone = new MultilineScanState(InsideDataDivision, DecimalPointIsComma, WithDebuggingMode, EncodingForAlphanumericLiterals); #if EUROINFO_LEGACY_REPLACING_SYNTAX clone.InsideRemarksDirective = InsideRemarksDirective; clone.InsideRemarksParentheses = InsideRemarksParentheses; if(CopyTextNamesVariations != null) { clone.CopyTextNamesVariations = CopyTextNamesVariations; } #endif if (SymbolicCharacters != null) { clone.SymbolicCharacters = new List<string>(SymbolicCharacters); } clone.KeywordsState = KeywordsState; if (LastToken != null) clone.LastToken = LastToken; if (LastKeywordOrSymbolToken != null) clone.LastKeywordOrSymbolToken = LastKeywordOrSymbolToken; return clone; }
private static void ScanTokensLineWithMultilineScanState(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IList<DocumentChange<ITokensLine>> tokensLinesChanges, MultilineScanState initialScanState, out int nextLineToScanIndex, out TokensLine nextLineToScan) { // Scan the current line (or continuation lines group) MultilineScanState scanState = ScanTokensLineWithContinuations(lineToScanIndex, lineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, initialScanState, out nextLineToScanIndex, out nextLineToScan); // Scan the following lines until we find that the scan state at the beginning of the next line has been updated while(nextLineToScan != null && nextLineToScan.InitialScanState != null && !nextLineToScan.InitialScanState.Equals(scanState)) { scanState = ScanTokensLineWithContinuations(nextLineToScanIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, scanState, out nextLineToScanIndex, out nextLineToScan); } }
internal void InitializeScanState(MultilineScanState initialScanState) { InitialScanState = initialScanState; ScanState = initialScanState != null?initialScanState.Clone() : null; }
private static MultilineScanState ScanTokensLineWithContinuations(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList <TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, List <RemarksDirective.TextNameVariation> copyTextNameVariations, IList <DocumentChange <ITokensLine> > tokensLinesChanges, [CanBeNull] MultilineScanState scanState, out int nextLineToScanIndex, out TokensLine nextLineToScan) { // Initialize out parameters if (lineToScanIndex < (documentLines.Count - 1)) { nextLineToScanIndex = lineToScanIndex + 1; nextLineToScan = documentLines[nextLineToScanIndex]; } else { nextLineToScanIndex = -1; nextLineToScan = null; } // Check if the line to scan participates in a multiline continuation // - because it is itself a continuation line bool partOfMultilineContinuation = lineToScan.Type == CobolTextLineType.Continuation; // - or because the following line is a continuation line if (!partOfMultilineContinuation && lineToScanIndex < (documentLines.Count - 1)) { nextLineToScanIndex = lineToScanIndex + 1; nextLineToScan = documentLines[nextLineToScanIndex]; partOfMultilineContinuation = nextLineToScan.Type == CobolTextLineType.Continuation; } // ^-- LIMITATION : we don't support the case where one or more comment lines follow the current line before a continuation line // Case 1 : the line is not part of a multiline continuation => we can parse it as as a standalone line if (!partOfMultilineContinuation) { // Create a new copy of the line before the update if necessary lineToScan = (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner); if (lineToScanIndex == 0) { if (scanState != null) { // Scan the first line of the document Scanner.ScanFirstLine(lineToScan, scanState.InsideDataDivision, scanState.DecimalPointIsComma, scanState.WithDebuggingMode, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations); } else { Scanner.ScanFirstLine(lineToScan, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations); } } else { // Get the scan state at the end of the previous line TokensLine previousLine = documentLines[lineToScanIndex - 1]; // Scan the current line with this initial scan state Scanner.ScanTokensLine(lineToScan, previousLine.ScanState, compilerOptions, copyTextNameVariations); } tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.LineUpdated, lineToScanIndex, lineToScan)); return(lineToScan.ScanState); } // Case 2 : the line is part of a multiline continuation => we must parse all continuation lines as a group else { // Build a list of the lines we will have to scan as a group : IList <TokensLine> continuationLinesGroup = new List <TokensLine>(); int firstLineIndex = lineToScanIndex; continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); // Navigate backwards to the start of the multiline continuation if (lineToScan.Type == CobolTextLineType.Continuation && lineToScanIndex > 0) { int revLineToScanIndex = lineToScanIndex; IEnumerator <TokensLine> reversedEnumerator = documentLines.GetEnumerator(lineToScanIndex - 1, -1, true); while (reversedEnumerator.MoveNext()) { // Get the previous line until a non continuation and non comment line is encountered revLineToScanIndex--; lineToScan = reversedEnumerator.Current; if (lineToScan.Type != CobolTextLineType.Continuation /*&& <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line * lineToScan.Type != CobolTextLineType.Comment && lineToScan.Type != CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines { firstLineIndex = revLineToScanIndex; continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); } else { break; } } } // Reuse our knowledge of the next line if it is available and if it is a continuation if (nextLineToScan != null && nextLineToScan.Type == CobolTextLineType.Continuation) { lineToScanIndex++; lineToScan = nextLineToScan; continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); nextLineToScanIndex = -1; nextLineToScan = null; } // Navigate forwards to the end of the multiline continuation if (lineToScanIndex < (documentLines.Count - 1)) { IEnumerator <TokensLine> enumerator = documentLines.GetEnumerator(lineToScanIndex + 1, -1, false); while (enumerator.MoveNext()) { // Get the next line until a non continuation and non comment line is encountered lineToScanIndex++; lineToScan = enumerator.Current; if (lineToScan.Type == CobolTextLineType.Continuation /*|| <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line * lineToScan.Type == CobolTextLineType.Comment || lineToScan.Type == CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines { // Add this line at the end of the list of continuation lines continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); } else { // Save the knowledge of the next line and exit the loop nextLineToScanIndex = lineToScanIndex; nextLineToScan = lineToScan; break; } } } // Scan the group of continuation lines if (firstLineIndex == 0) { // Scan the first line group of the document Scanner.ScanFirstLineContinuationGroup(continuationLinesGroup, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations); } else { // Get the scan state at the end of the previous line TokensLine previousLine = documentLines[firstLineIndex - 1]; // Scan the current line group with this initial scan state Scanner.ScanTokensLineContinuationGroup(continuationLinesGroup, previousLine.ScanState, compilerOptions, copyTextNameVariations); } int updatedLineIndex = firstLineIndex; foreach (TokensLine updatedLine in continuationLinesGroup) { tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.LineUpdated, updatedLineIndex, updatedLine)); updatedLineIndex++; } return(continuationLinesGroup[continuationLinesGroup.Count - 1].ScanState); } }
public static void ScanDocument(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <TokensLine> documentLines, TypeCobolOptions compilerOptions, List <RemarksDirective.TextNameVariation> copyTextNameVariations, [CanBeNull] MultilineScanState initialScanState) { TokensLine tokensLine = null; TokensLine nextTokensLine = null; MultilineScanState lastScanState = initialScanState; // Get the first line IEnumerator <TokensLine> documentLinesEnumerator = documentLines.GetEnumerator(); if (documentLinesEnumerator.MoveNext()) { tokensLine = documentLinesEnumerator.Current; } while (tokensLine != null) { // Peek the next line to look for continuations if (documentLinesEnumerator.MoveNext()) { nextTokensLine = documentLinesEnumerator.Current; } else { nextTokensLine = null; } // If no continuation is found, scan the current line if (nextTokensLine == null || nextTokensLine.Type != CobolTextLineType.Continuation) { if (lastScanState == null) { Scanner.ScanFirstLine(tokensLine, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations); } else { Scanner.ScanTokensLine(tokensLine, lastScanState, compilerOptions, copyTextNameVariations); } } // If a continuation is found on the next line, scan the continuation lines as a group else { // Build a list of the lines we will have to scan as a group : IList <TokensLine> continuationLinesGroup = new List <TokensLine>(); // Add current line continuationLinesGroup.Add(tokensLine); // Add next line tokensLine = nextTokensLine; nextTokensLine = null; continuationLinesGroup.Add(tokensLine); // Navigate forwards to the end of the multiline continuation while (documentLinesEnumerator.MoveNext()) { nextTokensLine = documentLinesEnumerator.Current; if (nextTokensLine.Type == CobolTextLineType.Continuation /*|| <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line * lineToScan.Type == CobolTextLineType.Comment || lineToScan.Type == CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines { // Add this line at the end of the list of continuation lines tokensLine = nextTokensLine; nextTokensLine = null; continuationLinesGroup.Add(tokensLine); } else { // Exit the loop break; } } // Scan the whole group of continuation lines if (lastScanState == null) { Scanner.ScanFirstLineContinuationGroup(continuationLinesGroup, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions, copyTextNameVariations); } else { Scanner.ScanTokensLineContinuationGroup(continuationLinesGroup, lastScanState, compilerOptions, copyTextNameVariations); } } lastScanState = tokensLine.ScanState; tokensLine = nextTokensLine; } }
internal static IList <DocumentChange <ITokensLine> > ScanTextLinesChanges(TextSourceInfo textSourceInfo, ISearchableReadOnlyList <TokensLine> documentLines, IList <DocumentChange <ICobolTextLine> > textLinesChanges, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, List <RemarksDirective.TextNameVariation> copyTextNameVariations, [CanBeNull] MultilineScanState scanState) { // Collect all changes applied to the tokens lines during the incremental scan IList <DocumentChange <ITokensLine> > tokensLinesChanges = new List <DocumentChange <ITokensLine> >(); // There are 3 reasons to scan a line after a text change : // 1. New text lines which were just inserted or updated must be scanned for the first time // 2. Text lines must be scanned again if their initial scan state changed : a new scan of the previous line can alter the scan state at the beginning of the following line // 3. Continuation lines and multiline tokens : if a line participates in a continuation on several lines, scan the group of lines as a whole // IMPORTANT : the text changes are ordered in increasing order of line index for (int textChangeIndex = 0; textChangeIndex < textLinesChanges.Count; textChangeIndex++) { // Local variables used to optimize navigation in the document int nextLineToScanIndex = -1; TokensLine nextLineToScan = null; // Update tokens depending on the current text change DocumentChange <ICobolTextLine> textChange = textLinesChanges[textChangeIndex]; if (textChange.Type == DocumentChangeType.DocumentCleared) { tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.DocumentCleared, 0, null)); continue; } else if (textChange.Type == DocumentChangeType.LineInserted || textChange.Type == DocumentChangeType.LineUpdated) { // We update lines as a group below, but we remember here which lines were inserted if (textChange.Type == DocumentChangeType.LineInserted) { tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.LineInserted, textChange.LineIndex, (ITokensLine)textChange.NewLine)); } // Text lines which were inserted or updated must be scanned again ScanTokensLineWithMultilineScanState(textChange.LineIndex, (TokensLine)textChange.NewLine, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, scanState, out nextLineToScanIndex, out nextLineToScan); } else if (textChange.Type == DocumentChangeType.LineRemoved) { tokensLinesChanges.Add(new DocumentChange <ITokensLine>(DocumentChangeType.LineRemoved, textChange.LineIndex, (ITokensLine)textChange.NewLine)); // Get the last line just before the line that was removed TokensLine previousLine = null; if (textChange.LineIndex > 0) { previousLine = documentLines[textChange.LineIndex - 1]; } // When a text line is removed : // - the previous line must be scanned again if the line which was removed was a member of a multiline continuation group if (previousLine != null && previousLine.HasTokenContinuedOnNextLine) { ScanTokensLineWithMultilineScanState(textChange.LineIndex - 1, previousLine, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, scanState, out nextLineToScanIndex, out nextLineToScan); } if (nextLineToScan == null && textChange.LineIndex < documentLines.Count) { nextLineToScanIndex = textChange.LineIndex; nextLineToScan = documentLines[nextLineToScanIndex]; } // - the next line must be scanned again if the scan state at the end of the previous line is different from the scan state at the beginning of the next line if (nextLineToScan != null && nextLineToScanIndex == textChange.LineIndex && previousLine != null && nextLineToScan.InitialScanState.Equals(previousLine.ScanState)) { ScanTokensLineWithMultilineScanState(textChange.LineIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, scanState, out nextLineToScanIndex, out nextLineToScan); } } // We can skip all text changes with an index smaller than the index of the last line which was already scanned if (nextLineToScan == null) { break; } else if (textChangeIndex < (textLinesChanges.Count - 1)) { int nextTextChangeIndex = textChangeIndex; DocumentChange <ICobolTextLine> nextTextChange = null; do { nextTextChangeIndex++; nextTextChange = textLinesChanges[nextTextChangeIndex]; }while (nextTextChangeIndex < (textLinesChanges.Count - 1) && nextTextChange.LineIndex <= nextLineToScanIndex); textChangeIndex = nextTextChangeIndex - 1; } } return(tokensLinesChanges); }
private static void ScanTokensLineWithMultilineScanState(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList <TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, List <RemarksDirective.TextNameVariation> copyTextNameVariations, IList <DocumentChange <ITokensLine> > tokensLinesChanges, MultilineScanState initialScanState, out int nextLineToScanIndex, out TokensLine nextLineToScan) { // Scan the current line (or continuation lines group) MultilineScanState scanState = ScanTokensLineWithContinuations(lineToScanIndex, lineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, initialScanState, out nextLineToScanIndex, out nextLineToScan); // Scan the following lines until we find that the scan state at the beginning of the next line has been updated while (nextLineToScan != null && nextLineToScan.InitialScanState != null && !nextLineToScan.InitialScanState.Equals(scanState)) { scanState = ScanTokensLineWithContinuations(nextLineToScanIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, copyTextNameVariations, tokensLinesChanges, scanState, out nextLineToScanIndex, out nextLineToScan); } }
internal void InitializeScanState(MultilineScanState initialScanState) { InitialScanState = initialScanState; ScanState = initialScanState.Clone(); }
/// <summary> /// Use a pre-existing text document, already initialized from a Cobol file /// </summary> public FileCompiler(string libraryName, string textName, SourceFileProvider sourceFileProvider, IProcessedTokensDocumentProvider documentProvider, ColumnsLayout columnsLayout, TypeCobolOptions compilerOptions, CodeModel.SymbolTable customSymbols, bool isCopyFile, MultilineScanState scanState) : this(libraryName, textName, null, sourceFileProvider, documentProvider, columnsLayout, null, compilerOptions, customSymbols, isCopyFile, scanState) { }
/// <summary> /// Scan a group of continuation lines when no previous scan state object is available /// </summary> public static void ScanFirstLineContinuationGroup(IList<TokensLine> continuationLinesGroup, bool insideDataDivision, bool decimalPointIsComma, bool withDebuggingMode, Encoding encodingForAlphanumericLiterals, TypeCobolOptions compilerOptions) { MultilineScanState initialScanState = new MultilineScanState(insideDataDivision, decimalPointIsComma, withDebuggingMode, encodingForAlphanumericLiterals); ScanTokensLineContinuationGroup(continuationLinesGroup, initialScanState, compilerOptions); }
/// <summary> /// Scan a line of a document /// </summary> public static void ScanTokensLine(TokensLine tokensLine, MultilineScanState initialScanState, TypeCobolOptions compilerOptions) { // Updates are forbidden after a snapshot of a specific version of a line if(!tokensLine.CanStillBeUpdatedBy(CompilationStep.Scanner)) { throw new InvalidOperationException("Can not update this TokensLine because it was already frozen by compilation step : " + tokensLine.CompilationStep.ToString()); } // Set the initial scan state for the line tokensLine.InitializeScanState(initialScanState); // Alias to refer to Cobol text line properties ICobolTextLine textLine = tokensLine; // The source section of this line of text must be split into tokens string line = tokensLine.Text; int startIndex = textLine.Source.StartIndex; int lastIndex = textLine.Source.EndIndex; #if EUROINFO_LEGACY_REPLACING_SYNTAX if (IsInsideRemarks(textLine.Type, tokensLine.SourceText)) tokensLine.ScanState.InsideRemarksDirective = true; // Try to scan REMARKS compiler directive parameters inside the comment or non-comment line if (tokensLine.ScanState.InsideRemarksDirective) { string remarksLine = textLine.SourceText; int startIndexForSignificantPart = GetStartIndexOfSignificantPart(remarksLine, tokensLine.ScanState); int firstPeriodIndex = remarksLine.IndexOf('.', startIndexForSignificantPart); int endIndexForSignificantPart = GetEndIndexOfSignificantPart(remarksLine, tokensLine.ScanState, firstPeriodIndex); string significantPart = remarksLine.Substring(startIndexForSignificantPart, endIndexForSignificantPart - startIndexForSignificantPart + 1).Trim(); if (firstPeriodIndex >= 0 || (!tokensLine.ScanState.InsideRemarksParentheses && !remarksLine.Contains("COPY"))) { tokensLine.ScanState.InsideRemarksDirective = false; // indicates the end of the REMARKS compiler directive } RemarksDirective remarksDirective = CreateRemarksDirective(significantPart, tokensLine.ScanState); if (remarksDirective != null && remarksDirective.CopyTextNamesVariations.Count > 0) { // A non empty remarks directive will replace the comment line tokensLine.AddToken(CreateCompilerDirectiveToken(remarksDirective, tokensLine, startIndex, lastIndex)); return; } } #endif // Comment line => return only one token with type CommentLine // Debug line => treated as a comment line if debugging mode was not activated if (textLine.Type == CobolTextLineType.Comment || (textLine.Type == CobolTextLineType.Debug && !tokensLine.InitialScanState.WithDebuggingMode)) { Token commentToken = new Token(TokenType.CommentLine, startIndex, lastIndex, tokensLine); tokensLine.AddToken(commentToken); return; } // Invalid indicator, the line type is unknown => the whole line text is handled as a single invalid token else if (textLine.Type == CobolTextLineType.Invalid) { // Invalid indicator => register an error tokensLine.AddDiagnostic(MessageCode.InvalidIndicatorCharacter, textLine.Indicator.StartIndex, textLine.Indicator.EndIndex, textLine.Indicator); Token invalidToken = new Token(TokenType.InvalidToken, startIndex, lastIndex, tokensLine); tokensLine.AddToken(invalidToken); return; } // Empty line => return immediately an empty list of tokens // Blank line => return only one token with type SpaceSeparator if(textLine.Type == CobolTextLineType.Blank) { if(!String.IsNullOrEmpty(line)) { Token whitespaceToken = new Token(TokenType.SpaceSeparator, startIndex, lastIndex, tokensLine); tokensLine.AddToken(whitespaceToken); } return; } // Create a stateful line scanner, and iterate over the tokens Scanner scanner = new Scanner(line, startIndex, lastIndex, tokensLine, compilerOptions); Token nextToken = null; while((nextToken = scanner.GetNextToken()) != null) { // Resolve DELETE ambiguity : DELETE + InterLiteral => DELETE_CD // Warning : DELETE and the sequence-number-field must be on the same line if(nextToken.TokenType == TokenType.IntegerLiteral && tokensLine.ScanState.KeywordsState == KeywordsSequenceState.After_DELETE) { tokensLine.ScanState.LastKeywordOrSymbolToken.CorrectType(TokenType.DELETE_CD); } tokensLine.AddToken(nextToken); } }
private static int GetStartIndexOfSignificantPart(string line, MultilineScanState state) { int start = Math.Max(line.IndexOf(' ') +1, line.IndexOf('=') +1); if (!state.InsideRemarksParentheses) { int firstLParenIndex = line.IndexOf('('); state.InsideRemarksParentheses = (firstLParenIndex >= 0); start = Math.Max(start, firstLParenIndex +1); } return start; }
private static int GetEndIndexOfSignificantPart(string line, MultilineScanState state, int firstPeriodIndex) { int end = line.Length -1; if (state.InsideRemarksParentheses) { int firstRParenIndex = line.IndexOf(')'); if (firstRParenIndex >= 0) { end = firstRParenIndex -1; state.InsideRemarksParentheses = false; } if (firstPeriodIndex >= 0 && firstPeriodIndex < firstRParenIndex) end = firstPeriodIndex - 1; } return end; }
private static RemarksDirective CreateRemarksDirective(string significantPart, MultilineScanState state) { if (significantPart.Length < 1) return null; var remarksDirective = new RemarksDirective(); foreach (string candidateName in significantPart.Split(' ')) { if (candidateName.Length == 7 || candidateName.Length == 8) { RemarksDirective.TextNameVariation textName = new RemarksDirective.TextNameVariation(candidateName); remarksDirective.CopyTextNamesVariations.Add(textName); } else if (!String.IsNullOrWhiteSpace(candidateName)) { // A string which is not a text name is an error : stop scanning here remarksDirective = null; state.InsideRemarksDirective = false; break; } } return remarksDirective; }
/// <summary> /// Scan a group of continuation lines /// </summary> public static void ScanTokensLineContinuationGroup(IList<TokensLine> continuationLinesGroup, MultilineScanState initialScanState, TypeCobolOptions compilerOptions) { // p54: Continuation lines // Any sentence, entry, clause, or phrase that requires more than one line can be // continued in Area B of the next line that is neither a comment line nor a blank line. // The line being continued is a continued line; the succeeding lines are continuation // lines. // Track the length of text contributed to the continuation text by each individual line TextArea[] textAreasForOriginalLinesInConcatenatedLine = new TextArea[continuationLinesGroup.Count]; int[] startIndexForTextAreasInOriginalLines = new int[continuationLinesGroup.Count]; int[] offsetForLiteralContinuationInOriginalLines = new int[continuationLinesGroup.Count]; // Initialize the continuation text with the complete source text of the first line TokensLine firstLine = continuationLinesGroup[0]; string concatenatedLine = null; if (firstLine.Type == CobolTextLineType.Source || (firstLine.Type == CobolTextLineType.Debug && initialScanState.WithDebuggingMode)) { concatenatedLine = firstLine.SourceText; } else { concatenatedLine = String.Empty; Scanner.ScanTokensLine(firstLine, initialScanState, compilerOptions); } textAreasForOriginalLinesInConcatenatedLine[0] = new TextArea(TextAreaType.Source, 0, concatenatedLine.Length -1); startIndexForTextAreasInOriginalLines[0] = firstLine.Source.StartIndex; offsetForLiteralContinuationInOriginalLines[0] = 0; // All the following lines are continuation lines // => build a character string representing the complete continuation text along the way for (int i = 1; i < continuationLinesGroup.Count; i++) { TokensLine continuationLine = continuationLinesGroup[i]; int startIndex = continuationLine.Source.StartIndex; int lastIndex = continuationLine.Source.EndIndex; string line = continuationLine.Text; // 1. Match and remove all blank characters at the beginning of the continuation line int startOfContinuationIndex = startIndex; for (; startOfContinuationIndex <= lastIndex && line[startOfContinuationIndex] == ' '; startOfContinuationIndex++) { } if (startOfContinuationIndex > startIndex) { Token whitespaceToken = new Token(TokenType.SpaceSeparator, startIndex, startOfContinuationIndex - 1, continuationLine); continuationLine.SourceTokens.Add(whitespaceToken); startIndex = startOfContinuationIndex; } if (startOfContinuationIndex <= lastIndex) { if (startOfContinuationIndex < 4) { continuationLine.AddDiagnostic(MessageCode.AreaAOfContinuationLineMustBeBlank, startOfContinuationIndex, startOfContinuationIndex); } } else { // Nothing but spaces on the continuation line continue; } // p55: Continuation of alphanumeric and national literals // Alphanumeric and national literals can be continued only when there are no DBCS // characters in the content of the literal. // The following rules apply to alphanumeric and national literals that do not contain // DBCS characters: // - If the continued line contains an alphanumeric or national literal without a // closing quotation mark, all spaces at the end of the continued line (through // column 72) are considered to be part of the literal. The continuation line must // contain a hyphen in the indicator area, and the first nonblank character must be // a quotation mark. The continuation of the literal begins with the character // immediately following the quotation mark. // - If an alphanumeric or national literal that is to be continued on the next line has // as its last character a quotation mark in column 72, the continuation line must // start with two consecutive quotation marks. This will result in a single quotation // mark as part of the value of the literal. // - If the last character on the continued line of an alphanumeric or national literal // is a single quotation mark in Area B, the continuation line can start with a single // quotation mark. This will result in two consecutive literals instead of one // continued literal. // The rules are the same when an apostrophe is used instead of a quotation mark in // delimiters. // ... p55 -> p56: examples of continuations and expected behavior ... int offsetForLiteralContinuation = 0; if (concatenatedLine.Length > 0) { // Scan the continuation text, and get its last token so far TokensLine temporaryTokensLine = TokensLine.CreateVirtualLineForInsertedToken(firstLine.InitialLineIndex, concatenatedLine); Scanner.ScanTokensLine(temporaryTokensLine, initialScanState, compilerOptions); Token lastTokenOfConcatenatedLineSoFar = temporaryTokensLine.SourceTokens[temporaryTokensLine.SourceTokens.Count - 1]; // Check if the last token so far is an alphanumeric or national literal if (lastTokenOfConcatenatedLineSoFar.TokenFamily == TokenFamily.AlphanumericLiteral) { // The continuation line must contain a hyphen in the indicator area, and the first nonblank character must be a quotation mark if (line[startOfContinuationIndex] != lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter) { continuationLine.AddDiagnostic(MessageCode.InvalidFirstCharForContinuationLine, startOfContinuationIndex, startOfContinuationIndex, lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter); } // The continuation of the literal begins with the character immediately following the quotation mark. else { offsetForLiteralContinuation = 1; // If an alphanumeric literal that is to be continued on the next line has as its last character a quotation mark in column 72, // the continuation line must start with two consecutive quotation marks. if (lastTokenOfConcatenatedLineSoFar.HasClosingDelimiter) { if ((startOfContinuationIndex + 1) > lastIndex || line[startOfContinuationIndex + 1] != lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter) { continuationLine.AddDiagnostic(MessageCode.InvalidFirstTwoCharsForContinuationLine, startOfContinuationIndex, startOfContinuationIndex + 1, lastTokenOfConcatenatedLineSoFar.ExpectedClosingDelimiter); // Use the first quotation mark to avoid a complete mess while scanning the rest of the line offsetForLiteralContinuation = 0; } } } } // Check if the last token so far is a floating comment else if (lastTokenOfConcatenatedLineSoFar.TokenType == TokenType.FloatingComment) { // => remove the floating comment from the text of the continuation concatenatedLine = concatenatedLine.Substring(0, concatenatedLine.Length - lastTokenOfConcatenatedLineSoFar.Length); textAreasForOriginalLinesInConcatenatedLine[i - 1] = new TextArea(TextAreaType.Source, textAreasForOriginalLinesInConcatenatedLine[i - 1].StartIndex, textAreasForOriginalLinesInConcatenatedLine[i - 1].EndIndex - lastTokenOfConcatenatedLineSoFar.Length); TokensLine lineWithFloatingComment = continuationLinesGroup[i - 1]; Token floatingCommentToken = new Token(TokenType.FloatingComment, lineWithFloatingComment.Length - lastTokenOfConcatenatedLineSoFar.Length, lineWithFloatingComment.Length - 1, lineWithFloatingComment); lineWithFloatingComment.SourceTokens.Add(floatingCommentToken); } // Check if the last token so far is a comment entry else if (lastTokenOfConcatenatedLineSoFar.TokenType == TokenType.CommentEntry) { // p105: A hyphen in the indicator area (column 7) is not permitted in comment - entries. // => impossible to ignore the continuation indicator here, it is too late // (we can not know there is a comment entry before scanning the continuation lines groups) // => register an error message continuationLine.AddDiagnostic(MessageCode.HyphenIndicatorNotPermittedInCommenEntries, continuationLine.Indicator.StartIndex + 1, continuationLine.Indicator.EndIndex + 1); } } // p54: If there is no hyphen (-) in the indicator area (column 7) of a line, the last character // of the preceding line is assumed to be followed by a space. // If there is a hyphen in the indicator area of a line, the first nonblank character of // the continuation line immediately follows the last nonblank character of the // continued line without an intervening space. // Concatenate the continuation text so far with the text of the current continuation line int startIndexOfContinuationStringInContinuationLine = startOfContinuationIndex + offsetForLiteralContinuation; int lengthOfContinuationStringInContinuationLine = lastIndex - startIndexOfContinuationStringInContinuationLine + 1; textAreasForOriginalLinesInConcatenatedLine[i] = new TextArea(TextAreaType.Source, concatenatedLine.Length, concatenatedLine.Length + lengthOfContinuationStringInContinuationLine - 1); startIndexForTextAreasInOriginalLines[i] = startIndexOfContinuationStringInContinuationLine; offsetForLiteralContinuationInOriginalLines[i] = offsetForLiteralContinuation; concatenatedLine += line.Substring(startIndexOfContinuationStringInContinuationLine, lengthOfContinuationStringInContinuationLine); } // Scan the complete continuation text as a whole TokensLine virtualContinuationTokensLine = TokensLine.CreateVirtualLineForInsertedToken(firstLine.InitialLineIndex, concatenatedLine); Scanner.ScanTokensLine(virtualContinuationTokensLine, initialScanState, compilerOptions); // Then attribute each token and diagnostic to its corresponding tokens line MultilineScanState scanState = initialScanState; for (int i = 0; i < continuationLinesGroup.Count; i++) { TokensLine originalLine = continuationLinesGroup[i]; originalLine.InitializeScanState(scanState); TextArea textAreaForOriginalLine = textAreasForOriginalLinesInConcatenatedLine[i]; int concatenatedLineToOriginalLineOffset = startIndexForTextAreasInOriginalLines[i] - textAreaForOriginalLine.StartIndex; foreach (Token token in virtualContinuationTokensLine.SourceTokens) { // Token located after the current line if(token.StartIndex > textAreaForOriginalLine.EndIndex) { break; } // Token located before the current line else if(token.StopIndex < textAreaForOriginalLine.StartIndex) { continue; } // Token completely completely included inside the current line else if(token.StartIndex >= textAreaForOriginalLine.StartIndex && token.StopIndex <= textAreaForOriginalLine.EndIndex) { int startIndexInOriginalLine = token.StartIndex + concatenatedLineToOriginalLineOffset; int stopIndexInOriginalLine = token.StopIndex + concatenatedLineToOriginalLineOffset; token.CorrectTokensLine(originalLine, startIndexInOriginalLine, stopIndexInOriginalLine); originalLine.AddToken(token); foreach(Diagnostic diag in virtualContinuationTokensLine.GetDiagnosticsForToken(token)) { originalLine.AddDiagnostic((MessageCode)diag.Info.Code, token, diag.MessageArgs); } } // Multiline continuation token only partially located on this line else { bool isContinuationFromPreviousLine = token.StartIndex < textAreaForOriginalLine.StartIndex; bool isContinuedOnNextLine = token.StopIndex > textAreaForOriginalLine.EndIndex; int startIndexInOriginalLine = 0; if (isContinuationFromPreviousLine) { startIndexInOriginalLine = startIndexForTextAreasInOriginalLines[i] - offsetForLiteralContinuationInOriginalLines[i]; } else { startIndexInOriginalLine = token.StartIndex + concatenatedLineToOriginalLineOffset; } int stopIndexInOriginalLine = 0; if (isContinuedOnNextLine) { stopIndexInOriginalLine = originalLine.Source.EndIndex; // If a continued line ends with a floating comment, the continued token ends just before the floating comment if (originalLine.SourceTokens.Count > 0 && originalLine.SourceTokens[originalLine.SourceTokens.Count - 1].TokenType == TokenType.FloatingComment) { stopIndexInOriginalLine -= originalLine.SourceTokens[originalLine.SourceTokens.Count - 1].Length; } } else { stopIndexInOriginalLine = token.StopIndex + concatenatedLineToOriginalLineOffset; } ContinuationToken continuationToken = new ContinuationToken(token, startIndexInOriginalLine, stopIndexInOriginalLine, originalLine, isContinuationFromPreviousLine, isContinuedOnNextLine); originalLine.AddToken(continuationToken); // Copy diagnostics on the first line only if(!isContinuationFromPreviousLine) { foreach (Diagnostic diag in virtualContinuationTokensLine.GetDiagnosticsForToken(token)) { originalLine.AddDiagnostic((MessageCode)diag.Info.Code, token, diag.MessageArgs); } } } } scanState = originalLine.ScanState; } }
public CompilationDocument(TextSourceInfo textSourceInfo, IEnumerable<ITextLine> initialTextLines, TypeCobolOptions compilerOptions, IProcessedTokensDocumentProvider processedTokensDocumentProvider, [CanBeNull] MultilineScanState scanState) { TextSourceInfo = textSourceInfo; CompilerOptions = compilerOptions; this.processedTokensDocumentProvider = processedTokensDocumentProvider; // Initialize the compilation document lines compilationDocumentLines = ImmutableList<CodeElementsLine>.Empty.ToBuilder(); // ... with the initial list of text lines received as a parameter if (initialTextLines != null) { // Insert Cobol text lines in the internal tree structure compilationDocumentLines.AddRange(initialTextLines.Select(textLine => CreateNewDocumentLine(textLine, textSourceInfo.ColumnsLayout))); } // Initialize document views versions currentTextLinesVersion = new DocumentVersion<ICobolTextLine>(this); currentTokensLinesVersion = new DocumentVersion<ITokensLine>(this); // Initialize performance stats PerfStatsForText = new PerfStatsForCompilationStep(CompilationStep.Text); PerfStatsForScanner = new PerfStatsForCompilationStep(CompilationStep.Scanner); PerfStatsForPreprocessor = new PerfStatsForCompilationStep(CompilationStep.Preprocessor); initialScanStateForCopy = scanState; }