private Scanner(string line, int startIndex, int lastIndex, TokensLine tokensLine, TypeCobolOptions compilerOptions) { this.tokensLine = tokensLine; this.line = line; this.currentIndex = startIndex; this.lastIndex = lastIndex; this.compilerOptions = compilerOptions; }
public static string ScanLines(string[] testLines) { TokensLine[] tokensLines = new TokensLine[testLines.Length]; for (int i = 0; i < testLines.Length; i++) { TokensLine tokensLine = new TestTokensLine(testLines[i]); tokensLines[i] = tokensLine; } return(ScanLines(tokensLines)); }
public static string ScanLines(string[] testLines) { TokensLine[] tokensLines = new TokensLine[testLines.Length]; for(int i = 0; i < testLines.Length; i++) { TokensLine tokensLine = new TestTokensLine(testLines[i]); tokensLines[i] = tokensLine; } return ScanLines(tokensLines); }
private static IList <Tuple <string, bool> > Split(string line, int max, int min) { var lines = new List <Tuple <string, bool> >(); int nLine = (line.Length / max) + ((line.Length % max) != 0 ? 1 : 0); if (nLine == 1) { lines.Add(new Tuple <string, bool>(line, false)); return(lines); } if (line.Length < 1) { lines.Add(new Tuple <string, bool>(line, false)); } else { for (int i = 0; i < line.Length; i += (i == 0 ? max : min)) { lines.Add(new Tuple <string, bool>(line.Substring(i, Math.Min((i == 0 ? max : min), line.Length - i)), true)); } } TokensLine tempTokensLine = TokensLine.CreateVirtualLineForInsertedToken(0, line); tempTokensLine.InitializeScanState(new MultilineScanState(false, false, false, IBMCodePages.GetDotNetEncodingFromIBMCCSID(1147))); Scanner.Scanner scanner = new Scanner.Scanner(line, 0, line.Length - 1, tempTokensLine, null, false); Token t = null; int nCurLength = 0; int nSpan = max; int index = 0; bool bNextNoIndicator = false; while ((t = scanner.GetNextToken()) != null) { nCurLength += t.Length; if (nCurLength >= nSpan) { if (t.TokenFamily == TokenFamily.Whitespace || (nCurLength == nSpan)) { bNextNoIndicator = true; } nSpan += min; index++; } else if (bNextNoIndicator) { lines[index] = new Tuple <string, bool>(lines[index].Item1, false); bNextNoIndicator = false; } } return(lines); }
public static string ScanLines(TokensLine[] tokensLines) { ImmutableList<TokensLine>.Builder tokensLinesList = ImmutableList<TokensLine>.Empty.ToBuilder(); tokensLinesList.AddRange(tokensLines); ScannerStep.ScanDocument(TextSourceInfo, tokensLinesList, CompilerOptions); StringBuilder sbResult = new StringBuilder(); for (int i = 0; i < tokensLines.Length; i++) { sbResult.AppendLine("-- Line " + (i + 1) + " --"); sbResult.AppendLine(BuildResultString(tokensLines[i])); } return sbResult.ToString(); }
private ReplaceStatus TryAndReplace(Token nextToken, ReplaceOperation replaceOperation) { ReplaceStatus status = new ReplaceStatus(); IList <Token> originalMatchingTokens; #if EUROINFO_RULES if (CompilerOptions.UseEuroInformationLegacyReplacingSyntax) { // Support for legacy replacing syntax semantics : // Insert Suffix before the first '-' in all user defined words found in the COPY text // before copying it into the main program if (CopyReplacingDirective != null && CopyReplacingDirective.InsertSuffixChar && nextToken.TokenType == TokenType.UserDefinedWord) { string originalText = nextToken.Text; if (originalText.Contains(CopyReplacingDirective.PreSuffix)) { string replacedText = originalText.Replace(CopyReplacingDirective.PreSuffix, CopyReplacingDirective.PreSuffix.Insert(3, CopyReplacingDirective.Suffix)); TokensLine virtualTokensLine = TokensLine.CreateVirtualLineForInsertedToken(0, replacedText); Token replacementToken = new Token(TokenType.UserDefinedWord, 0, replacedText.Length - 1, virtualTokensLine); status.replacedToken = new ReplacedToken(replacementToken, nextToken); currentPosition.CurrentToken = status.replacedToken; } } } #endif if (replaceOperation != null && TryMatchReplaceOperation(nextToken, replaceOperation, out originalMatchingTokens)) { status.replacedToken = CreateReplacedTokens(nextToken, replaceOperation, originalMatchingTokens); if (status.replacedToken != null) { // REPLACE pattern matched => return the first replaced token currentPosition.CurrentToken = status.replacedToken; } else { // If the replacement token set is empty (REPLACE == ... = BY == ==), get next token and try again status.tryAgain = true; } } return(status); }
private static Nodes.TypeDefinition CreateCurrency() { var entry = new DataTypeDescriptionEntry(); entry.LevelNumber = new GeneratedIntegerValue(1); entry.DataName = new SymbolDefinition(new GeneratedAlphanumericValue("CURRENCY"), SymbolType.DataName); entry.Picture = new GeneratedAlphanumericValue(string.Format("{0}({1})", 'X', 3)); entry.DataType = DataType.Currency; var tokenLine = TokensLine.CreateVirtualLineForInsertedToken(entry.Line, "01 CURRENCY TYPEDEF STRICT PUBLIC PIC X(03)."); entry.ConsumedTokens.Add(new Token(TokenType.LevelNumber, 0, 1, tokenLine)); entry.ConsumedTokens.Add(new Token(TokenType.UserDefinedWord, 3, 10, tokenLine)); entry.ConsumedTokens.Add(new Token(TokenType.TYPEDEF, 12, 18, tokenLine)); entry.ConsumedTokens.Add(new Token(TokenType.STRICT, 20, 25, tokenLine)); entry.ConsumedTokens.Add(new Token(TokenType.PUBLIC, 27, 32, tokenLine)); entry.ConsumedTokens.Add(new Token(TokenType.PIC, 34, 36, tokenLine)); entry.ConsumedTokens.Add(new Token(TokenType.PictureCharacterString, 38, 42, tokenLine)); entry.ConsumedTokens.Add(new Token(TokenType.PeriodSeparator, 43, 43, tokenLine)); return(new Nodes.TypeDefinition(entry)); }
public static string ScanTextLine(TokensLine tokensLine) { TypeCobol.Compiler.Scanner.Scanner.ScanFirstLine(tokensLine, false, false, false, TextSourceInfo.EncodingForAlphanumericLiterals, CompilerOptions, CopyTextNameVariations); return(BuildResultString(tokensLine)); }
public static string ScanTextLine(TokensLine tokensLine) { TypeCobol.Compiler.Scanner.Scanner.ScanFirstLine(tokensLine, false, false, false, TextSourceInfo.EncodingForAlphanumericLiterals, CompilerOptions); return BuildResultString(tokensLine); }
private static void ScanTokensLineWithMultilineScanState(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IList<DocumentChange<ITokensLine>> tokensLinesChanges, out int nextLineToScanIndex, out TokensLine nextLineToScan) { // Scan the current line (or continuation lines group) MultilineScanState scanState = ScanTokensLineWithContinuations(lineToScanIndex, lineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan); // Scan the following lines until we find that the scan state at the beginning of the next line has been updated while(nextLineToScan != null && nextLineToScan.InitialScanState != null && !nextLineToScan.InitialScanState.Equals(scanState)) { scanState = ScanTokensLineWithContinuations(nextLineToScanIndex, nextLineToScan, textSourceInfo, documentLines, prepareDocumentLineForUpdate, compilerOptions, tokensLinesChanges, out nextLineToScanIndex, out nextLineToScan); } }
private static MultilineScanState ScanTokensLineWithContinuations(int lineToScanIndex, TokensLine lineToScan, TextSourceInfo textSourceInfo, ISearchableReadOnlyList<TokensLine> documentLines, PrepareDocumentLineForUpdate prepareDocumentLineForUpdate, TypeCobolOptions compilerOptions, IList<DocumentChange<ITokensLine>> tokensLinesChanges, out int nextLineToScanIndex, out TokensLine nextLineToScan) { // Initialize out parameters if (lineToScanIndex < (documentLines.Count - 1)) { nextLineToScanIndex = lineToScanIndex + 1; nextLineToScan = documentLines[nextLineToScanIndex]; } else { nextLineToScanIndex = -1; nextLineToScan = null; } // Check if the line to scan participates in a multiline continuation // - because it is itself a continuation line bool partOfMultilineContinuation = lineToScan.Type == CobolTextLineType.Continuation; // - or because the following line is a continuation line if (!partOfMultilineContinuation && lineToScanIndex < (documentLines.Count - 1)) { nextLineToScanIndex = lineToScanIndex + 1; nextLineToScan = documentLines[nextLineToScanIndex]; partOfMultilineContinuation = nextLineToScan.Type == CobolTextLineType.Continuation; } // ^-- LIMITATION : we don't support the case where one or more comment lines follow the current line before a continuation line // Case 1 : the line is not part of a multiline continuation => we can parse it as as a standalone line if (!partOfMultilineContinuation) { // Create a new copy of the line before the update if necessary lineToScan = (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner); if (lineToScanIndex == 0) { // Scan the first line of the document Scanner.ScanFirstLine(lineToScan, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions); } else { // Get the scan state at the end of the previous line TokensLine previousLine = documentLines[lineToScanIndex - 1]; // Scan the current line with this initial scan state Scanner.ScanTokensLine(lineToScan, previousLine.ScanState, compilerOptions); } tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineUpdated, lineToScanIndex, lineToScan)); return lineToScan.ScanState; } // Case 2 : the line is part of a multiline continuation => we must parse all continuation lines as a group else { // Build a list of the lines we will have to scan as a group : IList<TokensLine> continuationLinesGroup = new List<TokensLine>(); int firstLineIndex = lineToScanIndex; continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); // Navigate backwards to the start of the multiline continuation if (lineToScan.Type == CobolTextLineType.Continuation && lineToScanIndex > 0) { int revLineToScanIndex = lineToScanIndex; IEnumerator<TokensLine> reversedEnumerator = documentLines.GetEnumerator(lineToScanIndex - 1, -1, true); while(reversedEnumerator.MoveNext()) { // Get the previous line until a non continuation and non comment line is encountered revLineToScanIndex--; lineToScan = reversedEnumerator.Current; if(lineToScan.Type != CobolTextLineType.Continuation /*&& <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line lineToScan.Type != CobolTextLineType.Comment && lineToScan.Type != CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines { firstLineIndex = revLineToScanIndex; continuationLinesGroup.Insert(0, (TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); } else { break; } } } // Reuse our knowledge of the next line if it is available and if it is a continuation if (nextLineToScan != null && nextLineToScan.Type == CobolTextLineType.Continuation) { lineToScanIndex++; lineToScan = nextLineToScan; continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); nextLineToScanIndex = -1; nextLineToScan = null; } // Navigate forwards to the end of the multiline continuation if (lineToScanIndex < (documentLines.Count - 1)) { IEnumerator<TokensLine> enumerator = documentLines.GetEnumerator(lineToScanIndex + 1, -1, false); while (enumerator.MoveNext()) { // Get the next line until a non continuation and non comment line is encountered lineToScanIndex++; lineToScan = enumerator.Current; if (lineToScan.Type == CobolTextLineType.Continuation /*|| <-- LIMITATION : this compiler does not support comment or blank lines between two continuation line lineToScan.Type == CobolTextLineType.Comment || lineToScan.Type == CobolTextLineType.Blank*/) // see p54 : for continuation, blank lines are treated like comment lines { // Add this line at the end of the list of continuation lines continuationLinesGroup.Add((TokensLine)prepareDocumentLineForUpdate(lineToScanIndex, lineToScan, CompilationStep.Scanner)); } else { // Save the knowledge of the next line and exit the loop nextLineToScanIndex = lineToScanIndex; nextLineToScan = lineToScan; break; } } } // Scan the group of continuation lines if (firstLineIndex == 0) { // Scan the first line group of the document Scanner.ScanFirstLineContinuationGroup(continuationLinesGroup, false, false, false, textSourceInfo.EncodingForAlphanumericLiterals, compilerOptions); } else { // Get the scan state at the end of the previous line TokensLine previousLine = documentLines[firstLineIndex - 1]; // Scan the current line group with this initial scan state Scanner.ScanTokensLineContinuationGroup(continuationLinesGroup, previousLine.ScanState, compilerOptions); } int updatedLineIndex = firstLineIndex; foreach(TokensLine updatedLine in continuationLinesGroup) { tokensLinesChanges.Add(new DocumentChange<ITokensLine>(DocumentChangeType.LineUpdated, updatedLineIndex, updatedLine)); updatedLineIndex++; } return continuationLinesGroup[continuationLinesGroup.Count - 1].ScanState; } }
/// <summary> /// Scan a line of a document /// </summary> public static void ScanTokensLine(TokensLine tokensLine, MultilineScanState initialScanState, TypeCobolOptions compilerOptions) { // Updates are forbidden after a snapshot of a specific version of a line if(!tokensLine.CanStillBeUpdatedBy(CompilationStep.Scanner)) { throw new InvalidOperationException("Can not update this TokensLine because it was already frozen by compilation step : " + tokensLine.CompilationStep.ToString()); } // Set the initial scan state for the line tokensLine.InitializeScanState(initialScanState); // Alias to refer to Cobol text line properties ICobolTextLine textLine = tokensLine; // The source section of this line of text must be split into tokens string line = tokensLine.Text; int startIndex = textLine.Source.StartIndex; int lastIndex = textLine.Source.EndIndex; #if EUROINFO_LEGACY_REPLACING_SYNTAX if (IsInsideRemarks(textLine.Type, tokensLine.SourceText)) tokensLine.ScanState.InsideRemarksDirective = true; // Try to scan REMARKS compiler directive parameters inside the comment or non-comment line if (tokensLine.ScanState.InsideRemarksDirective) { string remarksLine = textLine.SourceText; int startIndexForSignificantPart = GetStartIndexOfSignificantPart(remarksLine, tokensLine.ScanState); int firstPeriodIndex = remarksLine.IndexOf('.', startIndexForSignificantPart); int endIndexForSignificantPart = GetEndIndexOfSignificantPart(remarksLine, tokensLine.ScanState, firstPeriodIndex); string significantPart = remarksLine.Substring(startIndexForSignificantPart, endIndexForSignificantPart - startIndexForSignificantPart + 1).Trim(); if (firstPeriodIndex >= 0 || (!tokensLine.ScanState.InsideRemarksParentheses && !remarksLine.Contains("COPY"))) { tokensLine.ScanState.InsideRemarksDirective = false; // indicates the end of the REMARKS compiler directive } RemarksDirective remarksDirective = CreateRemarksDirective(significantPart, tokensLine.ScanState); if (remarksDirective != null && remarksDirective.CopyTextNamesVariations.Count > 0) { // A non empty remarks directive will replace the comment line tokensLine.AddToken(CreateCompilerDirectiveToken(remarksDirective, tokensLine, startIndex, lastIndex)); return; } } #endif // Comment line => return only one token with type CommentLine // Debug line => treated as a comment line if debugging mode was not activated if (textLine.Type == CobolTextLineType.Comment || (textLine.Type == CobolTextLineType.Debug && !tokensLine.InitialScanState.WithDebuggingMode)) { Token commentToken = new Token(TokenType.CommentLine, startIndex, lastIndex, tokensLine); tokensLine.AddToken(commentToken); return; } // Invalid indicator, the line type is unknown => the whole line text is handled as a single invalid token else if (textLine.Type == CobolTextLineType.Invalid) { // Invalid indicator => register an error tokensLine.AddDiagnostic(MessageCode.InvalidIndicatorCharacter, textLine.Indicator.StartIndex, textLine.Indicator.EndIndex, textLine.Indicator); Token invalidToken = new Token(TokenType.InvalidToken, startIndex, lastIndex, tokensLine); tokensLine.AddToken(invalidToken); return; } // Empty line => return immediately an empty list of tokens // Blank line => return only one token with type SpaceSeparator if(textLine.Type == CobolTextLineType.Blank) { if(!String.IsNullOrEmpty(line)) { Token whitespaceToken = new Token(TokenType.SpaceSeparator, startIndex, lastIndex, tokensLine); tokensLine.AddToken(whitespaceToken); } return; } // Create a stateful line scanner, and iterate over the tokens Scanner scanner = new Scanner(line, startIndex, lastIndex, tokensLine, compilerOptions); Token nextToken = null; while((nextToken = scanner.GetNextToken()) != null) { // Resolve DELETE ambiguity : DELETE + InterLiteral => DELETE_CD // Warning : DELETE and the sequence-number-field must be on the same line if(nextToken.TokenType == TokenType.IntegerLiteral && tokensLine.ScanState.KeywordsState == KeywordsSequenceState.After_DELETE) { tokensLine.ScanState.LastKeywordOrSymbolToken.CorrectType(TokenType.DELETE_CD); } tokensLine.AddToken(nextToken); } }
/// <summary> /// Scan a line of a document when no previous scan state object is available /// </summary> public static void ScanFirstLine(TokensLine tokensLine, bool insideDataDivision, bool decimalPointIsComma, bool withDebuggingMode, Encoding encodingForAlphanumericLiterals, TypeCobolOptions compilerOptions) { MultilineScanState initialScanState = new MultilineScanState(insideDataDivision, decimalPointIsComma, withDebuggingMode, encodingForAlphanumericLiterals); ScanTokensLine(tokensLine, initialScanState, compilerOptions); }
private static Token CreateCompilerDirectiveToken(RemarksDirective remarksDirective, TokensLine tokensLine, int start, int end) { tokensLine.ScanState.AddCopyTextNamesVariations(remarksDirective.CopyTextNamesVariations); IList<Token> originalTokens = new List<Token>(1); originalTokens.Add(new Token(TokenType.CommentLine, start,end, tokensLine)); return new CompilerDirectiveToken(remarksDirective, originalTokens, false); }