Example #1
0
        /// <summary>
        /// Scan an isolated token in the following "default" context :
        /// - insideDataDivision = true
        /// - decimalPointIsComma = false
        /// - withDebuggingMode = false
        /// - encodingForAlphanumericLiterals = IBM 1147
        /// - default compiler options
        /// </summary>
        public static Token ScanIsolatedTokenInDefaultContext(string tokenText, out Diagnostic error)
        {
            TokensLine tempTokensLine = TokensLine.CreateVirtualLineForInsertedToken(0, tokenText);
            tempTokensLine.InitializeScanState(new MultilineScanState(true, false, false, IBMCodePages.GetDotNetEncodingFromIBMCCSID(1147)));

            Scanner tempScanner = new Scanner(tokenText, 0, tokenText.Length - 1, tempTokensLine, new TypeCobolOptions());
            Token candidateToken = tempScanner.GetNextToken();

            if(tempTokensLine.ScannerDiagnostics.Count > 0)
            {
                error = tempTokensLine.ScannerDiagnostics[0];
            }
            else
            {
                error = null;
            }
            return candidateToken;
        }
Example #2
0
        /// <summary>
        /// Scan a line of a document
        /// </summary>
        public static void ScanTokensLine(TokensLine tokensLine, MultilineScanState initialScanState, TypeCobolOptions compilerOptions)
        {
            // Updates are forbidden after a snapshot of a specific version of a line
            if(!tokensLine.CanStillBeUpdatedBy(CompilationStep.Scanner))
            {
                throw new InvalidOperationException("Can not update this TokensLine because it was already frozen by compilation step : " + tokensLine.CompilationStep.ToString());
            }

            // Set the initial scan state for the line
            tokensLine.InitializeScanState(initialScanState);

            // Alias to refer to Cobol text line properties
            ICobolTextLine textLine = tokensLine;

            // The source section of this line of text must be split into tokens
            string line = tokensLine.Text;
            int startIndex = textLine.Source.StartIndex;
            int lastIndex = textLine.Source.EndIndex;

            #if EUROINFO_LEGACY_REPLACING_SYNTAX
            if (IsInsideRemarks(textLine.Type, tokensLine.SourceText)) tokensLine.ScanState.InsideRemarksDirective = true;
            // Try to scan REMARKS compiler directive parameters inside the comment or non-comment line
            if (tokensLine.ScanState.InsideRemarksDirective) {
                string remarksLine = textLine.SourceText;

                int startIndexForSignificantPart = GetStartIndexOfSignificantPart(remarksLine, tokensLine.ScanState);
                int firstPeriodIndex = remarksLine.IndexOf('.', startIndexForSignificantPart);
                int endIndexForSignificantPart = GetEndIndexOfSignificantPart(remarksLine, tokensLine.ScanState, firstPeriodIndex);
                string significantPart = remarksLine.Substring(startIndexForSignificantPart, endIndexForSignificantPart - startIndexForSignificantPart + 1).Trim();

                if (firstPeriodIndex >= 0 || (!tokensLine.ScanState.InsideRemarksParentheses && !remarksLine.Contains("COPY"))) {
                    tokensLine.ScanState.InsideRemarksDirective = false; // indicates the end of the REMARKS compiler directive
                }

                RemarksDirective remarksDirective = CreateRemarksDirective(significantPart, tokensLine.ScanState);
                if (remarksDirective != null && remarksDirective.CopyTextNamesVariations.Count > 0) {
                    // A non empty remarks directive will replace the comment line
                    tokensLine.AddToken(CreateCompilerDirectiveToken(remarksDirective, tokensLine, startIndex, lastIndex));
                    return;
                }
            }

            #endif

            // Comment line => return only one token with type CommentLine
            // Debug line => treated as a comment line if debugging mode was not activated
            if (textLine.Type == CobolTextLineType.Comment ||
               (textLine.Type == CobolTextLineType.Debug && !tokensLine.InitialScanState.WithDebuggingMode))
            {
                Token commentToken = new Token(TokenType.CommentLine, startIndex, lastIndex, tokensLine);
                tokensLine.AddToken(commentToken);
                return;
            }
            // Invalid indicator, the line type is unknown => the whole line text is handled as a single invalid token
            else if (textLine.Type == CobolTextLineType.Invalid)
            {
                // Invalid indicator => register an error
                tokensLine.AddDiagnostic(MessageCode.InvalidIndicatorCharacter, textLine.Indicator.StartIndex, textLine.Indicator.EndIndex, textLine.Indicator);

                Token invalidToken = new Token(TokenType.InvalidToken, startIndex, lastIndex, tokensLine);
                tokensLine.AddToken(invalidToken);
                return;
            }
            // Empty line => return immediately an empty list of tokens
            // Blank line => return only one token with type SpaceSeparator
            if(textLine.Type == CobolTextLineType.Blank)
            {
                if(!String.IsNullOrEmpty(line))
                {
                    Token whitespaceToken = new Token(TokenType.SpaceSeparator, startIndex, lastIndex, tokensLine);
                    tokensLine.AddToken(whitespaceToken);
                }
                return;
            }

            // Create a stateful line scanner, and iterate over the tokens
            Scanner scanner = new Scanner(line, startIndex, lastIndex, tokensLine, compilerOptions);
            Token nextToken = null;
            while((nextToken = scanner.GetNextToken()) != null)
            {
                // Resolve DELETE ambiguity : DELETE + InterLiteral => DELETE_CD
                // Warning : DELETE and the sequence-number-field must be on the same line
                if(nextToken.TokenType == TokenType.IntegerLiteral && tokensLine.ScanState.KeywordsState == KeywordsSequenceState.After_DELETE)
                {
                    tokensLine.ScanState.LastKeywordOrSymbolToken.CorrectType(TokenType.DELETE_CD);
                }
                tokensLine.AddToken(nextToken);
            }
        }