示例#1
0
        public void ReturnCharsInCorrectOrder()
        {
            var text = "abc";
            var sut  = new TextWindow(text);

            Assert.Equal('a', sut.PeekChar());
            sut.ConsumeChar();
            Assert.Equal('b', sut.PeekChar());
            sut.ConsumeChar();
            Assert.Equal('c', sut.PeekChar());
            sut.ConsumeChar();
            Assert.True(sut.IsEof());
        }
示例#2
0
 private void ScanISLNestedVerbatimString(ref SyntaxDiagnosticInfo error)
 {
     Debug.Assert(TextWindow.PeekChar() == '@');
     TextWindow.AdvanceChar();
     Debug.Assert(TextWindow.PeekChar() == '\"');
     TextWindow.AdvanceChar(); // move past quote
     while (true)
     {
         if (IsAtEnd())
         {
             // we'll get an error in the enclosing construct
             return;
         }
         char ch = TextWindow.PeekChar();
         TextWindow.AdvanceChar();
         if (ch == '\"')
         {
             if (TextWindow.PeekChar(1) == '\"')
             {
                 TextWindow.AdvanceChar(); // move past escaped quote
             }
             else
             {
                 return;
             }
         }
     }
 }
示例#3
0
        private void ScanSingleLineRawStringLiteral(ref TokenInfo info, int startingQuoteCount)
        {
            info.Kind = SyntaxKind.SingleLineRawStringLiteralToken;

            while (true)
            {
                var currentChar = TextWindow.PeekChar();

                // See if we reached the end of the line or file before hitting the end.
                if (SyntaxFacts.IsNewLine(currentChar))
                {
                    this.AddError(TextWindow.Position, width: TextWindow.GetNewLineWidth(), ErrorCode.ERR_UnterminatedRawString);
                    return;
                }
                else if (IsAtEndOfText(currentChar))
                {
                    this.AddError(TextWindow.Position, width: 0, ErrorCode.ERR_UnterminatedRawString);
                    return;
                }

                if (currentChar != '"')
                {
                    // anything not a quote sequence just moves it forward.
                    TextWindow.AdvanceChar();
                    continue;
                }

                var beforeEndDelimiter = TextWindow.Position;
                var currentQuoteCount  = ConsumeQuoteSequence();

                // A raw string literal starting with some number of quotes can contain a quote sequence with fewer quotes.
                if (currentQuoteCount < startingQuoteCount)
                {
                    continue;
                }

                // A raw string could never be followed by another string.  So once we've consumed all the closing quotes
                // if we have any more closing quotes then that's an error we can give a message for.
                if (currentQuoteCount > startingQuoteCount)
                {
                    var excessQuoteCount = currentQuoteCount - startingQuoteCount;
                    this.AddError(
                        position: TextWindow.Position - excessQuoteCount,
                        width: excessQuoteCount,
                        ErrorCode.ERR_TooManyQuotesForRawString);
                }

                // We have enough quotes to finish this string at this point.
                var afterStartDelimiter = TextWindow.LexemeStartPosition + startingQuoteCount;
                var valueLength         = beforeEndDelimiter - afterStartDelimiter;

                info.StringValue = TextWindow.GetText(
                    position: afterStartDelimiter,
                    length: valueLength,
                    intern: true);
                return;
            }
        }
示例#4
0
        private void AddMultiLineRawStringLiteralLineContents(
            StringBuilder indentationWhitespace,
            StringBuilder currentLineWhitespace,
            bool firstContentLine)
        {
            Debug.Assert(SyntaxFacts.IsNewLine(TextWindow.PeekChar()));

            var newLineWidth = TextWindow.GetNewLineWidth();

            for (var i = 0; i < newLineWidth; i++)
            {
                // the initial newline in `"""   \r\n` is not added to the contents.
                if (!firstContentLine)
                {
                    _builder.Append(TextWindow.PeekChar());
                }

                TextWindow.AdvanceChar();
            }

            var lineStartPosition = TextWindow.Position;

            currentLineWhitespace.Clear();
            ConsumeWhitespace(currentLineWhitespace);

            if (!StartsWith(currentLineWhitespace, indentationWhitespace))
            {
                // We have a line where the indentation of that line isn't a prefix of indentation
                // whitespace.
                //
                // If we're not on a blank line then this is bad.  That's a content line that doesn't start
                // with the indentation whitespace.  If we are on a blank line then it's ok if the whitespace
                // we do have is a prefix of the indentation whitespace.
                var isBlankLine      = SyntaxFacts.IsNewLine(TextWindow.PeekChar());
                var isLegalBlankLine = isBlankLine && StartsWith(indentationWhitespace, currentLineWhitespace);
                if (!isLegalBlankLine)
                {
                    // Specialized error message if this is a spacing difference.
                    if (CheckForSpaceDifference(
                            currentLineWhitespace, indentationWhitespace,
                            out var currentLineWhitespaceChar, out var indentationWhitespaceChar))
                    {
                        this.AddError(
                            lineStartPosition,
                            width: TextWindow.Position - lineStartPosition,
                            ErrorCode.ERR_LineContainsDifferentWhitespace,
                            currentLineWhitespaceChar, indentationWhitespaceChar);
                    }
                    else
                    {
                        this.AddError(
                            lineStartPosition,
                            width: TextWindow.Position - lineStartPosition,
                            ErrorCode.ERR_LineDoesNotStartWithSameWhitespace);
                    }
                    return;
                }
            }
示例#5
0
        private void ScanISLContents(ArrayBuilder <Interpolation> interpolations, ref SyntaxDiagnosticInfo error)
        {
            while (true)
            {
                if (IsAtEnd())
                {
                    // error: end of line before end of string
                    return;
                }
                switch (TextWindow.PeekChar())
                {
                case '\"':
                    // found the end of the string
                    return;

                case '\\':
                    TextWindow.AdvanceChar();
                    if (IsAtEnd())
                    {
                        // the caller will complain about unclosed quote
                        return;
                    }
                    else if (TextWindow.PeekChar() == '{')
                    {
                        int interpolationStart = TextWindow.Position;
                        TextWindow.AdvanceChar();
                        ScanISLHoleBalancedText('}', true, ref error);
                        int end = TextWindow.Position;
                        if (TextWindow.PeekChar() == '}')
                        {
                            TextWindow.AdvanceChar();
                        }
                        else
                        {
                            if (error == null)
                            {
                                error = MakeError(interpolationStart - 1, 2, ErrorCode.ERR_UnclosedExpressionHole);
                            }
                        }
                        if (interpolations != null)
                        {
                            interpolations.Add(new Interpolation(interpolationStart, end));
                        }
                    }
                    else
                    {
                        TextWindow.AdvanceChar();     // skip past a single escaped character
                    }
                    continue;

                default:
                    // found some other character in the string portion
                    TextWindow.AdvanceChar();
                    continue;
                }
            }
        }
示例#6
0
        /// <returns>The number of quotes that were consumed</returns>
        private int ConsumeCharSequence(char ch)
        {
            var start = TextWindow.Position;

            while (TextWindow.PeekChar() == ch)
            {
                TextWindow.AdvanceChar();
            }

            return(TextWindow.Position - start);
        }
示例#7
0
        /// <summary>
        /// Returns an appropriate error code if scanning this verbatim literal ran into an error.
        /// </summary>
        private ErrorCode?ScanVerbatimStringLiteral(ref TokenInfo info, bool allowNewlines)
        {
            _builder.Length = 0;

            Debug.Assert(TextWindow.PeekChar() == '@' && TextWindow.PeekChar(1) == '"');
            TextWindow.AdvanceChar(2);

            ErrorCode?error = null;

            while (true)
            {
                var ch = TextWindow.PeekChar();
                if (ch == '"')
                {
                    TextWindow.AdvanceChar();
                    if (TextWindow.PeekChar() == '"')
                    {
                        // Doubled quote -- skip & put the single quote in the string and keep going.
                        TextWindow.AdvanceChar();
                        _builder.Append(ch);
                        continue;
                    }

                    // otherwise, the string is finished.
                    break;
                }

                if (ch == SlidingTextWindow.InvalidCharacter && TextWindow.IsReallyAtEnd())
                {
                    // Reached the end of the source without finding the end-quote.  Give an error back at the
                    // starting point. And finish lexing this string.
                    error ??= ErrorCode.ERR_UnterminatedStringLit;
                    break;
                }

                // If we hit a new line when it's not allowed.  Give an error at that new line, but keep on consuming
                // the verbatim literal to the end to avoid the contents of the string being lexed as C# (which will
                // cause a ton of cascaded errors).  Only need to do this on the first newline we hit.
                if (!allowNewlines && SyntaxFacts.IsNewLine(ch))
                {
                    error ??= ErrorCode.ERR_NewlinesAreNotAllowedInsideANonVerbatimInterpolatedString;
                }

                TextWindow.AdvanceChar();
                _builder.Append(ch);
            }

            info.Kind        = SyntaxKind.StringLiteralToken;
            info.Text        = TextWindow.GetText(intern: false);
            info.StringValue = _builder.ToString();

            return(error);
        }
示例#8
0
        private char ScanEscapeSequence()
        {
            var start = TextWindow.Position;

            TextWindow.AdvanceChar();
            var ch = TextWindow.PeekChar();

            switch (ch)
            {
            case '\'':
            case '\"':
            case '\\':
                break;

            case 'a':
                ch = '\u0007';
                break;

            case 'b':
                ch = '\u0008';
                break;

            case 'f':
                ch = '\u000c';
                break;

            case 'n':
                ch = '\u000a';
                break;

            case 'r':
                ch = '\u000d';
                break;

            case 't':
                ch = '\u0009';
                break;

            case 'v':
                ch = '\u000b';
                break;

            case '0':
                ch = '\u0000';
                break;

            default:
                throw new Exception();
            }

            return(ch);
        }
示例#9
0
 void ScanISLHoleBracketed(char start, char end, ref SyntaxDiagnosticInfo error)
 {
     Debug.Assert(start == TextWindow.PeekChar());
     TextWindow.AdvanceChar();
     ScanISLHoleBalancedText(end, false, ref error);
     if (TextWindow.PeekChar() == end)
     {
         TextWindow.AdvanceChar();
     }
     else
     {
         // an error was given by the caller
     }
 }
示例#10
0
        private void ConsumeWhitespace(StringBuilder?builder)
        {
            while (true)
            {
                var ch = TextWindow.PeekChar();
                if (!SyntaxFacts.IsWhitespace(ch))
                {
                    break;
                }

                builder?.Append(ch);
                TextWindow.AdvanceChar();
            }
        }
示例#11
0
        private void ScanRawStringLiteral(ref TokenInfo info)
        {
            _builder.Length = 0;

            var startingQuoteCount = ConsumeQuoteSequence();

            Debug.Assert(startingQuoteCount >= 3);

            // Keep consuming whitespace after the initial quote sequence.
            ConsumeWhitespace(builder: null);

            if (SyntaxFacts.IsNewLine(TextWindow.PeekChar()))
            {
                // Past the initial whitespace, and we hit a newline, this is a multi line raw string literal.
                ScanMultiLineRawStringLiteral(ref info, startingQuoteCount);
            }
            else
            {
                // Past the initial whitespace, and we hit anything else, this is a single line raw string literal.
                ScanSingleLineRawStringLiteral(ref info, startingQuoteCount);
            }

            // If we encounter any errors while scanning this raw string then we can't really determine the true
            // value of the string.  So just do what we do with the normal strings and treat the contents as the
            // value from after the starting quote to the current position.  Note that for normal strings this will
            // have interpreted things like escape sequences.  However, as we're a raw string and there are no
            // escapes, we can just grab the text block directly.  This does mean that things like leading indentation
            // will not be stripped, and that multiline raw strings will contain the contents of their first line.
            // However, as this is error code anyways, the interpretation of the value is fine for us to define
            // however we want.  The user can (and should) check for the presence of diagnostics before blindly
            // trusting the contents.
            if (this.HasErrors)
            {
                var afterStartDelimiter = TextWindow.LexemeStartPosition + startingQuoteCount;
                var valueLength         = TextWindow.Position - afterStartDelimiter;

                info.StringValue = TextWindow.GetText(
                    position: afterStartDelimiter,
                    length: valueLength,
                    intern: true);
            }
            else
            {
                // If we didn't have an error, the subroutines better have set the string value for this literal.
                Debug.Assert(info.StringValue != null);
            }

            info.Text = TextWindow.GetText(intern: true);
        }
示例#12
0
        private int ConsumeDecimalDigits(StringBuilder?builder)
        {
            var  start = TextWindow.Position;
            char digit;

            while (CharUtils.IsDecimal(digit = TextWindow.PeekChar()) || digit == '_')
            {
                if (digit != '_')
                {
                    builder?.Append(digit);
                }
                TextWindow.AdvanceChar();
            }
            return(TextWindow.Position - start);
        }
示例#13
0
        private int ConsumeHexDigits()
        {
            var  start = TextWindow.Position;
            char digit;

            while (CharUtils.IsHexadecimal(digit = TextWindow.PeekChar()) || digit == '_')
            {
                if (digit != '_')
                {
                    _builder.Append(digit);
                }
                TextWindow.AdvanceChar();
            }
            return(TextWindow.Position - start);
        }
示例#14
0
        private void ScanNumericLiteral(TokenInfo info)
        {
            builder.Clear();

            while (true)
            {
                var ch = TextWindow.PeekChar();
                if (!(ch >= '0' && ch <= '9'))
                {
                    break;
                }
                TextWindow.AdvanceChar();
                builder.Append(ch);
            }
            info.IntValue = Int32.Parse(builder.ToString());
            info.Kind     = SyntaxKind.NumericLiteralToken;
        }
示例#15
0
        private void ScanISLNestedString(char quote, ref SyntaxDiagnosticInfo error)
        {
            Debug.Assert(TextWindow.PeekChar() == quote);
            TextWindow.AdvanceChar(); // move past quote
            while (true)
            {
                if (IsAtEnd())
                {
                    // we'll get an error in the enclosing construct
                    return;
                }
                char ch = TextWindow.PeekChar();
                TextWindow.AdvanceChar();
                switch (ch)
                {
                case '\"':
                case '\'':
                    if (ch == quote)
                    {
                        return;
                    }
                    break;

                case '\\':
                    ch = TextWindow.PeekChar();
                    if (IsAtEnd())
                    {
                        return;
                    }
                    else if (ch == '{' && quote == '"')
                    {
                        TextWindow.AdvanceChar();     // move past {
                        ScanISLHoleBalancedText('}', true, ref error);
                        if (TextWindow.PeekChar() == '}')
                        {
                            TextWindow.AdvanceChar();
                        }
                    }
                    else
                    {
                        TextWindow.AdvanceChar();     // move past one escaped character
                    }
                    break;
                }
            }
        }
示例#16
0
        /// <summary>
        /// Returns an appropriate error code if scanning this verbatim literal ran into an error.
        /// </summary>
        private ErrorCode?ScanVerbatimStringLiteral(ref TokenInfo info)
        {
            _builder.Length = 0;

            Debug.Assert(TextWindow.PeekChar() == '@' && TextWindow.PeekChar(1) == '"');
            TextWindow.AdvanceChar(2);

            ErrorCode?error = null;

            while (true)
            {
                var ch = TextWindow.PeekChar();
                if (ch == '"')
                {
                    TextWindow.AdvanceChar();
                    if (TextWindow.PeekChar() == '"')
                    {
                        // Doubled quote -- skip & put the single quote in the string and keep going.
                        TextWindow.AdvanceChar();
                        _builder.Append(ch);
                        continue;
                    }

                    // otherwise, the string is finished.
                    break;
                }

                if (ch == SlidingTextWindow.InvalidCharacter && TextWindow.IsReallyAtEnd())
                {
                    // Reached the end of the source without finding the end-quote.  Give an error back at the
                    // starting point. And finish lexing this string.
                    error ??= ErrorCode.ERR_UnterminatedStringLit;
                    break;
                }

                TextWindow.AdvanceChar();
                _builder.Append(ch);
            }

            info.Kind        = SyntaxKind.StringLiteralToken;
            info.Text        = TextWindow.GetText(intern: false);
            info.StringValue = _builder.ToString();

            return(error);
        }
示例#17
0
        internal void ScanISLTop(ArrayBuilder <Interpolation> interpolations, ref TokenInfo info, ref SyntaxDiagnosticInfo error)
        {
            Debug.Assert(TextWindow.PeekChar() == '\"');
            TextWindow.AdvanceChar(); // "
            ScanISLContents(interpolations, ref error);
            if (IsAtEnd() || TextWindow.PeekChar() != '\"')
            {
                if (error == null)
                {
                    error = MakeError(TextWindow.Position, 1, ErrorCode.ERR_NewlineInConst);
                }
            }
            else
            {
                // found the closing quote
                TextWindow.AdvanceChar(); // "
            }

            info.Kind = SyntaxKind.InterpolatedStringToken;
        }
示例#18
0
 private void ScanISLNestedComment()
 {
     Debug.Assert(TextWindow.PeekChar() == '/');
     TextWindow.AdvanceChar();
     Debug.Assert(TextWindow.PeekChar() == '*');
     TextWindow.AdvanceChar();
     while (true)
     {
         if (IsAtEnd())
         {
             return; // let the caller complain about the unterminated quote
         }
         var ch = TextWindow.PeekChar();
         TextWindow.AdvanceChar();
         if (ch == '*' && TextWindow.PeekChar() == '/')
         {
             TextWindow.AdvanceChar(); // skip */
             return;
         }
     }
 }
示例#19
0
        private void ScanStringLiteral(TokenInfo info)
        {
            builder.Clear();
            var quoteCharacter = TextWindow.PeekChar();

            TextWindow.AdvanceChar();

            while (true)
            {
                var ch = TextWindow.PeekChar();
                if (ch == quoteCharacter)
                {
                    TextWindow.AdvanceChar();
                    break;
                }
                else if (ch == '\\')
                {
                    ch = ScanEscapeSequence();
                    builder.Append(ch);
                }
                else
                {
                    TextWindow.AdvanceChar();
                    builder.Append(ch);
                }
            }

            if (quoteCharacter == '\'')
            {
                info.Kind      = SyntaxKind.CharacterLiteralToken;
                info.CharValue = builder[0];
            }
            else
            {
                info.Kind        = SyntaxKind.StringLiteralToken;
                info.StringValue = builder.ToString();
            }
        }
示例#20
0
        private void ParseBinaryNumber(ref TokenInfo info)
        {
            var  num            = 0UL;
            var  digits         = 0;
            var  hasUnderscores = false;
            var  hasOverflown   = false;
            char digit;

            while (CharUtils.IsBinary(digit = TextWindow.PeekChar()) || digit == '_')
            {
                TextWindow.AdvanceChar();
                if (digit == '_')
                {
                    hasUnderscores = true;
                    continue;
                }
                // Next shift will overflow if 63rd bit is set
                if ((num & 0x8000_0000_0000_0000) != 0)
                {
                    hasOverflown = true;
                }
                num = (num << 1) | (ulong)CharUtils.DecimalValue(digit);
                digits++;
            }

            var(isUnsignedLong, isSignedLong, isComplex) = (false, false, false);

            if (TextWindow.AdvanceIfMatches("ull", true))
            {
                isUnsignedLong = true;
            }
            else if (TextWindow.AdvanceIfMatches("ll", true))
            {
                isSignedLong = true;
            }
            else if (TextWindow.AdvanceIfMatches("i", true))
            {
                isComplex = true;
            }

            if (!_options.SyntaxOptions.AcceptBinaryNumbers)
            {
                AddError(ErrorCode.ERR_BinaryNumericLiteralNotSupportedInVersion);
            }
            if (!_options.SyntaxOptions.AcceptUnderscoreInNumberLiterals && hasUnderscores)
            {
                AddError(ErrorCode.ERR_UnderscoreInNumericLiteralNotSupportedInVersion);
            }
            if (!Options.SyntaxOptions.AcceptLuaJITNumberSuffixes && (isUnsignedLong || isSignedLong || isComplex))
            {
                AddError(ErrorCode.ERR_NumberSuffixNotSupportedInVersion);
            }

            if (digits < 1)
            {
                num = 0; // Safe default
                AddError(ErrorCode.ERR_InvalidNumber);
            }
            if (hasOverflown || (num > long.MaxValue && !isUnsignedLong))
            {
                num = 0; // Safe default
                AddError(ErrorCode.ERR_NumericLiteralTooLarge);
            }

            info.Text = TextWindow.GetText(intern: true);
            if (isUnsignedLong)
            {
                info.ValueKind  = ValueKind.ULong;
                info.ULongValue = num;
            }
            else if (isSignedLong)
            {
                info.ValueKind = ValueKind.Long;
                info.LongValue = unchecked ((long)num);
            }
            else if (isComplex)
            {
                info.ValueKind    = ValueKind.Complex;
                info.ComplexValue = new Complex(0, num);
            }
            else
            {
                switch (_options.SyntaxOptions.BinaryIntegerFormat)
                {
                case IntegerFormats.NotSupported:
                case IntegerFormats.Double:
                    info.ValueKind   = ValueKind.Double;
                    info.DoubleValue = num;
                    break;

                case IntegerFormats.Int64:
                    info.ValueKind = ValueKind.Long;
                    info.LongValue = unchecked ((long)num);
                    break;

                default:
                    throw ExceptionUtilities.UnexpectedValue(_options.SyntaxOptions.BinaryIntegerFormat);
                }
            }
        }
示例#21
0
        private bool ScanMultiLineRawStringLiteralLine(
            int startingQuoteCount, StringBuilder indentationWhitespace)
        {
            TextWindow.AdvancePastNewLine();

            indentationWhitespace.Clear();
            ConsumeWhitespace(indentationWhitespace);

            // after the whitespace see if this the line that ends the multiline literal.
            var currentQuoteCount = ConsumeQuoteSequence();

            if (currentQuoteCount >= startingQuoteCount)
            {
                // A raw string could never be followed by another string.  So once we've consumed all the closing quotes
                // if we have any more closing quotes then that's an error we can give a message for.
                if (currentQuoteCount > startingQuoteCount)
                {
                    var excessQuoteCount = currentQuoteCount - startingQuoteCount;
                    this.AddError(
                        position: TextWindow.Position - excessQuoteCount,
                        width: excessQuoteCount,
                        ErrorCode.ERR_TooManyQuotesForRawString);
                }

                // Done scanning lines.
                return(false);
            }

            // We're not on the terminating line. Consume a normal content line.  Eat to the end of line (or file in the
            // case of errors).
            while (true)
            {
                var currentChar = TextWindow.PeekChar();
                if (IsAtEndOfText(currentChar))
                {
                    this.AddError(TextWindow.Position, width: 0, ErrorCode.ERR_UnterminatedRawString);
                    return(false);
                }

                if (SyntaxFacts.IsNewLine(currentChar))
                {
                    return(true);
                }

                if (currentChar == '"')
                {
                    // Don't allow a content line to contain a quote sequence that looks like a delimiter (or longer)
                    currentQuoteCount = ConsumeQuoteSequence();
                    if (currentQuoteCount >= startingQuoteCount)
                    {
                        this.AddError(
                            position: TextWindow.Position - currentQuoteCount,
                            width: currentQuoteCount,
                            ErrorCode.ERR_RawStringDelimiterOnOwnLine);
                        return(false);
                    }
                }
                else
                {
                    TextWindow.AdvanceChar();
                }
            }
        }
        /// <summary>
        /// This method is essentially the same as ScanIdentifier_SlowPath,
        /// except that it can handle XML entities.  Since ScanIdentifier
        /// is hot code and since this method does extra work, it seem
        /// worthwhile to separate it from the common case.
        /// </summary>
        /// <param name="info"></param>
        /// <returns></returns>
        private bool ScanIdentifier_CrefSlowPath(ref TokenInfo info)
        {
            Debug.Assert(InXmlCrefOrNameAttributeValue);

            int start = TextWindow.Position;

            this.ResetIdentBuffer();

            if (AdvanceIfMatches('@'))
            {
                // In xml name _annotation values, the '@' is part of the value text of the identifier
                // (to match dev11).
                if (InXmlNameAttributeValue)
                {
                    AddIdentChar('@');
                }
                else
                {
                    info.IsVerbatim = true;
                }
            }

            while (true)
            {
                int  beforeConsumed = TextWindow.Position;
                char consumedChar;
                char consumedSurrogate;

                if (TextWindow.PeekChar() == '&')
                {
                    if (!TextWindow.TryScanXmlEntity(out consumedChar, out consumedSurrogate))
                    {
                        // If it's not a valid entity, then it's not part of the identifier.
                        TextWindow.Reset(beforeConsumed);
                        goto LoopExit;
                    }
                }
                else
                {
                    consumedChar      = TextWindow.NextChar();
                    consumedSurrogate = SlidingTextWindow.InvalidCharacter;
                }

                // NOTE: If the surrogate is non-zero, then consumedChar won't match
                // any of the cases below (UTF-16 guarantees that members of surrogate
                // pairs aren't separately valid).

                bool isEscaped = false;
top:
                switch (consumedChar)
                {
                case '\\':
                    // NOTE: For completeness, we should allow xml entities in unicode escape
                    // sequences (DevDiv #16321).  Since it is not currently a priority, we will
                    // try to make the interim behavior sensible: we will only attempt to scan
                    // a unicode escape if NONE of the characters are XML entities (including
                    // the backslash, which we have already consumed).
                    // When we're ready to implement this behavior, we can drop the position
                    // check and use AdvanceIfMatches instead of PeekChar.
                    if (!isEscaped && (TextWindow.Position == beforeConsumed + 1) &&
                        (TextWindow.PeekChar() == 'u' || TextWindow.PeekChar() == 'U'))
                    {
                        Debug.Assert(consumedSurrogate == SlidingTextWindow.InvalidCharacter, "Since consumedChar == '\\'");

                        info.HasIdentifierEscapeSequence = true;

                        TextWindow.Reset(beforeConsumed);
                        // ^^^^^^^ otherwise \u005Cu1234 looks just like \u1234! (i.e. escape within escape)
                        isEscaped = true;
                        SyntaxDiagnosticInfo error;
                        consumedChar = TextWindow.NextUnicodeEscape(out consumedSurrogate, out error);
                        AddCrefError(error);
                        goto top;
                    }

                    goto default;

                case '_':
                case 'A':
                case 'B':
                case 'C':
                case 'D':
                case 'E':
                case 'F':
                case 'G':
                case 'H':
                case 'I':
                case 'J':
                case 'K':
                case 'L':
                case 'M':
                case 'N':
                case 'O':
                case 'P':
                case 'Q':
                case 'R':
                case 'S':
                case 'T':
                case 'U':
                case 'V':
                case 'W':
                case 'X':
                case 'Y':
                case 'Z':
                case 'a':
                case 'b':
                case 'c':
                case 'd':
                case 'e':
                case 'f':
                case 'g':
                case 'h':
                case 'i':
                case 'j':
                case 'k':
                case 'l':
                case 'm':
                case 'n':
                case 'o':
                case 'p':
                case 'q':
                case 'r':
                case 's':
                case 't':
                case 'u':
                case 'v':
                case 'w':
                case 'x':
                case 'y':
                case 'z':
                {
                    // Again, these are the 'common' identifier characters...
                    break;
                }

                case '0':
                case '1':
                case '2':
                case '3':
                case '4':
                case '5':
                case '6':
                case '7':
                case '8':
                case '9':
                {
                    if (this._identLen == 0)
                    {
                        TextWindow.Reset(beforeConsumed);
                        goto LoopExit;
                    }

                    // Again, these are the 'common' identifier characters...
                    break;
                }

                case ' ':
                case '$':
                case '\t':
                case '.':
                case ';':
                case '(':
                case ')':
                case ',':
                case '<':
                    // ...and these are the 'common' stop characters.
                    TextWindow.Reset(beforeConsumed);
                    goto LoopExit;

                case SlidingTextWindow.InvalidCharacter:
                    if (!TextWindow.IsReallyAtEnd())
                    {
                        goto default;
                    }

                    TextWindow.Reset(beforeConsumed);
                    goto LoopExit;

                default:
                {
                    // This is the 'expensive' call
                    if (this._identLen == 0 && consumedChar > 127 && SyntaxKindFacts.IsIdentifierStartCharacter(consumedChar))
                    {
                        break;
                    }
                    else if (this._identLen > 0 && consumedChar > 127 && SyntaxKindFacts.IsIdentifierPartCharacter(consumedChar))
                    {
                        //// BUG 424819 : Handle identifier chars > 0xFFFF via surrogate pairs
                        if (SyntaxKindFacts.IsFormattingChar(consumedChar))
                        {
                            continue;                                             // Ignore formatting characters
                        }

                        break;
                    }
                    else
                    {
                        // Not a valid identifier character, so bail.
                        TextWindow.Reset(beforeConsumed);
                        goto LoopExit;
                    }
                }
                }

                this.AddIdentChar(consumedChar);
                if (consumedSurrogate != SlidingTextWindow.InvalidCharacter)
                {
                    this.AddIdentChar(consumedSurrogate);
                }
            }

LoopExit:
            if (this._identLen > 0)
            {
                // NOTE: If we don't intern the string value, then we won't get a hit
                // in the keyword dictionary!  (It searches for a key using identity.)
                // The text does not have to be interned (and probalbly shouldn't be
                // if it contains entities (else-case).

                var width = TextWindow.Width;                 // exact size of input characters

                // id buffer is identical to width in input
                if (this._identLen == width)
                {
                    info.StringValue = TextWindow.GetInternedText();
                    info.Text        = info.StringValue;
                }
                else
                {
                    info.StringValue = TextWindow.Intern(this._identBuffer, 0, this._identLen);
                    info.Text        = TextWindow.GetText(intern: false);
                }

                return(true);
            }
            else
            {
                info.Text        = null;
                info.StringValue = null;
                TextWindow.Reset(start);
                return(false);
            }
        }
示例#23
0
#pragma warning restore IDE0079 // Remove unnecessary suppression
        private void ParseHexadecimalNumber(ref TokenInfo info)
        {
            _builder.Clear();
            ConsumeHexDigits();
            var isHexFloat = false;

            if (TextWindow.PeekChar() == '.')
            {
                TextWindow.AdvanceChar();
                isHexFloat = true;
                _builder.Append('.');
                ConsumeHexDigits();
            }

            if (CharUtils.AsciiLowerCase(TextWindow.PeekChar()) == 'p')
            {
                TextWindow.AdvanceChar();
                isHexFloat = true;
                _builder.Append('p');
                if (TextWindow.PeekChar() is '+' or '-')
                {
                    _builder.Append(TextWindow.NextChar());
                }
                ConsumeDecimalDigits(_builder);
            }

            var(isUnsignedLong, isSignedLong, isComplex) = (false, false, false);

            if (TextWindow.AdvanceIfMatches("ull", true))
            {
                if (isHexFloat)
                {
                    AddError(ErrorCode.ERR_LuajitSuffixInFloat);
                }
                else
                {
                    isUnsignedLong = true;
                }
            }
            else if (TextWindow.AdvanceIfMatches("ll", true))
            {
                if (isHexFloat)
                {
                    AddError(ErrorCode.ERR_LuajitSuffixInFloat);
                }
                else
                {
                    isSignedLong = true;
                }
            }
            else if (TextWindow.AdvanceIfMatches("i", true))
            {
                isComplex = true;
            }

            info.Text = TextWindow.GetText(intern: true);
            if (!_options.SyntaxOptions.AcceptUnderscoreInNumberLiterals && info.Text.IndexOf('_') >= 0)
            {
                AddError(ErrorCode.ERR_UnderscoreInNumericLiteralNotSupportedInVersion);
            }
            if (!Options.SyntaxOptions.AcceptLuaJITNumberSuffixes && (isUnsignedLong || isSignedLong || isComplex))
            {
                AddError(ErrorCode.ERR_NumberSuffixNotSupportedInVersion);
            }

            if (isUnsignedLong)
            {
                if (!ulong.TryParse(TextWindow.Intern(_builder), NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out var result))
                {
                    AddError(ErrorCode.ERR_NumericLiteralTooLarge);
                }

                info.ValueKind  = ValueKind.ULong;
                info.ULongValue = result;
            }
            else if (isSignedLong)
            {
                if (!long.TryParse(TextWindow.Intern(_builder), NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out var result))
                {
                    AddError(ErrorCode.ERR_NumericLiteralTooLarge);
                }

                info.ValueKind = ValueKind.Long;
                info.LongValue = result;
            }
            else if (isComplex)
            {
                var result = 0d;
                try
                {
                    result = HexFloat.DoubleFromHexString(TextWindow.Intern(_builder));
                }
                catch (OverflowException)
                {
                    AddError(ErrorCode.ERR_DoubleOverflow);
                }

                info.ValueKind    = ValueKind.Complex;
                info.ComplexValue = new Complex(0, result);
            }
            else if (isHexFloat || _options.SyntaxOptions.HexIntegerFormat == IntegerFormats.NotSupported)
            {
                if (!_options.SyntaxOptions.AcceptHexFloatLiterals)
                {
                    AddError(ErrorCode.ERR_HexFloatLiteralNotSupportedInVersion);
                }

                var result = 0d;
                try
                {
                    result = HexFloat.DoubleFromHexString(TextWindow.Intern(_builder));
                }
                catch (OverflowException)
                {
                    AddError(ErrorCode.ERR_DoubleOverflow);
                }
                info.ValueKind   = ValueKind.Double;
                info.DoubleValue = result;
            }
            else
            {
                if (!long.TryParse(TextWindow.Intern(_builder), NumberStyles.AllowHexSpecifier, CultureInfo.InvariantCulture, out var result))
                {
                    AddError(ErrorCode.ERR_NumericLiteralTooLarge);
                }
                switch (_options.SyntaxOptions.HexIntegerFormat)
                {
                case IntegerFormats.Double:
                    info.ValueKind   = ValueKind.Double;
                    info.DoubleValue = result;
                    break;

                case IntegerFormats.Int64:
                    info.ValueKind = ValueKind.Long;
                    info.LongValue = result;
                    break;

                default:
                    throw ExceptionUtilities.UnexpectedValue(_options.SyntaxOptions.HexIntegerFormat);
                }
            }
        }
示例#24
0
        private void ParseOctalNumber(ref TokenInfo info)
        {
            var  num            = 0L;
            var  digits         = 0;
            var  hasUnderscores = false;
            var  hasOverflown   = false;
            char digit;

            while (CharUtils.IsOctal(digit = TextWindow.PeekChar()) || digit == '_')
            {
                TextWindow.AdvanceChar();
                if (digit == '_')
                {
                    hasUnderscores = true;
                    continue;
                }
                // If any of these bits are set, we'll overflow
                if ((num & 0x7000_0000_0000_0000) != 0)
                {
                    hasOverflown = true;
                }
                num = (num << 3) | CharUtils.DecimalValue(digit);
                digits++;
            }

            if (!_options.SyntaxOptions.AcceptOctalNumbers)
            {
                AddError(ErrorCode.ERR_OctalNumericLiteralNotSupportedInVersion);
            }
            if (!_options.SyntaxOptions.AcceptUnderscoreInNumberLiterals && hasUnderscores)
            {
                AddError(ErrorCode.ERR_UnderscoreInNumericLiteralNotSupportedInVersion);
            }

            if (digits < 1)
            {
                num = 0; // Safe default
                AddError(ErrorCode.ERR_InvalidNumber);
            }
            if (hasOverflown)
            {
                num = 0; // Safe default
                AddError(ErrorCode.ERR_NumericLiteralTooLarge);
            }

            info.Text = TextWindow.GetText(intern: true);
            switch (_options.SyntaxOptions.OctalIntegerFormat)
            {
            case IntegerFormats.NotSupported:
            case IntegerFormats.Double:
                info.ValueKind   = ValueKind.Double;
                info.DoubleValue = num;
                break;

            case IntegerFormats.Int64:
                info.ValueKind = ValueKind.Long;
                info.LongValue = num;
                break;

            default:
                throw ExceptionUtilities.UnexpectedValue(_options.SyntaxOptions.OctalIntegerFormat);
            }
        }
示例#25
0
        private void ScanMultiLineRawStringLiteral(ref TokenInfo info, int startingQuoteCount)
        {
            info.Kind = SyntaxKind.MultiLineRawStringLiteralToken;

            // The indentation-whitespace computed from the very last line of the raw string literal
            var indentationWhitespace = PooledStringBuilder.GetInstance();

            // The leading whitespace of whatever line we are currently on.
            var currentLineWhitespace = PooledStringBuilder.GetInstance();

            try
            {
                // Do the first pass, finding the end of the raw string, and determining the 'indentation whitespace'
                // that must be complimentary with all content lines of the raw string literal.
                var afterStartDelimiter = TextWindow.Position;
                Debug.Assert(SyntaxFacts.IsNewLine(TextWindow.PeekChar()));

                var contentLineCount = 0;
                while (ScanMultiLineRawStringLiteralLine(startingQuoteCount, indentationWhitespace.Builder))
                {
                    contentLineCount++;
                }

                // If the initial scan failed then just bail out without a constant value.
                if (this.HasErrors)
                {
                    return;
                }

                // The trivial raw string literal is not legal in the language.
                if (contentLineCount == 0)
                {
                    this.AddError(
                        position: TextWindow.Position - startingQuoteCount,
                        width: startingQuoteCount,
                        ErrorCode.ERR_RawStringMustContainContent);
                    return;
                }

                // Now, do the second pass, building up the literal value.  This may produce an error as well if the
                // indentation whitespace of the lines isn't complimentary.

                // Reset us to right after the starting delimiter.  Note: if we fail to generate a constant value we'll
                // ensure that we reset back to the original end we scanned to above.
                var tokenEnd = TextWindow.Position;
                TextWindow.Reset(afterStartDelimiter);
                Debug.Assert(SyntaxFacts.IsNewLine(TextWindow.PeekChar()));

                for (var currentLine = 0; currentLine < contentLineCount; currentLine++)
                {
                    AddMultiLineRawStringLiteralLineContents(
                        indentationWhitespace.Builder,
                        currentLineWhitespace.Builder,
                        firstContentLine: currentLine == 0);

                    // If processing the line produced errors, then bail out from continued processing.
                    if (this.HasErrors)
                    {
                        break;
                    }
                }

                info.StringValue = this.HasErrors ? "" : TextWindow.Intern(_builder);

                // Make sure that even if we fail to determine the constant content value of the string that
                // we still consume all the way to original end that we computed.
                TextWindow.Reset(tokenEnd);
            }
            finally
            {
                indentationWhitespace.Free();
                currentLineWhitespace.Free();
            }
        }
示例#26
0
        private string ParseShortString()
        {
            _builder.Clear();
            var delim = TextWindow.NextChar();

            LorettaDebug.Assert(delim is '"' or '\'' or '`');

            char ch;

            while (!IsAtEnd(ch = TextWindow.PeekChar()) && ch != delim)
            {
                var charStart = TextWindow.Position;
                switch (ch)
                {
                    #region Escapes

                case '\\':
                {
                    var escapeStart = TextWindow.Position;
                    TextWindow.AdvanceChar();

                    switch (ch = TextWindow.PeekChar())
                    {
                    case '\n':
                    case '\r':
                    {
                        _builder.Append(TextWindow.NextChar());
                        char ch2;
                        if (CharUtils.IsNewLine(ch2 = TextWindow.PeekChar()) &&
                            ch != ch2)
                        {
                            _builder.Append(TextWindow.NextChar());
                        }
                        break;
                    }

                    case 'a':
                        TextWindow.AdvanceChar();
                        _builder.Append('\a');
                        break;

                    case 'b':
                        TextWindow.AdvanceChar();
                        _builder.Append('\b');
                        break;

                    case 'f':
                        TextWindow.AdvanceChar();
                        _builder.Append('\f');
                        break;

                    case 'n':
                        TextWindow.AdvanceChar();
                        _builder.Append('\n');
                        break;

                    case 'r':
                        TextWindow.AdvanceChar();
                        _builder.Append('\r');
                        break;

                    case 't':
                        TextWindow.AdvanceChar();
                        _builder.Append('\t');
                        break;

                    case 'v':
                        TextWindow.AdvanceChar();
                        _builder.Append('\v');
                        break;

                    case '\\':
                        TextWindow.AdvanceChar();
                        _builder.Append('\\');
                        break;

                    case '\'':
                        TextWindow.AdvanceChar();
                        _builder.Append('\'');
                        break;

                    case '"':
                        TextWindow.AdvanceChar();
                        _builder.Append('"');
                        break;

                    case 'z':
                        TextWindow.AdvanceChar();

                        while (CharUtils.IsWhitespace(TextWindow.PeekChar()))
                        {
                            TextWindow.AdvanceChar();
                        }

                        if (!_options.SyntaxOptions.AcceptWhitespaceEscape)
                        {
                            AddError(escapeStart, TextWindow.Position - escapeStart, ErrorCode.ERR_WhitespaceEscapeNotSupportedInVersion);
                        }
                        break;

                    case '0':
                    case '1':
                    case '2':
                    case '3':
                    case '4':
                    case '5':
                    case '6':
                    case '7':
                    case '8':
                    case '9':
                    {
                        var parsedCharInteger = parseDecimalInteger(escapeStart);
                        if (parsedCharInteger != char.MaxValue)
                        {
                            _builder.Append(parsedCharInteger);
                        }
                        break;
                    }

                    case 'x':
                    {
                        TextWindow.AdvanceChar();
                        var parsedCharInteger = parseHexadecimalEscapeInteger(escapeStart);
                        if (parsedCharInteger != char.MaxValue)
                        {
                            _builder.Append(parsedCharInteger);
                        }

                        if (!_options.SyntaxOptions.AcceptHexEscapesInStrings)
                        {
                            AddError(escapeStart, TextWindow.Position - escapeStart, ErrorCode.ERR_HexStringEscapesNotSupportedInVersion);
                        }
                    }
                    break;

                    case 'u':
                    {
                        TextWindow.AdvanceChar();
                        var parsed = parseUnicodeEscape(escapeStart);
                        _builder.Append(parsed);

                        if (!_options.SyntaxOptions.AcceptUnicodeEscape)
                        {
                            AddError(escapeStart, TextWindow.Position - escapeStart, ErrorCode.ERR_UnicodeEscapesNotSupportedLuaInVersion);
                        }
                    }
                    break;

                    default:
                        if (!_options.SyntaxOptions.AcceptInvalidEscapes)
                        {
                            // Skip the character after the escape.
                            TextWindow.AdvanceChar();
                            AddError(escapeStart, TextWindow.Position - escapeStart, ErrorCode.ERR_InvalidStringEscape);
                        }
                        break;
                    }
                }
                break;

                    #endregion Escapes

                case '\n':
                case '\r':
                {
                    _builder.Append(TextWindow.NextChar());
                    char ch2;
                    if (CharUtils.IsNewLine(ch2 = TextWindow.PeekChar()) &&
                        ch != ch2)
                    {
                        _builder.Append(TextWindow.NextChar());
                    }

                    AddError(charStart, TextWindow.Position - charStart, ErrorCode.ERR_UnescapedLineBreakInString);
                }
                break;

                default:
                    _builder.Append(TextWindow.NextChar());
                    break;
                }
            }

            if (TextWindow.PeekChar() == delim)
            {
                TextWindow.AdvanceChar();
            }
            else
            {
                AddError(ErrorCode.ERR_UnfinishedString);
            }

            return(TextWindow.Intern(_builder));

            char parseDecimalInteger(int start)
            {
                var  readChars = 0;
                var  num       = 0;
                char ch;

                while (readChars < 3 && CharUtils.IsDecimal(ch = TextWindow.PeekChar()))
                {
                    TextWindow.AdvanceChar();
                    num = (num * 10) + (ch - '0');
                    readChars++;
                }

                if (readChars < 1 || num > 255)
                {
                    AddError(start, TextWindow.Position - start, ErrorCode.ERR_InvalidStringEscape);
                    return(char.MaxValue);
                }

                return((char)num);
            }

            ulong parseHexadecimalNumber(int start, int maxDigits, ErrorCode lessThanZeroErrorCode)
            {
                var readChars = 0;
                var num       = 0L;

                while (readChars < maxDigits)
                {
                    var peek = TextWindow.PeekChar();
                    if (CharUtils.IsDecimal(peek))
                    {
                        TextWindow.AdvanceChar();
                        num = (num << 4) | (uint)(peek - '0');
                    }
                    else if (CharUtils.IsHexadecimal(peek))
                    {
                        TextWindow.AdvanceChar();
                        num = (num << 4) | (uint)(10 + CharUtils.AsciiLowerCase(peek) - 'a');
                    }
                    else
                    {
                        break;
                    }
                    readChars++;
                }

                if (readChars < 1)
                {
                    AddError(start, TextWindow.Position - start, lessThanZeroErrorCode);
                    return(0UL);
                }

                return((ulong)num);
            }

            char parseHexadecimalEscapeInteger(int start) =>
            (char)parseHexadecimalNumber(start, 2, ErrorCode.ERR_InvalidStringEscape);

            string parseUnicodeEscape(int start)
            {
                var missingOpeningBrace = TextWindow.PeekChar() is not '{';

                if (!missingOpeningBrace)
                {
                    TextWindow.AdvanceChar();
                }

                var codepoint = parseHexadecimalNumber(start, 16, ErrorCode.ERR_HexDigitExpected);

                var missingClosingBrace = TextWindow.PeekChar() is not '}';

                if (!missingClosingBrace)
                {
                    TextWindow.AdvanceChar();
                }

                if (missingOpeningBrace)
                {
                    AddError(start, TextWindow.Position - start, ErrorCode.ERR_UnicodeEscapeMissingOpenBrace);
                }
                if (missingClosingBrace)
                {
                    AddError(start, TextWindow.Position - start, ErrorCode.ERR_UnicodeEscapeMissingCloseBrace);
                }
                if (codepoint > 0x10FFFF)
                {
                    AddError(start, TextWindow.Position - start, ErrorCode.ERR_EscapeTooLarge, "10FFFF");
                    codepoint = 0x10FFFF;
                }

                // Return the codepoint itself if it's in the BMP.
                // NOTE: It *is* technically incorrect to consider a surrogate
                // an Unicode codepoint but Lua accepts it so we do it as well.
                if (codepoint <= 0xFFFF)
                {
                    return(char.ToString((char)codepoint));
                }

                return(char.ConvertFromUtf32((int)codepoint));
            }
        }
示例#27
0
        private void ScanStringLiteral(ref TokenInfo info, bool allowEscapes = true)
        {
            var quoteCharacter = TextWindow.PeekChar();

            if (quoteCharacter == '\'' || quoteCharacter == '"')
            {
                TextWindow.AdvanceChar();
                _builder.Length = 0;
                while (true)
                {
                    char ch = TextWindow.PeekChar();
                    if (ch == '\\' && allowEscapes)
                    {
                        // normal string & char constants can have escapes
                        char c2;
                        ch = this.ScanEscapeSequence(out c2);
                        _builder.Append(ch);
                        if (c2 != SlidingTextWindow.InvalidCharacter)
                        {
                            _builder.Append(c2);
                        }
                    }
                    else if (ch == quoteCharacter)
                    {
                        TextWindow.AdvanceChar();
                        break;
                    }
                    else if (SyntaxFacts.IsNewLine(ch) ||
                             (ch == SlidingTextWindow.InvalidCharacter && TextWindow.IsReallyAtEnd()))
                    {
                        //String and character literals can contain any Unicode character. They are not limited
                        //to valid UTF-16 characters. So if we get the SlidingTextWindow's sentinel value,
                        //double check that it was not real user-code contents. This will be rare.
                        Debug.Assert(TextWindow.Width > 0);
                        this.AddError(ErrorCode.ERR_NewlineInConst);
                        break;
                    }
                    else
                    {
                        TextWindow.AdvanceChar();
                        _builder.Append(ch);
                    }
                }

                info.Text = TextWindow.GetText(true);
                if (quoteCharacter == '\'')
                {
                    info.Kind = SyntaxKind.CharacterLiteralToken;
                    if (_builder.Length != 1)
                    {
                        this.AddError((_builder.Length != 0) ? ErrorCode.ERR_TooManyCharsInConst : ErrorCode.ERR_EmptyCharConst);
                    }

                    if (_builder.Length > 0)
                    {
                        info.StringValue = TextWindow.Intern(_builder);
                        info.CharValue   = info.StringValue[0];
                    }
                    else
                    {
                        info.StringValue = string.Empty;
                        info.CharValue   = SlidingTextWindow.InvalidCharacter;
                    }
                }
                else
                {
                    info.Kind = SyntaxKind.StringLiteralToken;
                    if (_builder.Length > 0)
                    {
                        info.StringValue = TextWindow.Intern(_builder);
                    }
                    else
                    {
                        info.StringValue = string.Empty;
                    }
                }
            }
            else
            {
                info.Kind = SyntaxKind.None;
                info.Text = null;
            }
        }
示例#28
0
        /// <summary>
        /// Scan past the hole inside an interpolated string literal, leaving the current character on the '}' (if any)
        /// </summary>
        private void ScanISLHoleBalancedText(char endingChar, bool isHole, ref SyntaxDiagnosticInfo error)
        {
            while (true)
            {
                if (IsAtEnd())
                {
                    // the caller will complain
                    return;
                }
                char ch = TextWindow.PeekChar();
                switch (ch)
                {
                case '}':
                case ')':
                case ']':
                    if (ch == endingChar)
                    {
                        return;
                    }
                    if (error == null)
                    {
                        error = MakeError(TextWindow.Position, 1, ErrorCode.ERR_SyntaxError, endingChar.ToString());
                    }
                    goto default;

                case '\"':
                case '\'':
                    // handle string or character literal inside an expression hole.
                    ScanISLNestedString(ch, ref error);
                    continue;

                case '@':
                    if (TextWindow.PeekChar(1) == '\"')
                    {
                        // check for verbatim string inside an expression hole.
                        ScanISLNestedVerbatimString(ref error);
                    }
                    goto default;

                case '/':
                    switch (TextWindow.PeekChar(1))
                    {
                    case '/':
                        // error: single-line comment not allowed in an interpolated string
                        if (error == null)
                        {
                            error = MakeError(TextWindow.Position, 2, ErrorCode.ERR_SingleLineCommentInExpressionHole);
                        }
                        TextWindow.AdvanceChar();
                        TextWindow.AdvanceChar();
                        continue;

                    case '*':
                        // check for and scan /* comment */
                        ScanISLNestedComment();
                        continue;

                    default:
                        TextWindow.AdvanceChar();
                        continue;
                    }

                case '{':
                    ScanISLHoleBracketed('{', '}', ref error);
                    continue;

                case '(':
                    ScanISLHoleBracketed('(', ')', ref error);
                    continue;

                case '[':
                    ScanISLHoleBracketed('[', ']', ref error);
                    continue;

                default:
                    // part of code in the expression hole
                    TextWindow.AdvanceChar();
                    continue;
                }
            }
        }
示例#29
0
        private void ScanVerbatimStringLiteral(ref TokenInfo info, bool allowNewlines = true)
        {
            _builder.Length = 0;

            if (TextWindow.PeekChar() == '@' && TextWindow.PeekChar(1) == '"')
            {
                TextWindow.AdvanceChar(2);
                bool done = false;
                char ch;
                _builder.Length = 0;
                while (!done)
                {
                    switch (ch = TextWindow.PeekChar())
                    {
                    case '"':
                        TextWindow.AdvanceChar();
                        if (TextWindow.PeekChar() == '"')
                        {
                            // Doubled quote -- skip & put the single quote in the string
                            TextWindow.AdvanceChar();
                            _builder.Append(ch);
                        }
                        else
                        {
                            done = true;
                        }

                        break;

                    case SlidingTextWindow.InvalidCharacter:
                        if (!TextWindow.IsReallyAtEnd())
                        {
                            goto default;
                        }

                        // Reached the end of the source without finding the end-quote.  Give
                        // an error back at the starting point.
                        this.AddError(ErrorCode.ERR_UnterminatedStringLit);
                        done = true;
                        break;

                    default:
                        if (!allowNewlines && SyntaxFacts.IsNewLine(ch))
                        {
                            this.AddError(ErrorCode.ERR_UnterminatedStringLit);
                            done = true;
                            break;
                        }

                        TextWindow.AdvanceChar();
                        _builder.Append(ch);
                        break;
                    }
                }

                info.Kind        = SyntaxKind.StringLiteralToken;
                info.Text        = TextWindow.GetText(false);
                info.StringValue = _builder.ToString();
            }
            else
            {
                info.Kind        = SyntaxKind.None;
                info.Text        = null;
                info.StringValue = null;
            }
        }
示例#30
0
        private bool IsAtEnd()
        {
            char ch = TextWindow.PeekChar();

            return(SyntaxFacts.IsNewLine(ch) || (ch == SlidingTextWindow.InvalidCharacter && TextWindow.IsReallyAtEnd()));
        }