Example #1
0
        private Option <IToken> FlushBuffer(
            StringBuilder buffer,
            ref uint absolutePosition,
            uint lineNumber,
            ref uint lexemeStartPositionInLine
            )
        {
            if (buffer.Length > 0)
            {
                var lexeme = buffer.ToString();

                var result =
                    KeywordToken.FromString(
                        lexeme,
                        absolutePosition,
                        lineNumber,
                        lexemeStartPositionInLine
                        ) ||
                    IdentifierToken.FromString(
                        lexeme,
                        absolutePosition,
                        lineNumber,
                        lexemeStartPositionInLine
                        ) ||
                    IntegerLiteralToken.FromString(
                        lexeme,
                        absolutePosition,
                        lineNumber,
                        lexemeStartPositionInLine
                        ) ||
                    RealLiteralToken.FromString(
                        lexeme,
                        absolutePosition,
                        lineNumber,
                        lexemeStartPositionInLine
                        ) ||
                    new UnrecognizedToken(
                        lexeme,
                        absolutePosition,
                        lineNumber,
                        lexemeStartPositionInLine
                        )
                ;

                buffer.Clear();

                absolutePosition          += (uint)lexeme.Length;
                lexemeStartPositionInLine += (uint)lexeme.Length;

                return(result);
            }
            else
            {
                return(Option <IToken> .None);
            }
        }
Example #2
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="stream"></param>
        /// <returns></returns>
        /// <remarks>
        ///
        /// See https://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.1.pdf
        ///
        /// 7.6.1.1 Integer value
        ///
        /// No whitespace is allowed between the elements of the rules in this ABNF section.
        ///
        ///     integerValue         = binaryValue / octalValue / hexValue / decimalValue
        ///
        ///     binaryValue          = [ "+" / "-" ] 1*binaryDigit ( "b" / "B" )
        ///     binaryDigit          = "0" / "1"
        ///
        ///     octalValue           = [ "+" / "-" ] unsignedOctalValue
        ///     unsignedOctalValue   = "0" 1*octalDigit
        ///     octalDigit           = "0" / "1" / "2" / "3" / "4" / "5" / "6" / "7"
        ///
        ///     hexValue             = [ "+" / "-" ] ( "0x" / "0X" ) 1*hexDigit
        ///     hexDigit             = decimalDigit / "a" / "A" / "b" / "B" / "c" / "C" /
        ///                                           "d" / "D" / "e" / "E" / "f" / "F"
        ///
        ///     decimalValue         = [ "+" / "-" ] unsignedDecimalValue
        ///     unsignedDecimalValue = "0" / positiveDecimalDigit *decimalDigit
        ///
        ///     decimalDigit         = "0" / positiveDecimalDigit
        ///     positiveDecimalDigit = "1"..."9"
        ///
        /// 7.6.1.2 Real value
        ///
        /// No whitespace is allowed between the elements of the rules in this ABNF section.
        ///
        ///     realValue            = [ "+" / "-" ] *decimalDigit "." 1*decimalDigit
        ///                            [ ( "e" / "E" ) [ "+" / "-" ] 1*decimalDigit ]
        ///
        ///     decimalDigit         = "0" / positiveDecimalDigit
        ///     positiveDecimalDigit = "1"..."9"
        ///
        /// </remarks>
        public static (Token, Lexer) ReadNumericLiteralToken(SourceReader reader)
        {
            int ParseBinaryValueDigits(IEnumerable <SourceChar> binaryChars, SourceChar sign)
            {
                return(ParseIntegerValueDigits(new Dictionary <char, int>
                {
                    { '0', 0 }, { '1', 1 }
                }, 2, binaryChars, sign));
            }

            int ParseOctalValueDigits(IEnumerable <SourceChar> octalChars, SourceChar sign)
            {
                return(ParseIntegerValueDigits(new Dictionary <char, int>
                {
                    { '0', 0 }, { '1', 1 }, { '2', 2 }, { '3', 3 }, { '4', 4 }, { '5', 5 }, { '6', 6 }, { '7', 7 }
                }, 8, octalChars, sign));
            }

            int ParseHexValueDigits(IEnumerable <SourceChar> hexChars, SourceChar sign)
            {
                return(ParseIntegerValueDigits(new Dictionary <char, int>
                {
                    { '0', 0 }, { '1', 1 }, { '2', 2 }, { '3', 3 }, { '4', 4 }, { '5', 5 }, { '6', 6 }, { '7', 7 }, { '8', 8 }, { '9', 9 },
                    { 'a', 10 }, { 'b', 11 }, { 'c', 12 }, { 'd', 13 }, { 'e', 14 }, { 'f', 15 },
                    { 'A', 10 }, { 'B', 11 }, { 'C', 12 }, { 'D', 13 }, { 'E', 14 }, { 'F', 15 }
                }, 16, hexChars, sign));
            }

            int ParseDecimalValueDigits(IEnumerable <SourceChar> decimalChars, SourceChar sign)
            {
                return(ParseIntegerValueDigits(new Dictionary <char, int>
                {
                    { '0', 0 }, { '1', 1 }, { '2', 2 }, { '3', 3 }, { '4', 4 }, { '5', 5 }, { '6', 6 }, { '7', 7 }, { '8', 8 }, { '9', 9 }
                }, 10, decimalChars, sign));
            }

            int ParseIntegerValueDigits(Dictionary <char, int> alphabet, int radix, IEnumerable <SourceChar> chars, SourceChar sign)
            {
                var literalValue = 0;

                foreach (var digit in chars)
                {
                    var digitValue = alphabet[digit.Value];
                    literalValue = (literalValue * radix) + digitValue;
                }
                if (sign?.Value == '-')
                {
                    literalValue = -literalValue;
                }
                return(literalValue);
            }

            const int stateLeadingSign         = 1;
            const int stateFirstDigitBlock     = 2;
            const int stateOctalOrDecimalValue = 3;
            const int stateBinaryValue         = 4;
            const int stateOctalValue          = 5;
            const int stateHexValue            = 6;
            const int stateDecimalValue        = 7;
            const int stateRealValue           = 8;
            const int stateRealValueFraction   = 9;
            const int stateRealValueExponent   = 10;
            const int stateDone = 99;

            var thisReader  = reader;
            var sourceChar  = default(SourceChar);
            var sourceChars = new List <SourceChar>();

            var token           = default(Token);
            var signChar        = default(SourceChar);
            var firstDigitBlock = new List <SourceChar>();

            var currentState = stateLeadingSign;

            while (currentState != stateDone)
            {
                switch (currentState)
                {
                case stateLeadingSign:
                    // we're reading the initial optional leading sign
                    // [ "+" / "-" ]
                    sourceChar = thisReader.Peek();
                    switch (sourceChar.Value)
                    {
                    case '+':
                    case '-':
                        (signChar, thisReader) = thisReader.Read();
                        sourceChars.Add(signChar);
                        break;
                    }
                    currentState = stateFirstDigitBlock;
                    break;

                case stateFirstDigitBlock:
                    // we're reading the first block of digits in the value,
                    // but we won't necessarily know which type we're reading
                    // until we've consumed more of the input stream
                    //
                    //     binaryValue  => 1*binaryDigit
                    //     octalValue   => "0" 1*octalDigit
                    //     hexValue     => ( "0x" / "0X" )
                    //     decimalValue => positiveDecimalDigit *decimalDigit
                    //     realValue    => *decimalDigit
                    //
                    if (thisReader.Peek('.'))
                    {
                        // we're reading a realValue with no "*decimalDigit" characters before the "."
                        // e.g. ".45", "+.45", "-.45", so consume the decimal point
                        (sourceChar, thisReader) = thisReader.Read();
                        sourceChars.Add(sourceChar);
                        // and go to the next state
                        currentState = stateRealValueFraction;
                        break;
                    }
                    // we don't know which base the value is in yet, but if it's hexadecimal them
                    // we should be reading the "0x" part here, so restrict digits to decimal in
                    // all cases
                    (firstDigitBlock, thisReader) = thisReader.ReadWhile(StringValidator.IsDecimalDigit);
                    sourceChars.AddRange(firstDigitBlock);
                    // now we can do some validation
                    if (firstDigitBlock.Count == 0)
                    {
                        // only realValue allows no digits in the first block, and
                        // we've already handled that at the start of this case
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    // if we've reached the end of the stream then there's no suffix
                    // (e.g. b, B, x, X, .) so this must be an octalValue or decimalValue
                    if (thisReader.Eof())
                    {
                        currentState = stateOctalOrDecimalValue;
                        break;
                    }
                    // check the next character to see if it tells us anything
                    // about which type of literal we're reading
                    sourceChar = thisReader.Peek();
                    switch (sourceChar.Value)
                    {
                    case 'b':
                    case 'B':
                        // binaryValue
                        currentState = stateBinaryValue;
                        break;

                    case 'x':
                    case 'X':
                        // hexValue
                        currentState = stateHexValue;
                        break;

                    case '.':
                        // realValue
                        currentState = stateRealValue;
                        break;

                    default:
                        // by elmination, this must be an octalValue or decimalValue
                        currentState = stateOctalOrDecimalValue;
                        break;
                    }
                    break;

                case stateOctalOrDecimalValue:
                    // we're reading an octalValue or decimalValue, but we're not sure which yet...
                    if ((firstDigitBlock.First().Value == '0') && (firstDigitBlock.Count > 1))
                    {
                        currentState = stateOctalValue;
                    }
                    else
                    {
                        currentState = stateDecimalValue;
                    }
                    break;

                case stateBinaryValue:
                    // we're trying to read a binaryValue, so check all the characters in the digit block are valid,
                    // i.e. 1*binaryDigit
                    if (firstDigitBlock.Any(c => !StringValidator.IsBinaryDigit(c.Value)))
                    {
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    // all the characters are valid, so consume the suffix
                    (sourceChar, thisReader) = thisReader.Read(c => (c == 'b') || (c == 'B'));
                    sourceChars.Add(sourceChar);
                    // now build the return value
                    var binaryValue = ParseBinaryValueDigits(firstDigitBlock, signChar);
                    token = new IntegerLiteralToken(SourceExtent.From(sourceChars), IntegerKind.BinaryValue, binaryValue);
                    // and we're done
                    currentState = stateDone;
                    break;

                case stateOctalValue:
                    // we're trying to read an octalValue (since decimalValue can't start with a
                    // leading '0') so check all the characters in the digit block are valid,
                    // i.e. "0" 1*octalDigit
                    if ((firstDigitBlock.Count < 2) || (firstDigitBlock.First().Value != '0'))
                    {
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    if (firstDigitBlock.Skip(1).Any(c => !StringValidator.IsOctalDigit(c.Value)))
                    {
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    // now build the return value
                    var octalValue = ParseOctalValueDigits(firstDigitBlock, signChar);
                    token = new IntegerLiteralToken(SourceExtent.From(sourceChars), IntegerKind.OctalValue, octalValue);
                    // and we're done
                    currentState = stateDone;
                    break;

                case stateHexValue:
                    // we're trying to read a hexValue, so we should have just read a leading zero
                    if ((firstDigitBlock.Count != 1) || (firstDigitBlock.First().Value != '0'))
                    {
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    // all the characters are valid, so consume the suffix
                    (sourceChar, thisReader) = thisReader.Read(c => (c == 'x') || (c == 'X'));
                    sourceChars.Add(sourceChar);
                    // 1*hexDigit
                    var hexDigits = default(List <SourceChar>);
                    (hexDigits, thisReader) = thisReader.ReadWhile(StringValidator.IsHexDigit);
                    if (hexDigits.Count == 0)
                    {
                        throw new UnexpectedCharacterException(thisReader.Peek());
                    }
                    sourceChars.AddRange(hexDigits);
                    // build the return value
                    var hexValue = ParseHexValueDigits(hexDigits, signChar);
                    token = new IntegerLiteralToken(SourceExtent.From(sourceChars), IntegerKind.HexValue, hexValue);
                    // and we're done
                    currentState = stateDone;
                    break;

                case stateDecimalValue:
                    // we're trying to read a decimalValue (since that's the only remaining option),
                    // so check all the characters in the digit block are valid,
                    // i.e. "0" / positiveDecimalDigit *decimalDigit
                    if ((firstDigitBlock.Count == 1) && (firstDigitBlock.First().Value == '0'))
                    {
                        // "0"
                    }
                    else if (!StringValidator.IsPositiveDecimalDigit(firstDigitBlock.First().Value))
                    {
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    else if (firstDigitBlock.Skip(1).Any(c => !StringValidator.IsDecimalDigit(c.Value)))
                    {
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    // build the return value
                    var decimalValue = ParseDecimalValueDigits(firstDigitBlock, signChar);
                    token = new IntegerLiteralToken(SourceExtent.From(sourceChars), IntegerKind.DecimalValue, decimalValue);
                    // and we're done
                    currentState = stateDone;
                    break;

                case stateRealValue:
                    // we're trying to read a realValue, so check all the characters in the digit block are valid,
                    // i.e. *decimalDigit
                    if (firstDigitBlock.Any(c => !StringValidator.IsDecimalDigit(c.Value)))
                    {
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    // all the characters are valid, so consume the decimal point
                    (sourceChar, thisReader) = thisReader.Read('.');
                    sourceChars.Add(sourceChar);
                    // and go to the next state
                    currentState = stateRealValueFraction;
                    break;

                case stateRealValueFraction:
                    // 1*decimalDigit
                    var realFractionDigits = default(List <SourceChar>);
                    (realFractionDigits, thisReader) = thisReader.ReadWhile(StringValidator.IsHexDigit);
                    if (realFractionDigits.Count == 0)
                    {
                        throw new UnexpectedCharacterException(thisReader.Peek());
                    }
                    sourceChars.AddRange(realFractionDigits);
                    // ( "e" / "E" )
                    if (!thisReader.Eof())
                    {
                        sourceChar = thisReader.Peek();
                        if ((sourceChar.Value == 'e') || (sourceChar.Value == 'E'))
                        {
                            currentState = stateRealValueExponent;
                            break;
                        }
                    }
                    // build the return value
                    var realIntegerValue  = ParseDecimalValueDigits(firstDigitBlock, signChar);
                    var realFractionValue = (double)ParseDecimalValueDigits(realFractionDigits, signChar);
                    if (realFractionDigits.Any())
                    {
                        realFractionValue = realFractionValue / Math.Pow(10, realFractionDigits.Count);
                    }
                    token = new RealLiteralToken(
                        SourceExtent.From(sourceChars),
                        realIntegerValue + realFractionValue
                        );
                    // and we're done
                    currentState = stateDone;
                    break;

                case stateRealValueExponent:
                    throw new InvalidOperationException();

                case stateDone:
                    // the main while loop should exit before we ever get here
                    throw new InvalidOperationException();

                default:
                    throw new InvalidOperationException();
                }
            }

            return(token, new Lexer(thisReader));
        }
        private static void AssertAreEqualInternal(Token expectedToken, Token actualToken, int index = -1)
        {
            if ((expectedToken == null) && (actualToken == null))
            {
                return;
            }
            if (expectedToken == null)
            {
                Assert.Fail(LexerHelper.GetAssertErrorMessage("expected is null, but actual is not null", index));
            }
            if (actualToken == null)
            {
                Assert.Fail(LexerHelper.GetAssertErrorMessage("expected is not null, but actual is null", index));
            }
            Assert.AreEqual(expectedToken.GetType(), actualToken.GetType(),
                            LexerHelper.GetAssertErrorMessage($"actual type does not match expected value", index));
            Assert.AreEqual(expectedToken.Extent.StartPosition.Position, actualToken.Extent.StartPosition.Position,
                            LexerHelper.GetAssertErrorMessage($"actual Start Position does not match expected value", index));
            Assert.AreEqual(expectedToken.Extent.StartPosition.LineNumber, actualToken.Extent.StartPosition.LineNumber,
                            LexerHelper.GetAssertErrorMessage($"actual Start Line does not match expected value", index));
            Assert.AreEqual(expectedToken.Extent.StartPosition.ColumnNumber, actualToken.Extent.StartPosition.ColumnNumber,
                            LexerHelper.GetAssertErrorMessage($"actual Start Column does not match expected value", index));
            Assert.AreEqual(expectedToken.Extent.EndPosition.Position, actualToken.Extent.EndPosition.Position,
                            LexerHelper.GetAssertErrorMessage($"actual End Position does not match expected value", index));
            Assert.AreEqual(expectedToken.Extent.EndPosition.LineNumber, actualToken.Extent.EndPosition.LineNumber,
                            LexerHelper.GetAssertErrorMessage($"actual End Line does not match expected value", index));
            Assert.AreEqual(expectedToken.Extent.EndPosition.ColumnNumber, actualToken.Extent.EndPosition.ColumnNumber,
                            LexerHelper.GetAssertErrorMessage($"actual End Column does not match expected value", index));
            Assert.AreEqual(expectedToken.Extent.Text, actualToken.Extent.Text,
                            LexerHelper.GetAssertErrorMessage($"actual Text does not match expected value", index));
            switch (expectedToken)
            {
            case AliasIdentifierToken token:
                Assert.IsTrue(
                    AliasIdentifierToken.AreEqual((AliasIdentifierToken)expectedToken, (AliasIdentifierToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case AttributeCloseToken token:
                Assert.IsTrue(
                    AttributeCloseToken.AreEqual((AttributeCloseToken)expectedToken, (AttributeCloseToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case AttributeOpenToken token:
                Assert.IsTrue(
                    AttributeOpenToken.AreEqual((AttributeOpenToken)expectedToken, (AttributeOpenToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case BlockCloseToken token:
                Assert.IsTrue(
                    BlockCloseToken.AreEqual((BlockCloseToken)expectedToken, (BlockCloseToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case BlockOpenToken token:
                Assert.IsTrue(
                    BlockOpenToken.AreEqual((BlockOpenToken)expectedToken, (BlockOpenToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case BooleanLiteralToken token:
                Assert.IsTrue(
                    BooleanLiteralToken.AreEqual((BooleanLiteralToken)expectedToken, (BooleanLiteralToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case ColonToken token:
                Assert.IsTrue(
                    ColonToken.AreEqual((ColonToken)expectedToken, (ColonToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case CommaToken token:
                Assert.IsTrue(
                    CommaToken.AreEqual((CommaToken)expectedToken, (CommaToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case CommentToken token:
                Assert.IsTrue(
                    CommentToken.AreEqual((CommentToken)expectedToken, (CommentToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case DotOperatorToken token:
                Assert.IsTrue(
                    DotOperatorToken.AreEqual((DotOperatorToken)expectedToken, (DotOperatorToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case EqualsOperatorToken token:
                Assert.IsTrue(
                    EqualsOperatorToken.AreEqual((EqualsOperatorToken)expectedToken, (EqualsOperatorToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case IdentifierToken token:
                Assert.IsTrue(
                    IdentifierToken.AreEqual((IdentifierToken)expectedToken, (IdentifierToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case IntegerLiteralToken token:
                Assert.IsTrue(
                    IntegerLiteralToken.AreEqual((IntegerLiteralToken)expectedToken, (IntegerLiteralToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case NullLiteralToken token:
                Assert.IsTrue(
                    NullLiteralToken.AreEqual((NullLiteralToken)expectedToken, (NullLiteralToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case ParenthesisCloseToken token:
                Assert.IsTrue(
                    ParenthesisCloseToken.AreEqual((ParenthesisCloseToken)expectedToken, (ParenthesisCloseToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case ParenthesisOpenToken token:
                Assert.IsTrue(
                    ParenthesisOpenToken.AreEqual((ParenthesisOpenToken)expectedToken, (ParenthesisOpenToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case PragmaToken token:
                Assert.IsTrue(
                    PragmaToken.AreEqual((PragmaToken)expectedToken, (PragmaToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case RealLiteralToken token:
                Assert.IsTrue(
                    RealLiteralToken.AreEqual((RealLiteralToken)expectedToken, (RealLiteralToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case StatementEndToken token:
                Assert.IsTrue(
                    StatementEndToken.AreEqual((StatementEndToken)expectedToken, (StatementEndToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case StringLiteralToken token:
                Assert.IsTrue(
                    StringLiteralToken.AreEqual((StringLiteralToken)expectedToken, (StringLiteralToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            case WhitespaceToken token:
                Assert.IsTrue(
                    WhitespaceToken.AreEqual((WhitespaceToken)expectedToken, (WhitespaceToken)actualToken),
                    LexerHelper.GetAssertErrorMessage($"actual token does not match expected token", index)
                    );
                break;

            default:
                throw new NotImplementedException($"Cannot compare type '{expectedToken.GetType().Name}'");
            }
        }
 public RealValueAst(RealLiteralToken realLiteralToken)
 {
     this.RealLiteralToken = realLiteralToken ?? throw new ArgumentNullException(nameof(RealLiteralToken));
     this.Value            = realLiteralToken.Value;
 }