Пример #1
0
        public static (CommaToken, Lexer) ReadCommaToken(SourceReader reader)
        {
            (var sourceChar, var nextReader) = reader.Read(',');
            var extent = SourceExtent.From(sourceChar);

            return(new CommaToken(extent), new Lexer(nextReader));
        }
Пример #2
0
        public static (ParenthesisOpenToken, Lexer) ReadParenthesisOpenToken(SourceReader reader)
        {
            (var sourceChar, var nextReader) = reader.Read('(');
            var extent = SourceExtent.From(sourceChar);

            return(new ParenthesisOpenToken(extent), new Lexer(nextReader));
        }
Пример #3
0
        public static (EqualsOperatorToken, Lexer) ReadEqualsOperatorToken(SourceReader reader)
        {
            (var sourceChar, var nextReader) = reader.Read('=');
            var extent = SourceExtent.From(sourceChar);

            return(new EqualsOperatorToken(extent), new Lexer(nextReader));
        }
Пример #4
0
        public static (StatementEndToken, Lexer) ReadStatementEndToken(SourceReader reader)
        {
            (var sourceChar, var nextReader) = reader.Read(';');
            var extent = SourceExtent.From(sourceChar);

            return(new StatementEndToken(extent), new Lexer(nextReader));
        }
Пример #5
0
        /// <summary>
        /// </summary>
        /// <param name="stream"></param>
        /// <returns></returns>
        /// <remarks>
        ///
        /// See https://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.1.pdf
        ///
        /// 7.3 Compiler directives
        ///
        /// Compiler directives direct the processing of MOF files. Compiler directives do not create, modify, or
        /// annotate the language elements.
        ///
        /// Compiler directives shall conform to the format defined by ABNF rule compilerDirective (whitespace
        /// as defined in 5.2 is allowed between the elements of the rules in this ABNF section):
        ///
        ///     compilerDirective = PRAGMA ( pragmaName / standardPragmaName )
        ///                         "(" pragmaParameter ")"
        ///
        ///     pragmaName         = directiveName
        ///     standardPragmaName = INCLUDE
        ///     pragmaParameter    = stringValue ; if the pragma is INCLUDE,
        ///                                      ; the parameter value
        ///                                      ; shall represent a relative
        ///                                      ; or full file path
        ///     PRAGMA             = "#pragma"  ; keyword: case insensitive
        ///     INCLUDE            = "include"  ; keyword: case insensitive
        ///
        ///     directiveName      = org-id "_" IDENTIFIER
        ///
        /// </remarks>
        public static (PragmaToken, Lexer) ReadPragmaToken(SourceReader reader)
        {
            (var sourceChars, var thisReader) = reader.ReadString(Constants.PRAGMA, true);
            var extent = SourceExtent.From(sourceChars.ToList());

            return(new PragmaToken(extent), new Lexer(thisReader));
        }
Пример #6
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="stream"></param>
        /// <returns></returns>
        /// <remarks>
        ///
        /// See https://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.1.pdf
        ///
        /// 7.7.3 Alias identifier
        ///
        /// An aliasIdentifier identifies an Instance or Value within the context of a MOF compilation unit.
        ///
        /// No whitespace is allowed between the elements of this rule.
        ///
        ///     aliasIdentifier     = "$" IDENTIFIER
        ///
        ///     IDENTIFIER          = firstIdentifierChar *( nextIdentifierChar )
        ///     firstIdentifierChar = UPPERALPHA / LOWERALPHA / UNDERSCORE
        ///     nextIdentifierChar  = firstIdentifierChar / decimalDigit
        ///     elementName         = localName / schemaQualifiedName
        ///     localName           = IDENTIFIER
        ///
        /// </remarks>
        public static (AliasIdentifierToken, Lexer) ReadAliasIdentifierToken(SourceReader reader)
        {
            var thisReader  = reader;
            var sourceChar  = default(SourceChar);
            var sourceChars = new List <SourceChar>();
            var nameChars   = new StringBuilder();

            // "$"
            (sourceChar, thisReader) = thisReader.Read('$');
            sourceChars.Add(sourceChar);
            // firstIdentifierChar
            (sourceChar, thisReader) = thisReader.Read(StringValidator.IsFirstIdentifierChar);
            sourceChars.Add(sourceChar);
            nameChars.Append(sourceChar.Value);
            // *( nextIdentifierChar )
            while (!thisReader.Eof() && thisReader.Peek(StringValidator.IsNextIdentifierChar))
            {
                (sourceChar, thisReader) = thisReader.Read();
                sourceChars.Add(sourceChar);
                nameChars.Append(sourceChar.Value);
            }
            // return the result
            var extent = SourceExtent.From(sourceChars);
            var name   = nameChars.ToString();

            return(new AliasIdentifierToken(extent, name), new Lexer(thisReader));
        }
Пример #7
0
        public static (BlockOpenToken, Lexer) ReadBlockOpenToken(SourceReader reader)
        {
            (var sourceChar, var nextReader) = reader.Read('{');
            var extent = SourceExtent.From(sourceChar);

            return(new BlockOpenToken(extent), new Lexer(nextReader));
        }
Пример #8
0
        public static (AttributeOpenToken, Lexer) ReadAttributeOpenToken(SourceReader reader)
        {
            (var sourceChar, var nextReader) = reader.Read('[');
            var extent = SourceExtent.From(sourceChar);

            return(new AttributeOpenToken(extent), new Lexer(nextReader));
        }
 internal static AliasIdentifierToken Read(ILexerStream stream)
 {
     var extent = new SourceExtent(stream);
     var sourceChars = new List<char>();
     // read the first character
     sourceChars.Add(stream.ReadChar('$'));
     // read the name
     var nameChars = new List<char>();
     while (!stream.Eof)
     {
         extent = extent.WithEndExtent(stream);
         var peek = stream.Peek();
         if (char.IsLetterOrDigit(peek) || ("_".IndexOf(peek) != -1) )
         {
             var @char = stream.Read();
             sourceChars.Add(@char);
             nameChars.Add(@char);
         }
         else
         {
             break;
         }
     }
     // return the result
     extent = extent.WithText(sourceChars);
     var name = new string(nameChars.ToArray());
     return new AliasIdentifierToken(extent, name);
 }
 internal static StringLiteralToken Read(ILexerStream stream)
 {
     var extent = new SourceExtent(stream);
     var sourceChars = new List<char>();
     // read the first character
     sourceChars.Add(stream.ReadChar('"'));
     // read the remaining characters
     var parser = new StringParser();
     while (!stream.Eof)
     {
         var peek = stream.Peek();
         sourceChars.Add(peek);
         if ((peek == '"') && !parser.IsEscaped)
         {
             parser.ConsumeEof();
             break;
         }
         else
         {
             parser.ConsumeChar(stream.Read());
         }
     }
     // read the last character
     sourceChars.Add(stream.ReadChar('"'));
     // process any escape sequences in the string
     var unescaped = parser.OutputString.ToString();
     // return the result
     extent = extent.WithText(sourceChars);
     return new StringLiteralToken(extent, unescaped);
 }
Пример #11
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="stream"></param>
        /// <returns></returns>
        /// <remarks>
        ///
        /// See http://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.0.pdf
        ///
        /// A.13 Names
        ///
        /// MOF names are identifiers with the format defined by the IDENTIFIER rule.
        /// No whitespace is allowed between the elements of the rules in this ABNF section.
        ///
        ///     IDENTIFIER          = firstIdentifierChar *( nextIdentifierChar )
        ///     firstIdentifierChar = UPPERALPHA / LOWERALPHA / UNDERSCORE
        ///     nextIdentifierChar  = firstIdentifierChar / decimalDigit
        ///     elementName         = localName / schemaQualifiedName
        ///     localName           = IDENTIFIER
        ///
        /// </remarks>
        private static IdentifierToken ReadIdentifierToken(ILexerStream stream)
        {
            var sourceChars = new List <SourceChar>();
            var nameChars   = new List <char>();
            // firstIdentifierChar
            var peek = stream.Peek();

            if (!StringValidator.IsFirstIdentifierChar(peek.Value))
            {
                throw new InvalidOperationException(
                          string.Format("Unexpected character '{0}' encountered", peek.Value));
            }
            // *( nextIdentifierChar )
            while (!stream.Eof)
            {
                peek = stream.Peek();
                if (StringValidator.IsNextIdentifierChar(peek.Value))
                {
                    var @char = stream.Read();
                    sourceChars.Add(@char);
                    nameChars.Add(@char.Value);
                }
                else
                {
                    break;
                }
            }
            // return the result
            var extent = new SourceExtent(sourceChars);
            var name   = new string(nameChars.ToArray());

            return(new IdentifierToken(extent, name));
        }
Пример #12
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="stream"></param>
        /// <returns></returns>
        /// <remarks>
        ///
        /// See http://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.0a.pdf
        ///
        /// A.17.3 String values
        ///
        /// Unless explicitly specified via ABNF rule WS, no whitespace is allowed between the elements of the rules
        /// in this ABNF section.
        ///
        ///     stringValue   = DOUBLEQUOTE *stringChar DOUBLEQUOTE
        ///                     *( *WS DOUBLEQUOTE *stringChar DOUBLEQUOTE )
        ///     stringChar    = stringUCSchar / stringEscapeSequence
        ///
        /// </remarks>
        private static StringLiteralToken ReadStringLiteralToken(ILexerStream stream)
        {
            // BUGBUG - no support for *( *WS DOUBLEQUOTE *stringChar DOUBLEQUOTE )
            // BUGBUG - incomplete escape sequences
            // BUGBUG - no support for UCS characters
            var sourceChars = new List <SourceChar>();

            // read the first character
            sourceChars.Add(stream.ReadChar('"'));
            // read the remaining characters
            var parser = new StringLiteralParser();

            while (!stream.Eof)
            {
                var peek = stream.Peek();
                if (StringValidator.IsDoubleQuote(peek.Value) && !parser.IsEscaped)
                {
                    parser.ConsumeEos();
                    break;
                }
                else
                {
                    sourceChars.Add(peek);
                    parser.ConsumeChar(stream.Read());
                }
            }
            // read the last character
            sourceChars.Add(stream.ReadChar('"'));
            // process any escape sequences in the string
            var unescaped = parser.OutputString.ToString();
            // return the result
            var extent = new SourceExtent(sourceChars);

            return(new StringLiteralToken(extent, unescaped));
        }
Пример #13
0
        private static StatementEndToken ReadStatementEndToken(ILexerStream stream)
        {
            var sourceChars = new List <SourceChar>();

            sourceChars.Add(stream.ReadChar(';'));
            var extent = new SourceExtent(sourceChars);

            return(new StatementEndToken(extent));
        }
Пример #14
0
        private static ParenthesesOpenToken ReadParenthesesOpenToken(ILexerStream stream)
        {
            var sourceChars = new List <SourceChar>();

            sourceChars.Add(stream.ReadChar('('));
            var extent = new SourceExtent(sourceChars);

            return(new ParenthesesOpenToken(extent));
        }
Пример #15
0
        private static EqualsOperatorToken ReadEqualsOperatorToken(ILexerStream stream)
        {
            var sourceChars = new List <SourceChar>();

            sourceChars.Add(stream.ReadChar('='));
            var extent = new SourceExtent(sourceChars);

            return(new EqualsOperatorToken(extent));
        }
Пример #16
0
        private static CommaToken ReadCommaToken(ILexerStream stream)
        {
            var sourceChars = new List <SourceChar>();

            sourceChars.Add(stream.ReadChar(','));
            var extent = new SourceExtent(sourceChars);

            return(new CommaToken(extent));
        }
Пример #17
0
        private static BlockOpenToken ReadBlockOpenToken(ILexerStream stream)
        {
            var sourceChars = new List <SourceChar>();

            sourceChars.Add(stream.ReadChar('{'));
            var extent = new SourceExtent(sourceChars);

            return(new BlockOpenToken(extent));
        }
Пример #18
0
        /// <summary>
        /// </summary>
        /// <param name="stream"></param>
        /// <returns></returns>
        /// <remarks>
        ///
        /// See http://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.0.pdf
        ///
        /// A.3 Compiler directive
        ///
        /// compilerDirective = PRAGMA ( pragmaName / standardPragmaName )
        ///                     "(" pragmaParameter ")"
        ///
        /// pragmaName         = IDENTIFIER
        /// standardPragmaName = INCLUDE
        /// pragmaParameter    = stringValue ; if the pragma is INCLUDE,
        ///                                  ; the parameter value
        ///                                  ; shall represent a relative
        ///                                  ; or full file path
        /// PRAGMA             = "#pragma"  ; keyword: case insensitive
        /// INCLUDE            = "include"  ; keyword: case insensitive
        ///
        /// </remarks>
        private static PragmaToken ReadPragmaToken(ILexerStream stream)
        {
            var sourceChars = new List <SourceChar>();

            sourceChars.AddRange(stream.ReadString(Keywords.PRAGMA));
            var extent = new SourceExtent(sourceChars);

            return(new PragmaToken(extent));
        }
Пример #19
0
        private static AttributeOpenToken ReadAttributeOpenToken(ILexerStream stream)
        {
            var sourceChars = new List <SourceChar>();

            sourceChars.Add(stream.ReadChar('['));
            var extent = new SourceExtent(sourceChars);

            return(new AttributeOpenToken(extent));
        }
Пример #20
0
 internal static PragmaToken Read(ILexerStream stream)
 {
     var extent = new SourceExtent(stream);
     var sourceChars = new List<char>();
     sourceChars.Add(stream.ReadChar('#'));
     sourceChars.AddRange(stream.ReadString("pragma"));
     extent = extent.WithText(sourceChars);
     return new PragmaToken(extent);
 }
 internal static CloseParenthesesToken Read(ILexerStream stream)
 {
     var extent = new SourceExtent(stream);
     var sourceChars = new List<char>();
     // read the character
     sourceChars.Add(stream.ReadChar(')'));
     // return the result
     extent = extent.WithText(sourceChars);
     return new CloseParenthesesToken(extent);
 }
 internal static AttributeOpenToken Read(ILexerStream stream)
 {
     var extent = new SourceExtent(stream);
     var sourceChars = new List<char>();
     // read the character
     sourceChars.Add(stream.ReadChar('['));
     // return the result
     extent = extent.WithText(sourceChars);
     return new AttributeOpenToken(extent);
 }
Пример #23
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="stream"></param>
        /// <returns></returns>
        /// <remarks>
        ///
        /// See https://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.1.pdf
        ///
        /// 5.4 Comments
        ///
        /// Comments in a MOF file do not create, modify, or annotate language elements. They shall be treated as if
        /// they were whitespace.
        ///
        /// Comments may appear anywhere in MOF syntax where whitespace is allowed and are indicated by either
        /// a leading double slash (//) or a pair of matching /* and */ character sequences. Occurrences of these
        /// character sequences in string literals shall not be treated as comments.
        ///
        /// A // comment is terminated by the end of line (see 5.3), as shown in the example below.
        ///
        ///     Integer MyProperty; // This is an example of a single-line comment
        ///
        /// A comment that begins with /* is terminated by the next */ sequence, or by the end of the MOF file,
        /// whichever comes first.
        ///
        ///     /* example of a comment between property definition tokens and a multi-line comment */
        ///     Integer /* 16-bit integer property */ MyProperty; /* and a multi-line
        ///                             comment */
        ///
        /// </remarks>
        public static (CommentToken, Lexer) ReadCommentToken(SourceReader reader)
        {
            var thisReader  = reader;
            var sourceChar  = default(SourceChar);
            var sourceChars = new List <SourceChar>();

            // read the starting '/'
            (sourceChar, thisReader) = thisReader.Read('/');
            sourceChars.Add(sourceChar);
            switch (thisReader.Peek().Value)
            {
            case '/':     // single-line
                (sourceChar, thisReader) = thisReader.Read('/');
                sourceChars.Add(sourceChar);
                // read the comment text
                while (!thisReader.Eof() && !thisReader.Peek(StringValidator.IsLineTerminator))
                {
                    (sourceChar, thisReader) = thisReader.Read();
                    sourceChars.Add(sourceChar);
                }
                ;
                break;

            case '*':     // multi-line
                (sourceChar, thisReader) = thisReader.Read('*');
                sourceChars.Add(sourceChar);
                // read the comment text
                while (!thisReader.Eof())
                {
                    (sourceChar, thisReader) = thisReader.Read();
                    sourceChars.Add(sourceChar);
                    if ((sourceChar.Value == '*') && thisReader.Peek('/'))
                    {
                        // read the closing sequence
                        (sourceChar, thisReader) = thisReader.Read('/');
                        sourceChars.Add(sourceChar);
                        break;
                    }
                }
                break;

            default:
                throw new UnexpectedCharacterException(reader.Peek());
            }
            // return the result
            var extent = SourceExtent.From(sourceChars);

            return(new CommentToken(extent), new Lexer(thisReader));
        }
Пример #24
0
 /// <summary>
 /// 
 /// </summary>
 /// <param name="stream"></param>
 /// <returns></returns>
 /// <remarks>
 /// 
 /// See http://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.0a.pdf
 /// 5.2 - Whitespace
 /// 
 /// Whitespace in a MOF file is any combination of the following characters:
 /// 
 ///     Space (U+0020),
 ///     Horizontal Tab (U+0009),
 ///     Carriage Return (U+000D) and
 ///     Line Feed (U+000A).
 /// 
 /// </remarks>
 internal static WhitespaceToken Read(ILexerStream stream)
 {
     var extent = new SourceExtent(stream);
     var sourceChars = new List<char>();
     // read the first whitespace character
     sourceChars.Add(stream.ReadWhitespace());
     // read the remaining whitespace
     while (!stream.Eof && char.IsWhiteSpace(stream.Peek()))
     {
         sourceChars.Add(stream.Read());
     }
     // return the result
     extent = extent.WithText(sourceChars);
     return new WhitespaceToken(extent);
 }
 internal static IntegerLiteralToken Read(ILexerStream stream)
 {
     var extent = new SourceExtent(stream);
     var sourceChars = new List<char>();
     // read the first character
     sourceChars.Add(stream.ReadDigit());
     // read the remaining characters
     while (stream.PeekDigit())
     {
         sourceChars.Add(stream.ReadDigit());
     }
     // return the result
     extent = extent.WithText(sourceChars);
     return new IntegerLiteralToken(extent, int.Parse(extent.Text));
 }
Пример #26
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="stream"></param>
        /// <returns></returns>
        /// <remarks>
        ///
        /// See http://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.0a.pdf
        ///
        /// 5.2 - Whitespace
        ///
        /// Whitespace in a MOF file is any combination of the following characters:
        ///
        ///     Space (U+0020),
        ///     Horizontal Tab (U+0009),
        ///     Carriage Return (U+000D) and
        ///     Line Feed (U+000A).
        ///
        /// The WS ABNF rule represents any one of these whitespace characters:
        ///
        ///     WS = U+0020 / U+0009 / U+000D / U+000A
        ///
        /// </remarks>
        private static WhitespaceToken ReadWhitespaceToken(ILexerStream stream)
        {
            var sourceChars = new List <SourceChar>();

            // read the first whitespace character
            sourceChars.Add(stream.ReadWhitespace());
            // read the remaining whitespace
            while (!stream.Eof && StringValidator.IsWhitespace(stream.Peek().Value))
            {
                sourceChars.Add(stream.Read());
            }
            // return the result
            var extent = new SourceExtent(sourceChars);

            return(new WhitespaceToken(extent));
        }
Пример #27
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="stream"></param>
        /// <returns></returns>
        /// <remarks>
        ///
        /// See http://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.0.pdf
        ///
        /// 5.4 Comments
        /// Comments in a MOF file do not create, modify, or annotate language elements. They shall be treated as if
        /// they were whitespace.
        ///
        /// Comments may appear anywhere in MOF syntax where whitespace is allowed and are indicated by either
        /// a leading double slash( // ) or a pair of matching /* and */ character sequences. Occurrences of these
        /// character sequences in string literals shall not be treated as comments.
        ///
        /// A // comment is terminated by the end of line (see 5.3), as shown in the example below.
        ///
        ///     uint16 MyProperty; // This is an example of a single-line comment
        ///
        /// A comment that begins with /* is terminated by the next */ sequence, or by the end of the MOF file,
        /// whichever comes first.
        ///
        ///     /* example of a comment between property definition tokens and a multi-line comment */
        ///     uint16 /* 16-bit integer property */ MyProperty; /* and a multi-line
        ///                             comment */
        ///
        /// </remarks>
        private static CommentToken ReadCommentToken(ILexerStream stream)
        {
            var sourceChars = new List <SourceChar>();

            sourceChars.Add(stream.ReadChar('/'));
            switch (stream.Peek().Value)
            {
            case '/':     // single-line
                sourceChars.Add(stream.ReadChar('/'));
                // read the comment text
                while (!stream.Eof && !StringValidator.IsLineTerminator(stream.Peek().Value))
                {
                    sourceChars.Add(stream.Read());
                }
                ;
                break;

            case '*':     // multi-line
                sourceChars.Add(stream.ReadChar('*'));
                // read the comment text
                while (!stream.Eof)
                {
                    var @char = stream.Read();
                    sourceChars.Add(@char);
                    if ((@char.Value == '*') && stream.PeekChar('/'))
                    {
                        // read the closing sequence
                        sourceChars.Add(stream.ReadChar('/'));
                        break;
                    }
                }
                break;

            default:
                throw new InvalidOperationException(
                          string.Format("Unexpected character '{0}'.", stream.Peek()));
            }
            // return the result
            var extent = new SourceExtent(sourceChars);

            return(new CommentToken(extent));
        }
Пример #28
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="stream"></param>
        /// <returns></returns>
        /// <remarks>
        ///
        /// See http://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.0a.pdf
        ///
        /// A.17.1 Integer value
        ///
        /// No whitespace is allowed between the elements of the rules in this ABNF section.
        ///
        ///     integerValue         = binaryValue / octalValue / hexValue / decimalValue
        ///
        ///     binaryValue          = [ "+" / "-" ] 1*binaryDigit ( "b" / "B" )
        ///     binaryDigit          = "0" / "1"
        ///
        ///     octalValue           = [ "+" / "-" ] unsignedOctalValue
        ///     unsignedOctalValue   = "0" 1*octalDigit
        ///     octalDigit           = "0" / "1" / "2" / "3" / "4" / "5" / "6" / "7"
        ///
        ///     hexValue             = [ "+" / "-" ] ( "0x" / "0X" ) 1*hexDigit
        ///     hexDigit             = decimalDigit / "a" / "A" / "b" / "B" / "c" / "C" /
        ///                                           "d" / "D" / "e" / "E" / "f" / "F"
        ///
        ///     decimalValue         = [ "+" / "-" ] unsignedDecimalValue
        ///     unsignedDecimalValue = positiveDecimalDigit *decimalDigit
        ///     decimalDigit         = "0" / positiveDecimalDigit
        ///     positiveDecimalDigit = "1"..."9"
        ///
        /// </remarks>
        private static NumericLiteralToken ReadNumericLiteralToken(ILexerStream stream)
        {
            var sourceChars = new List <SourceChar>();
            // read the sign (if there is one)
            var sign = 0;
            var peek = stream.Peek();

            switch (peek.Value)
            {
            case '+':
                sign = 1;
                sourceChars.Add(stream.Read());
                break;

            case '-':
                sign = -1;
                sourceChars.Add(stream.Read());
                break;
            }
            // read the remaining characters
            sourceChars.Add(stream.ReadDigit());
            while (!stream.Eof && stream.PeekDigit())
            {
                sourceChars.Add(stream.ReadDigit());
            }
            // return the result
            var extent = new SourceExtent(sourceChars);

            long    longValue;
            decimal decimalValue;

            if (long.TryParse(extent.Text, out longValue))
            {
                return(new IntegerLiteralToken(extent, longValue));
            }
            if (decimal.TryParse(extent.Text, out decimalValue))
            {
                return(new RealLiteralToken(extent, decimalValue));
            }

            throw new NotSupportedException($"{extent.Text} is not a valid number");
        }
Пример #29
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="stream"></param>
        /// <returns></returns>
        /// <remarks>
        ///
        /// See https://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.1.pdf
        ///
        /// 5.2 Whitespace
        ///
        /// Whitespace in a MOF file is any combination of the following characters:
        ///
        ///     * Space (U+0020),
        ///     * Horizontal Tab (U+0009),
        ///     * Carriage Return (U+000D) and
        ///     * Line Feed (U+000A).
        ///
        /// The WS ABNF rule represents any one of these whitespace characters:
        ///
        ///     WS = U+0020 / U+0009 / U+000D / U+000A
        ///
        /// </remarks>
        public static (WhitespaceToken, Lexer) ReadWhitespaceToken(SourceReader reader)
        {
            var thisReader  = reader;
            var sourceChar  = default(SourceChar);
            var sourceChars = new List <SourceChar>();

            // read the first whitespace character
            (sourceChar, thisReader) = thisReader.Read(StringValidator.IsWhitespace);
            sourceChars.Add(sourceChar);
            // read the remaining whitespace
            while (!thisReader.Eof() && thisReader.Peek(StringValidator.IsWhitespace))
            {
                (sourceChar, thisReader) = thisReader.Read();
                sourceChars.Add(sourceChar);
            }
            // return the result
            var extent = SourceExtent.From(sourceChars);

            return(new WhitespaceToken(extent), new Lexer(thisReader));
        }
        internal static MultilineCommentToken Read(ILexerStream stream)
        {
            var extent = new SourceExtent(stream);
            var sourceChars = new List<char>();
            // read the opening sequence
            sourceChars.Add(stream.ReadChar('/'));
            sourceChars.Add(stream.ReadChar('*'));
            // read the comment text
            while (!stream.Eof)
            {
                var c = stream.Read();
                sourceChars.Add(c);

                if (c == '*' && stream.PeekChar('/'))
                    break;
            }
            // read the closing  sequence
            sourceChars.Add(stream.ReadChar('/'));
            // return the result
            extent = extent.WithText(sourceChars);
            return new MultilineCommentToken(extent);
        }
 internal IntegerLiteralToken(SourceExtent extent, int value)
     : base(extent)
 {
     this.Value = value;
 }
 internal CloseParenthesesToken(SourceExtent extent)
     : base(extent)
 {
 }
Пример #33
0
 internal StatementEndToken(SourceExtent extent)
     : base(extent)
 {
 }
Пример #34
0
 internal WhitespaceToken(SourceExtent extent)
     : base(extent)
 {
 }
 internal OpenParenthesesToken(SourceExtent extent)
     : base(extent)
 {
 }
 internal BooleanLiteralToken(SourceExtent extent, bool value)
     : base(extent)
 {
     this.Value = value;
 }
Пример #37
0
 internal CommaToken(SourceExtent extent)
     : base(extent)
 {
 }
Пример #38
0
 internal PragmaToken(SourceExtent extent)
     : base(extent)
 {
 }
 internal AttributeCloseToken(SourceExtent extent)
     : base(extent)
 {
 }
 internal StringLiteralToken(SourceExtent extent, string value)
     : base(extent)
 {
     this.Value = value;
 }
 internal AliasIdentifierToken(SourceExtent extent, string name)
     : base(extent)
 {
     this.Name = name;
 }
 internal AttributeOpenToken(SourceExtent extent)
     : base(extent)
 {
 }
Пример #43
0
        /// <summary>
        ///
        /// </summary>
        /// <param name="stream"></param>
        /// <returns></returns>
        /// <remarks>
        ///
        /// See https://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.1.pdf
        ///
        /// 7.6.1.1 Integer value
        ///
        /// No whitespace is allowed between the elements of the rules in this ABNF section.
        ///
        ///     integerValue         = binaryValue / octalValue / hexValue / decimalValue
        ///
        ///     binaryValue          = [ "+" / "-" ] 1*binaryDigit ( "b" / "B" )
        ///     binaryDigit          = "0" / "1"
        ///
        ///     octalValue           = [ "+" / "-" ] unsignedOctalValue
        ///     unsignedOctalValue   = "0" 1*octalDigit
        ///     octalDigit           = "0" / "1" / "2" / "3" / "4" / "5" / "6" / "7"
        ///
        ///     hexValue             = [ "+" / "-" ] ( "0x" / "0X" ) 1*hexDigit
        ///     hexDigit             = decimalDigit / "a" / "A" / "b" / "B" / "c" / "C" /
        ///                                           "d" / "D" / "e" / "E" / "f" / "F"
        ///
        ///     decimalValue         = [ "+" / "-" ] unsignedDecimalValue
        ///     unsignedDecimalValue = "0" / positiveDecimalDigit *decimalDigit
        ///
        ///     decimalDigit         = "0" / positiveDecimalDigit
        ///     positiveDecimalDigit = "1"..."9"
        ///
        /// 7.6.1.2 Real value
        ///
        /// No whitespace is allowed between the elements of the rules in this ABNF section.
        ///
        ///     realValue            = [ "+" / "-" ] *decimalDigit "." 1*decimalDigit
        ///                            [ ( "e" / "E" ) [ "+" / "-" ] 1*decimalDigit ]
        ///
        ///     decimalDigit         = "0" / positiveDecimalDigit
        ///     positiveDecimalDigit = "1"..."9"
        ///
        /// </remarks>
        public static (Token, Lexer) ReadNumericLiteralToken(SourceReader reader)
        {
            int ParseBinaryValueDigits(IEnumerable <SourceChar> binaryChars, SourceChar sign)
            {
                return(ParseIntegerValueDigits(new Dictionary <char, int>
                {
                    { '0', 0 }, { '1', 1 }
                }, 2, binaryChars, sign));
            }

            int ParseOctalValueDigits(IEnumerable <SourceChar> octalChars, SourceChar sign)
            {
                return(ParseIntegerValueDigits(new Dictionary <char, int>
                {
                    { '0', 0 }, { '1', 1 }, { '2', 2 }, { '3', 3 }, { '4', 4 }, { '5', 5 }, { '6', 6 }, { '7', 7 }
                }, 8, octalChars, sign));
            }

            int ParseHexValueDigits(IEnumerable <SourceChar> hexChars, SourceChar sign)
            {
                return(ParseIntegerValueDigits(new Dictionary <char, int>
                {
                    { '0', 0 }, { '1', 1 }, { '2', 2 }, { '3', 3 }, { '4', 4 }, { '5', 5 }, { '6', 6 }, { '7', 7 }, { '8', 8 }, { '9', 9 },
                    { 'a', 10 }, { 'b', 11 }, { 'c', 12 }, { 'd', 13 }, { 'e', 14 }, { 'f', 15 },
                    { 'A', 10 }, { 'B', 11 }, { 'C', 12 }, { 'D', 13 }, { 'E', 14 }, { 'F', 15 }
                }, 16, hexChars, sign));
            }

            int ParseDecimalValueDigits(IEnumerable <SourceChar> decimalChars, SourceChar sign)
            {
                return(ParseIntegerValueDigits(new Dictionary <char, int>
                {
                    { '0', 0 }, { '1', 1 }, { '2', 2 }, { '3', 3 }, { '4', 4 }, { '5', 5 }, { '6', 6 }, { '7', 7 }, { '8', 8 }, { '9', 9 }
                }, 10, decimalChars, sign));
            }

            int ParseIntegerValueDigits(Dictionary <char, int> alphabet, int radix, IEnumerable <SourceChar> chars, SourceChar sign)
            {
                var literalValue = 0;

                foreach (var digit in chars)
                {
                    var digitValue = alphabet[digit.Value];
                    literalValue = (literalValue * radix) + digitValue;
                }
                if (sign?.Value == '-')
                {
                    literalValue = -literalValue;
                }
                return(literalValue);
            }

            const int stateLeadingSign         = 1;
            const int stateFirstDigitBlock     = 2;
            const int stateOctalOrDecimalValue = 3;
            const int stateBinaryValue         = 4;
            const int stateOctalValue          = 5;
            const int stateHexValue            = 6;
            const int stateDecimalValue        = 7;
            const int stateRealValue           = 8;
            const int stateRealValueFraction   = 9;
            const int stateRealValueExponent   = 10;
            const int stateDone = 99;

            var thisReader  = reader;
            var sourceChar  = default(SourceChar);
            var sourceChars = new List <SourceChar>();

            var token           = default(Token);
            var signChar        = default(SourceChar);
            var firstDigitBlock = new List <SourceChar>();

            var currentState = stateLeadingSign;

            while (currentState != stateDone)
            {
                switch (currentState)
                {
                case stateLeadingSign:
                    // we're reading the initial optional leading sign
                    // [ "+" / "-" ]
                    sourceChar = thisReader.Peek();
                    switch (sourceChar.Value)
                    {
                    case '+':
                    case '-':
                        (signChar, thisReader) = thisReader.Read();
                        sourceChars.Add(signChar);
                        break;
                    }
                    currentState = stateFirstDigitBlock;
                    break;

                case stateFirstDigitBlock:
                    // we're reading the first block of digits in the value,
                    // but we won't necessarily know which type we're reading
                    // until we've consumed more of the input stream
                    //
                    //     binaryValue  => 1*binaryDigit
                    //     octalValue   => "0" 1*octalDigit
                    //     hexValue     => ( "0x" / "0X" )
                    //     decimalValue => positiveDecimalDigit *decimalDigit
                    //     realValue    => *decimalDigit
                    //
                    if (thisReader.Peek('.'))
                    {
                        // we're reading a realValue with no "*decimalDigit" characters before the "."
                        // e.g. ".45", "+.45", "-.45", so consume the decimal point
                        (sourceChar, thisReader) = thisReader.Read();
                        sourceChars.Add(sourceChar);
                        // and go to the next state
                        currentState = stateRealValueFraction;
                        break;
                    }
                    // we don't know which base the value is in yet, but if it's hexadecimal them
                    // we should be reading the "0x" part here, so restrict digits to decimal in
                    // all cases
                    (firstDigitBlock, thisReader) = thisReader.ReadWhile(StringValidator.IsDecimalDigit);
                    sourceChars.AddRange(firstDigitBlock);
                    // now we can do some validation
                    if (firstDigitBlock.Count == 0)
                    {
                        // only realValue allows no digits in the first block, and
                        // we've already handled that at the start of this case
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    // if we've reached the end of the stream then there's no suffix
                    // (e.g. b, B, x, X, .) so this must be an octalValue or decimalValue
                    if (thisReader.Eof())
                    {
                        currentState = stateOctalOrDecimalValue;
                        break;
                    }
                    // check the next character to see if it tells us anything
                    // about which type of literal we're reading
                    sourceChar = thisReader.Peek();
                    switch (sourceChar.Value)
                    {
                    case 'b':
                    case 'B':
                        // binaryValue
                        currentState = stateBinaryValue;
                        break;

                    case 'x':
                    case 'X':
                        // hexValue
                        currentState = stateHexValue;
                        break;

                    case '.':
                        // realValue
                        currentState = stateRealValue;
                        break;

                    default:
                        // by elmination, this must be an octalValue or decimalValue
                        currentState = stateOctalOrDecimalValue;
                        break;
                    }
                    break;

                case stateOctalOrDecimalValue:
                    // we're reading an octalValue or decimalValue, but we're not sure which yet...
                    if ((firstDigitBlock.First().Value == '0') && (firstDigitBlock.Count > 1))
                    {
                        currentState = stateOctalValue;
                    }
                    else
                    {
                        currentState = stateDecimalValue;
                    }
                    break;

                case stateBinaryValue:
                    // we're trying to read a binaryValue, so check all the characters in the digit block are valid,
                    // i.e. 1*binaryDigit
                    if (firstDigitBlock.Any(c => !StringValidator.IsBinaryDigit(c.Value)))
                    {
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    // all the characters are valid, so consume the suffix
                    (sourceChar, thisReader) = thisReader.Read(c => (c == 'b') || (c == 'B'));
                    sourceChars.Add(sourceChar);
                    // now build the return value
                    var binaryValue = ParseBinaryValueDigits(firstDigitBlock, signChar);
                    token = new IntegerLiteralToken(SourceExtent.From(sourceChars), IntegerKind.BinaryValue, binaryValue);
                    // and we're done
                    currentState = stateDone;
                    break;

                case stateOctalValue:
                    // we're trying to read an octalValue (since decimalValue can't start with a
                    // leading '0') so check all the characters in the digit block are valid,
                    // i.e. "0" 1*octalDigit
                    if ((firstDigitBlock.Count < 2) || (firstDigitBlock.First().Value != '0'))
                    {
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    if (firstDigitBlock.Skip(1).Any(c => !StringValidator.IsOctalDigit(c.Value)))
                    {
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    // now build the return value
                    var octalValue = ParseOctalValueDigits(firstDigitBlock, signChar);
                    token = new IntegerLiteralToken(SourceExtent.From(sourceChars), IntegerKind.OctalValue, octalValue);
                    // and we're done
                    currentState = stateDone;
                    break;

                case stateHexValue:
                    // we're trying to read a hexValue, so we should have just read a leading zero
                    if ((firstDigitBlock.Count != 1) || (firstDigitBlock.First().Value != '0'))
                    {
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    // all the characters are valid, so consume the suffix
                    (sourceChar, thisReader) = thisReader.Read(c => (c == 'x') || (c == 'X'));
                    sourceChars.Add(sourceChar);
                    // 1*hexDigit
                    var hexDigits = default(List <SourceChar>);
                    (hexDigits, thisReader) = thisReader.ReadWhile(StringValidator.IsHexDigit);
                    if (hexDigits.Count == 0)
                    {
                        throw new UnexpectedCharacterException(thisReader.Peek());
                    }
                    sourceChars.AddRange(hexDigits);
                    // build the return value
                    var hexValue = ParseHexValueDigits(hexDigits, signChar);
                    token = new IntegerLiteralToken(SourceExtent.From(sourceChars), IntegerKind.HexValue, hexValue);
                    // and we're done
                    currentState = stateDone;
                    break;

                case stateDecimalValue:
                    // we're trying to read a decimalValue (since that's the only remaining option),
                    // so check all the characters in the digit block are valid,
                    // i.e. "0" / positiveDecimalDigit *decimalDigit
                    if ((firstDigitBlock.Count == 1) && (firstDigitBlock.First().Value == '0'))
                    {
                        // "0"
                    }
                    else if (!StringValidator.IsPositiveDecimalDigit(firstDigitBlock.First().Value))
                    {
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    else if (firstDigitBlock.Skip(1).Any(c => !StringValidator.IsDecimalDigit(c.Value)))
                    {
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    // build the return value
                    var decimalValue = ParseDecimalValueDigits(firstDigitBlock, signChar);
                    token = new IntegerLiteralToken(SourceExtent.From(sourceChars), IntegerKind.DecimalValue, decimalValue);
                    // and we're done
                    currentState = stateDone;
                    break;

                case stateRealValue:
                    // we're trying to read a realValue, so check all the characters in the digit block are valid,
                    // i.e. *decimalDigit
                    if (firstDigitBlock.Any(c => !StringValidator.IsDecimalDigit(c.Value)))
                    {
                        throw new UnexpectedCharacterException(sourceChar);
                    }
                    // all the characters are valid, so consume the decimal point
                    (sourceChar, thisReader) = thisReader.Read('.');
                    sourceChars.Add(sourceChar);
                    // and go to the next state
                    currentState = stateRealValueFraction;
                    break;

                case stateRealValueFraction:
                    // 1*decimalDigit
                    var realFractionDigits = default(List <SourceChar>);
                    (realFractionDigits, thisReader) = thisReader.ReadWhile(StringValidator.IsHexDigit);
                    if (realFractionDigits.Count == 0)
                    {
                        throw new UnexpectedCharacterException(thisReader.Peek());
                    }
                    sourceChars.AddRange(realFractionDigits);
                    // ( "e" / "E" )
                    if (!thisReader.Eof())
                    {
                        sourceChar = thisReader.Peek();
                        if ((sourceChar.Value == 'e') || (sourceChar.Value == 'E'))
                        {
                            currentState = stateRealValueExponent;
                            break;
                        }
                    }
                    // build the return value
                    var realIntegerValue  = ParseDecimalValueDigits(firstDigitBlock, signChar);
                    var realFractionValue = (double)ParseDecimalValueDigits(realFractionDigits, signChar);
                    if (realFractionDigits.Any())
                    {
                        realFractionValue = realFractionValue / Math.Pow(10, realFractionDigits.Count);
                    }
                    token = new RealLiteralToken(
                        SourceExtent.From(sourceChars),
                        realIntegerValue + realFractionValue
                        );
                    // and we're done
                    currentState = stateDone;
                    break;

                case stateRealValueExponent:
                    throw new InvalidOperationException();

                case stateDone:
                    // the main while loop should exit before we ever get here
                    throw new InvalidOperationException();

                default:
                    throw new InvalidOperationException();
                }
            }

            return(token, new Lexer(thisReader));
        }
Пример #44
0
 internal BlockCloseToken(SourceExtent extent)
     : base(extent)
 {
 }
Пример #45
0
        /// <summary>
        /// </summary>
        /// <param name="stream"></param>
        /// <returns></returns>
        /// <remarks>
        ///
        /// See https://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.1.pdf
        ///
        /// 7.6.1.3 String values
        ///
        /// Unless explicitly specified via ABNF rule WS, no whitespace is allowed between the elements of the rules
        /// in this ABNF section.
        ///
        ///     singleStringValue = DOUBLEQUOTE *stringChar DOUBLEQUOTE
        ///     stringValue       = singleStringValue *( *WS singleStringValue )
        ///
        ///     stringChar        = stringUCSchar / stringEscapeSequence
        ///     stringUCSchar     = U+0020...U+0021 / U+0023...U+D7FF /
        ///                         U+E000...U+FFFD / U+10000...U+10FFFF
        ///                         ; Note that these UCS characters can be
        ///                         ; represented in XML without any escaping
        ///                         ; (see W3C XML).
        ///
        ///     stringEscapeSequence = BACKSLASH ( BACKSLASH / DOUBLEQUOTE / SINGLEQUOTE /
        ///                            BACKSPACE_ESC / TAB_ESC / LINEFEED_ESC /
        ///                            FORMFEED_ESC / CARRIAGERETURN_ESC /
        ///                            escapedUCSchar )
        ///
        ///     BACKSPACE_ESC      = "b" ; escape for back space (U+0008)
        ///     TAB_ESC            = "t" ; escape for horizontal tab(U+0009)
        ///     LINEFEED_ESC       = "n" ; escape for line feed(U+000A)
        ///     FORMFEED_ESC       = "f" ; escape for form feed(U+000C)
        ///     CARRIAGERETURN_ESC = "r" ; escape for carriage return (U+000D)
        ///
        ///     escapedUCSchar     = ( "x" / "X" ) 1*6(hexDigit ) ; escaped UCS
        ///                          ; character with a UCS code position that is
        ///                          ; the numeric value of the hex number
        ///
        /// The following special characters are also used in other ABNF rules in this specification:
        ///
        ///     BACKSLASH   = U+005C ; \
        ///     DOUBLEQUOTE = U+0022 ; "
        ///     SINGLEQUOTE = U+0027 ; '
        ///     UPPERALPHA  = U+0041...U+005A ; A ... Z
        ///     LOWERALPHA  = U+0061...U+007A ; a ... z
        ///     UNDERSCORE  = U+005F ; _
        ///
        /// </remarks>
        public static (StringLiteralToken, Lexer) ReadStringLiteralToken(SourceReader reader)
        {
            // BUGBUG - no support for *( *WS DOUBLEQUOTE *stringChar DOUBLEQUOTE )
            // BUGBUG - incomplete escape sequences
            // BUGBUG - no support for UCS characters
            var thisReader  = reader;
            var sourceChar  = default(SourceChar);
            var sourceChars = new List <SourceChar>();
            var stringChars = new StringBuilder();

            // read the first double-quote character
            (sourceChar, thisReader) = thisReader.Read(Constants.DOUBLEQUOTE);
            sourceChars.Add(sourceChar);
            // read the remaining characters
            bool isEscaped    = false;
            bool isTerminated = false;

            while (!thisReader.Eof())
            {
                var peek = thisReader.Peek();
                if (isEscaped)
                {
                    // read the second character in an escape sequence
                    var escapedChar = default(char);
                    switch (peek.Value)
                    {
                    case Constants.BACKSLASH:
                        escapedChar = Constants.BACKSLASH;
                        break;

                    case Constants.DOUBLEQUOTE:
                        escapedChar = Constants.DOUBLEQUOTE;
                        break;

                    case Constants.SINGLEQUOTE:
                        escapedChar = Constants.SINGLEQUOTE;
                        break;

                    case Constants.BACKSPACE_ESC:
                        escapedChar = '\b';
                        break;

                    case Constants.TAB_ESC:
                        escapedChar = '\t';
                        break;

                    case Constants.LINEFEED_ESC:
                        escapedChar = '\n';
                        break;

                    case Constants.FORMFEED_ESC:
                        escapedChar = '\f';
                        break;

                    case Constants.CARRIAGERETURN_ESC:
                        escapedChar = '\r';
                        break;

                    default:
                        throw new UnexpectedCharacterException(peek);
                    }
                    thisReader = thisReader.Next();
                    sourceChars.Add(peek);
                    stringChars.Append(escapedChar);
                    isEscaped = false;
                }
                else if (peek.Value == Constants.BACKSLASH)
                {
                    // read the first character in an escape sequence
                    thisReader = thisReader.Next();
                    sourceChars.Add(peek);
                    isEscaped = true;
                }
                else if (peek.Value == Constants.DOUBLEQUOTE)
                {
                    // read the last double-quote character
                    (_, thisReader) = thisReader.Read(Constants.DOUBLEQUOTE);
                    sourceChars.Add(peek);
                    isTerminated = true;
                    break;
                }
                else
                {
                    // read a literal string character
                    thisReader = thisReader.Next();
                    sourceChars.Add(peek);
                    stringChars.Append(peek.Value);
                }
            }
            // make sure we found the end of the string
            if (!isTerminated)
            {
                throw new InvalidOperationException("Unterminated string found.");
            }
            // return the result
            var extent      = SourceExtent.From(sourceChars);
            var stringValue = stringChars.ToString();

            return(new StringLiteralToken(extent, stringValue), new Lexer(thisReader));
        }
Пример #46
0
 internal ColonToken(SourceExtent extent)
     : base(extent)
 {
 }
 internal MultilineCommentToken(SourceExtent extent)
     : base(extent)
 {
 }
Пример #48
0
 internal BlockOpenToken(SourceExtent extent)
     : base(extent)
 {
 }