/// <summary>
///
/// </summary>
/// <param name="stream"></param>
/// <returns></returns>
/// <remarks>
///
/// See http://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.0a.pdf
///
/// A.17.3 String values
///
/// Unless explicitly specified via ABNF rule WS, no whitespace is allowed between the elements of the rules
/// in this ABNF section.
///
/// stringValue = DOUBLEQUOTE *stringChar DOUBLEQUOTE
/// *( *WS DOUBLEQUOTE *stringChar DOUBLEQUOTE )
/// stringChar = stringUCSchar / stringEscapeSequence
///
/// </remarks>
private static StringLiteralToken ReadStringLiteralToken(ILexerStream stream)
{
// BUGBUG - no support for *( *WS DOUBLEQUOTE *stringChar DOUBLEQUOTE )
// BUGBUG - incomplete escape sequences
// BUGBUG - no support for UCS characters
var sourceChars = new List <SourceChar>();
// read the first character
sourceChars.Add(stream.ReadChar('"'));
// read the remaining characters
var parser = new StringLiteralParser();
while (!stream.Eof)
{
var peek = stream.Peek();
if (StringValidator.IsDoubleQuote(peek.Value) && !parser.IsEscaped)
{
parser.ConsumeEos();
break;
}
else
{
sourceChars.Add(peek);
parser.ConsumeChar(stream.Read());
}
}
// read the last character
sourceChars.Add(stream.ReadChar('"'));
// process any escape sequences in the string
var unescaped = parser.OutputString.ToString();
// return the result
var extent = new SourceExtent(sourceChars);
return(new StringLiteralToken(extent, unescaped));
}
/// <summary>
///
/// </summary>
/// <param name="stream"></param>
/// <returns></returns>
/// <remarks>
///
/// See http://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.0.pdf
///
/// A.13 Names
///
/// MOF names are identifiers with the format defined by the IDENTIFIER rule.
/// No whitespace is allowed between the elements of the rules in this ABNF section.
///
/// IDENTIFIER = firstIdentifierChar *( nextIdentifierChar )
/// firstIdentifierChar = UPPERALPHA / LOWERALPHA / UNDERSCORE
/// nextIdentifierChar = firstIdentifierChar / decimalDigit
/// elementName = localName / schemaQualifiedName
/// localName = IDENTIFIER
///
/// </remarks>
private static IdentifierToken ReadIdentifierToken(ILexerStream stream)
{
var sourceChars = new List <SourceChar>();
var nameChars = new List <char>();
// firstIdentifierChar
var peek = stream.Peek();
if (!StringValidator.IsFirstIdentifierChar(peek.Value))
{
throw new InvalidOperationException(
string.Format("Unexpected character '{0}' encountered", peek.Value));
}
// *( nextIdentifierChar )
while (!stream.Eof)
{
peek = stream.Peek();
if (StringValidator.IsNextIdentifierChar(peek.Value))
{
var @char = stream.Read();
sourceChars.Add(@char);
nameChars.Add(@char.Value);
}
else
{
break;
}
}
// return the result
var extent = new SourceExtent(sourceChars);
var name = new string(nameChars.ToArray());
return(new IdentifierToken(extent, name));
}
internal static AliasIdentifierToken Read(ILexerStream stream)
{
var extent = new SourceExtent(stream);
var sourceChars = new List<char>();
// read the first character
sourceChars.Add(stream.ReadChar('$'));
// read the name
var nameChars = new List<char>();
while (!stream.Eof)
{
extent = extent.WithEndExtent(stream);
var peek = stream.Peek();
if (char.IsLetterOrDigit(peek) || ("_".IndexOf(peek) != -1) )
{
var @char = stream.Read();
sourceChars.Add(@char);
nameChars.Add(@char);
}
else
{
break;
}
}
// return the result
extent = extent.WithText(sourceChars);
var name = new string(nameChars.ToArray());
return new AliasIdentifierToken(extent, name);
}
internal static StringLiteralToken Read(ILexerStream stream)
{
var extent = new SourceExtent(stream);
var sourceChars = new List<char>();
// read the first character
sourceChars.Add(stream.ReadChar('"'));
// read the remaining characters
var parser = new StringParser();
while (!stream.Eof)
{
var peek = stream.Peek();
sourceChars.Add(peek);
if ((peek == '"') && !parser.IsEscaped)
{
parser.ConsumeEof();
break;
}
else
{
parser.ConsumeChar(stream.Read());
}
}
// read the last character
sourceChars.Add(stream.ReadChar('"'));
// process any escape sequences in the string
var unescaped = parser.OutputString.ToString();
// return the result
extent = extent.WithText(sourceChars);
return new StringLiteralToken(extent, unescaped);
}
public Lexer(ILexerStream stream)
{
if (stream == null)
{
throw new ArgumentNullException("stream");
}
this.Stream = stream;
}
private static ParenthesesOpenToken ReadParenthesesOpenToken(ILexerStream stream)
{
var sourceChars = new List <SourceChar>();
sourceChars.Add(stream.ReadChar('('));
var extent = new SourceExtent(sourceChars);
return(new ParenthesesOpenToken(extent));
}
private static EqualsOperatorToken ReadEqualsOperatorToken(ILexerStream stream)
{
var sourceChars = new List <SourceChar>();
sourceChars.Add(stream.ReadChar('='));
var extent = new SourceExtent(sourceChars);
return(new EqualsOperatorToken(extent));
}
private static CommaToken ReadCommaToken(ILexerStream stream)
{
var sourceChars = new List <SourceChar>();
sourceChars.Add(stream.ReadChar(','));
var extent = new SourceExtent(sourceChars);
return(new CommaToken(extent));
}
private static BlockOpenToken ReadBlockOpenToken(ILexerStream stream)
{
var sourceChars = new List <SourceChar>();
sourceChars.Add(stream.ReadChar('{'));
var extent = new SourceExtent(sourceChars);
return(new BlockOpenToken(extent));
}
private static AttributeOpenToken ReadAttributeOpenToken(ILexerStream stream)
{
var sourceChars = new List <SourceChar>();
sourceChars.Add(stream.ReadChar('['));
var extent = new SourceExtent(sourceChars);
return(new AttributeOpenToken(extent));
}
internal static PragmaToken Read(ILexerStream stream)
{
var extent = new SourceExtent(stream);
var sourceChars = new List<char>();
sourceChars.Add(stream.ReadChar('#'));
sourceChars.AddRange(stream.ReadString("pragma"));
extent = extent.WithText(sourceChars);
return new PragmaToken(extent);
}
/// <summary>
/// </summary>
/// <param name="stream"></param>
/// <returns></returns>
/// <remarks>
///
/// See http://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.0.pdf
///
/// A.3 Compiler directive
///
/// compilerDirective = PRAGMA ( pragmaName / standardPragmaName )
/// "(" pragmaParameter ")"
///
/// pragmaName = IDENTIFIER
/// standardPragmaName = INCLUDE
/// pragmaParameter = stringValue ; if the pragma is INCLUDE,
/// ; the parameter value
/// ; shall represent a relative
/// ; or full file path
/// PRAGMA = "#pragma" ; keyword: case insensitive
/// INCLUDE = "include" ; keyword: case insensitive
///
/// </remarks>
private static PragmaToken ReadPragmaToken(ILexerStream stream)
{
var sourceChars = new List <SourceChar>();
sourceChars.AddRange(stream.ReadString(Keywords.PRAGMA));
var extent = new SourceExtent(sourceChars);
return(new PragmaToken(extent));
}
private static StatementEndToken ReadStatementEndToken(ILexerStream stream)
{
var sourceChars = new List <SourceChar>();
sourceChars.Add(stream.ReadChar(';'));
var extent = new SourceExtent(sourceChars);
return(new StatementEndToken(extent));
}
internal static CloseParenthesesToken Read(ILexerStream stream)
{
var extent = new SourceExtent(stream);
var sourceChars = new List<char>();
// read the character
sourceChars.Add(stream.ReadChar(')'));
// return the result
extent = extent.WithText(sourceChars);
return new CloseParenthesesToken(extent);
}
internal static AttributeOpenToken Read(ILexerStream stream)
{
var extent = new SourceExtent(stream);
var sourceChars = new List<char>();
// read the character
sourceChars.Add(stream.ReadChar('['));
// return the result
extent = extent.WithText(sourceChars);
return new AttributeOpenToken(extent);
}
internal static IntegerLiteralToken Read(ILexerStream stream)
{
var extent = new SourceExtent(stream);
var sourceChars = new List<char>();
// read the first character
sourceChars.Add(stream.ReadDigit());
// read the remaining characters
while (stream.PeekDigit())
{
sourceChars.Add(stream.ReadDigit());
}
// return the result
extent = extent.WithText(sourceChars);
return new IntegerLiteralToken(extent, int.Parse(extent.Text));
}
/// <summary>
///
/// </summary>
/// <param name="stream"></param>
/// <returns></returns>
/// <remarks>
///
/// See http://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.0a.pdf
/// 5.2 - Whitespace
///
/// Whitespace in a MOF file is any combination of the following characters:
///
/// Space (U+0020),
/// Horizontal Tab (U+0009),
/// Carriage Return (U+000D) and
/// Line Feed (U+000A).
///
/// </remarks>
internal static WhitespaceToken Read(ILexerStream stream)
{
var extent = new SourceExtent(stream);
var sourceChars = new List<char>();
// read the first whitespace character
sourceChars.Add(stream.ReadWhitespace());
// read the remaining whitespace
while (!stream.Eof && char.IsWhiteSpace(stream.Peek()))
{
sourceChars.Add(stream.Read());
}
// return the result
extent = extent.WithText(sourceChars);
return new WhitespaceToken(extent);
}
/// <summary>
///
/// </summary>
/// <param name="stream"></param>
/// <returns></returns>
/// <remarks>
///
/// See http://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.0a.pdf
///
/// 5.2 - Whitespace
///
/// Whitespace in a MOF file is any combination of the following characters:
///
/// Space (U+0020),
/// Horizontal Tab (U+0009),
/// Carriage Return (U+000D) and
/// Line Feed (U+000A).
///
/// The WS ABNF rule represents any one of these whitespace characters:
///
/// WS = U+0020 / U+0009 / U+000D / U+000A
///
/// </remarks>
private static WhitespaceToken ReadWhitespaceToken(ILexerStream stream)
{
var sourceChars = new List <SourceChar>();
// read the first whitespace character
sourceChars.Add(stream.ReadWhitespace());
// read the remaining whitespace
while (!stream.Eof && StringValidator.IsWhitespace(stream.Peek().Value))
{
sourceChars.Add(stream.Read());
}
// return the result
var extent = new SourceExtent(sourceChars);
return(new WhitespaceToken(extent));
}
/// <summary>
///
/// </summary>
/// <param name="stream"></param>
/// <returns></returns>
/// <remarks>
///
/// See http://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.0a.pdf
///
/// A.17.1 Integer value
///
/// No whitespace is allowed between the elements of the rules in this ABNF section.
///
/// integerValue = binaryValue / octalValue / hexValue / decimalValue
///
/// binaryValue = [ "+" / "-" ] 1*binaryDigit ( "b" / "B" )
/// binaryDigit = "0" / "1"
///
/// octalValue = [ "+" / "-" ] unsignedOctalValue
/// unsignedOctalValue = "0" 1*octalDigit
/// octalDigit = "0" / "1" / "2" / "3" / "4" / "5" / "6" / "7"
///
/// hexValue = [ "+" / "-" ] ( "0x" / "0X" ) 1*hexDigit
/// hexDigit = decimalDigit / "a" / "A" / "b" / "B" / "c" / "C" /
/// "d" / "D" / "e" / "E" / "f" / "F"
///
/// decimalValue = [ "+" / "-" ] unsignedDecimalValue
/// unsignedDecimalValue = positiveDecimalDigit *decimalDigit
/// decimalDigit = "0" / positiveDecimalDigit
/// positiveDecimalDigit = "1"..."9"
///
/// </remarks>
private static NumericLiteralToken ReadNumericLiteralToken(ILexerStream stream)
{
var sourceChars = new List <SourceChar>();
// read the sign (if there is one)
var sign = 0;
var peek = stream.Peek();
switch (peek.Value)
{
case '+':
sign = 1;
sourceChars.Add(stream.Read());
break;
case '-':
sign = -1;
sourceChars.Add(stream.Read());
break;
}
// read the remaining characters
sourceChars.Add(stream.ReadDigit());
while (!stream.Eof && stream.PeekDigit())
{
sourceChars.Add(stream.ReadDigit());
}
// return the result
var extent = new SourceExtent(sourceChars);
long longValue;
decimal decimalValue;
if (long.TryParse(extent.Text, out longValue))
{
return(new IntegerLiteralToken(extent, longValue));
}
if (decimal.TryParse(extent.Text, out decimalValue))
{
return(new RealLiteralToken(extent, decimalValue));
}
throw new NotSupportedException($"{extent.Text} is not a valid number");
}
/// <summary>
///
/// </summary>
/// <param name="stream"></param>
/// <returns></returns>
/// <remarks>
///
/// See http://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.0.pdf
///
/// 5.4 Comments
/// Comments in a MOF file do not create, modify, or annotate language elements. They shall be treated as if
/// they were whitespace.
///
/// Comments may appear anywhere in MOF syntax where whitespace is allowed and are indicated by either
/// a leading double slash( // ) or a pair of matching /* and */ character sequences. Occurrences of these
/// character sequences in string literals shall not be treated as comments.
///
/// A // comment is terminated by the end of line (see 5.3), as shown in the example below.
///
/// uint16 MyProperty; // This is an example of a single-line comment
///
/// A comment that begins with /* is terminated by the next */ sequence, or by the end of the MOF file,
/// whichever comes first.
///
/// /* example of a comment between property definition tokens and a multi-line comment */
/// uint16 /* 16-bit integer property */ MyProperty; /* and a multi-line
/// comment */
///
/// </remarks>
private static CommentToken ReadCommentToken(ILexerStream stream)
{
var sourceChars = new List <SourceChar>();
sourceChars.Add(stream.ReadChar('/'));
switch (stream.Peek().Value)
{
case '/': // single-line
sourceChars.Add(stream.ReadChar('/'));
// read the comment text
while (!stream.Eof && !StringValidator.IsLineTerminator(stream.Peek().Value))
{
sourceChars.Add(stream.Read());
}
;
break;
case '*': // multi-line
sourceChars.Add(stream.ReadChar('*'));
// read the comment text
while (!stream.Eof)
{
var @char = stream.Read();
sourceChars.Add(@char);
if ((@char.Value == '*') && stream.PeekChar('/'))
{
// read the closing sequence
sourceChars.Add(stream.ReadChar('/'));
break;
}
}
break;
default:
throw new InvalidOperationException(
string.Format("Unexpected character '{0}'.", stream.Peek()));
}
// return the result
var extent = new SourceExtent(sourceChars);
return(new CommentToken(extent));
}
internal static MultilineCommentToken Read(ILexerStream stream)
{
var extent = new SourceExtent(stream);
var sourceChars = new List<char>();
// read the opening sequence
sourceChars.Add(stream.ReadChar('/'));
sourceChars.Add(stream.ReadChar('*'));
// read the comment text
while (!stream.Eof)
{
var c = stream.Read();
sourceChars.Add(c);
if (c == '*' && stream.PeekChar('/'))
break;
}
// read the closing sequence
sourceChars.Add(stream.ReadChar('/'));
// return the result
extent = extent.WithText(sourceChars);
return new MultilineCommentToken(extent);
}
internal SourceExtent WithEndExtent(ILexerStream stream)
{
return(this.WithEndExtent(stream.Position, stream.LineNumber, stream.ColumnNumber));
}
/// <summary>
/// Initializes a new SourceExtent with the start position at the current position in the stream.
/// </summary>
/// <param name="stream"></param>
internal SourceExtent(ILexerStream stream)
: this(stream.Position, stream.LineNumber, stream.ColumnNumber, 0, 0, 0, string.Empty)
{
}
public static List<Token> Lex(ILexerStream stream)
{
var lexTokens = new List<Token>();
while (!stream.Eof)
{
var peek = stream.Peek();
switch (peek)
{
case '/':
lexTokens.Add(MultilineCommentToken.Read(stream));
break;
case '$':
lexTokens.Add(AliasIdentifierToken.Read(stream));
break;
case '{':
lexTokens.Add(BlockOpenToken.Read(stream));
break;
case '}':
lexTokens.Add(BlockCloseToken.Read(stream));
break;
case '(':
lexTokens.Add(OpenParenthesesToken.Read(stream));
break;
case ')':
lexTokens.Add(CloseParenthesesToken.Read(stream));
break;
case '[':
lexTokens.Add(AttributeOpenToken.Read(stream));
break;
case ']':
lexTokens.Add(AttributeCloseToken.Read(stream));
break;
case '=':
lexTokens.Add(EqualsOperatorToken.Read(stream));
break;
case '"':
lexTokens.Add(StringLiteralToken.Read(stream));
break;
case ',':
lexTokens.Add(CommaToken.Read(stream));
break;
case ';':
lexTokens.Add(StatementEndToken.Read(stream));
break;
case ':':
lexTokens.Add(ColonToken.Read(stream));
break;
case '#':
lexTokens.Add(PragmaToken.Read(stream));
break;
default:
if (char.IsWhiteSpace(peek))
{
lexTokens.Add(WhitespaceToken.Read(stream));
break;
}
else if (char.IsLetter(peek) || peek == '_')
{
var identifier = IdentifierToken.Read(stream);
if (identifier.Name.Equals("True", StringComparison.InvariantCultureIgnoreCase))
{
lexTokens.Add(new BooleanLiteralToken(identifier.Extent, true));
}
else if (identifier.Name.Equals("False", StringComparison.InvariantCultureIgnoreCase))
{
lexTokens.Add(new BooleanLiteralToken(identifier.Extent, false));
}
else
{
lexTokens.Add(identifier);
}
break;
}
else if (char.IsDigit(peek))
{
lexTokens.Add(IntegerLiteralToken.Read(stream));
break;
}
else
{
throw new InvalidOperationException(
string.Format("Unexpected character '{0}'", peek));
}
}
}
return lexTokens;
}
internal SourceExtent(ILexerStream stream)
: this(stream.Position, stream.LineNumber, stream.Column, 0, 0, 0, string.Empty)
{
}
internal SourceExtent WithEndExtent(ILexerStream stream)
{
return this.WithEndExtent(stream.Position, stream.LineNumber, stream.Column);
}
public static List <Token> Lex(ILexerStream stream)
{
var lexer = new Lexer(stream);
return(lexer.AllTokens().ToList());
}
