/// <summary> /// Reads the next character off of the input stream and advances the current position. /// Throws an exception if the character is not a letter. /// </summary> /// <returns></returns> public SourceChar ReadDigit() { var peek = this.Peek(); if (!StringValidator.IsDecimalDigit(peek.Value)) { throw new UnexpectedCharacterException(peek); } return(this.Read()); }
public static void IsDecimalDigitTest(char value, bool expectedResult) { var result = StringValidator.IsDecimalDigit(value); Assert.AreEqual(expectedResult, result); }
/// <summary> /// /// </summary> /// <param name="stream"></param> /// <returns></returns> /// <remarks> /// /// See https://www.dmtf.org/sites/default/files/standards/documents/DSP0221_3.0.1.pdf /// /// 7.6.1.1 Integer value /// /// No whitespace is allowed between the elements of the rules in this ABNF section. /// /// integerValue = binaryValue / octalValue / hexValue / decimalValue /// /// binaryValue = [ "+" / "-" ] 1*binaryDigit ( "b" / "B" ) /// binaryDigit = "0" / "1" /// /// octalValue = [ "+" / "-" ] unsignedOctalValue /// unsignedOctalValue = "0" 1*octalDigit /// octalDigit = "0" / "1" / "2" / "3" / "4" / "5" / "6" / "7" /// /// hexValue = [ "+" / "-" ] ( "0x" / "0X" ) 1*hexDigit /// hexDigit = decimalDigit / "a" / "A" / "b" / "B" / "c" / "C" / /// "d" / "D" / "e" / "E" / "f" / "F" /// /// decimalValue = [ "+" / "-" ] unsignedDecimalValue /// unsignedDecimalValue = "0" / positiveDecimalDigit *decimalDigit /// /// decimalDigit = "0" / positiveDecimalDigit /// positiveDecimalDigit = "1"..."9" /// /// 7.6.1.2 Real value /// /// No whitespace is allowed between the elements of the rules in this ABNF section. /// /// realValue = [ "+" / "-" ] *decimalDigit "." 1*decimalDigit /// [ ( "e" / "E" ) [ "+" / "-" ] 1*decimalDigit ] /// /// decimalDigit = "0" / positiveDecimalDigit /// positiveDecimalDigit = "1"..."9" /// /// </remarks> public static (Token, Lexer) ReadNumericLiteralToken(SourceReader reader) { int ParseBinaryValueDigits(IEnumerable <SourceChar> binaryChars, SourceChar sign) { return(ParseIntegerValueDigits(new Dictionary <char, int> { { '0', 0 }, { '1', 1 } }, 2, binaryChars, sign)); } int ParseOctalValueDigits(IEnumerable <SourceChar> octalChars, SourceChar sign) { return(ParseIntegerValueDigits(new Dictionary <char, int> { { '0', 0 }, { '1', 1 }, { '2', 2 }, { '3', 3 }, { '4', 4 }, { '5', 5 }, { '6', 6 }, { '7', 7 } }, 8, octalChars, sign)); } int ParseHexValueDigits(IEnumerable <SourceChar> hexChars, SourceChar sign) { return(ParseIntegerValueDigits(new Dictionary <char, int> { { '0', 0 }, { '1', 1 }, { '2', 2 }, { '3', 3 }, { '4', 4 }, { '5', 5 }, { '6', 6 }, { '7', 7 }, { '8', 8 }, { '9', 9 }, { 'a', 10 }, { 'b', 11 }, { 'c', 12 }, { 'd', 13 }, { 'e', 14 }, { 'f', 15 }, { 'A', 10 }, { 'B', 11 }, { 'C', 12 }, { 'D', 13 }, { 'E', 14 }, { 'F', 15 } }, 16, hexChars, sign)); } int ParseDecimalValueDigits(IEnumerable <SourceChar> decimalChars, SourceChar sign) { return(ParseIntegerValueDigits(new Dictionary <char, int> { { '0', 0 }, { '1', 1 }, { '2', 2 }, { '3', 3 }, { '4', 4 }, { '5', 5 }, { '6', 6 }, { '7', 7 }, { '8', 8 }, { '9', 9 } }, 10, decimalChars, sign)); } int ParseIntegerValueDigits(Dictionary <char, int> alphabet, int radix, IEnumerable <SourceChar> chars, SourceChar sign) { var literalValue = 0; foreach (var digit in chars) { var digitValue = alphabet[digit.Value]; literalValue = (literalValue * radix) + digitValue; } if (sign?.Value == '-') { literalValue = -literalValue; } return(literalValue); } const int stateLeadingSign = 1; const int stateFirstDigitBlock = 2; const int stateOctalOrDecimalValue = 3; const int stateBinaryValue = 4; const int stateOctalValue = 5; const int stateHexValue = 6; const int stateDecimalValue = 7; const int stateRealValue = 8; const int stateRealValueFraction = 9; const int stateRealValueExponent = 10; const int stateDone = 99; var thisReader = reader; var sourceChar = default(SourceChar); var sourceChars = new List <SourceChar>(); var token = default(Token); var signChar = default(SourceChar); var firstDigitBlock = new List <SourceChar>(); var currentState = stateLeadingSign; while (currentState != stateDone) { switch (currentState) { case stateLeadingSign: // we're reading the initial optional leading sign // [ "+" / "-" ] sourceChar = thisReader.Peek(); switch (sourceChar.Value) { case '+': case '-': (signChar, thisReader) = thisReader.Read(); sourceChars.Add(signChar); break; } currentState = stateFirstDigitBlock; break; case stateFirstDigitBlock: // we're reading the first block of digits in the value, // but we won't necessarily know which type we're reading // until we've consumed more of the input stream // // binaryValue => 1*binaryDigit // octalValue => "0" 1*octalDigit // hexValue => ( "0x" / "0X" ) // decimalValue => positiveDecimalDigit *decimalDigit // realValue => *decimalDigit // if (thisReader.Peek('.')) { // we're reading a realValue with no "*decimalDigit" characters before the "." // e.g. ".45", "+.45", "-.45", so consume the decimal point (sourceChar, thisReader) = thisReader.Read(); sourceChars.Add(sourceChar); // and go to the next state currentState = stateRealValueFraction; break; } // we don't know which base the value is in yet, but if it's hexadecimal them // we should be reading the "0x" part here, so restrict digits to decimal in // all cases (firstDigitBlock, thisReader) = thisReader.ReadWhile(StringValidator.IsDecimalDigit); sourceChars.AddRange(firstDigitBlock); // now we can do some validation if (firstDigitBlock.Count == 0) { // only realValue allows no digits in the first block, and // we've already handled that at the start of this case throw new UnexpectedCharacterException(sourceChar); } // if we've reached the end of the stream then there's no suffix // (e.g. b, B, x, X, .) so this must be an octalValue or decimalValue if (thisReader.Eof()) { currentState = stateOctalOrDecimalValue; break; } // check the next character to see if it tells us anything // about which type of literal we're reading sourceChar = thisReader.Peek(); switch (sourceChar.Value) { case 'b': case 'B': // binaryValue currentState = stateBinaryValue; break; case 'x': case 'X': // hexValue currentState = stateHexValue; break; case '.': // realValue currentState = stateRealValue; break; default: // by elmination, this must be an octalValue or decimalValue currentState = stateOctalOrDecimalValue; break; } break; case stateOctalOrDecimalValue: // we're reading an octalValue or decimalValue, but we're not sure which yet... if ((firstDigitBlock.First().Value == '0') && (firstDigitBlock.Count > 1)) { currentState = stateOctalValue; } else { currentState = stateDecimalValue; } break; case stateBinaryValue: // we're trying to read a binaryValue, so check all the characters in the digit block are valid, // i.e. 1*binaryDigit if (firstDigitBlock.Any(c => !StringValidator.IsBinaryDigit(c.Value))) { throw new UnexpectedCharacterException(sourceChar); } // all the characters are valid, so consume the suffix (sourceChar, thisReader) = thisReader.Read(c => (c == 'b') || (c == 'B')); sourceChars.Add(sourceChar); // now build the return value var binaryValue = ParseBinaryValueDigits(firstDigitBlock, signChar); token = new IntegerLiteralToken(SourceExtent.From(sourceChars), IntegerKind.BinaryValue, binaryValue); // and we're done currentState = stateDone; break; case stateOctalValue: // we're trying to read an octalValue (since decimalValue can't start with a // leading '0') so check all the characters in the digit block are valid, // i.e. "0" 1*octalDigit if ((firstDigitBlock.Count < 2) || (firstDigitBlock.First().Value != '0')) { throw new UnexpectedCharacterException(sourceChar); } if (firstDigitBlock.Skip(1).Any(c => !StringValidator.IsOctalDigit(c.Value))) { throw new UnexpectedCharacterException(sourceChar); } // now build the return value var octalValue = ParseOctalValueDigits(firstDigitBlock, signChar); token = new IntegerLiteralToken(SourceExtent.From(sourceChars), IntegerKind.OctalValue, octalValue); // and we're done currentState = stateDone; break; case stateHexValue: // we're trying to read a hexValue, so we should have just read a leading zero if ((firstDigitBlock.Count != 1) || (firstDigitBlock.First().Value != '0')) { throw new UnexpectedCharacterException(sourceChar); } // all the characters are valid, so consume the suffix (sourceChar, thisReader) = thisReader.Read(c => (c == 'x') || (c == 'X')); sourceChars.Add(sourceChar); // 1*hexDigit var hexDigits = default(List <SourceChar>); (hexDigits, thisReader) = thisReader.ReadWhile(StringValidator.IsHexDigit); if (hexDigits.Count == 0) { throw new UnexpectedCharacterException(thisReader.Peek()); } sourceChars.AddRange(hexDigits); // build the return value var hexValue = ParseHexValueDigits(hexDigits, signChar); token = new IntegerLiteralToken(SourceExtent.From(sourceChars), IntegerKind.HexValue, hexValue); // and we're done currentState = stateDone; break; case stateDecimalValue: // we're trying to read a decimalValue (since that's the only remaining option), // so check all the characters in the digit block are valid, // i.e. "0" / positiveDecimalDigit *decimalDigit if ((firstDigitBlock.Count == 1) && (firstDigitBlock.First().Value == '0')) { // "0" } else if (!StringValidator.IsPositiveDecimalDigit(firstDigitBlock.First().Value)) { throw new UnexpectedCharacterException(sourceChar); } else if (firstDigitBlock.Skip(1).Any(c => !StringValidator.IsDecimalDigit(c.Value))) { throw new UnexpectedCharacterException(sourceChar); } // build the return value var decimalValue = ParseDecimalValueDigits(firstDigitBlock, signChar); token = new IntegerLiteralToken(SourceExtent.From(sourceChars), IntegerKind.DecimalValue, decimalValue); // and we're done currentState = stateDone; break; case stateRealValue: // we're trying to read a realValue, so check all the characters in the digit block are valid, // i.e. *decimalDigit if (firstDigitBlock.Any(c => !StringValidator.IsDecimalDigit(c.Value))) { throw new UnexpectedCharacterException(sourceChar); } // all the characters are valid, so consume the decimal point (sourceChar, thisReader) = thisReader.Read('.'); sourceChars.Add(sourceChar); // and go to the next state currentState = stateRealValueFraction; break; case stateRealValueFraction: // 1*decimalDigit var realFractionDigits = default(List <SourceChar>); (realFractionDigits, thisReader) = thisReader.ReadWhile(StringValidator.IsHexDigit); if (realFractionDigits.Count == 0) { throw new UnexpectedCharacterException(thisReader.Peek()); } sourceChars.AddRange(realFractionDigits); // ( "e" / "E" ) if (!thisReader.Eof()) { sourceChar = thisReader.Peek(); if ((sourceChar.Value == 'e') || (sourceChar.Value == 'E')) { currentState = stateRealValueExponent; break; } } // build the return value var realIntegerValue = ParseDecimalValueDigits(firstDigitBlock, signChar); var realFractionValue = (double)ParseDecimalValueDigits(realFractionDigits, signChar); if (realFractionDigits.Any()) { realFractionValue = realFractionValue / Math.Pow(10, realFractionDigits.Count); } token = new RealLiteralToken( SourceExtent.From(sourceChars), realIntegerValue + realFractionValue ); // and we're done currentState = stateDone; break; case stateRealValueExponent: throw new InvalidOperationException(); case stateDone: // the main while loop should exit before we ever get here throw new InvalidOperationException(); default: throw new InvalidOperationException(); } } return(token, new Lexer(thisReader)); }
public static (Token Token, Lexer NextLexer) ReadToken(Lexer lexer) { var reader = lexer.Reader; var peek = reader.Peek(); switch (peek.Value) { case '$': return(LexerEngine.ReadAliasIdentifierToken(reader)); case ']': return(LexerEngine.ReadAttributeCloseToken(reader)); case '[': return(LexerEngine.ReadAttributeOpenToken(reader)); case '}': return(LexerEngine.ReadBlockCloseToken(reader)); case '{': return(LexerEngine.ReadBlockOpenToken(reader)); case ':': return(LexerEngine.ReadColonToken(reader)); case ',': return(LexerEngine.ReadCommaToken(reader)); case '/': return(LexerEngine.ReadCommentToken(reader)); case '=': return(LexerEngine.ReadEqualsOperatorToken(reader)); case ')': return(LexerEngine.ReadParenthesisCloseToken(reader)); case '(': return(LexerEngine.ReadParenthesisOpenToken(reader)); case '#': return(LexerEngine.ReadPragmaToken(reader)); case ';': return(LexerEngine.ReadStatementEndToken(reader)); case '"': return(LexerEngine.ReadStringLiteralToken(reader)); case '+': case '-': return(LexerEngine.ReadNumericLiteralToken(reader)); case '.': // if the next character is a decimalDigit then we're reading a RealLiteralToken with // no leading digits before the decimal point (e.g. ".45"), otherwise we're reading // a DotOperatorToken (e.g. the "." in "MyPropertyValue = MyEnum.Value;") var readAhead = reader.Read().NextReader; if (!readAhead.Eof() && StringValidator.IsDecimalDigit(readAhead.Peek().Value)) { return(LexerEngine.ReadNumericLiteralToken(reader)); } else { return(LexerEngine.ReadDotOperatorToken(reader)); } case 'a': case 'b': case 'c': case 'd': case 'e': case 'f': case 'g': case 'h': case 'i': case 'j': case 'k': case 'l': case 'm': case 'n': case 'o': case 'p': case 'q': case 'r': case 's': case 't': case 'u': case 'v': case 'w': case 'x': case 'y': case 'z': case 'A': case 'B': case 'C': case 'D': case 'E': case 'F': case 'G': case 'H': case 'I': case 'J': case 'K': case 'L': case 'M': case 'N': case 'O': case 'P': case 'Q': case 'R': case 'S': case 'T': case 'U': case 'V': case 'W': case 'X': case 'Y': case 'Z': case '_': // firstIdentifierChar var(identifierToken, nextLexer) = LexerEngine.ReadIdentifierToken(reader); var normalized = identifierToken.Name.ToLowerInvariant(); switch (normalized) { case "false": var falseToken = new BooleanLiteralToken(identifierToken.Extent, false); return(falseToken, nextLexer); case "true": var trueToken = new BooleanLiteralToken(identifierToken.Extent, true); return(trueToken, nextLexer); case "null": var nullLiteralToken = new NullLiteralToken(identifierToken.Extent); return(nullLiteralToken, nextLexer); default: return(identifierToken, nextLexer); } case '0': case '1': case '2': case '3': case '4': case '5': case '6': case '7': case '8': case '9': // decimalDigit return(LexerEngine.ReadNumericLiteralToken(reader)); case '\u0020': // space case '\u0009': // horizontal tab case '\u000D': // carriage return case '\u000A': // line feed // WS return(LexerEngine.ReadWhitespaceToken(reader)); default: throw new UnexpectedCharacterException(peek); } }
public Token NextToken() { var stream = this.Stream; var peek = stream.Peek(); switch (peek.Value) { case '$': return(Lexer.ReadAliasIdentifierToken(stream)); case ']': return(Lexer.ReadAttributeCloseToken(stream)); case '[': return(Lexer.ReadAttributeOpenToken(stream)); case '}': return(Lexer.ReadBlockCloseToken(stream)); case '{': return(Lexer.ReadBlockOpenToken(stream)); case ':': return(Lexer.ReadColonToken(stream)); case ',': return(Lexer.ReadCommaToken(stream)); case '/': return(Lexer.ReadCommentToken(stream)); case '=': return(Lexer.ReadEqualsOperatorToken(stream)); case ')': return(Lexer.ReadParenthesesCloseToken(stream)); case '(': return(Lexer.ReadParenthesesOpenToken(stream)); case '#': return(Lexer.ReadPragmaToken(stream)); case ';': return(Lexer.ReadStatementEndToken(stream)); case '"': return(Lexer.ReadStringLiteralToken(stream)); default: if (StringValidator.IsWhitespace(peek.Value)) { return(Lexer.ReadWhitespaceToken(stream)); } else if (StringValidator.IsFirstIdentifierChar(peek.Value)) { var identifier = Lexer.ReadIdentifierToken(stream); if (StringValidator.IsFalse(identifier.Name)) { /// A.17.6 Boolean value /// /// booleanValue = TRUE / FALSE /// /// FALSE = "false" ; keyword: case insensitive /// TRUE = "true" ; keyword: case insensitive return(new BooleanLiteralToken(identifier.Extent, false)); } else if (StringValidator.IsTrue(identifier.Name)) { /// A.17.6 Boolean value /// /// booleanValue = TRUE / FALSE /// /// FALSE = "false" ; keyword: case insensitive /// TRUE = "true" ; keyword: case insensitive return(new BooleanLiteralToken(identifier.Extent, true)); } else if (StringValidator.IsNull(identifier.Name)) { /// A.17.7 Null value /// /// nullValue = NULL /// /// NULL = "null" ; keyword: case insensitive /// ; second return(new NullLiteralToken(identifier.Extent)); } else { return(identifier); } } else if ((peek.Value == '+') || (peek.Value == '-') || (StringValidator.IsDecimalDigit(peek.Value))) { return(Lexer.ReadNumericLiteralToken(stream)); } else { throw new UnexpectedCharacterException(peek); } } }
/// <summary> /// Returns true if the next character off of the input stream is a digit. /// </summary> /// <returns></returns> public bool PeekDigit() { var peek = this.Peek(); return(StringValidator.IsDecimalDigit(peek.Value)); }