public void Test3CharacterString()
 {
     var json = "\t \"123\"x";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.String, token.Type);
     Assert.AreEqual("123", token.Lexeme);
     Assert.AreEqual('x', buffer.Read());
 }
 public void TestBeginArray()
 {
     var json = "\t [x";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.BeginArray, token.Type);
     Assert.AreEqual("[", token.Lexeme);
     Assert.AreEqual('x', buffer.Read());
 }
        private static JsonToken GetNumberToken(
            BsonJsonBuffer buffer,
            int c // first character
        )
        {
            // leading digit or '-' has already been read
            var start = buffer.Position - 1;
            NumberState state;
            switch (c) {
                case '-': state = NumberState.SawLeadingMinus; break;
                case '0': state = NumberState.SawLeadingZero; break;
                default: state = NumberState.SawIntegerDigits; break;
            }
            var type = JsonTokenType.Integer; // assume integer until proved otherwise

            while (true) {
                c = buffer.Read();
                switch (state) {
                    case NumberState.SawLeadingMinus:
                        switch (c) {
                            case '0':
                                state = NumberState.SawLeadingZero;
                                break;
                            default:
                                if (char.IsDigit((char) c)) {
                                    state = NumberState.SawIntegerDigits;
                                } else {
                                    state = NumberState.Invalid;
                                }
                                break;
                        }
                        break;
                    case NumberState.SawLeadingZero:
                        switch (c) {
                            case '.':
                                state = NumberState.SawDecimalPoint;
                                break;
                            case 'e':
                            case 'E':
                                state = NumberState.SawExponentLetter;
                                break;
                            case ',':
                            case '}':
                            case ']':
                            case -1:
                                state = NumberState.Done;
                                break;
                            default:
                                if (char.IsWhiteSpace((char) c)) {
                                    state = NumberState.Done;
                                } else {
                                    state = NumberState.Invalid;
                                }
                                break;
                        }
                        break;
                    case NumberState.SawIntegerDigits:
                        switch (c) {
                            case '.':
                                state = NumberState.SawDecimalPoint;
                                break;
                            case 'e':
                            case 'E':
                                state = NumberState.SawExponentLetter;
                                break;
                            case ',':
                            case '}':
                            case ']':
                            case -1:
                                state = NumberState.Done;
                                break;
                            default:
                                if (char.IsDigit((char) c)) {
                                    state = NumberState.SawIntegerDigits;
                                } else if (char.IsWhiteSpace((char) c)) {
                                    state = NumberState.Done;
                                } else {
                                    state = NumberState.Invalid;
                                }
                                break;
                        }
                        break;
                    case NumberState.SawDecimalPoint:
                        type = JsonTokenType.FloatingPoint;
                        if (char.IsDigit((char) c)) {
                            state = NumberState.SawFractionDigits;
                        } else {
                            state = NumberState.Invalid;
                        }
                        break;
                    case NumberState.SawFractionDigits:
                        switch (c) {
                            case 'e':
                            case 'E':
                                state = NumberState.SawExponentLetter;
                                break;
                            case ',':
                            case '}':
                            case ']':
                            case -1:
                                state = NumberState.Done;
                                break;
                            default:
                                if (char.IsDigit((char) c)) {
                                    state = NumberState.SawFractionDigits;
                                } else if (char.IsWhiteSpace((char) c)) {
                                    state = NumberState.Done;
                                } else {
                                    state = NumberState.Invalid;
                                }
                                break;
                        }
                        break;
                    case NumberState.SawExponentLetter:
                        type = JsonTokenType.FloatingPoint;
                        switch (c) {
                            case '+':
                            case '-':
                                state = NumberState.SawExponentSign;
                                break;
                            default:
                                if (char.IsDigit((char) c)) {
                                    state = NumberState.SawExponentDigits;
                                } else {
                                    state = NumberState.Invalid;
                                }
                                break;
                        }
                        break;
                    case NumberState.SawExponentSign:
                        if (char.IsDigit((char) c)) {
                            state = NumberState.SawExponentDigits;
                        } else {
                            state = NumberState.Invalid;
                        }
                        break;
                    case NumberState.SawExponentDigits:
                        switch (c) {
                            case ',':
                            case '}':
                            case ']':
                            case -1:
                                state = NumberState.Done;
                                break;
                            default:
                                if (char.IsDigit((char) c)) {
                                    state = NumberState.SawExponentDigits;
                                } else if (char.IsWhiteSpace((char) c)) {
                                    state = NumberState.Done;
                                } else {
                                    state = NumberState.Invalid;
                                }
                                break;
                        }
                        break;
                }

                switch (state) {
                    case NumberState.Done:
                        buffer.UnRead(c);
                        return new JsonToken(type, buffer.Substring(start, buffer.Position - start));
                    case NumberState.Invalid:
                        throw new FileFormatException(FormatMessage("Invalid JSON number", buffer, start));
                }
            }
        }
 public void TestValueSeparator()
 {
     var json = "\t ,x";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.Comma, token.Type);
     Assert.AreEqual(",", token.Lexeme);
     Assert.AreEqual('x', buffer.Read());
 }
 public void TestZeroPointZero()
 {
     var json = "\t 0.0,";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.FloatingPoint, token.Type);
     Assert.AreEqual("0.0", token.Lexeme);
     Assert.AreEqual(',', buffer.Read());
 }
 private static JsonToken GetUnquotedStringToken(
     BsonJsonBuffer buffer
 )
 {
     // opening letter or $ has already been read
     var start = buffer.Position - 1;
     while (true) {
         var c = buffer.Read();
         switch (c) {
             case ':':
             case ',':
             case '}':
             case ']':
             case -1:
                 buffer.UnRead(c);
                 return new JsonToken(JsonTokenType.UnquotedString, buffer.Substring(start, buffer.Position - start));
             default:
                 if (c == '$' || char.IsLetterOrDigit((char) c)) {
                     // continue
                 } else if (char.IsWhiteSpace((char) c)) {
                     buffer.UnRead(c);
                     return new JsonToken(JsonTokenType.UnquotedString, buffer.Substring(start, buffer.Position - start));
                 } else {
                     throw new FileFormatException(FormatMessage("Invalid JSON unquoted string", buffer, start));
                 }
                 break;
         }
     }
 }
 public void TestEndOfFile()
 {
     var json = "\t ";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.EndOfFile, token.Type);
     Assert.AreEqual("<eof>", token.Lexeme);
     Assert.AreEqual(-1, buffer.Read());
 }
 public void TestRegularExpressionPatternAndOptions()
 {
     var json = "\t /pattern/gim,";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.RegularExpression, token.Type);
     Assert.AreEqual("/pattern/gim", token.Lexeme);
     Assert.AreEqual(',', buffer.Read());
 }
 public void TestTrue()
 {
     var json = "\t true,";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.UnquotedString, token.Type);
     Assert.AreEqual("true", token.Lexeme);
     Assert.AreEqual(',', buffer.Read());
 }
 public void TestOneExponentTwelve()
 {
     var json = "\t 1e12,";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.FloatingPoint, token.Type);
     Assert.AreEqual("1e12", token.Lexeme);
     Assert.AreEqual(',', buffer.Read());
 }
 public void TestRegularExpressionEmpty()
 {
     var json = "\t //,";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.RegularExpression, token.Type);
     Assert.AreEqual("//", token.Lexeme);
     Assert.AreEqual(',', buffer.Read());
 }
 public void TestMinusZeroExponentOne()
 {
     var json = "\t -0e1,";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.FloatingPoint, token.Type);
     Assert.AreEqual("-0e1", token.Lexeme);
     Assert.AreEqual(',', buffer.Read());
 }
 public void TestMinusOnePointTwo()
 {
     var json = "\t -1.2,";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.FloatingPoint, token.Type);
     Assert.AreEqual("-1.2", token.Lexeme);
     Assert.AreEqual(',', buffer.Read());
 }
 public void TestEscapeSequences()
 {
     var json = "\t \"x\\\"\\\\\\/\\b\\f\\n\\r\\t\\u0030y\"x";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.String, token.Type);
     Assert.AreEqual("x\"\\/\b\f\n\r\t0y", token.Lexeme);
     Assert.AreEqual('x', buffer.Read());
 }
        private static JsonToken GetRegularExpressionToken(
            BsonJsonBuffer buffer
        )
        {
            // opening slash has already been read
            var start = buffer.Position - 1;
            var state = RegularExpressionState.InPattern;
            while (true) {
                var c = buffer.Read();
                switch (state) {
                    case RegularExpressionState.InPattern:
                        switch (c) {
                            case '/': state = RegularExpressionState.InOptions; break;
                            case '\\': state = RegularExpressionState.InEscapeSequence; break;
                            default: state = RegularExpressionState.InPattern; break;
                        }
                        break;
                    case RegularExpressionState.InEscapeSequence:
                        state = RegularExpressionState.InPattern;
                        break;
                    case RegularExpressionState.InOptions:
                        switch (c) {
                            case 'g':
                            case 'i':
                            case 'm':
                                state = RegularExpressionState.InOptions;
                                break;
                            case ',':
                            case '}':
                            case ']':
                            case -1:
                                state = RegularExpressionState.Done;
                                break;
                            default:
                                if (char.IsWhiteSpace((char) c)) {
                                    state = RegularExpressionState.Done;
                                } else {
                                    state = RegularExpressionState.Invalid;
                                }
                                break;
                        }
                        break;
                }

                switch (state) {
                    case RegularExpressionState.Done:
                        buffer.UnRead(c);
                        var count = buffer.Position - start;
                        return new JsonToken(JsonTokenType.RegularExpression, buffer.Substring(start, count));
                    case RegularExpressionState.Invalid:
                        throw new FileFormatException(FormatMessage("Invalid JSON regular expression", buffer, start));
                }
            }
        }
 public void TestTwelve()
 {
     var json = "\t 12,";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.Integer, token.Type);
     Assert.AreEqual("12", token.Lexeme);
     Assert.AreEqual(',', buffer.Read());
 }
 private static JsonToken GetStringToken(
     BsonJsonBuffer buffer
 )
 {
     // opening quote has already been read
     var start = buffer.Position - 1;
     var sb = new StringBuilder();
     while (true) {
         var c = buffer.Read();
         switch (c) {
             case '\\':
                 c = buffer.Read();
                 switch (c) {
                     case '"': sb.Append('"'); break;
                     case '\\': sb.Append('\\'); break;
                     case '/': sb.Append('/'); break;
                     case 'b': sb.Append('\b'); break;
                     case 'f': sb.Append('\f'); break;
                     case 'n': sb.Append('\n'); break;
                     case 'r': sb.Append('\r'); break;
                     case 't': sb.Append('\t'); break;
                     case 'u':
                         var u1 = buffer.Read();
                         var u2 = buffer.Read();
                         var u3 = buffer.Read();
                         var u4 = buffer.Read();
                         if (u4 != -1) {
                             var hex = new string(new char[] { (char) u1, (char) u2, (char) u3, (char) u4 });
                             var n = Convert.ToInt32(hex, 16);
                             sb.Append((char) n);
                         }
                         break;
                     default:
                         if (c != -1) {
                             var message = string.Format("Invalid escape sequence in JSON string: '\\{0}'", (char) c);
                             throw new FileFormatException(message);
                         }
                         break;
                 }
                 break;
             case '"':
                 return new JsonToken(JsonTokenType.String, sb.ToString());
             default:
                 if (c != -1) {
                     sb.Append((char) c);
                 }
                 break;
         }
         if (c == -1) {
             throw new FileFormatException(FormatMessage("End of file in JSON string", buffer, start));
         }
     }
 }
 public void TestUnquotedString()
 {
     var json = "\t name123:1";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.UnquotedString, token.Type);
     Assert.AreEqual("name123", token.Lexeme);
     Assert.AreEqual(':', buffer.Read());
 }
        public static JsonToken GetNextToken(
            BsonJsonBuffer buffer
        )
        {
            // skip leading whitespace
            var c = buffer.Read();
            while (c != -1 && char.IsWhiteSpace((char) c)) {
                c = buffer.Read();
            }
            if (c == -1) {
                return new JsonToken(JsonTokenType.EndOfFile, "<eof>");
            }

            // leading character determines token type
            switch (c) {
                case '{': return new JsonToken(JsonTokenType.BeginObject, "{");
                case '}': return new JsonToken(JsonTokenType.EndObject, "}");
                case '[': return new JsonToken(JsonTokenType.BeginArray, "[");
                case ']': return new JsonToken(JsonTokenType.EndArray, "]");
                case ':': return new JsonToken(JsonTokenType.Colon, ":");
                case ',': return new JsonToken(JsonTokenType.Comma, ",");
                case '"': return GetStringToken(buffer);
                case '/': return GetRegularExpressionToken(buffer);
                default:
                    if (c == '-' || char.IsDigit((char) c)) {
                        return GetNumberToken(buffer, c);
                    } else if (c == '$' || char.IsLetter((char) c)) {
                        return GetUnquotedStringToken(buffer);
                    } else {
                        buffer.UnRead(c);
                        throw new FileFormatException(FormatMessage("Invalid JSON input", buffer, buffer.Position));
                    }
            }
        }
 public void TestEndObject()
 {
     var json = "\t }x";
     var buffer = new BsonJsonBuffer(json);
     var token = BsonJsonScanner.GetNextToken(buffer);
     Assert.AreEqual(JsonTokenType.EndObject, token.Type);
     Assert.AreEqual("}", token.Lexeme);
     Assert.AreEqual('x', buffer.Read());
 }