/// <summary>
/// Gets the next JsonToken from a JsonBuffer.
/// </summary>
/// <param name="buffer">The buffer.</param>
/// <returns>The next token.</returns>
public static JsonToken GetNextToken(
JsonBuffer buffer
) {
// skip leading whitespace
var c = buffer.Read();
while (c != -1 && char.IsWhiteSpace((char) c)) {
c = buffer.Read();
}
if (c == -1) {
return new JsonToken(JsonTokenType.EndOfFile, "<eof>");
}
// leading character determines token type
switch (c) {
case '{': return new JsonToken(JsonTokenType.BeginObject, "{");
case '}': return new JsonToken(JsonTokenType.EndObject, "}");
case '[': return new JsonToken(JsonTokenType.BeginArray, "[");
case ']': return new JsonToken(JsonTokenType.EndArray, "]");
case ':': return new JsonToken(JsonTokenType.Colon, ":");
case ',': return new JsonToken(JsonTokenType.Comma, ",");
case '"': return GetStringToken(buffer);
case '/': return GetRegularExpressionToken(buffer);
default:
if (c == '-' || char.IsDigit((char) c)) {
return GetNumberToken(buffer, c);
} else if (c == '$' || char.IsLetter((char) c)) {
return GetUnquotedStringToken(buffer);
} else {
buffer.UnRead(c);
throw new FileFormatException(FormatMessage("Invalid JSON input", buffer, buffer.Position));
}
}
}
public void TestNameSeparator() {
var json = "\t :x";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.Colon, token.Type);
Assert.AreEqual(":", token.Lexeme);
Assert.AreEqual('x', buffer.Read());
}
public void TestEndArray() {
var json = "\t ]x";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.EndArray, token.Type);
Assert.AreEqual("]", token.Lexeme);
Assert.AreEqual('x', buffer.Read());
}
public void TestBeginObject() {
var json = "\t {x";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.BeginObject, token.Type);
Assert.AreEqual("{", token.Lexeme);
Assert.AreEqual('x', buffer.Read());
}
public void TestEndOfFile() {
var json = "\t ";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.EndOfFile, token.Type);
Assert.AreEqual("<eof>", token.Lexeme);
Assert.AreEqual(-1, buffer.Read());
}
public void Test3CharacterString()
{
var json = "\t \"123\"x";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.String, token.Type);
Assert.AreEqual("123", token.StringValue);
Assert.AreEqual('x', buffer.Read());
}
private static JsonToken GetUnquotedStringToken(
JsonBuffer buffer
)
{
// opening letter or $ has already been read
var start = buffer.Position - 1;
string lexeme;
while (true)
{
var c = buffer.Read();
switch (c)
{
case ':':
case ',':
case '}':
case ']':
case -1:
buffer.UnRead(c);
lexeme = buffer.Substring(start, buffer.Position - start);
return(new StringJsonToken(JsonTokenType.UnquotedString, lexeme, lexeme));
default:
if (c == '$' || char.IsLetterOrDigit((char)c))
{
// continue
}
else if (char.IsWhiteSpace((char)c))
{
buffer.UnRead(c);
lexeme = buffer.Substring(start, buffer.Position - start);
return(new StringJsonToken(JsonTokenType.UnquotedString, lexeme, lexeme));
}
else
{
if (c == '(')
{
var value = buffer.Substring(start, buffer.Position - 1 - start);
switch (value)
{
case "Date": return(GetTenGenDate(buffer, start));
case "NumberLong": return(GetTenGenInt64(buffer, start));
case "ObjectId": return(GetTenGenObjectId(buffer, start));
}
}
throw new FileFormatException(FormatMessage("Invalid JSON unquoted string", buffer, start));
}
break;
}
}
}
private static JsonToken GetTenGenInt64(
JsonBuffer buffer,
int start
)
{
var firstDigit = buffer.Position;
while (true)
{
var c = buffer.Read();
if (c == ')')
{
var lexeme = buffer.Substring(start, buffer.Position - start);
var digits = buffer.Substring(firstDigit, buffer.Position - firstDigit - 1);
var value = XmlConvert.ToInt64(digits);
return(new Int64JsonToken(lexeme, value));
}
if (c == -1 || !char.IsDigit((char)c))
{
throw new FileFormatException(FormatMessage("Invalid JSON NumberLong value", buffer, start));
}
}
}
private static JsonToken GetTenGenDate(
JsonBuffer buffer,
int start
)
{
var firstDigit = buffer.Position;
while (true)
{
var c = buffer.Read();
if (c == ')')
{
var lexeme = buffer.Substring(start, buffer.Position - start);
var digits = buffer.Substring(firstDigit, buffer.Position - firstDigit - 1);
var ms = XmlConvert.ToInt64(digits);
var value = BsonConstants.UnixEpoch.AddMilliseconds(ms);
return(new DateTimeJsonToken(lexeme, value));
}
if (c == -1 || !char.IsDigit((char)c))
{
throw new FileFormatException(FormatMessage("Invalid JSON Date value", buffer, start));
}
}
}
public void TestRegularExpressionPatternAndOptions()
{
var json = "\t /pattern/gim,";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.RegularExpression, token.Type);
Assert.AreEqual("/pattern/gim", token.Lexeme);
Assert.AreEqual(',', buffer.Read());
}
public void TestOneExponentMinuesTwelve()
{
var json = "\t 1e-12,";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.Double, token.Type);
Assert.AreEqual("1e-12", token.Lexeme);
Assert.AreEqual(',', buffer.Read());
}
public void TestRegularExpressionEmpty()
{
var json = "\t //,";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.RegularExpression, token.Type);
Assert.AreEqual("//", token.Lexeme);
Assert.AreEqual(',', buffer.Read());
}
private static JsonToken GetTenGenInt64(
JsonBuffer buffer,
int start
) {
var firstDigit = buffer.Position;
while (true) {
var c = buffer.Read();
if (c == ')') {
var lexeme = buffer.Substring(start, buffer.Position - start);
var digits = buffer.Substring(firstDigit, buffer.Position - firstDigit - 1);
var value = XmlConvert.ToInt64(digits);
return new Int64JsonToken(lexeme, value);
}
if (c == -1 || !char.IsDigit((char) c)) {
throw new FileFormatException(FormatMessage("Invalid JSON NumberLong value", buffer, start));
}
}
}
public void TestMinusOnePointTwo()
{
var json = "\t -1.2,";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.Double, token.Type);
Assert.AreEqual("-1.2", token.Lexeme);
Assert.AreEqual(',', buffer.Read());
}
public void TestUnquotedString()
{
var json = "\t name123:1";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.UnquotedString, token.Type);
Assert.AreEqual("name123", token.StringValue);
Assert.AreEqual(':', buffer.Read());
}
public void TestZeroPointZero()
{
var json = "\t 0.0,";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.Double, token.Type);
Assert.AreEqual("0.0", token.Lexeme);
Assert.AreEqual(',', buffer.Read());
}
private static JsonToken GetNumberToken(
JsonBuffer buffer,
int c // first character
) {
// leading digit or '-' has already been read
var start = buffer.Position - 1;
NumberState state;
switch (c) {
case '-': state = NumberState.SawLeadingMinus; break;
case '0': state = NumberState.SawLeadingZero; break;
default: state = NumberState.SawIntegerDigits; break;
}
var type = JsonTokenType.Int64; // assume integer until proved otherwise
while (true) {
c = buffer.Read();
switch (state) {
case NumberState.SawLeadingMinus:
switch (c) {
case '0':
state = NumberState.SawLeadingZero;
break;
case 'I':
state = NumberState.SawMinusI;
break;
default:
if (char.IsDigit((char) c)) {
state = NumberState.SawIntegerDigits;
} else {
state = NumberState.Invalid;
}
break;
}
break;
case NumberState.SawLeadingZero:
switch (c) {
case '.':
state = NumberState.SawDecimalPoint;
break;
case 'e':
case 'E':
state = NumberState.SawExponentLetter;
break;
case ',':
case '}':
case ']':
case ')':
case -1:
state = NumberState.Done;
break;
default:
if (char.IsWhiteSpace((char) c)) {
state = NumberState.Done;
} else {
state = NumberState.Invalid;
}
break;
}
break;
case NumberState.SawIntegerDigits:
switch (c) {
case '.':
state = NumberState.SawDecimalPoint;
break;
case 'e':
case 'E':
state = NumberState.SawExponentLetter;
break;
case ',':
case '}':
case ']':
case ')':
case -1:
state = NumberState.Done;
break;
default:
if (char.IsDigit((char) c)) {
state = NumberState.SawIntegerDigits;
} else if (char.IsWhiteSpace((char) c)) {
state = NumberState.Done;
} else {
state = NumberState.Invalid;
}
break;
}
break;
case NumberState.SawDecimalPoint:
type = JsonTokenType.Double;
if (char.IsDigit((char) c)) {
state = NumberState.SawFractionDigits;
} else {
state = NumberState.Invalid;
}
break;
case NumberState.SawFractionDigits:
switch (c) {
case 'e':
case 'E':
state = NumberState.SawExponentLetter;
break;
case ',':
case '}':
case ']':
case ')':
case -1:
state = NumberState.Done;
break;
default:
if (char.IsDigit((char) c)) {
state = NumberState.SawFractionDigits;
} else if (char.IsWhiteSpace((char) c)) {
state = NumberState.Done;
} else {
state = NumberState.Invalid;
}
break;
}
break;
case NumberState.SawExponentLetter:
type = JsonTokenType.Double;
switch (c) {
case '+':
case '-':
state = NumberState.SawExponentSign;
break;
default:
if (char.IsDigit((char) c)) {
state = NumberState.SawExponentDigits;
} else {
state = NumberState.Invalid;
}
break;
}
break;
case NumberState.SawExponentSign:
if (char.IsDigit((char) c)) {
state = NumberState.SawExponentDigits;
} else {
state = NumberState.Invalid;
}
break;
case NumberState.SawExponentDigits:
switch (c) {
case ',':
case '}':
case ']':
case ')':
case -1:
state = NumberState.Done;
break;
default:
if (char.IsDigit((char) c)) {
state = NumberState.SawExponentDigits;
} else if (char.IsWhiteSpace((char) c)) {
state = NumberState.Done;
} else {
state = NumberState.Invalid;
}
break;
}
break;
case NumberState.SawMinusI:
var sawMinusInfinity = true;
var nfinity = new char[] { 'n', 'f', 'i', 'n', 'i', 't', 'y' };
for (var i = 0; i < nfinity.Length; i++) {
if (c != nfinity[i]) {
sawMinusInfinity = false;
break;
}
c = buffer.Read();
}
if (sawMinusInfinity) {
type = JsonTokenType.Double;
switch (c) {
case ',':
case '}':
case ']':
case ')':
case -1:
state = NumberState.Done;
break;
default:
if (char.IsWhiteSpace((char) c)) {
state = NumberState.Done;
} else {
state = NumberState.Invalid;
}
break;
}
} else {
state = NumberState.Invalid;
}
break;
}
switch (state) {
case NumberState.Done:
buffer.UnRead(c);
var lexeme = buffer.Substring(start, buffer.Position - start);
if (type == JsonTokenType.Double) {
var value = XmlConvert.ToDouble(lexeme);
return new DoubleJsonToken(lexeme, value);
} else {
var value = XmlConvert.ToInt64(lexeme);
if (value < int.MinValue || value > int.MaxValue) {
return new Int64JsonToken(lexeme, value);
} else {
return new Int32JsonToken(lexeme, (int) value);
}
}
case NumberState.Invalid:
throw new FileFormatException(FormatMessage("Invalid JSON number", buffer, start));
}
}
}
private static JsonToken GetNumberToken(
JsonBuffer buffer,
int c // first character
)
{
// leading digit or '-' has already been read
var start = buffer.Position - 1;
NumberState state;
switch (c)
{
case '-': state = NumberState.SawLeadingMinus; break;
case '0': state = NumberState.SawLeadingZero; break;
default: state = NumberState.SawIntegerDigits; break;
}
var type = JsonTokenType.Int64; // assume integer until proved otherwise
while (true)
{
c = buffer.Read();
switch (state)
{
case NumberState.SawLeadingMinus:
switch (c)
{
case '0':
state = NumberState.SawLeadingZero;
break;
case 'I':
state = NumberState.SawMinusI;
break;
default:
if (char.IsDigit((char)c))
{
state = NumberState.SawIntegerDigits;
}
else
{
state = NumberState.Invalid;
}
break;
}
break;
case NumberState.SawLeadingZero:
switch (c)
{
case '.':
state = NumberState.SawDecimalPoint;
break;
case 'e':
case 'E':
state = NumberState.SawExponentLetter;
break;
case ',':
case '}':
case ']':
case ')':
case -1:
state = NumberState.Done;
break;
default:
if (char.IsWhiteSpace((char)c))
{
state = NumberState.Done;
}
else
{
state = NumberState.Invalid;
}
break;
}
break;
case NumberState.SawIntegerDigits:
switch (c)
{
case '.':
state = NumberState.SawDecimalPoint;
break;
case 'e':
case 'E':
state = NumberState.SawExponentLetter;
break;
case ',':
case '}':
case ']':
case ')':
case -1:
state = NumberState.Done;
break;
default:
if (char.IsDigit((char)c))
{
state = NumberState.SawIntegerDigits;
}
else if (char.IsWhiteSpace((char)c))
{
state = NumberState.Done;
}
else
{
state = NumberState.Invalid;
}
break;
}
break;
case NumberState.SawDecimalPoint:
type = JsonTokenType.Double;
if (char.IsDigit((char)c))
{
state = NumberState.SawFractionDigits;
}
else
{
state = NumberState.Invalid;
}
break;
case NumberState.SawFractionDigits:
switch (c)
{
case 'e':
case 'E':
state = NumberState.SawExponentLetter;
break;
case ',':
case '}':
case ']':
case ')':
case -1:
state = NumberState.Done;
break;
default:
if (char.IsDigit((char)c))
{
state = NumberState.SawFractionDigits;
}
else if (char.IsWhiteSpace((char)c))
{
state = NumberState.Done;
}
else
{
state = NumberState.Invalid;
}
break;
}
break;
case NumberState.SawExponentLetter:
type = JsonTokenType.Double;
switch (c)
{
case '+':
case '-':
state = NumberState.SawExponentSign;
break;
default:
if (char.IsDigit((char)c))
{
state = NumberState.SawExponentDigits;
}
else
{
state = NumberState.Invalid;
}
break;
}
break;
case NumberState.SawExponentSign:
if (char.IsDigit((char)c))
{
state = NumberState.SawExponentDigits;
}
else
{
state = NumberState.Invalid;
}
break;
case NumberState.SawExponentDigits:
switch (c)
{
case ',':
case '}':
case ']':
case ')':
case -1:
state = NumberState.Done;
break;
default:
if (char.IsDigit((char)c))
{
state = NumberState.SawExponentDigits;
}
else if (char.IsWhiteSpace((char)c))
{
state = NumberState.Done;
}
else
{
state = NumberState.Invalid;
}
break;
}
break;
case NumberState.SawMinusI:
var sawMinusInfinity = true;
var nfinity = new char[] { 'n', 'f', 'i', 'n', 'i', 't', 'y' };
for (var i = 0; i < nfinity.Length; i++)
{
if (c != nfinity[i])
{
sawMinusInfinity = false;
break;
}
c = buffer.Read();
}
if (sawMinusInfinity)
{
type = JsonTokenType.Double;
switch (c)
{
case ',':
case '}':
case ']':
case ')':
case -1:
state = NumberState.Done;
break;
default:
if (char.IsWhiteSpace((char)c))
{
state = NumberState.Done;
}
else
{
state = NumberState.Invalid;
}
break;
}
}
else
{
state = NumberState.Invalid;
}
break;
}
switch (state)
{
case NumberState.Done:
buffer.UnRead(c);
var lexeme = buffer.Substring(start, buffer.Position - start);
if (type == JsonTokenType.Double)
{
var value = XmlConvert.ToDouble(lexeme);
return(new DoubleJsonToken(lexeme, value));
}
else
{
var value = XmlConvert.ToInt64(lexeme);
if (value < int.MinValue || value > int.MaxValue)
{
return(new Int64JsonToken(lexeme, value));
}
else
{
return(new Int32JsonToken(lexeme, (int)value));
}
}
case NumberState.Invalid:
throw new FileFormatException(FormatMessage("Invalid JSON number", buffer, start));
}
}
}
private static JsonToken GetStringToken(
JsonBuffer buffer,
char quoteCharacter // either single or double quote
) {
// opening quote has already been read
var start = buffer.Position - 1;
var sb = new StringBuilder();
while (true) {
var c = buffer.Read();
switch (c) {
case '\\':
c = buffer.Read();
switch (c) {
case '\'': sb.Append('\''); break;
case '"': sb.Append('"'); break;
case '\\': sb.Append('\\'); break;
case '/': sb.Append('/'); break;
case 'b': sb.Append('\b'); break;
case 'f': sb.Append('\f'); break;
case 'n': sb.Append('\n'); break;
case 'r': sb.Append('\r'); break;
case 't': sb.Append('\t'); break;
case 'u':
var u1 = buffer.Read();
var u2 = buffer.Read();
var u3 = buffer.Read();
var u4 = buffer.Read();
if (u4 != -1) {
var hex = new string(new char[] { (char) u1, (char) u2, (char) u3, (char) u4 });
var n = Convert.ToInt32(hex, 16);
sb.Append((char) n);
}
break;
default:
if (c != -1) {
var message = string.Format("Invalid escape sequence in JSON string '\\{0}'.", (char) c);
throw new FileFormatException(message);
}
break;
}
break;
default:
if (c == quoteCharacter) {
var lexeme = buffer.Substring(start, buffer.Position - start);
return new StringJsonToken(JsonTokenType.String, lexeme, sb.ToString());
}
if (c != -1) {
sb.Append((char) c);
}
break;
}
if (c == -1) {
throw new FileFormatException(FormatMessage("End of file in JSON string.", buffer, start));
}
}
}
internal static IEnumerable<object> ReadDocumentsAs(Type documentType, string filePath, FileFormat format)
{
if (format == FileFormat.Auto)
format = filePath.EndsWith(".json", StringComparison.OrdinalIgnoreCase) ? FileFormat.Json : FileFormat.Bson;
var serializer = BsonSerializer.LookupSerializer(documentType);
var options = DocumentSerializationOptions.Defaults;
if (format == FileFormat.Json)
{
var jb = new JsonBuffer(File.ReadAllText(filePath));
bool array = false;
for (; ; )
{
// skip white
int c;
for (; ; )
{
// end or white?
if ((c = jb.Read()) <= 32)
{
// end?
if (c < 0)
goto end;
// white
continue;
}
// document
if (c == '{')
{
jb.UnRead(c);
break;
}
// array
if (c == ',')
{
if (array)
continue;
}
else if (c == ']')
{
if (array)
{
array = false;
continue;
}
}
else if (c == '[')
{
if (!array)
{
array = true;
continue;
}
}
throw new FormatException(string.Format(null, "Unexpected character '{0}' at position {1}.", (char)c, jb.Position - 1));
}
using (var bsonReader = BsonReader.Create(jb))
yield return serializer.Deserialize(bsonReader, documentType, options);
}
end: ;
}
else
{
using (var stream = File.OpenRead(filePath))
{
long length = stream.Length;
while (stream.Position < length)
using (var bsonReader = BsonReader.Create(stream))
yield return serializer.Deserialize(bsonReader, documentType, options);
}
}
}
private static JsonToken GetRegularExpressionToken(
JsonBuffer buffer
) {
// opening slash has already been read
var start = buffer.Position - 1;
var state = RegularExpressionState.InPattern;
while (true) {
var c = buffer.Read();
switch (state) {
case RegularExpressionState.InPattern:
switch (c) {
case '/': state = RegularExpressionState.InOptions; break;
case '\\': state = RegularExpressionState.InEscapeSequence; break;
default: state = RegularExpressionState.InPattern; break;
}
break;
case RegularExpressionState.InEscapeSequence:
state = RegularExpressionState.InPattern;
break;
case RegularExpressionState.InOptions:
switch (c) {
case 'g':
case 'i':
case 'm':
state = RegularExpressionState.InOptions;
break;
case ',':
case '}':
case ']':
case ')':
case -1:
state = RegularExpressionState.Done;
break;
default:
if (char.IsWhiteSpace((char) c)) {
state = RegularExpressionState.Done;
} else {
state = RegularExpressionState.Invalid;
}
break;
}
break;
}
switch (state) {
case RegularExpressionState.Done:
buffer.UnRead(c);
var lexeme = buffer.Substring(start, buffer.Position - start);
var regex = BsonRegularExpression.Create(lexeme);
return new RegularExpressionJsonToken(lexeme, regex);
case RegularExpressionState.Invalid:
throw new FileFormatException(FormatMessage("Invalid JSON regular expression", buffer, start));
}
}
}
private static JsonToken GetUnquotedStringToken(
JsonBuffer buffer
) {
// opening letter or $ has already been read
var start = buffer.Position - 1;
string lexeme;
while (true) {
var c = buffer.Read();
switch (c) {
case ':':
case ',':
case '}':
case ']':
case -1:
buffer.UnRead(c);
lexeme = buffer.Substring(start, buffer.Position - start);
return new StringJsonToken(JsonTokenType.UnquotedString, lexeme, lexeme);
default:
if (c == '$' || char.IsLetterOrDigit((char) c)) {
// continue
} else if (char.IsWhiteSpace((char) c)) {
buffer.UnRead(c);
lexeme = buffer.Substring(start, buffer.Position - start);
return new StringJsonToken(JsonTokenType.UnquotedString, lexeme, lexeme);
} else {
if (c == '(') {
var value = buffer.Substring(start, buffer.Position - 1 - start);
switch (value) {
case "Date": return GetTenGenDate(buffer, start);
case "NumberLong": return GetTenGenInt64(buffer, start);
case "ObjectId": return GetTenGenObjectId(buffer, start);
}
}
throw new FileFormatException(FormatMessage("Invalid JSON unquoted string", buffer, start));
}
break;
}
}
}
private static JsonToken GetTenGenObjectId(
JsonBuffer buffer,
int start
) {
var c = buffer.Read();
if (c != '"') {
throw new FileFormatException(FormatMessage("Invalid JSON ObjectId value", buffer, start));
}
var firstHexDigit = buffer.Position;
c = buffer.Read();
while (c != '"') {
if (
!char.IsDigit((char) c) &&
!(c >= 'a' && c <= 'f') &&
!(c >= 'A' && c <= 'F')
) {
throw new FileFormatException(FormatMessage("Invalid JSON ObjectId value", buffer, start));
}
c = buffer.Read();
}
var count = buffer.Position - 1 - firstHexDigit;
if (count != 24) {
throw new FileFormatException(FormatMessage("Invalid JSON ObjectId value", buffer, start));
}
var hexDigits = buffer.Substring(firstHexDigit, count);
var objectId = ObjectId.Parse(hexDigits);
c = buffer.Read();
if (c != ')') {
throw new FileFormatException(FormatMessage("Invalid JSON ObjectId value", buffer, start));
}
var lexeme = buffer.Substring(start, buffer.Position - start);
return new ObjectIdJsonToken(lexeme, objectId);
}
public void TestTrue()
{
var json = "\t true,";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.UnquotedString, token.Type);
Assert.AreEqual("true", token.StringValue);
Assert.AreEqual(',', buffer.Read());
}
private static JsonToken GetRegularExpressionToken(
JsonBuffer buffer
)
{
// opening slash has already been read
var start = buffer.Position - 1;
var state = RegularExpressionState.InPattern;
while (true)
{
var c = buffer.Read();
switch (state)
{
case RegularExpressionState.InPattern:
switch (c)
{
case '/': state = RegularExpressionState.InOptions; break;
case '\\': state = RegularExpressionState.InEscapeSequence; break;
default: state = RegularExpressionState.InPattern; break;
}
break;
case RegularExpressionState.InEscapeSequence:
state = RegularExpressionState.InPattern;
break;
case RegularExpressionState.InOptions:
switch (c)
{
case 'g':
case 'i':
case 'm':
state = RegularExpressionState.InOptions;
break;
case ',':
case '}':
case ']':
case ')':
case -1:
state = RegularExpressionState.Done;
break;
default:
if (char.IsWhiteSpace((char)c))
{
state = RegularExpressionState.Done;
}
else
{
state = RegularExpressionState.Invalid;
}
break;
}
break;
}
switch (state)
{
case RegularExpressionState.Done:
buffer.UnRead(c);
var lexeme = buffer.Substring(start, buffer.Position - start);
var regex = BsonRegularExpression.Create(lexeme);
return(new RegularExpressionJsonToken(lexeme, regex));
case RegularExpressionState.Invalid:
throw new FileFormatException(FormatMessage("Invalid JSON regular expression", buffer, start));
}
}
}
public void TestTwelve()
{
var json = "\t 12,";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.Int32, token.Type);
Assert.AreEqual("12", token.Lexeme);
Assert.AreEqual(',', buffer.Read());
}
private static JsonToken GetStringToken(
JsonBuffer buffer,
char quoteCharacter // either single or double quote
)
{
// opening quote has already been read
var start = buffer.Position - 1;
var sb = new StringBuilder();
while (true)
{
var c = buffer.Read();
switch (c)
{
case '\\':
c = buffer.Read();
switch (c)
{
case '\'': sb.Append('\''); break;
case '"': sb.Append('"'); break;
case '\\': sb.Append('\\'); break;
case '/': sb.Append('/'); break;
case 'b': sb.Append('\b'); break;
case 'f': sb.Append('\f'); break;
case 'n': sb.Append('\n'); break;
case 'r': sb.Append('\r'); break;
case 't': sb.Append('\t'); break;
case 'u':
var u1 = buffer.Read();
var u2 = buffer.Read();
var u3 = buffer.Read();
var u4 = buffer.Read();
if (u4 != -1)
{
var hex = new string(new char[] { (char)u1, (char)u2, (char)u3, (char)u4 });
var n = Convert.ToInt32(hex, 16);
sb.Append((char)n);
}
break;
default:
if (c != -1)
{
var message = string.Format("Invalid escape sequence in JSON string '\\{0}'.", (char)c);
throw new FileFormatException(message);
}
break;
}
break;
default:
if (c == quoteCharacter)
{
var lexeme = buffer.Substring(start, buffer.Position - start);
return(new StringJsonToken(JsonTokenType.String, lexeme, sb.ToString()));
}
if (c != -1)
{
sb.Append((char)c);
}
break;
}
if (c == -1)
{
throw new FileFormatException(FormatMessage("End of file in JSON string.", buffer, start));
}
}
}
public void TestZeroExponentMinusOne()
{
var json = "\t 0e-1,";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.Double, token.Type);
Assert.AreEqual("0e-1", token.Lexeme);
Assert.AreEqual(',', buffer.Read());
}
public void TestEscapeSequences()
{
var json = "\t \"x\\\"\\\\\\/\\b\\f\\n\\r\\t\\u0030y\"x";
var buffer = new JsonBuffer(json);
var token = JsonScanner.GetNextToken(buffer);
Assert.AreEqual(JsonTokenType.String, token.Type);
Assert.AreEqual("x\"\\/\b\f\n\r\t0y", token.StringValue);
Assert.AreEqual('x', buffer.Read());
}
private static JsonToken GetUnquotedStringToken(
JsonBuffer buffer
) {
// opening letter or $ has already been read
var start = buffer.Position - 1;
var c = buffer.Read();
while (c == '$' || char.IsLetterOrDigit((char) c)) {
c = buffer.Read();
}
buffer.UnRead(c);
var lexeme = buffer.Substring(start, buffer.Position - start);
return new StringJsonToken(JsonTokenType.UnquotedString, lexeme, lexeme);
}
private static JsonToken GetTenGenDate(
JsonBuffer buffer,
int start
) {
var firstDigit = buffer.Position;
while (true) {
var c = buffer.Read();
if (c == ')') {
var lexeme = buffer.Substring(start, buffer.Position - start);
var digits = buffer.Substring(firstDigit, buffer.Position - firstDigit - 1);
var ms = XmlConvert.ToInt64(digits);
var value = BsonConstants.UnixEpoch.AddMilliseconds(ms);
return new DateTimeJsonToken(lexeme, value);
}
if (c == -1 || !char.IsDigit((char) c)) {
throw new FileFormatException(FormatMessage("Invalid JSON Date value", buffer, start));
}
}
}
