static unsafe void Main(string[] args)
{
string helloWorldJson = @"{ ""answer"": 42, ""name"": ""Egor"" }";
ReadOnlySpan <byte> bytes = Encoding.UTF8.GetBytes(helloWorldJson);
// SimdJson is UTF8 only
fixed(byte *ptr = bytes)
{
using (ParsedJson doc = SimdJson.ParseJson(ptr, bytes.Length))
{
Console.WriteLine($"Is json valid:{doc.IsValid}\n");
// open iterator:
using (var iterator = new ParsedJsonIterator(doc))
{
while (iterator.MoveForward())
{
switch (iterator.GetTokenType())
{
case JsonTokenType.Number:
Console.WriteLine("integer: " + iterator.GetInteger());
break;
case JsonTokenType.String:
Console.WriteLine("string: " + iterator.GetUtf16String());
break;
}
}
}
}
}
}
static unsafe void Main(string[] args)
{
string helloWorldJson = "{ \"answer\": 42, \"name\": \"Egor\" }";
ReadOnlySpan <byte> bytes = Encoding.UTF8.GetBytes(helloWorldJson);
// SimdJson is UTF8 only
using (ParsedJson doc = SimdJson.build_parsed_json(bytes))
{
Console.WriteLine("Is json valid:" + doc.isValid());
// open iterator:
using (iterator iterator = new iterator(&doc))
{
while (iterator.move_forward())
{
switch (iterator.GetTokenType())
{
case JsonTokenType.Number:
Console.WriteLine("integer: " + iterator.get_integer());
break;
case JsonTokenType.String:
Console.WriteLine("string: " + iterator.GetUtf16String());
break;
}
}
}
}
}
public unsafe ulong _SimdJson(byte[] data, string fileName, string fileSize)
{
ulong wordsCount = 0;
fixed(byte *dataPtr = data)
{
using (ParsedJson doc = SimdJson.ParseJson(dataPtr, data.Length))
using (var iterator = new ParsedJsonIterator(doc))
{
while (iterator.MoveForward())
{
if (iterator.IsString)
{
// count all strings starting with 'a'
// NOTE: it could be much faster with direct UTF8 API: (*iterator.GetUtf8String()) == (byte)'a')
if (iterator.GetUtf16String().StartsWith('a'))
{
wordsCount++;
}
}
}
}
}
return(wordsCount);
}
//[Fact]
public void ValidateMinifier()
{
string json = @"{
""Egor"": ""Bogatov""
}
";
string minifiedJson = SimdJson.MinifyJson(json);
Assert.Equal(@"{""Egor"":""Bogatov""}", minifiedJson);
// TODO: more tests
}
static void Main(string[] args)
{
ReadOnlySpan <byte> beforeData = LoadEmbeddedFile("MinifyJson.sample.json");
string beforeString = Encoding.UTF8.GetString(beforeData);
Console.WriteLine($"Before:\n{beforeString}\nLength={beforeString.Length}");
string afterString = SimdJson.MinifyJson(beforeString); // there is also Span API
Console.WriteLine($"\n\nAfter:\n{afterString}.\nLength={afterString.Length}");
}
unsafe protected override BookShelf Deserialize(Stream stream)
{
// A small Json like {"Books":[{"Title":"Book 1","Id":1}]}
// will be translated to the following JsonTokenTypes by the iterator:
// String = "Books"
// StartArray [
// StartObject {
// String "Title"
// String "Book 1"
// String "Id"
// Number 1
// EndObject }
// EndArray ]
// EndObject }
BookShelf lret = new BookShelf()
{
Books = new List <Book>()
};
if (stream is MemoryStream mem)
{
byte[] buffer = mem.GetBuffer();
fixed(byte *ptr = buffer)
{
using (ParsedJson doc = SimdJson.ParseJson(ptr, (int)stream.Length))
{
// open iterator:
myIterator = new ParsedJsonIterator(doc);
try
{
while (myIterator.MoveForward() && myIterator.GetTokenType() == JsonTokenType.Comment)
{
if (myIterator.GetUtf16String() == "Books")
{
if (myIterator.MoveForward() && myIterator.GetTokenType() == JsonTokenType.StartArray)
{
ReadBooks(lret.Books);
}
}
}
}
finally
{
myIterator.Dispose();
}
}
}
}
return(lret);
}
public unsafe void ValidateAllFiles()
{
string[] files = Directory.GetFiles(testDataDir, "*.json", SearchOption.AllDirectories);
// 20 files, ~15Mb of JSON
Assert.NotEmpty(files);
foreach (string file in files)
{
byte[] fileData = File.ReadAllBytes(file);
fixed(byte *ptr = fileData)
using (ParsedJson doc = SimdJson.ParseJson(ptr, (ulong)fileData.Length))
Assert.True(doc.IsValid);
}
}
public unsafe void ValidateStrings()
{
string invalidJson = @"{ ""name"": ""\udc00\ud800\uggggxy"" }";
var bytes = Encoding.UTF8.GetBytes(invalidJson);
fixed(byte *ptr = bytes)
{
using (ParsedJson doc = SimdJson.ParseJson(ptr, (ulong)bytes.Length))
{
Assert.False(doc.IsValid);
Assert.Throws <InvalidOperationException>(() => doc.CreateIterator());
}
}
}
public unsafe void ValidateAllFiles()
{
string currentDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string testDataDir = Path.Combine(currentDir, @"../../../../external/simdjson/jsonexamples");
string[] files = Directory.GetFiles(testDataDir, "*.json", SearchOption.AllDirectories);
// 20 files, ~15Mb of JSON
Assert.NotEmpty(files);
foreach (string file in files)
{
byte[] fileData = File.ReadAllBytes(file);
fixed(byte *ptr = fileData)
using (ParsedJson doc = SimdJson.ParseJson(ptr, fileData.Length))
Assert.True(doc.IsValid);
}
}
public unsafe string SimdJson_Validation(byte[] jsonData, string fileName, string fileSize)
{
string json = Encoding.UTF8.GetString(jsonData);
// Validate json first
// this step is not required for minification, it's here because JSON.NET also does validation
fixed(byte *dataPtr = jsonData)
{
using (var doc = SimdJson.ParseJson(dataPtr, jsonData.Length))
if (!doc.IsValid)
{
throw new InvalidOperationException("Json is invalid");
}
}
return(SimdJson.MinifyJson(json));
}
public unsafe void ValidateMinimizedJson()
{
string[] files = Directory.GetFiles(testDataDir, "*.json", SearchOption.AllDirectories);
// 20 files, ~15Mb of JSON
Assert.NotEmpty(files);
foreach (string file in files)
{
ReadOnlySpan <byte> fileData = File.ReadAllBytes(file);
Span <byte> output = new byte[fileData.Length];
SimdJson.MinifyJson(fileData, output, out int bytesWritten);
output = output.Slice(0, bytesWritten);
fixed(byte *outPtr = output)
using (ParsedJson doc = SimdJson.ParseJson(outPtr, (ulong)output.Length))
Assert.True(doc.IsValid);
}
}
public void ValidateAllFiles()
{
string currentDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string testDataDir = Path.Combine(currentDir, @"../../../../../external/simdjson/jsonexamples");
string[] files = Directory.GetFiles(testDataDir, "*.json", SearchOption.AllDirectories);
// 20 files, ~15Mb of JSON
Assert.NotEmpty(files);
foreach (string file in files)
{
ReadOnlySpan <byte> fileData = File.ReadAllBytes(file);
using (ParsedJson doc = SimdJson.build_parsed_json(fileData))
{
Assert.True(doc.isValid());
}
}
}
//[Fact]
public unsafe void ParseDoubles()
{
byte[] fileData = File.ReadAllBytes(Path.Combine(testDataDir, "canada.json"));
var simdDoubles = new List <double>();
var referenceDoubles = new List <double>();
fixed(byte *ptr = fileData)
{
using (ParsedJson doc = SimdJson.ParseJson(ptr, fileData.Length))
{
using (var iterator = doc.CreateIterator())
{
while (iterator.MoveForward())
{
if (iterator.IsDouble || iterator.IsInteger)
{
simdDoubles.Add(iterator.GetDouble());
}
}
}
}
}
// compare with doubles from Utf8JsonReader
Utf8JsonReader reader = new Utf8JsonReader(fileData, true, default);
while (reader.Read())
{
if (reader.TokenType == JsonTokenType.Number) //Utf8JsonReader doesn't have a token type for Double/Float
{
referenceDoubles.Add(reader.GetDouble());
}
}
for (int i = 0; i < simdDoubles.Count; i++)
{
var doubleSimd = simdDoubles[i];
var doubleRef = referenceDoubles[i];
// TODO: compare
}
}
public unsafe void ValidateMinimizedJson()
{
string currentDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string testDataDir = Path.Combine(currentDir, @"../../../../jsonexamples");
string[] files = Directory.GetFiles(testDataDir, "*.json", SearchOption.AllDirectories);
// 20 files, ~15Mb of JSON
Assert.NotEmpty(files);
foreach (string file in files)
{
ReadOnlySpan <byte> fileData = File.ReadAllBytes(file);
Span <byte> output = new byte[fileData.Length];
SimdJson.MinifyJson(fileData, output, out int bytesWritten);
output = output.Slice(0, bytesWritten);
fixed(byte *outPtr = output)
using (ParsedJson doc = SimdJson.ParseJson(outPtr, output.Length))
Assert.True(doc.IsValid);
}
}
public unsafe ulong SimdJsonSharpApi(byte[] data, string fileName, string fileSize)
{
ulong numbersCount = 0;
using (ParsedJson doc = SimdJson.build_parsed_json(data))
{
using (var iterator = new iterator(&doc))
{
while (iterator.move_forward())
{
if (iterator.is_integer() || iterator.is_double())
{
numbersCount++;
}
}
}
}
return(numbersCount);
}
public unsafe ulong _SimdJson(byte[] data, string fileName, string fileSize)
{
ulong numbersCount = 0;
fixed(byte *dataPtr = data)
{
using (ParsedJson doc = SimdJson.ParseJson(dataPtr, data.Length))
using (var iterator = new ParsedJsonIterator(doc))
{
while (iterator.MoveForward())
{
if (iterator.IsInteger || iterator.IsDouble)
{
numbersCount++;
}
}
}
}
return(numbersCount);
}
static unsafe void Main(string[] args)
{
string helloWorldJson = @"{ ""answer"": 42, ""name"": ""Egor"" }";
ReadOnlySpan <byte> bytes = Encoding.UTF8.GetBytes(helloWorldJson);
fixed(byte *ptr = bytes)
{
// SimdJsonN -- N stands for Native, it means we are using Bindings for simdjson native lib
// SimdJson -- fully managed .NET Core 3.0 port
using (ParsedJson doc = SimdJson.ParseJson(ptr, (ulong)bytes.Length))
{
if (!doc.IsValid)
{
Console.WriteLine("Error: " + doc.ErrorCode);
return;
}
Console.WriteLine("Json is valid!");
//open iterator:
using (var iterator = new ParsedJsonIterator(doc))
{
while (iterator.MoveForward())
{
if (iterator.IsInteger)
{
Console.WriteLine("integer: " + iterator.GetInteger());
}
if (iterator.IsString)
{
Console.WriteLine("string: " + iterator.GetUtf16String());
}
}
}
}
Console.WriteLine("Done");
}
}
public unsafe ulong SimdJsonUtf8(byte[] data, string fileName, string fileSize)
{
ulong wordsCount = 0;
fixed(byte *dataPtr = data)
{
using (ParsedJson doc = SimdJson.ParseJson(dataPtr, data.Length))
using (var iterator = new ParsedJsonIterator(doc))
{
while (iterator.MoveForward())
{
if (iterator.IsString)
{
if (*iterator.GetUtf8String() == (byte)'a')
{
wordsCount++;
}
}
}
}
}
return(wordsCount);
}
public string SimdJson_NoValidation(byte[] jsonData, string fileName, string fileSize)
{
string json = Encoding.UTF8.GetString(jsonData);
return(SimdJson.MinifyJson(json));
}