public unsafe void ValidateStrings()
{
string invalidJson = @"{ ""name"": ""\udc00\ud800\uggggxy"" }";
var bytes = Encoding.UTF8.GetBytes(invalidJson);
fixed(byte *ptr = bytes)
{
using (ParsedJson doc = SimdJson.ParseJson(ptr, bytes.Length))
{
Assert.False(doc.IsValid);
Assert.Throws <InvalidOperationException>(() => doc.CreateIterator());
}
}
}
public unsafe void ValidateAllFiles()
{
string[] files = Directory.GetFiles(testDataDir, "*.json", SearchOption.AllDirectories);
// 20 files, ~15Mb of JSON
Assert.NotEmpty(files);
foreach (string file in files)
{
byte[] fileData = File.ReadAllBytes(file);
fixed(byte *ptr = fileData)
using (ParsedJson doc = SimdJson.ParseJson(ptr, fileData.Length))
Assert.True(doc.IsValid);
}
}
public unsafe string SimdJson_Validation(byte[] jsonData, string fileName, string fileSize)
{
string json = Encoding.UTF8.GetString(jsonData);
// Validate json first
// this step is not required for minification, it's here because JSON.NET also does validation
fixed(byte *dataPtr = jsonData)
{
using (var doc = SimdJson.ParseJson(dataPtr, jsonData.Length))
if (!doc.IsValid)
{
throw new InvalidOperationException("Json is invalid");
}
}
return(SimdJson.MinifyJson(json));
}
public unsafe void ValidateMinimizedJson()
{
string[] files = Directory.GetFiles(testDataDir, "*.json", SearchOption.AllDirectories);
// 20 files, ~15Mb of JSON
Assert.NotEmpty(files);
foreach (string file in files)
{
ReadOnlySpan <byte> fileData = File.ReadAllBytes(file);
Span <byte> output = new byte[fileData.Length];
SimdJson.MinifyJson(fileData, output, out int bytesWritten);
output = output.Slice(0, bytesWritten);
fixed(byte *outPtr = output)
using (ParsedJson doc = SimdJson.ParseJson(outPtr, (ulong)output.Length))
Assert.True(doc.IsValid);
}
}
public unsafe void ValidateAllFiles()
{
string currentDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string testDataDir = Path.Combine(currentDir, @"../../../../external/simdjson/jsonexamples");
string[] files = Directory.GetFiles(testDataDir, "*.json", SearchOption.AllDirectories);
// 20 files, ~15Mb of JSON
Assert.NotEmpty(files);
foreach (string file in files)
{
byte[] fileData = File.ReadAllBytes(file);
fixed(byte *ptr = fileData)
using (ParsedJson doc = SimdJson.ParseJson(ptr, fileData.Length))
Assert.True(doc.IsValid);
}
}
//[Fact]
public unsafe void ParseDoubles()
{
byte[] fileData = File.ReadAllBytes(Path.Combine(testDataDir, "canada.json"));
var simdDoubles = new List <double>();
var referenceDoubles = new List <double>();
fixed(byte *ptr = fileData)
{
using (ParsedJson doc = SimdJson.ParseJson(ptr, fileData.Length))
{
using (var iterator = doc.CreateIterator())
{
while (iterator.MoveForward())
{
if (iterator.IsDouble || iterator.IsInteger)
{
simdDoubles.Add(iterator.GetDouble());
}
}
}
}
}
// compare with doubles from Utf8JsonReader
Utf8JsonReader reader = new Utf8JsonReader(fileData, true, default);
while (reader.Read())
{
if (reader.TokenType == JsonTokenType.Number) //Utf8JsonReader doesn't have a token type for Double/Float
{
referenceDoubles.Add(reader.GetDouble());
}
}
for (int i = 0; i < simdDoubles.Count; i++)
{
var doubleSimd = simdDoubles[i];
var doubleRef = referenceDoubles[i];
// TODO: compare
}
}
public unsafe void ValidateMinimizedJson()
{
string currentDir = Path.GetDirectoryName(Assembly.GetExecutingAssembly().Location);
string testDataDir = Path.Combine(currentDir, @"../../../../jsonexamples");
string[] files = Directory.GetFiles(testDataDir, "*.json", SearchOption.AllDirectories);
// 20 files, ~15Mb of JSON
Assert.NotEmpty(files);
foreach (string file in files)
{
ReadOnlySpan <byte> fileData = File.ReadAllBytes(file);
Span <byte> output = new byte[fileData.Length];
SimdJson.MinifyJson(fileData, output, out int bytesWritten);
output = output.Slice(0, bytesWritten);
fixed(byte *outPtr = output)
using (ParsedJson doc = SimdJson.ParseJson(outPtr, output.Length))
Assert.True(doc.IsValid);
}
}
public unsafe ulong _SimdJson(byte[] data, string fileName, string fileSize)
{
ulong numbersCount = 0;
fixed(byte *dataPtr = data)
{
using (ParsedJson doc = SimdJson.ParseJson(dataPtr, data.Length))
using (var iterator = new ParsedJsonIterator(doc))
{
while (iterator.MoveForward())
{
if (iterator.IsInteger || iterator.IsDouble)
{
numbersCount++;
}
}
}
}
return(numbersCount);
}
static unsafe void Main(string[] args)
{
string helloWorldJson = @"{ ""answer"": 42, ""name"": ""Egor"" }";
ReadOnlySpan <byte> bytes = Encoding.UTF8.GetBytes(helloWorldJson);
fixed(byte *ptr = bytes)
{
// SimdJsonN -- N stands for Native, it means we are using Bindings for simdjson native lib
// SimdJson -- fully managed .NET Core 3.0 port
using (ParsedJson doc = SimdJson.ParseJson(ptr, (ulong)bytes.Length))
{
if (!doc.IsValid)
{
Console.WriteLine("Error: " + doc.ErrorCode);
return;
}
Console.WriteLine("Json is valid!");
//open iterator:
using (var iterator = new ParsedJsonIterator(doc))
{
while (iterator.MoveForward())
{
if (iterator.IsInteger)
{
Console.WriteLine("integer: " + iterator.GetInteger());
}
if (iterator.IsString)
{
Console.WriteLine("string: " + iterator.GetUtf16String());
}
}
}
}
Console.WriteLine("Done");
}
}
public unsafe ulong SimdJsonUtf8(byte[] data, string fileName, string fileSize)
{
ulong wordsCount = 0;
fixed(byte *dataPtr = data)
{
using (ParsedJson doc = SimdJson.ParseJson(dataPtr, data.Length))
using (var iterator = new ParsedJsonIterator(doc))
{
while (iterator.MoveForward())
{
if (iterator.IsString)
{
if (*iterator.GetUtf8String() == (byte)'a')
{
wordsCount++;
}
}
}
}
}
return(wordsCount);
}
