public unsafe void DeserializeFromString_UsingSerializedObjectReaderWithNoActualization_BasicDataBatch(string json, int batchSize)
{
var jsonBatch = $"{{\"batch\":[{string.Join(",", Enumerable.Repeat(json, batchSize))}]}}";
Measure.Method(() =>
{
var config = SerializedObjectReaderConfiguration.Default;
config.UseReadAsync = false;
config.ValidationType = JsonValidationType.None;
config.NodeBufferSize = 128;
config.TokenBufferSize = Math.Max(jsonBatch.Length / 2, 16);
config.BlockBufferSize = jsonBatch.Length;
config.OutputBufferSize = jsonBatch.Length;
fixed(char *ptr = jsonBatch)
{
using (var reader = new SerializedObjectReader(new UnsafeBuffer <char>(ptr, jsonBatch.Length), config))
{
var document = reader.ReadObject().AsUnsafe();
}
}
})
.WarmupCount(1)
.MeasurementCount(100)
.IterationsPerMeasurement(1)
.Run();
PerformanceTest.Active.CalculateStatisticalValues();
}
public void SerializedObjectReader_Read_MockManifest()
{
SetJson(@"
{
hash = 17bc734727d742b091491895812a6cea
files = [
""c:/path/to/assets/image.jpg""
""c:/path/to/assets/image_a.png""
]
}");
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation))
{
var obj = reader.ReadObject();
var hash = obj["hash"].AsStringView();
Assert.IsTrue(hash.Equals("17bc734727d742b091491895812a6cea"));
Assert.AreEqual("17bc734727d742b091491895812a6cea", hash.ToString());
var files = obj["files"].AsArrayView();
Assert.AreEqual(2, files.Count());
Assert.IsTrue(files.Select(view => view.AsStringView().ToString()).SequenceEqual(new [] { "c:/path/to/assets/image.jpg", "c:/path/to/assets/image_a.png" }));
}
}
public void FromJson_StreamWithPartialTokens()
{
const string kJson =
@"{
""Dependencies"": [
""GlobalObjectId_V1-1-4040109c1e5014e8abe57249dc3a759b-93214019566545601-0""
],
""Components"": [
{
""$type"": ""NetCodeConversionSettings, Unity.NetCode"",
""Target"": 2
}
]
}";
using (var stream = new MemoryStream(UTF8Encoding.Default.GetBytes(kJson)))
{
var configuration = SerializedObjectReaderConfiguration.Default;
configuration.UseReadAsync = stream.Length > 512 << 10;
configuration.ValidationType = JsonValidationType.Standard;
configuration.BlockBufferSize = (int)stream.Length; // 512 kb max
configuration.OutputBufferSize = (int)stream.Length; // 1 mb max
using (var reader = new SerializedObjectReader(stream, configuration))
{
var obj = reader.ReadObject();
Assert.That(obj.Count(), Is.EqualTo(2));
}
}
}
/**
* Deserialize the input string into a series of buffers
*/
internal TinyJsonData(string jsonString, Allocator allocator)
{
var config = JsonDataConfiguration.Default;
m_Allocator = allocator;
m_BoolBuffer = new NativeList <bool>(config.boolCapacity, m_Allocator);
m_IntBuffer = new NativeList <int>(config.intCapacity, m_Allocator);
m_FloatBuffer = new NativeList <float>(config.floatCapacity, m_Allocator);
m_StringBuffer = new NativeList <FixedString4096>(config.stringCapacity, m_Allocator);
m_ArrayBuffer = new NativeList <UnsafeList <JsonKeyHandle> >(config.arrayCapacity, m_Allocator);
m_ObjectBuffer = new NativeList <UnsafeHashMap <FixedString128, JsonKeyHandle> >(config.objectCapacity, m_Allocator);
m_JsonKeyBuffer = new NativeList <JsonKey>(config.intCapacity, m_Allocator);
m_ValueRefFreeList = new NativeList <int>(config.intCapacity, m_Allocator);
// parse existing json into a packed binary stream
SerializedObjectReader reader;
fixed(char *ptr = jsonString)
{
reader = new SerializedObjectReader(ptr, jsonString.Length, SerializedObjectReaderConfiguration.Default);
}
// determine the type and copy each field-value pair into the mutable map / buffers
m_RootObjMap = new UnsafeHashMap <FixedString128, JsonKeyHandle>(config.fieldCapacity, m_Allocator);
m_JsonKeyBuffer.Add(new JsonKey {
Type = JsonValueType.Object, Location = 0
});
ReadObjectViewIntoMap(reader.ReadObject(), ref m_RootObjMap);
reader.Dispose();
}
public void SerializedObjectReader_Step_MatchIfAtLeastOneFlag()
{
SetJson("{\"a\":\"b\"}");
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation, leaveInputOpen: false))
{
Assert.That(reader.Step(NodeType.ObjectKey), Is.EqualTo(NodeType.ObjectKey | NodeType.String));
}
}
public void SerializedObjectReader_Read_MockEditorAssetFile()
{
SetJson(@"
{
uxml_inspectors =
[
{
target = ""Unity.Tiny.Core2D.Translation, Unity.Tiny.Core2D, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null""
},
{
target = ""Unity.Tiny.Core2D.Camera2D, Unity.Tiny.Core2D, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null""
uxml = { guid = a384dee09c50b0c478ccc1064fc29d9a fileId = 9197481836555534336 type = 3 }
uss = { guid = a384dee09c50b0c478ccc1064fc29d9a fileId = 9197481836555534336 type = 3 }
}
],
code_inspectors =
[
{
target = ""Unity.Tiny.Core2D.Camera2D, Unity.Tiny.Core2D, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null""
type = ""Unity.Tiny.Core2D.Camera2DInspector, Unity.Tiny.Core2D, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null""
}
]
}");
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation))
{
var obj = reader.ReadObject();
var uxmlInspectors = obj["uxml_inspectors"].AsArrayView().ToArray();
Assert.AreEqual(2, uxmlInspectors.Length);
var localPositionUxmlInspector = uxmlInspectors[0].AsObjectView();
Assert.IsTrue(localPositionUxmlInspector.TryGetValue("target", out var target));
Assert.AreEqual("Unity.Tiny.Core2D.Translation, Unity.Tiny.Core2D, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null", target.AsStringView());
Assert.IsFalse(localPositionUxmlInspector.TryGetValue("uxml", out _));
Assert.IsFalse(localPositionUxmlInspector.TryGetValue("uss", out _));
var camera2DUxmlInspector = uxmlInspectors[1].AsObjectView();
Assert.IsTrue(camera2DUxmlInspector.TryGetValue("target", out target));
Assert.AreEqual("Unity.Tiny.Core2D.Camera2D, Unity.Tiny.Core2D, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null", target.AsStringView());
Assert.IsTrue(camera2DUxmlInspector.TryGetValue("uxml", out var uxml));
Assert.AreEqual(TokenType.Object, uxml.Type);
Assert.IsTrue(camera2DUxmlInspector.TryGetValue("uss", out var uss));
Assert.AreEqual(TokenType.Object, uss.Type);
var codeInspectors = obj["code_inspectors"].AsArrayView().ToArray();
Assert.AreEqual(1, codeInspectors.Length);
var camera2DCodeInspector = codeInspectors[0].AsObjectView();
Assert.IsTrue(camera2DCodeInspector.TryGetValue("target", out target));
Assert.AreEqual("Unity.Tiny.Core2D.Camera2D, Unity.Tiny.Core2D, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null", target.AsStringView());
Assert.IsTrue(camera2DCodeInspector.TryGetValue("type", out var type));
Assert.AreEqual("Unity.Tiny.Core2D.Camera2DInspector, Unity.Tiny.Core2D, Version=0.0.0.0, Culture=neutral, PublicKeyToken=null", type.AsStringView());
}
}
public void SerializedObjectReader_Read_JsonWithStringRoot()
{
SetJson("\"TestString\"");
using (var reader = new SerializedObjectReader(m_Stream))
{
reader.Read(out var view);
var str = view.ToString();
Assert.That(str, Is.EqualTo("TestString"));
}
}
public void SerializedObjectReader_Read_JsonWithBoolRoot()
{
SetJson("true");
using (var reader = new SerializedObjectReader(m_Stream))
{
reader.Read(out var view);
Assert.That(view.Type, Is.EqualTo(TokenType.Primitive));
Assert.AreEqual(true, view.AsBoolean());
}
}
public void SerializedObjectReader_Convert_UInt64(string json, ulong expected)
{
SetJson(json);
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation))
{
var obj = reader.ReadObject();
var value = obj["x"];
Assert.AreEqual(expected, value.AsUInt64());
}
}
public void SerializedObjectReader_Step_EmptyObject(string json)
{
SetJson(json);
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation))
{
Assert.AreEqual(NodeType.BeginObject, reader.Step(out var view));
Assert.AreEqual(TokenType.Object, view.Type);
Assert.AreEqual(NodeType.EndObject, reader.Step());
}
}
public void SerializedObjectReader_Read_JsonWithFloat32Root()
{
SetJson("123.4");
using (var reader = new SerializedObjectReader(m_Stream))
{
reader.Read(out var view);
Assert.That(view.Type, Is.EqualTo(TokenType.Primitive));
Assert.AreEqual(123.4f, view.AsFloat());
}
}
public void SerializedObjectReader_Convert_UInt64_Throws(string json)
{
SetJson(json);
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation))
{
var obj = reader.ReadObject();
var value = obj["x"];
Assert.Throws <ParseErrorException>(() => { value.AsUInt64(); });
}
}
public void SerializedObjectReader_Step_ObjectWithMember(string json, int expectedCount)
{
SetJson(json);
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation))
{
Assert.AreEqual(NodeType.BeginObject, reader.Step(out var view));
Assert.AreEqual(0, view.AsObjectView().Count());
Assert.AreEqual(NodeType.EndObject, reader.Step(NodeType.EndObject));
Assert.AreEqual(expectedCount, view.AsObjectView().Count());
}
}
public void SerializedObjectReader_Read_InvalidJsonWithStringPrimitiveRoot()
{
SetJson("\"TestString\"10");
using (var reader = new SerializedObjectReader(m_Stream))
{
Assert.Throws <InvalidJsonException>(() =>
{
reader.Read(out var view);
});
}
}
public void SerializedObjectReader_Read_Array(string json, int expectedCount)
{
SetJson(json);
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation))
{
reader.Read(out var view);
Assert.AreEqual(TokenType.Array, view.Type);
Assert.AreEqual(expectedCount, view.AsArrayView().Count());
}
}
public void SerializedObjectReader_Step_ObjectWithSingleMultiComment()
{
SetJson("{/*\nThis is a \nmulti-line comment\n*/\"a\":\"b\"}");
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation, leaveInputOpen: false))
{
Assert.That(reader.Step(), Is.EqualTo(NodeType.BeginObject));
Assert.That(reader.Step(), Is.EqualTo(NodeType.Comment));
Assert.That(reader.Step(), Is.EqualTo(NodeType.ObjectKey | NodeType.String));
Assert.That(reader.Step(), Is.EqualTo(NodeType.String));
Assert.That(reader.Step(), Is.EqualTo(NodeType.EndObject));
}
}
public void SerializedObjectReader_Read_ObjectWithSpecialsCharactersInStrings(string json, string expectedString)
{
SetJson(json);
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation, leaveInputOpen: false))
{
var view = reader.ReadObject();
var members = view.ToArray();
Assert.That(members.Length, Is.EqualTo(1));
Assert.That(members[0].Value().ToString(), Is.EqualTo(expectedString));
}
}
public unsafe void DeserializeFromString_UsingSerializedObjectReader_BasicDataBatch(string json, int batchSize)
{
var jsonBatch = $"{{\"batch\":[{string.Join(",", Enumerable.Repeat(json, batchSize))}]}}";
Measure.Method(() =>
{
var config = SerializedObjectReaderConfiguration.Default;
config.UseReadAsync = false;
config.ValidationType = JsonValidationType.None;
config.NodeBufferSize = 128;
config.TokenBufferSize = Math.Max(jsonBatch.Length / 2, 16);
config.BlockBufferSize = jsonBatch.Length;
config.OutputBufferSize = jsonBatch.Length;
var data = new BasicDataBatch();
fixed(char *ptr = jsonBatch)
{
using (var reader = new SerializedObjectReader(new UnsafeBuffer <char>(ptr, jsonBatch.Length), config))
{
var document = reader.ReadObject().AsUnsafe();
if (document.TryGetValue("batch", out var batch))
{
var array = batch.AsArrayView();
data.batch = new BasicData[array.Count()];
var index = 0;
foreach (var element in array)
{
var obj = element.AsObjectView();
data.batch[index++] = new BasicData
{
Value = obj["Value"].ToString()
};
}
}
}
}
})
.WarmupCount(1)
.MeasurementCount(100)
.IterationsPerMeasurement(1)
.Run();
PerformanceTest.Active.CalculateStatisticalValues();
}
private static SerializedObjectReader ReadSceneFile(string path, out SerializedObjectView headerInformations)
{
var config = new SerializedObjectReaderConfiguration
{
UseReadAsync = false,
BlockBufferSize = 256 << 10, // 256 KB
TokenBufferSize = 1024,
NodeBufferSize = JsonFrontEnd.EntityBatchSize,
ValidationType = JsonValidationType.Standard,
OutputBufferSize = 1024 << 10 // 1 MB
};
var stream = new FileStream(path, FileMode.Open, FileAccess.Read, FileShare.Read, config.BlockBufferSize);
var reader = new SerializedObjectReader(stream, config, leaveInputOpen: false);
try
{
if (reader.Step(out var root) != NodeType.BeginObject)
{
throw new Exception();
}
// ASSUMPTION: All members are written BEFORE the blob of entities.
// Read until we hit "Entities":
NodeType node;
while ((node = reader.Step(out var view)) != NodeType.None)
{
if ((node & NodeType.ObjectKey) == 0)
{
continue;
}
if (view.AsStringView().Equals("Entities"))
{
break;
}
}
// Unpack header information from whats been read so far.
headerInformations = root.AsObjectView();
return(reader);
}
catch
{
reader.Dispose();
throw;
}
}
public void SerializedObjectReader_Read_ObjectWithComments()
{
SetJson("{/*\nThis is a \nmulti-line comment\n*/\"a\": // This is a really bad place for a comment, but it should work\n \"b\"}");
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation, leaveInputOpen: false))
{
var view = reader.ReadObject();
var members = view.ToArray();
Assert.That(members.Length, Is.EqualTo(1));
Assert.That(members[0].Name().ToString(), Is.EqualTo("a"));
Assert.That(members[0].Value().ToString(), Is.EqualTo("b"));
}
}
public void SerializedObjectReader_Read_PartialToken_EscapedString(string json)
{
SetJson(json);
var config = SerializedObjectReaderConfiguration.Default;
config.ValidationType = JsonValidationType.None;
config.BlockBufferSize = 16; // 16 bytes | 8 characters
using (var reader = new SerializedObjectReader(m_Stream, config))
{
var obj = reader.ReadObject();
Assert.IsTrue(obj.TryGetValue("a", out var a));
Assert.AreEqual("a\"b", a.AsStringView().ToString());
}
}
public void SerializedObjectReader_ReadArrayElement_Numbers(string json, int[] expectedValues)
{
SetJson(json);
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation))
{
Assert.AreEqual(NodeType.BeginArray, reader.Step());
foreach (var expectedValue in expectedValues)
{
reader.ReadArrayElement(out var element);
Assert.AreEqual(expectedValue, element.AsDouble());
}
Assert.AreEqual(NodeType.EndArray, reader.Step());
}
}
public unsafe void PerformanceTest_SerializedObjectReader_Read_MockEntities(int count, int batchSize)
{
File.WriteAllText("test.json", JsonTestData.GetMockEntities(count));
try
{
Measure.Method(() =>
{
var views = stackalloc SerializedValueView[batchSize];
var config = SerializedObjectReaderConfiguration.Default;
config.BlockBufferSize = 512 << 10;
config.NodeBufferSize = batchSize;
config.ValidationType = JsonValidationType.None;
config.UseReadAsync = true;
config.OutputBufferSize = 4096 << 10;
using (var stream = new FileStream("test.json", FileMode.Open, FileAccess.Read, FileShare.Read, config.BlockBufferSize, FileOptions.Asynchronous))
using (var reader = new SerializedObjectReader(stream, config))
{
reader.Step();
while (reader.ReadArrayElementBatch(views, batchSize) != 0)
{
reader.DiscardCompleted();
}
reader.Step();
}
})
.Definition("SerializedObjectReaderRead")
.WarmupCount(1)
.MeasurementCount(100)
.Run();
PerformanceTest.Active.CalculateStatisticalValues();
var size = new FileInfo("test.json").Length / (double)1024 / 1024;
Debug.Log($"MB/s=[{size / (PerformanceTest.Active.SampleGroups.First().Median / 1000)}]");
}
finally
{
File.Delete("test.json");
}
}
static string[] GatherDependenciesFromSourceFile(string assetPath)
{
var dependencies = new List <string>();
try
{
using (var reader = new SerializedObjectReader(assetPath))
{
var root = reader.ReadObject();
if (!root.TryGetMember(nameof(BuildConfiguration.Dependencies), out var member))
{
return(Array.Empty <string>());
}
var valueView = member.Value();
if (valueView.Type != TokenType.Array)
{
return(Array.Empty <string>());
}
var arrayView = valueView.AsArrayView();
foreach (var value in arrayView)
{
if (!GlobalObjectId.TryParse(value.AsStringView().ToString(), out var id))
{
continue;
}
var dependencyPath = AssetDatabase.GUIDToAssetPath(id.assetGUID.ToString());
if (string.IsNullOrEmpty(dependencyPath))
{
continue;
}
dependencies.Add(dependencyPath);
}
}
}
catch
{
return(Array.Empty <string>());
}
return(dependencies.ToArray());
}
public void SerializedObjectReader_ReadArrayElement_Objects(string json)
{
SetJson(json);
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation))
{
Assert.AreEqual(NodeType.BeginArray, reader.Step());
while (reader.ReadArrayElement(out var element))
{
var obj = element.AsObjectView();
Assert.AreEqual(10, obj["foo"].AsInt64());
Assert.AreEqual("test", obj["bar"].AsStringView().ToString());
}
Assert.AreEqual(NodeType.EndArray, reader.Step());
}
}
public void SerializedObjectReader_ReadArrayElement_Numbers(string json, string expected)
{
var expectedValues = expected.Split(',').Where(x => !string.IsNullOrEmpty(x)).Select(int.Parse);
SetJson(json);
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation))
{
Assert.AreEqual(NodeType.BeginArray, reader.Step());
foreach (var expectedValue in expectedValues)
{
reader.ReadArrayElement(out var element);
Assert.AreEqual(expectedValue, element.AsDouble());
}
Assert.AreEqual(NodeType.EndArray, reader.Step());
}
}
public void SerializedObjectReader_Read_PartialToken(string json)
{
SetJson(json);
var config = SerializedObjectReaderConfiguration.Default;
config.ValidationType = JsonValidationType.None;
config.BlockBufferSize = 16; // 16 bytes | 8 characters
using (var reader = new SerializedObjectReader(m_Stream, config))
{
var obj = reader.ReadObject();
Assert.IsTrue(obj.TryGetValue("LongObjectKey", out var a));
Assert.AreEqual(10, a.AsPrimitiveView().AsInt64());
Assert.IsTrue(obj.TryGetValue("1234567890", out var b));
Assert.AreEqual("LongValueString", b.AsStringView().ToString());
}
}
public void SerializedObjectReader_Read_MockSettings()
{
using (var output = new PackedBinaryStream(Allocator.TempJob))
using (var members = new SerializedMemberViewCollection(Allocator.TempJob))
{
SetJson(@"
foo = {
a = 10
b = 20
c = ""hello""
}
bar = [1,2,3,4,5]
");
using (var reader = new SerializedObjectReader(m_Stream, output, m_ConfigWithNoValidation))
{
reader.ReadMember(members);
reader.ReadMember(members);
}
SetJson(@"
test = ""string""
color = { r = 1 g = 1 b = 1 a = 1 }
");
using (var reader = new SerializedObjectReader(m_Stream, output, m_ConfigWithNoValidation))
{
reader.ReadMember(members);
reader.ReadMember(members);
}
Assert.AreEqual(4, members.Count());
Assert.IsTrue(members.TryGetValue("foo", out _));
Assert.IsTrue(members.TryGetValue("bar", out _));
Assert.IsTrue(members.TryGetValue("test", out _));
Assert.IsTrue(members.TryGetValue("color", out _));
Assert.IsFalse(members.TryGetValue("baz", out _));
}
}
public override void OnImportAsset(AssetImportContext context)
{
var asset = ScriptableObject.CreateInstance <SceneAsset>();
var icon = GetThumbnailForAsset(context, asset);
asset.Icon = icon;
using (var reader = new SerializedObjectReader(context.assetPath))
{
reader.Step(out var root);
// @TODO This is not very future proof at all...
reader.Step(NodeType.BeginArray);
var obj = root.AsObjectView();
asset.Guid = obj["Guid"].AsStringView().ToString();
asset.SerializedVersion = (uint)obj["SerializedVersion"].AsUInt64();
}
context.AddObjectToAsset("asset", asset, icon);
context.SetMainObject(asset);
}
public unsafe void SerializedObjectReader_ReadArrayElementBatch(string json, int expectedCount, int batchSize)
{
SetJson(json);
using (var reader = new SerializedObjectReader(m_Stream, m_ConfigWithNoValidation))
{
Assert.AreEqual(NodeType.BeginArray, reader.Step());
var views = stackalloc SerializedValueView[batchSize];
var actualCount = 0;
int count;
while ((count = reader.ReadArrayElementBatch(views, batchSize)) != 0)
{
actualCount += count;
}
Assert.AreEqual(expectedCount, actualCount);
Assert.AreEqual(NodeType.EndArray, reader.Step());
}
}