public void RemapEntityMapsSourceToTarget() { var a = new Entity { Index = 1, Version = 2 }; var b = new Entity { Index = 3, Version = 5 }; EntityRemapUtility.AddEntityRemapping(ref m_Remapping, a, b); Assert.AreEqual(b, EntityRemapUtility.RemapEntity(ref m_Remapping, a)); }
public void RemapEntityMapsNonExistentSourceToNull() { var a = new Entity { Index = 1, Version = 2 }; var b = new Entity { Index = 3, Version = 5 }; var oldA = new Entity { Index = 1, Version = 1 }; EntityRemapUtility.AddEntityRemapping(ref m_Remapping, a, b); Assert.AreEqual(Entity.Null, EntityRemapUtility.RemapEntity(ref m_Remapping, oldA)); }
public static SceneData[] WriteEntityScene(Scene scene, Hash128 sceneGUID, ConversionFlags conversionFlags) { var world = new World("ConversionWorld"); var entityManager = world.EntityManager; ConvertScene(scene, sceneGUID, world, conversionFlags); EntitySceneOptimization.Optimize(world); var sceneSections = new List <SceneData>(); var subSectionList = new List <SceneSection>(); entityManager.GetAllUniqueSharedComponentData(subSectionList); var extRefInfoEntities = new NativeArray <Entity>(subSectionList.Count, Allocator.Temp); NativeArray <Entity> entitiesInMainSection; var sectionQuery = entityManager.CreateEntityQuery( new EntityQueryDesc { All = new[] { ComponentType.ReadWrite <SceneSection>() }, Options = EntityQueryOptions.IncludePrefab | EntityQueryOptions.IncludeDisabled } ); var sectionBoundsQuery = entityManager.CreateEntityQuery( new EntityQueryDesc { All = new[] { ComponentType.ReadWrite <SceneBoundingVolume>(), ComponentType.ReadWrite <SceneSection>() }, Options = EntityQueryOptions.IncludePrefab | EntityQueryOptions.IncludeDisabled } ); { var section = new SceneSection { SceneGUID = sceneGUID, Section = 0 }; sectionQuery.SetFilter(new SceneSection { SceneGUID = sceneGUID, Section = 0 }); sectionBoundsQuery.SetFilter(new SceneSection { SceneGUID = sceneGUID, Section = 0 }); entitiesInMainSection = sectionQuery.ToEntityArray(Allocator.TempJob); var bounds = GetBoundsAndDestroy(entityManager, sectionBoundsQuery); // Each section will be serialized in its own world, entities that don't have a section are part of the main scene. // An entity that holds the array of external references to the main scene is required for each section. // We need to create them all before we start moving entities to section scenes, // otherwise they would reuse entities that have been moved and mess up the remapping tables. for (int sectionIndex = 1; sectionIndex < subSectionList.Count; ++sectionIndex) { if (subSectionList[sectionIndex].Section == 0) { // Main section, the only one that doesn't need an external ref array continue; } var extRefInfoEntity = entityManager.CreateEntity(); entityManager.AddSharedComponentData(extRefInfoEntity, subSectionList[sectionIndex]); extRefInfoEntities[sectionIndex] = extRefInfoEntity; } // Public references array, only on the main section. var refInfoEntity = entityManager.CreateEntity(); entityManager.AddBuffer <PublicEntityRef>(refInfoEntity); entityManager.AddSharedComponentData(refInfoEntity, section); var publicRefs = entityManager.GetBuffer <PublicEntityRef>(refInfoEntity); // entityManager.Debug.CheckInternalConsistency(); //@TODO do we need to keep this index? doesn't carry any additional info for (int i = 0; i < entitiesInMainSection.Length; ++i) { PublicEntityRef.Add(ref publicRefs, new PublicEntityRef { entityIndex = i, targetEntity = entitiesInMainSection[i] }); } Debug.Assert(publicRefs.Length == entitiesInMainSection.Length); // Save main section var sectionWorld = new World("SectionWorld"); var sectionManager = sectionWorld.EntityManager; var entityRemapping = entityManager.CreateEntityRemapArray(Allocator.TempJob); sectionManager.MoveEntitiesFrom(entityManager, sectionQuery, entityRemapping); // The section component is only there to break the conversion world into different sections // We don't want to store that on the disk //@TODO: Component should be removed but currently leads to corrupt data file. Figure out why. //sectionManager.RemoveComponent(sectionManager.UniversalQuery, typeof(SceneSection)); var sectionFileSize = WriteEntityScene(sectionManager, sceneGUID, "0"); sceneSections.Add(new SceneData { FileSize = sectionFileSize, SceneGUID = sceneGUID, SharedComponentCount = sectionManager.GetSharedComponentCount() - 1, SubSectionIndex = 0, BoundingVolume = bounds }); entityRemapping.Dispose(); sectionWorld.Dispose(); } { // Index 0 is the default value of the shared component, not an actual section for (int subSectionIndex = 0; subSectionIndex < subSectionList.Count; ++subSectionIndex) { var subSection = subSectionList[subSectionIndex]; if (subSection.Section == 0) { continue; } sectionQuery.SetFilter(subSection); sectionBoundsQuery.SetFilter(subSection); var bounds = GetBoundsAndDestroy(entityManager, sectionBoundsQuery); var entitiesInSection = sectionQuery.ToEntityArray(Allocator.TempJob); if (entitiesInSection.Length > 0) { // Fetch back the external reference entity we created earlier to not disturb the mapping var refInfoEntity = extRefInfoEntities[subSectionIndex]; entityManager.AddBuffer <ExternalEntityRef>(refInfoEntity); var externRefs = entityManager.GetBuffer <ExternalEntityRef>(refInfoEntity); // Store the mapping to everything in the main section //@TODO maybe we don't need all that? is this worth worrying about? for (int i = 0; i < entitiesInMainSection.Length; ++i) { ExternalEntityRef.Add(ref externRefs, new ExternalEntityRef { entityIndex = i }); } var entityRemapping = entityManager.CreateEntityRemapArray(Allocator.TempJob); // Entities will be remapped to a contiguous range in the section world, but they will // also come with an unpredictable amount of meta entities. We have the guarantee that // the entities in the main section won't be moved over, so there's a free range of that // size at the end of the remapping table. So we use that range for external references. var externEntityIndexStart = entityRemapping.Length - entitiesInMainSection.Length; entityManager.AddComponentData(refInfoEntity, new ExternalEntityRefInfo { SceneGUID = sceneGUID, EntityIndexStart = externEntityIndexStart }); var sectionWorld = new World("SectionWorld"); var sectionManager = sectionWorld.EntityManager; // Insert mapping for external references, conversion world entity to virtual index in section for (int i = 0; i < entitiesInMainSection.Length; ++i) { EntityRemapUtility.AddEntityRemapping(ref entityRemapping, entitiesInMainSection[i], new Entity { Index = i + externEntityIndexStart, Version = 1 }); } sectionManager.MoveEntitiesFrom(entityManager, sectionQuery, entityRemapping); // Now that all the required entities have been moved over, we can get rid of the gap between // real entities and external references. This allows remapping during load to deal with a // smaller remap table, containing only useful entries. int highestEntityIndexInUse = 0; for (int i = 0; i < externEntityIndexStart; ++i) { var targetIndex = entityRemapping[i].Target.Index; if (targetIndex < externEntityIndexStart && targetIndex > highestEntityIndexInUse) { highestEntityIndexInUse = targetIndex; } } var oldExternEntityIndexStart = externEntityIndexStart; externEntityIndexStart = highestEntityIndexInUse + 1; sectionManager.SetComponentData ( EntityRemapUtility.RemapEntity(ref entityRemapping, refInfoEntity), new ExternalEntityRefInfo { SceneGUID = sceneGUID, EntityIndexStart = externEntityIndexStart } ); // When writing the scene, references to missing entities are set to Entity.Null by default // (but only if they have been used, otherwise they remain untouched) // We obviously don't want that to happen to our external references, so we add explicit mapping // And at the same time, we put them back at the end of the effective range of real entities. for (int i = 0; i < entitiesInMainSection.Length; ++i) { var src = new Entity { Index = i + oldExternEntityIndexStart, Version = 1 }; var dst = new Entity { Index = i + externEntityIndexStart, Version = 1 }; EntityRemapUtility.AddEntityRemapping(ref entityRemapping, src, dst); } // The section component is only there to break the conversion world into different sections // We don't want to store that on the disk //@TODO: Component should be removed but currently leads to corrupt data file. Figure out why. //sectionManager.RemoveComponent(sectionManager.UniversalQuery, typeof(SceneSection)); var fileSize = WriteEntityScene(sectionManager, sceneGUID, subSection.Section.ToString(), entityRemapping); sceneSections.Add(new SceneData { FileSize = fileSize, SceneGUID = sceneGUID, SharedComponentCount = sectionManager.GetSharedComponentCount() - 1, SubSectionIndex = subSection.Section, BoundingVolume = bounds }); entityRemapping.Dispose(); sectionWorld.Dispose(); } entitiesInSection.Dispose(); } } { var noSectionQuery = entityManager.CreateEntityQuery( new EntityQueryDesc { None = new[] { ComponentType.ReadWrite <SceneSection>() }, Options = EntityQueryOptions.IncludePrefab | EntityQueryOptions.IncludeDisabled } ); if (noSectionQuery.CalculateLength() != 0) { Debug.LogWarning($"{noSectionQuery.CalculateLength()} entities in the scene '{scene.path}' had no SceneSection and as a result were not serialized at all."); } } sectionQuery.Dispose(); sectionBoundsQuery.Dispose(); entitiesInMainSection.Dispose(); world.Dispose(); // Save the new header var header = ScriptableObject.CreateInstance <SubSceneHeader>(); header.Sections = sceneSections.ToArray(); WriteHeader(sceneGUID, header); return(sceneSections.ToArray()); }
public static unsafe void SerializeWorld(EntityManager entityManager, BinaryWriter writer, out int[] sharedComponentsToSerialize, NativeArray <EntityRemapUtility.EntityRemapInfo> entityRemapInfos) { writer.Write(CurrentFileFormatVersion); var archetypeManager = entityManager.ArchetypeManager; Dictionary <EntityArchetype, int> archetypeToIndex; EntityArchetype[] archetypeArray; GetAllArchetypes(archetypeManager, out archetypeToIndex, out archetypeArray); var typeindices = new HashSet <int>(); foreach (var archetype in archetypeArray) { for (int iType = 0; iType < archetype.Archetype->TypesCount; ++iType) { typeindices.Add(archetype.Archetype->Types[iType].TypeIndex & TypeManager.ClearFlagsMask); } } var typeArray = typeindices.Select(index => { var type = TypeManager.GetType(index); var name = TypeManager.GetType(index).AssemblyQualifiedName; var hash = TypeManager.GetTypeInfo(index).StableTypeHash; return(new { index, type, name, hash, utf8Name = Encoding.UTF8.GetBytes(name) }); }).OrderBy(t => t.name).ToArray(); int typeNameBufferSize = typeArray.Sum(t => t.utf8Name.Length + 1); writer.Write(typeArray.Length); foreach (var n in typeArray) { writer.Write(n.hash); } writer.Write(typeNameBufferSize); foreach (var n in typeArray) { writer.Write(n.utf8Name); writer.Write((byte)0); } var typeIndexMap = new Dictionary <int, int>(); for (int i = 0; i < typeArray.Length; ++i) { typeIndexMap[typeArray[i].index] = i; } WriteArchetypes(writer, archetypeArray, typeIndexMap); var sharedComponentMapping = GatherSharedComponents(archetypeArray, out var sharedComponentArraysTotalCount); var sharedComponentArrays = new NativeArray <int>(sharedComponentArraysTotalCount, Allocator.Temp); FillSharedComponentArrays(sharedComponentArrays, archetypeArray, sharedComponentMapping); writer.Write(sharedComponentArrays.Length); writer.WriteArray(sharedComponentArrays); sharedComponentArrays.Dispose(); //TODO: ensure chunks are defragged? var bufferPatches = new NativeList <BufferPatchRecord>(128, Allocator.Temp); var totalChunkCount = GenerateRemapInfo(entityManager, archetypeArray, entityRemapInfos); writer.Write(totalChunkCount); GatherAllUsedBlobAssets(archetypeArray, out var blobAssetRefs, out var blobAssets); var blobAssetOffsets = new NativeArray <int>(blobAssets.Length, Allocator.Temp); int totalBlobAssetSize = 0; int Align16(int x) => (x + 15) & ~15; for (int i = 0; i < blobAssets.Length; ++i) { totalBlobAssetSize += sizeof(BlobAssetHeader); blobAssetOffsets[i] = totalBlobAssetSize; totalBlobAssetSize += Align16(blobAssets[i].header->Length); } writer.Write(totalBlobAssetSize); var zeroBytes = int4.zero; for (int i = 0; i < blobAssets.Length; ++i) { var blobAssetLength = blobAssets[i].header->Length; BlobAssetHeader header = new BlobAssetHeader { ValidationPtr = null, Allocator = Allocator.None, Length = Align16(blobAssetLength) }; writer.WriteBytes(&header, sizeof(BlobAssetHeader)); writer.WriteBytes(blobAssets[i].header + 1, blobAssetLength); writer.WriteBytes(&zeroBytes, header.Length - blobAssetLength); } var tempChunk = (Chunk *)UnsafeUtility.Malloc(Chunk.kChunkSize, 16, Allocator.Temp); for (int archetypeIndex = 0; archetypeIndex < archetypeArray.Length; ++archetypeIndex) { var archetype = archetypeArray[archetypeIndex].Archetype; for (var ci = 0; ci < archetype->Chunks.Count; ++ci) { var chunk = archetype->Chunks.p[ci]; bufferPatches.Clear(); UnsafeUtility.MemCpy(tempChunk, chunk, Chunk.kChunkSize); tempChunk->metaChunkEntity = EntityRemapUtility.RemapEntity(ref entityRemapInfos, tempChunk->metaChunkEntity); // Prevent patching from touching buffers allocated memory BufferHeader.PatchAfterCloningChunk(tempChunk); byte *tempChunkBuffer = tempChunk->Buffer; EntityRemapUtility.PatchEntities(archetype->ScalarEntityPatches, archetype->ScalarEntityPatchCount, archetype->BufferEntityPatches, archetype->BufferEntityPatchCount, tempChunkBuffer, tempChunk->Count, ref entityRemapInfos); if (archetype->ContainsBlobAssetRefs) { PatchBlobAssetsInChunkBeforeSave(tempChunk, chunk, blobAssetOffsets, blobAssetRefs); } FillPatchRecordsForChunk(chunk, bufferPatches); ClearChunkHeaderComponents(tempChunk); ChunkDataUtility.MemsetUnusedChunkData(tempChunk, 0); tempChunk->Archetype = (Archetype *)archetypeIndex; if (archetype->NumManagedArrays != 0) { throw new ArgumentException("Serialization of GameObject components is not supported for pure entity scenes"); } writer.WriteBytes(tempChunk, Chunk.kChunkSize); writer.Write(bufferPatches.Length); if (bufferPatches.Length > 0) { writer.WriteList(bufferPatches); // Write heap backed data for each required patch. // TODO: PERF: Investigate static-only deserialization could manage one block and mark in pointers somehow that they are not indiviual for (int i = 0; i < bufferPatches.Length; ++i) { var patch = bufferPatches[i]; var header = (BufferHeader *)OffsetFromPointer(tempChunk->Buffer, patch.ChunkOffset); writer.WriteBytes(header->Pointer, patch.AllocSizeBytes); BufferHeader.Destroy(header); } } } } blobAssetRefs.Dispose(); blobAssets.Dispose(); bufferPatches.Dispose(); UnsafeUtility.Free(tempChunk, Allocator.Temp); sharedComponentsToSerialize = new int[sharedComponentMapping.Count - 1]; foreach (var i in sharedComponentMapping) { if (i.Key != 0) { sharedComponentsToSerialize[i.Value - 1] = i.Key; } } }
public void RemapEntityMapsNullSourceToNull() { Assert.AreEqual(Entity.Null, EntityRemapUtility.RemapEntity(ref m_Remapping, Entity.Null)); }
public static unsafe void SerializeWorld(EntityManager entityManager, BinaryWriter writer, out int[] sharedComponentsToSerialize, NativeArray <EntityRemapUtility.EntityRemapInfo> entityRemapInfos) { writer.Write(CurrentFileFormatVersion); var entityComponentStore = entityManager.EntityComponentStore; NativeHashMap <EntityArchetype, int> archetypeToIndex; EntityArchetype[] archetypeArray; GetAllArchetypes(entityComponentStore, out archetypeToIndex, out archetypeArray); var typeHashes = new NativeHashMap <ulong, int>(1024, Allocator.Temp); foreach (var archetype in archetypeArray) { for (int iType = 0; iType < archetype.Archetype->TypesCount; ++iType) { var typeIndex = archetype.Archetype->Types[iType].TypeIndex; var ti = TypeManager.GetTypeInfo(typeIndex); var hash = ti.StableTypeHash; typeHashes.TryAdd(hash, 0); } } var typeHashSet = typeHashes.GetKeyArray(Allocator.Temp); writer.Write(typeHashSet.Length); foreach (ulong hash in typeHashSet) { writer.Write(hash); } var typeHashToIndexMap = new NativeHashMap <ulong, int>(typeHashSet.Length, Allocator.Temp); for (int i = 0; i < typeHashes.Length; ++i) { typeHashToIndexMap.TryAdd(typeHashSet[i], i); } WriteArchetypes(writer, archetypeArray, typeHashToIndexMap); var sharedComponentMapping = GatherSharedComponents(archetypeArray, out var sharedComponentArraysTotalCount); var sharedComponentArrays = new NativeArray <int>(sharedComponentArraysTotalCount, Allocator.Temp); FillSharedComponentArrays(sharedComponentArrays, archetypeArray, sharedComponentMapping); writer.Write(sharedComponentArrays.Length); writer.WriteArray(sharedComponentArrays); sharedComponentArrays.Dispose(); //TODO: ensure chunks are defragged? var bufferPatches = new NativeList <BufferPatchRecord>(128, Allocator.Temp); var totalChunkCount = GenerateRemapInfo(entityManager, archetypeArray, entityRemapInfos); writer.Write(totalChunkCount); GatherAllUsedBlobAssets(archetypeArray, out var blobAssetRefs, out var blobAssets); var blobAssetOffsets = new NativeArray <int>(blobAssets.Length, Allocator.Temp); int totalBlobAssetSize = 0; int Align16(int x) => (x + 15) & ~15; for (int i = 0; i < blobAssets.Length; ++i) { totalBlobAssetSize += sizeof(BlobAssetHeader); blobAssetOffsets[i] = totalBlobAssetSize; totalBlobAssetSize += Align16(blobAssets[i].header->Length); } writer.Write(totalBlobAssetSize); var zeroBytes = int4.zero; for (int i = 0; i < blobAssets.Length; ++i) { var blobAssetLength = blobAssets[i].header->Length; BlobAssetHeader header = new BlobAssetHeader { ValidationPtr = null, Allocator = Allocator.None, Length = Align16(blobAssetLength) }; writer.WriteBytes(&header, sizeof(BlobAssetHeader)); writer.WriteBytes(blobAssets[i].header + 1, blobAssetLength); writer.WriteBytes(&zeroBytes, header.Length - blobAssetLength); } var tempChunk = (Chunk *)UnsafeUtility.Malloc(Chunk.kChunkSize, 16, Allocator.Temp); for (int archetypeIndex = 0; archetypeIndex < archetypeArray.Length; ++archetypeIndex) { var archetype = archetypeArray[archetypeIndex].Archetype; for (var ci = 0; ci < archetype->Chunks.Count; ++ci) { var chunk = archetype->Chunks.p[ci]; bufferPatches.Clear(); UnsafeUtility.MemCpy(tempChunk, chunk, Chunk.kChunkSize); tempChunk->metaChunkEntity = EntityRemapUtility.RemapEntity(ref entityRemapInfos, tempChunk->metaChunkEntity); // Prevent patching from touching buffers allocated memory BufferHeader.PatchAfterCloningChunk(tempChunk); byte *tempChunkBuffer = tempChunk->Buffer; EntityRemapUtility.PatchEntities(archetype->ScalarEntityPatches, archetype->ScalarEntityPatchCount, archetype->BufferEntityPatches, archetype->BufferEntityPatchCount, tempChunkBuffer, tempChunk->Count, ref entityRemapInfos); if (archetype->ContainsBlobAssetRefs) { PatchBlobAssetsInChunkBeforeSave(tempChunk, chunk, blobAssetOffsets, blobAssetRefs); } FillPatchRecordsForChunk(chunk, bufferPatches); ClearChunkHeaderComponents(tempChunk); ChunkDataUtility.MemsetUnusedChunkData(tempChunk, 0); tempChunk->Archetype = (Archetype *)archetypeIndex; if (archetype->NumManagedArrays != 0) { throw new ArgumentException("Serialization of GameObject components is not supported for pure entity scenes"); } writer.WriteBytes(tempChunk, Chunk.kChunkSize); writer.Write(bufferPatches.Length); if (bufferPatches.Length > 0) { writer.WriteList(bufferPatches); // Write heap backed data for each required patch. // TODO: PERF: Investigate static-only deserialization could manage one block and mark in pointers somehow that they are not indiviual for (int i = 0; i < bufferPatches.Length; ++i) { var patch = bufferPatches[i]; var header = (BufferHeader *)OffsetFromPointer(tempChunk->Buffer, patch.ChunkOffset); writer.WriteBytes(header->Pointer, patch.AllocSizeBytes); BufferHeader.Destroy(header); } } } } blobAssetRefs.Dispose(); blobAssets.Dispose(); bufferPatches.Dispose(); UnsafeUtility.Free(tempChunk, Allocator.Temp); sharedComponentsToSerialize = new int[sharedComponentMapping.Length - 1]; using (var keyArray = sharedComponentMapping.GetKeyArray(Allocator.Temp)) foreach (var key in keyArray) { if (key == 0) { continue; } if (sharedComponentMapping.TryGetValue(key, out var val)) { sharedComponentsToSerialize[val - 1] = key; } } archetypeToIndex.Dispose(); typeHashes.Dispose(); typeHashSet.Dispose(); typeHashToIndexMap.Dispose(); }
public static SceneSectionData[] WriteEntityScene(Scene scene, GameObjectConversionSettings settings, List <ReferencedUnityObjects> sectionRefObjs = null) { int framesToRetainBlobAssets = RetainBlobAssetsSetting.GetFramesToRetainBlobAssets(settings.BuildConfiguration); var world = new World("ConversionWorld"); var entityManager = world.EntityManager; settings.DestinationWorld = world; bool disposeBlobAssetCache = false; if (settings.BlobAssetStore == null) { settings.BlobAssetStore = new BlobAssetStore(); disposeBlobAssetCache = true; } List <(int, LogEventData)> journalData = null; settings.ConversionWorldPreDispose += conversionWorld => { var mappingSystem = conversionWorld.GetExistingSystem <GameObjectConversionMappingSystem>(); journalData = mappingSystem.JournalData.SelectLogEventsOrdered().ToList(); }; ConvertScene(scene, settings); EntitySceneOptimization.Optimize(world); if (settings.AssetImportContext != null) { using (var allTypes = new NativeHashMap <ComponentType, int>(100, Allocator.Temp)) using (var archetypes = new NativeList <EntityArchetype>(Allocator.Temp)) { entityManager.GetAllArchetypes(archetypes); foreach (var archetype in archetypes) { using (var componentTypes = archetype.GetComponentTypes()) foreach (var componentType in componentTypes) { if (allTypes.TryAdd(componentType, 0)) { TypeDependencyCache.AddDependency(settings.AssetImportContext, componentType); } } } } TypeDependencyCache.AddAllSystemsDependency(settings.AssetImportContext); } var sceneSections = new List <SceneSectionData>(); var subSectionList = new List <SceneSection>(); entityManager.GetAllUniqueSharedComponentData(subSectionList); var extRefInfoEntities = new NativeArray <Entity>(subSectionList.Count, Allocator.Temp); NativeArray <Entity> entitiesInMainSection; var sectionQuery = entityManager.CreateEntityQuery( new EntityQueryDesc { All = new[] { ComponentType.ReadWrite <SceneSection>() }, Options = EntityQueryOptions.IncludePrefab | EntityQueryOptions.IncludeDisabled } ); var sectionBoundsQuery = entityManager.CreateEntityQuery( new EntityQueryDesc { All = new[] { ComponentType.ReadWrite <SceneBoundingVolume>(), ComponentType.ReadWrite <SceneSection>() }, Options = EntityQueryOptions.IncludePrefab | EntityQueryOptions.IncludeDisabled } ); var sceneGUID = settings.SceneGUID; { var section = new SceneSection { SceneGUID = sceneGUID, Section = 0 }; sectionQuery.SetSharedComponentFilter(new SceneSection { SceneGUID = sceneGUID, Section = 0 }); sectionBoundsQuery.SetSharedComponentFilter(new SceneSection { SceneGUID = sceneGUID, Section = 0 }); entitiesInMainSection = sectionQuery.ToEntityArray(Allocator.TempJob); var bounds = GetBoundsAndRemove(entityManager, sectionBoundsQuery); // Each section will be serialized in its own world, entities that don't have a section are part of the main scene. // An entity that holds the array of external references to the main scene is required for each section. // We need to create them all before we start moving entities to section scenes, // otherwise they would reuse entities that have been moved and mess up the remapping tables. for (int sectionIndex = 1; sectionIndex < subSectionList.Count; ++sectionIndex) { if (subSectionList[sectionIndex].Section == 0) { // Main section, the only one that doesn't need an external ref array continue; } var extRefInfoEntity = entityManager.CreateEntity(); entityManager.AddSharedComponentData(extRefInfoEntity, subSectionList[sectionIndex]); extRefInfoEntities[sectionIndex] = extRefInfoEntity; } // Public references array, only on the main section. var refInfoEntity = entityManager.CreateEntity(); entityManager.AddBuffer <PublicEntityRef>(refInfoEntity); entityManager.AddSharedComponentData(refInfoEntity, section); var publicRefs = entityManager.GetBuffer <PublicEntityRef>(refInfoEntity); // entityManager.Debug.CheckInternalConsistency(); //@TODO do we need to keep this index? doesn't carry any additional info for (int i = 0; i < entitiesInMainSection.Length; ++i) { PublicEntityRef.Add(ref publicRefs, new PublicEntityRef { entityIndex = i, targetEntity = entitiesInMainSection[i] }); } UnityEngine.Debug.Assert(publicRefs.Length == entitiesInMainSection.Length); // Save main section var sectionWorld = new World("SectionWorld"); var sectionManager = sectionWorld.EntityManager; var entityRemapping = entityManager.CreateEntityRemapArray(Allocator.TempJob); sectionManager.MoveEntitiesFrom(entityManager, sectionQuery, entityRemapping); AddRetainBlobAssetsEntity(sectionManager, framesToRetainBlobAssets); // The section component is only there to break the conversion world into different sections // We don't want to store that on the disk //@TODO: Component should be removed but currently leads to corrupt data file. Figure out why. //sectionManager.RemoveComponent(sectionManager.UniversalQuery, typeof(SceneSection)); var sectionFileSize = WriteEntityScene(sectionManager, sceneGUID, "0", settings, out var objectRefCount, out var objRefs); sectionRefObjs?.Add(objRefs); sceneSections.Add(new SceneSectionData { FileSize = sectionFileSize, SceneGUID = sceneGUID, ObjectReferenceCount = objectRefCount, SubSectionIndex = 0, BoundingVolume = bounds }); entityRemapping.Dispose(); sectionWorld.Dispose(); } { // Index 0 is the default value of the shared component, not an actual section for (int subSectionIndex = 0; subSectionIndex < subSectionList.Count; ++subSectionIndex) { var subSection = subSectionList[subSectionIndex]; if (subSection.Section == 0) { continue; } sectionQuery.SetSharedComponentFilter(subSection); sectionBoundsQuery.SetSharedComponentFilter(subSection); var bounds = GetBoundsAndRemove(entityManager, sectionBoundsQuery); var entitiesInSection = sectionQuery.ToEntityArray(Allocator.TempJob); if (entitiesInSection.Length > 0) { // Fetch back the external reference entity we created earlier to not disturb the mapping var refInfoEntity = extRefInfoEntities[subSectionIndex]; entityManager.AddBuffer <ExternalEntityRef>(refInfoEntity); var externRefs = entityManager.GetBuffer <ExternalEntityRef>(refInfoEntity); // Store the mapping to everything in the main section //@TODO maybe we don't need all that? is this worth worrying about? for (int i = 0; i < entitiesInMainSection.Length; ++i) { ExternalEntityRef.Add(ref externRefs, new ExternalEntityRef { entityIndex = i }); } var entityRemapping = entityManager.CreateEntityRemapArray(Allocator.TempJob); // Entities will be remapped to a contiguous range in the section world, but they will // also come with an unpredictable amount of meta entities. We have the guarantee that // the entities in the main section won't be moved over, so there's a free range of that // size at the end of the remapping table. So we use that range for external references. var externEntityIndexStart = entityRemapping.Length - entitiesInMainSection.Length; entityManager.AddComponentData(refInfoEntity, new ExternalEntityRefInfo { SceneGUID = sceneGUID, EntityIndexStart = externEntityIndexStart }); var sectionWorld = new World("SectionWorld"); var sectionManager = sectionWorld.EntityManager; // Insert mapping for external references, conversion world entity to virtual index in section for (int i = 0; i < entitiesInMainSection.Length; ++i) { EntityRemapUtility.AddEntityRemapping(ref entityRemapping, entitiesInMainSection[i], new Entity { Index = i + externEntityIndexStart, Version = 1 }); } sectionManager.MoveEntitiesFrom(entityManager, sectionQuery, entityRemapping); AddRetainBlobAssetsEntity(sectionManager, framesToRetainBlobAssets); // Now that all the required entities have been moved over, we can get rid of the gap between // real entities and external references. This allows remapping during load to deal with a // smaller remap table, containing only useful entries. int highestEntityIndexInUse = 0; for (int i = 0; i < externEntityIndexStart; ++i) { var targetIndex = entityRemapping[i].Target.Index; if (targetIndex < externEntityIndexStart && targetIndex > highestEntityIndexInUse) { highestEntityIndexInUse = targetIndex; } } var oldExternEntityIndexStart = externEntityIndexStart; externEntityIndexStart = highestEntityIndexInUse + 1; sectionManager.SetComponentData ( EntityRemapUtility.RemapEntity(ref entityRemapping, refInfoEntity), new ExternalEntityRefInfo { SceneGUID = sceneGUID, EntityIndexStart = externEntityIndexStart } ); // When writing the scene, references to missing entities are set to Entity.Null by default // (but only if they have been used, otherwise they remain untouched) // We obviously don't want that to happen to our external references, so we add explicit mapping // And at the same time, we put them back at the end of the effective range of real entities. for (int i = 0; i < entitiesInMainSection.Length; ++i) { var src = new Entity { Index = i + oldExternEntityIndexStart, Version = 1 }; var dst = new Entity { Index = i + externEntityIndexStart, Version = 1 }; EntityRemapUtility.AddEntityRemapping(ref entityRemapping, src, dst); } // The section component is only there to break the conversion world into different sections // We don't want to store that on the disk //@TODO: Component should be removed but currently leads to corrupt data file. Figure out why. //sectionManager.RemoveComponent(sectionManager.UniversalQuery, typeof(SceneSection)); var fileSize = WriteEntityScene(sectionManager, sceneGUID, subSection.Section.ToString(), settings, out var objectRefCount, out var objRefs, entityRemapping); sectionRefObjs?.Add(objRefs); sceneSections.Add(new SceneSectionData { FileSize = fileSize, SceneGUID = sceneGUID, ObjectReferenceCount = objectRefCount, SubSectionIndex = subSection.Section, BoundingVolume = bounds }); entityRemapping.Dispose(); sectionWorld.Dispose(); } entitiesInSection.Dispose(); } } { var noSectionQuery = entityManager.CreateEntityQuery( new EntityQueryDesc { None = new[] { ComponentType.ReadWrite <SceneSection>() }, Options = EntityQueryOptions.IncludePrefab | EntityQueryOptions.IncludeDisabled } ); if (noSectionQuery.CalculateEntityCount() != 0) { Debug.LogWarning($"{noSectionQuery.CalculateEntityCount()} entities in the scene '{scene.path}' had no SceneSection and as a result were not serialized at all."); } } sectionQuery.Dispose(); sectionBoundsQuery.Dispose(); entitiesInMainSection.Dispose(); world.Dispose(); // Save the new header var sceneSectionsArray = sceneSections.ToArray(); WriteHeader(sceneGUID, sceneSectionsArray, scene.name, settings.AssetImportContext); // If we are writing assets to assets folder directly, then we need to make sure the asset database see them so they can be loaded. if (settings.AssetImportContext == null) { AssetDatabase.Refresh(); } if (disposeBlobAssetCache) { settings.BlobAssetStore.Dispose(); } // Save the log of issues that happened during conversion WriteConversionLog(sceneGUID, journalData, scene.name, settings.AssetImportContext); return(sceneSectionsArray); }
public static unsafe void SerializeWorld(EntityManager entityManager, BinaryWriter writer, out int[] sharedComponentsToSerialize, NativeArray <EntityRemapUtility.EntityRemapInfo> entityRemapInfos) { writer.Write(CurrentFileFormatVersion); var archetypeManager = entityManager.ArchetypeManager; Dictionary <EntityArchetype, int> archetypeToIndex; EntityArchetype[] archetypeArray; GetAllArchetypes(archetypeManager, out archetypeToIndex, out archetypeArray); var typeindices = new HashSet <int>(); foreach (var archetype in archetypeArray) { for (int iType = 0; iType < archetype.Archetype->TypesCount; ++iType) { typeindices.Add(archetype.Archetype->Types[iType].TypeIndex & ~TypeManager.ChunkComponentTypeFlag); } } var typeArray = typeindices.Select(index => { var type = TypeManager.GetType(index); var name = TypeManager.GetType(index).AssemblyQualifiedName; var hash = TypeManager.GetTypeInfo(index).StableTypeHash; return(new { index, type, name, hash, utf8Name = Encoding.UTF8.GetBytes(name) }); }).OrderBy(t => t.name).ToArray(); int typeNameBufferSize = typeArray.Sum(t => t.utf8Name.Length + 1); writer.Write(typeArray.Length); foreach (var n in typeArray) { writer.Write(n.hash); } writer.Write(typeNameBufferSize); foreach (var n in typeArray) { writer.Write(n.utf8Name); writer.Write((byte)0); } var typeIndexMap = new Dictionary <int, int>(); for (int i = 0; i < typeArray.Length; ++i) { typeIndexMap[typeArray[i].index] = i; } WriteArchetypes(writer, archetypeArray, typeIndexMap); //TODO: ensure chunks are defragged? var bufferPatches = new NativeList <BufferPatchRecord>(128, Allocator.Temp); var totalChunkCount = GenerateRemapInfo(entityManager, archetypeArray, entityRemapInfos); writer.Write(totalChunkCount); var tempChunk = (Chunk *)UnsafeUtility.Malloc(Chunk.kChunkSize, 16, Allocator.Temp); var sharedIndexToSerialize = new Dictionary <int, int>(); for (int archetypeIndex = 0; archetypeIndex < archetypeArray.Length; ++archetypeIndex) { var archetype = archetypeArray[archetypeIndex].Archetype; for (var ci = 0; ci < archetype->Chunks.Count; ++ci) { var chunk = archetype->Chunks.p[ci]; bufferPatches.Clear(); UnsafeUtility.MemCpy(tempChunk, chunk, Chunk.kChunkSize); tempChunk->SharedComponentValueArray = (int *)((byte *)(tempChunk) + Chunk.GetSharedComponentOffset(archetype->NumSharedComponents)); tempChunk->metaChunkEntity = EntityRemapUtility.RemapEntity(ref entityRemapInfos, tempChunk->metaChunkEntity); byte *tempChunkBuffer = tempChunk->Buffer; EntityRemapUtility.PatchEntities(archetype->ScalarEntityPatches, archetype->ScalarEntityPatchCount, archetype->BufferEntityPatches, archetype->BufferEntityPatchCount, tempChunkBuffer, tempChunk->Count, ref entityRemapInfos); // Find all buffer pointer locations and work out how much memory the deserializer must allocate on load. for (int ti = 0; ti < archetype->TypesCount; ++ti) { int index = archetype->TypeMemoryOrder[ti]; if (!archetype->Types[index].IsBuffer) { continue; } int subArrayOffset = archetype->Offsets[index]; BufferHeader *header = (BufferHeader *)OffsetFromPointer(tempChunkBuffer, subArrayOffset); int stride = archetype->SizeOfs[index]; int count = chunk->Count; var ct = TypeManager.GetTypeInfo(archetype->Types[index].TypeIndex); for (int bi = 0; bi < count; ++bi) { if (header->Pointer != null) { header->Pointer = null; bufferPatches.Add(new BufferPatchRecord { ChunkOffset = (int)(((byte *)header) - (byte *)tempChunkBuffer), AllocSizeBytes = ct.ElementSize * header->Capacity, }); } header = (BufferHeader *)OffsetFromPointer(header, stride); } } ClearChunkHeaderComponents(tempChunk); ClearUnusedChunkData(tempChunk); tempChunk->Archetype = (Archetype *)archetypeIndex; if (archetype->NumManagedArrays != 0) { throw new ArgumentException("Serialization of GameObject components is not supported for pure entity scenes"); } for (int i = 0; i != archetype->NumSharedComponents; i++) { int sharedComponentIndex = tempChunk->SharedComponentValueArray[i]; int newIndex; if (tempChunk->SharedComponentValueArray[i] != 0) { if (sharedIndexToSerialize.TryGetValue(sharedComponentIndex, out newIndex)) { tempChunk->SharedComponentValueArray[i] = newIndex; } else { // 0 is reserved for null types in shared components newIndex = sharedIndexToSerialize.Count + 1; sharedIndexToSerialize[sharedComponentIndex] = newIndex; tempChunk->SharedComponentValueArray[i] = newIndex; } } } writer.WriteBytes(tempChunk, Chunk.kChunkSize); writer.Write(bufferPatches.Length); if (bufferPatches.Length > 0) { writer.WriteList(bufferPatches); // Write heap backed data for each required patch. // TODO: PERF: Investigate static-only deserialization could manage one block and mark in pointers somehow that they are not indiviual for (int i = 0; i < bufferPatches.Length; ++i) { var patch = bufferPatches[i]; // NOTE that this reads the pointer from the original, unpatched chunk. // We have nulled out the pointer in the serialized data above. var header = (BufferHeader *)OffsetFromPointer(chunk->Buffer, patch.ChunkOffset); writer.WriteBytes(header->Pointer, patch.AllocSizeBytes); } } } } bufferPatches.Dispose(); UnsafeUtility.Free(tempChunk, Allocator.Temp); sharedComponentsToSerialize = new int[sharedIndexToSerialize.Count]; foreach (var i in sharedIndexToSerialize) { sharedComponentsToSerialize[i.Value - 1] = i.Key; } }