public void RemapEntityMapsSourceToTarget()
{
var a = new Entity {
Index = 1, Version = 2
};
var b = new Entity {
Index = 3, Version = 5
};
EntityRemapUtility.AddEntityRemapping(ref m_Remapping, a, b);
Assert.AreEqual(b, EntityRemapUtility.RemapEntity(ref m_Remapping, a));
}
public void RemapEntityMapsNonExistentSourceToNull()
{
var a = new Entity {
Index = 1, Version = 2
};
var b = new Entity {
Index = 3, Version = 5
};
var oldA = new Entity {
Index = 1, Version = 1
};
EntityRemapUtility.AddEntityRemapping(ref m_Remapping, a, b);
Assert.AreEqual(Entity.Null, EntityRemapUtility.RemapEntity(ref m_Remapping, oldA));
}
public static SceneData[] WriteEntityScene(Scene scene, Hash128 sceneGUID, ConversionFlags conversionFlags)
{
var world = new World("ConversionWorld");
var entityManager = world.EntityManager;
ConvertScene(scene, sceneGUID, world, conversionFlags);
EntitySceneOptimization.Optimize(world);
var sceneSections = new List <SceneData>();
var subSectionList = new List <SceneSection>();
entityManager.GetAllUniqueSharedComponentData(subSectionList);
var extRefInfoEntities = new NativeArray <Entity>(subSectionList.Count, Allocator.Temp);
NativeArray <Entity> entitiesInMainSection;
var sectionQuery = entityManager.CreateEntityQuery(
new EntityQueryDesc
{
All = new[] { ComponentType.ReadWrite <SceneSection>() },
Options = EntityQueryOptions.IncludePrefab | EntityQueryOptions.IncludeDisabled
}
);
var sectionBoundsQuery = entityManager.CreateEntityQuery(
new EntityQueryDesc
{
All = new[] { ComponentType.ReadWrite <SceneBoundingVolume>(), ComponentType.ReadWrite <SceneSection>() },
Options = EntityQueryOptions.IncludePrefab | EntityQueryOptions.IncludeDisabled
}
);
{
var section = new SceneSection {
SceneGUID = sceneGUID, Section = 0
};
sectionQuery.SetFilter(new SceneSection {
SceneGUID = sceneGUID, Section = 0
});
sectionBoundsQuery.SetFilter(new SceneSection {
SceneGUID = sceneGUID, Section = 0
});
entitiesInMainSection = sectionQuery.ToEntityArray(Allocator.TempJob);
var bounds = GetBoundsAndDestroy(entityManager, sectionBoundsQuery);
// Each section will be serialized in its own world, entities that don't have a section are part of the main scene.
// An entity that holds the array of external references to the main scene is required for each section.
// We need to create them all before we start moving entities to section scenes,
// otherwise they would reuse entities that have been moved and mess up the remapping tables.
for (int sectionIndex = 1; sectionIndex < subSectionList.Count; ++sectionIndex)
{
if (subSectionList[sectionIndex].Section == 0)
{
// Main section, the only one that doesn't need an external ref array
continue;
}
var extRefInfoEntity = entityManager.CreateEntity();
entityManager.AddSharedComponentData(extRefInfoEntity, subSectionList[sectionIndex]);
extRefInfoEntities[sectionIndex] = extRefInfoEntity;
}
// Public references array, only on the main section.
var refInfoEntity = entityManager.CreateEntity();
entityManager.AddBuffer <PublicEntityRef>(refInfoEntity);
entityManager.AddSharedComponentData(refInfoEntity, section);
var publicRefs = entityManager.GetBuffer <PublicEntityRef>(refInfoEntity);
// entityManager.Debug.CheckInternalConsistency();
//@TODO do we need to keep this index? doesn't carry any additional info
for (int i = 0; i < entitiesInMainSection.Length; ++i)
{
PublicEntityRef.Add(ref publicRefs,
new PublicEntityRef {
entityIndex = i, targetEntity = entitiesInMainSection[i]
});
}
Debug.Assert(publicRefs.Length == entitiesInMainSection.Length);
// Save main section
var sectionWorld = new World("SectionWorld");
var sectionManager = sectionWorld.EntityManager;
var entityRemapping = entityManager.CreateEntityRemapArray(Allocator.TempJob);
sectionManager.MoveEntitiesFrom(entityManager, sectionQuery, entityRemapping);
// The section component is only there to break the conversion world into different sections
// We don't want to store that on the disk
//@TODO: Component should be removed but currently leads to corrupt data file. Figure out why.
//sectionManager.RemoveComponent(sectionManager.UniversalQuery, typeof(SceneSection));
var sectionFileSize = WriteEntityScene(sectionManager, sceneGUID, "0");
sceneSections.Add(new SceneData
{
FileSize = sectionFileSize,
SceneGUID = sceneGUID,
SharedComponentCount = sectionManager.GetSharedComponentCount() - 1,
SubSectionIndex = 0,
BoundingVolume = bounds
});
entityRemapping.Dispose();
sectionWorld.Dispose();
}
{
// Index 0 is the default value of the shared component, not an actual section
for (int subSectionIndex = 0; subSectionIndex < subSectionList.Count; ++subSectionIndex)
{
var subSection = subSectionList[subSectionIndex];
if (subSection.Section == 0)
{
continue;
}
sectionQuery.SetFilter(subSection);
sectionBoundsQuery.SetFilter(subSection);
var bounds = GetBoundsAndDestroy(entityManager, sectionBoundsQuery);
var entitiesInSection = sectionQuery.ToEntityArray(Allocator.TempJob);
if (entitiesInSection.Length > 0)
{
// Fetch back the external reference entity we created earlier to not disturb the mapping
var refInfoEntity = extRefInfoEntities[subSectionIndex];
entityManager.AddBuffer <ExternalEntityRef>(refInfoEntity);
var externRefs = entityManager.GetBuffer <ExternalEntityRef>(refInfoEntity);
// Store the mapping to everything in the main section
//@TODO maybe we don't need all that? is this worth worrying about?
for (int i = 0; i < entitiesInMainSection.Length; ++i)
{
ExternalEntityRef.Add(ref externRefs, new ExternalEntityRef {
entityIndex = i
});
}
var entityRemapping = entityManager.CreateEntityRemapArray(Allocator.TempJob);
// Entities will be remapped to a contiguous range in the section world, but they will
// also come with an unpredictable amount of meta entities. We have the guarantee that
// the entities in the main section won't be moved over, so there's a free range of that
// size at the end of the remapping table. So we use that range for external references.
var externEntityIndexStart = entityRemapping.Length - entitiesInMainSection.Length;
entityManager.AddComponentData(refInfoEntity,
new ExternalEntityRefInfo
{
SceneGUID = sceneGUID,
EntityIndexStart = externEntityIndexStart
});
var sectionWorld = new World("SectionWorld");
var sectionManager = sectionWorld.EntityManager;
// Insert mapping for external references, conversion world entity to virtual index in section
for (int i = 0; i < entitiesInMainSection.Length; ++i)
{
EntityRemapUtility.AddEntityRemapping(ref entityRemapping, entitiesInMainSection[i],
new Entity {
Index = i + externEntityIndexStart, Version = 1
});
}
sectionManager.MoveEntitiesFrom(entityManager, sectionQuery, entityRemapping);
// Now that all the required entities have been moved over, we can get rid of the gap between
// real entities and external references. This allows remapping during load to deal with a
// smaller remap table, containing only useful entries.
int highestEntityIndexInUse = 0;
for (int i = 0; i < externEntityIndexStart; ++i)
{
var targetIndex = entityRemapping[i].Target.Index;
if (targetIndex < externEntityIndexStart && targetIndex > highestEntityIndexInUse)
{
highestEntityIndexInUse = targetIndex;
}
}
var oldExternEntityIndexStart = externEntityIndexStart;
externEntityIndexStart = highestEntityIndexInUse + 1;
sectionManager.SetComponentData
(
EntityRemapUtility.RemapEntity(ref entityRemapping, refInfoEntity),
new ExternalEntityRefInfo
{
SceneGUID = sceneGUID,
EntityIndexStart = externEntityIndexStart
}
);
// When writing the scene, references to missing entities are set to Entity.Null by default
// (but only if they have been used, otherwise they remain untouched)
// We obviously don't want that to happen to our external references, so we add explicit mapping
// And at the same time, we put them back at the end of the effective range of real entities.
for (int i = 0; i < entitiesInMainSection.Length; ++i)
{
var src = new Entity {
Index = i + oldExternEntityIndexStart, Version = 1
};
var dst = new Entity {
Index = i + externEntityIndexStart, Version = 1
};
EntityRemapUtility.AddEntityRemapping(ref entityRemapping, src, dst);
}
// The section component is only there to break the conversion world into different sections
// We don't want to store that on the disk
//@TODO: Component should be removed but currently leads to corrupt data file. Figure out why.
//sectionManager.RemoveComponent(sectionManager.UniversalQuery, typeof(SceneSection));
var fileSize = WriteEntityScene(sectionManager, sceneGUID, subSection.Section.ToString(), entityRemapping);
sceneSections.Add(new SceneData
{
FileSize = fileSize,
SceneGUID = sceneGUID,
SharedComponentCount = sectionManager.GetSharedComponentCount() - 1,
SubSectionIndex = subSection.Section,
BoundingVolume = bounds
});
entityRemapping.Dispose();
sectionWorld.Dispose();
}
entitiesInSection.Dispose();
}
}
{
var noSectionQuery = entityManager.CreateEntityQuery(
new EntityQueryDesc
{
None = new[] { ComponentType.ReadWrite <SceneSection>() },
Options = EntityQueryOptions.IncludePrefab | EntityQueryOptions.IncludeDisabled
}
);
if (noSectionQuery.CalculateLength() != 0)
{
Debug.LogWarning($"{noSectionQuery.CalculateLength()} entities in the scene '{scene.path}' had no SceneSection and as a result were not serialized at all.");
}
}
sectionQuery.Dispose();
sectionBoundsQuery.Dispose();
entitiesInMainSection.Dispose();
world.Dispose();
// Save the new header
var header = ScriptableObject.CreateInstance <SubSceneHeader>();
header.Sections = sceneSections.ToArray();
WriteHeader(sceneGUID, header);
return(sceneSections.ToArray());
}
public static unsafe void SerializeWorld(EntityManager entityManager, BinaryWriter writer, out int[] sharedComponentsToSerialize, NativeArray <EntityRemapUtility.EntityRemapInfo> entityRemapInfos)
{
writer.Write(CurrentFileFormatVersion);
var archetypeManager = entityManager.ArchetypeManager;
Dictionary <EntityArchetype, int> archetypeToIndex;
EntityArchetype[] archetypeArray;
GetAllArchetypes(archetypeManager, out archetypeToIndex, out archetypeArray);
var typeindices = new HashSet <int>();
foreach (var archetype in archetypeArray)
{
for (int iType = 0; iType < archetype.Archetype->TypesCount; ++iType)
{
typeindices.Add(archetype.Archetype->Types[iType].TypeIndex & TypeManager.ClearFlagsMask);
}
}
var typeArray = typeindices.Select(index =>
{
var type = TypeManager.GetType(index);
var name = TypeManager.GetType(index).AssemblyQualifiedName;
var hash = TypeManager.GetTypeInfo(index).StableTypeHash;
return(new
{
index,
type,
name,
hash,
utf8Name = Encoding.UTF8.GetBytes(name)
});
}).OrderBy(t => t.name).ToArray();
int typeNameBufferSize = typeArray.Sum(t => t.utf8Name.Length + 1);
writer.Write(typeArray.Length);
foreach (var n in typeArray)
{
writer.Write(n.hash);
}
writer.Write(typeNameBufferSize);
foreach (var n in typeArray)
{
writer.Write(n.utf8Name);
writer.Write((byte)0);
}
var typeIndexMap = new Dictionary <int, int>();
for (int i = 0; i < typeArray.Length; ++i)
{
typeIndexMap[typeArray[i].index] = i;
}
WriteArchetypes(writer, archetypeArray, typeIndexMap);
var sharedComponentMapping = GatherSharedComponents(archetypeArray, out var sharedComponentArraysTotalCount);
var sharedComponentArrays = new NativeArray <int>(sharedComponentArraysTotalCount, Allocator.Temp);
FillSharedComponentArrays(sharedComponentArrays, archetypeArray, sharedComponentMapping);
writer.Write(sharedComponentArrays.Length);
writer.WriteArray(sharedComponentArrays);
sharedComponentArrays.Dispose();
//TODO: ensure chunks are defragged?
var bufferPatches = new NativeList <BufferPatchRecord>(128, Allocator.Temp);
var totalChunkCount = GenerateRemapInfo(entityManager, archetypeArray, entityRemapInfos);
writer.Write(totalChunkCount);
GatherAllUsedBlobAssets(archetypeArray, out var blobAssetRefs, out var blobAssets);
var blobAssetOffsets = new NativeArray <int>(blobAssets.Length, Allocator.Temp);
int totalBlobAssetSize = 0;
int Align16(int x) => (x + 15) & ~15;
for (int i = 0; i < blobAssets.Length; ++i)
{
totalBlobAssetSize += sizeof(BlobAssetHeader);
blobAssetOffsets[i] = totalBlobAssetSize;
totalBlobAssetSize += Align16(blobAssets[i].header->Length);
}
writer.Write(totalBlobAssetSize);
var zeroBytes = int4.zero;
for (int i = 0; i < blobAssets.Length; ++i)
{
var blobAssetLength = blobAssets[i].header->Length;
BlobAssetHeader header = new BlobAssetHeader
{
ValidationPtr = null, Allocator = Allocator.None, Length = Align16(blobAssetLength)
};
writer.WriteBytes(&header, sizeof(BlobAssetHeader));
writer.WriteBytes(blobAssets[i].header + 1, blobAssetLength);
writer.WriteBytes(&zeroBytes, header.Length - blobAssetLength);
}
var tempChunk = (Chunk *)UnsafeUtility.Malloc(Chunk.kChunkSize, 16, Allocator.Temp);
for (int archetypeIndex = 0; archetypeIndex < archetypeArray.Length; ++archetypeIndex)
{
var archetype = archetypeArray[archetypeIndex].Archetype;
for (var ci = 0; ci < archetype->Chunks.Count; ++ci)
{
var chunk = archetype->Chunks.p[ci];
bufferPatches.Clear();
UnsafeUtility.MemCpy(tempChunk, chunk, Chunk.kChunkSize);
tempChunk->metaChunkEntity = EntityRemapUtility.RemapEntity(ref entityRemapInfos, tempChunk->metaChunkEntity);
// Prevent patching from touching buffers allocated memory
BufferHeader.PatchAfterCloningChunk(tempChunk);
byte *tempChunkBuffer = tempChunk->Buffer;
EntityRemapUtility.PatchEntities(archetype->ScalarEntityPatches, archetype->ScalarEntityPatchCount, archetype->BufferEntityPatches, archetype->BufferEntityPatchCount, tempChunkBuffer, tempChunk->Count, ref entityRemapInfos);
if (archetype->ContainsBlobAssetRefs)
{
PatchBlobAssetsInChunkBeforeSave(tempChunk, chunk, blobAssetOffsets, blobAssetRefs);
}
FillPatchRecordsForChunk(chunk, bufferPatches);
ClearChunkHeaderComponents(tempChunk);
ChunkDataUtility.MemsetUnusedChunkData(tempChunk, 0);
tempChunk->Archetype = (Archetype *)archetypeIndex;
if (archetype->NumManagedArrays != 0)
{
throw new ArgumentException("Serialization of GameObject components is not supported for pure entity scenes");
}
writer.WriteBytes(tempChunk, Chunk.kChunkSize);
writer.Write(bufferPatches.Length);
if (bufferPatches.Length > 0)
{
writer.WriteList(bufferPatches);
// Write heap backed data for each required patch.
// TODO: PERF: Investigate static-only deserialization could manage one block and mark in pointers somehow that they are not indiviual
for (int i = 0; i < bufferPatches.Length; ++i)
{
var patch = bufferPatches[i];
var header = (BufferHeader *)OffsetFromPointer(tempChunk->Buffer, patch.ChunkOffset);
writer.WriteBytes(header->Pointer, patch.AllocSizeBytes);
BufferHeader.Destroy(header);
}
}
}
}
blobAssetRefs.Dispose();
blobAssets.Dispose();
bufferPatches.Dispose();
UnsafeUtility.Free(tempChunk, Allocator.Temp);
sharedComponentsToSerialize = new int[sharedComponentMapping.Count - 1];
foreach (var i in sharedComponentMapping)
{
if (i.Key != 0)
{
sharedComponentsToSerialize[i.Value - 1] = i.Key;
}
}
}
public void RemapEntityMapsNullSourceToNull()
{
Assert.AreEqual(Entity.Null, EntityRemapUtility.RemapEntity(ref m_Remapping, Entity.Null));
}
public static unsafe void SerializeWorld(EntityManager entityManager, BinaryWriter writer, out int[] sharedComponentsToSerialize, NativeArray <EntityRemapUtility.EntityRemapInfo> entityRemapInfos)
{
writer.Write(CurrentFileFormatVersion);
var entityComponentStore = entityManager.EntityComponentStore;
NativeHashMap <EntityArchetype, int> archetypeToIndex;
EntityArchetype[] archetypeArray;
GetAllArchetypes(entityComponentStore, out archetypeToIndex, out archetypeArray);
var typeHashes = new NativeHashMap <ulong, int>(1024, Allocator.Temp);
foreach (var archetype in archetypeArray)
{
for (int iType = 0; iType < archetype.Archetype->TypesCount; ++iType)
{
var typeIndex = archetype.Archetype->Types[iType].TypeIndex;
var ti = TypeManager.GetTypeInfo(typeIndex);
var hash = ti.StableTypeHash;
typeHashes.TryAdd(hash, 0);
}
}
var typeHashSet = typeHashes.GetKeyArray(Allocator.Temp);
writer.Write(typeHashSet.Length);
foreach (ulong hash in typeHashSet)
{
writer.Write(hash);
}
var typeHashToIndexMap = new NativeHashMap <ulong, int>(typeHashSet.Length, Allocator.Temp);
for (int i = 0; i < typeHashes.Length; ++i)
{
typeHashToIndexMap.TryAdd(typeHashSet[i], i);
}
WriteArchetypes(writer, archetypeArray, typeHashToIndexMap);
var sharedComponentMapping = GatherSharedComponents(archetypeArray, out var sharedComponentArraysTotalCount);
var sharedComponentArrays = new NativeArray <int>(sharedComponentArraysTotalCount, Allocator.Temp);
FillSharedComponentArrays(sharedComponentArrays, archetypeArray, sharedComponentMapping);
writer.Write(sharedComponentArrays.Length);
writer.WriteArray(sharedComponentArrays);
sharedComponentArrays.Dispose();
//TODO: ensure chunks are defragged?
var bufferPatches = new NativeList <BufferPatchRecord>(128, Allocator.Temp);
var totalChunkCount = GenerateRemapInfo(entityManager, archetypeArray, entityRemapInfos);
writer.Write(totalChunkCount);
GatherAllUsedBlobAssets(archetypeArray, out var blobAssetRefs, out var blobAssets);
var blobAssetOffsets = new NativeArray <int>(blobAssets.Length, Allocator.Temp);
int totalBlobAssetSize = 0;
int Align16(int x) => (x + 15) & ~15;
for (int i = 0; i < blobAssets.Length; ++i)
{
totalBlobAssetSize += sizeof(BlobAssetHeader);
blobAssetOffsets[i] = totalBlobAssetSize;
totalBlobAssetSize += Align16(blobAssets[i].header->Length);
}
writer.Write(totalBlobAssetSize);
var zeroBytes = int4.zero;
for (int i = 0; i < blobAssets.Length; ++i)
{
var blobAssetLength = blobAssets[i].header->Length;
BlobAssetHeader header = new BlobAssetHeader
{
ValidationPtr = null, Allocator = Allocator.None, Length = Align16(blobAssetLength)
};
writer.WriteBytes(&header, sizeof(BlobAssetHeader));
writer.WriteBytes(blobAssets[i].header + 1, blobAssetLength);
writer.WriteBytes(&zeroBytes, header.Length - blobAssetLength);
}
var tempChunk = (Chunk *)UnsafeUtility.Malloc(Chunk.kChunkSize, 16, Allocator.Temp);
for (int archetypeIndex = 0; archetypeIndex < archetypeArray.Length; ++archetypeIndex)
{
var archetype = archetypeArray[archetypeIndex].Archetype;
for (var ci = 0; ci < archetype->Chunks.Count; ++ci)
{
var chunk = archetype->Chunks.p[ci];
bufferPatches.Clear();
UnsafeUtility.MemCpy(tempChunk, chunk, Chunk.kChunkSize);
tempChunk->metaChunkEntity = EntityRemapUtility.RemapEntity(ref entityRemapInfos, tempChunk->metaChunkEntity);
// Prevent patching from touching buffers allocated memory
BufferHeader.PatchAfterCloningChunk(tempChunk);
byte *tempChunkBuffer = tempChunk->Buffer;
EntityRemapUtility.PatchEntities(archetype->ScalarEntityPatches, archetype->ScalarEntityPatchCount, archetype->BufferEntityPatches, archetype->BufferEntityPatchCount, tempChunkBuffer, tempChunk->Count, ref entityRemapInfos);
if (archetype->ContainsBlobAssetRefs)
{
PatchBlobAssetsInChunkBeforeSave(tempChunk, chunk, blobAssetOffsets, blobAssetRefs);
}
FillPatchRecordsForChunk(chunk, bufferPatches);
ClearChunkHeaderComponents(tempChunk);
ChunkDataUtility.MemsetUnusedChunkData(tempChunk, 0);
tempChunk->Archetype = (Archetype *)archetypeIndex;
if (archetype->NumManagedArrays != 0)
{
throw new ArgumentException("Serialization of GameObject components is not supported for pure entity scenes");
}
writer.WriteBytes(tempChunk, Chunk.kChunkSize);
writer.Write(bufferPatches.Length);
if (bufferPatches.Length > 0)
{
writer.WriteList(bufferPatches);
// Write heap backed data for each required patch.
// TODO: PERF: Investigate static-only deserialization could manage one block and mark in pointers somehow that they are not indiviual
for (int i = 0; i < bufferPatches.Length; ++i)
{
var patch = bufferPatches[i];
var header = (BufferHeader *)OffsetFromPointer(tempChunk->Buffer, patch.ChunkOffset);
writer.WriteBytes(header->Pointer, patch.AllocSizeBytes);
BufferHeader.Destroy(header);
}
}
}
}
blobAssetRefs.Dispose();
blobAssets.Dispose();
bufferPatches.Dispose();
UnsafeUtility.Free(tempChunk, Allocator.Temp);
sharedComponentsToSerialize = new int[sharedComponentMapping.Length - 1];
using (var keyArray = sharedComponentMapping.GetKeyArray(Allocator.Temp))
foreach (var key in keyArray)
{
if (key == 0)
{
continue;
}
if (sharedComponentMapping.TryGetValue(key, out var val))
{
sharedComponentsToSerialize[val - 1] = key;
}
}
archetypeToIndex.Dispose();
typeHashes.Dispose();
typeHashSet.Dispose();
typeHashToIndexMap.Dispose();
}
public static SceneSectionData[] WriteEntityScene(Scene scene, GameObjectConversionSettings settings, List <ReferencedUnityObjects> sectionRefObjs = null)
{
int framesToRetainBlobAssets = RetainBlobAssetsSetting.GetFramesToRetainBlobAssets(settings.BuildConfiguration);
var world = new World("ConversionWorld");
var entityManager = world.EntityManager;
settings.DestinationWorld = world;
bool disposeBlobAssetCache = false;
if (settings.BlobAssetStore == null)
{
settings.BlobAssetStore = new BlobAssetStore();
disposeBlobAssetCache = true;
}
List <(int, LogEventData)> journalData = null;
settings.ConversionWorldPreDispose += conversionWorld =>
{
var mappingSystem = conversionWorld.GetExistingSystem <GameObjectConversionMappingSystem>();
journalData = mappingSystem.JournalData.SelectLogEventsOrdered().ToList();
};
ConvertScene(scene, settings);
EntitySceneOptimization.Optimize(world);
if (settings.AssetImportContext != null)
{
using (var allTypes = new NativeHashMap <ComponentType, int>(100, Allocator.Temp))
using (var archetypes = new NativeList <EntityArchetype>(Allocator.Temp))
{
entityManager.GetAllArchetypes(archetypes);
foreach (var archetype in archetypes)
{
using (var componentTypes = archetype.GetComponentTypes())
foreach (var componentType in componentTypes)
{
if (allTypes.TryAdd(componentType, 0))
{
TypeDependencyCache.AddDependency(settings.AssetImportContext, componentType);
}
}
}
}
TypeDependencyCache.AddAllSystemsDependency(settings.AssetImportContext);
}
var sceneSections = new List <SceneSectionData>();
var subSectionList = new List <SceneSection>();
entityManager.GetAllUniqueSharedComponentData(subSectionList);
var extRefInfoEntities = new NativeArray <Entity>(subSectionList.Count, Allocator.Temp);
NativeArray <Entity> entitiesInMainSection;
var sectionQuery = entityManager.CreateEntityQuery(
new EntityQueryDesc
{
All = new[] { ComponentType.ReadWrite <SceneSection>() },
Options = EntityQueryOptions.IncludePrefab | EntityQueryOptions.IncludeDisabled
}
);
var sectionBoundsQuery = entityManager.CreateEntityQuery(
new EntityQueryDesc
{
All = new[] { ComponentType.ReadWrite <SceneBoundingVolume>(), ComponentType.ReadWrite <SceneSection>() },
Options = EntityQueryOptions.IncludePrefab | EntityQueryOptions.IncludeDisabled
}
);
var sceneGUID = settings.SceneGUID;
{
var section = new SceneSection {
SceneGUID = sceneGUID, Section = 0
};
sectionQuery.SetSharedComponentFilter(new SceneSection {
SceneGUID = sceneGUID, Section = 0
});
sectionBoundsQuery.SetSharedComponentFilter(new SceneSection {
SceneGUID = sceneGUID, Section = 0
});
entitiesInMainSection = sectionQuery.ToEntityArray(Allocator.TempJob);
var bounds = GetBoundsAndRemove(entityManager, sectionBoundsQuery);
// Each section will be serialized in its own world, entities that don't have a section are part of the main scene.
// An entity that holds the array of external references to the main scene is required for each section.
// We need to create them all before we start moving entities to section scenes,
// otherwise they would reuse entities that have been moved and mess up the remapping tables.
for (int sectionIndex = 1; sectionIndex < subSectionList.Count; ++sectionIndex)
{
if (subSectionList[sectionIndex].Section == 0)
{
// Main section, the only one that doesn't need an external ref array
continue;
}
var extRefInfoEntity = entityManager.CreateEntity();
entityManager.AddSharedComponentData(extRefInfoEntity, subSectionList[sectionIndex]);
extRefInfoEntities[sectionIndex] = extRefInfoEntity;
}
// Public references array, only on the main section.
var refInfoEntity = entityManager.CreateEntity();
entityManager.AddBuffer <PublicEntityRef>(refInfoEntity);
entityManager.AddSharedComponentData(refInfoEntity, section);
var publicRefs = entityManager.GetBuffer <PublicEntityRef>(refInfoEntity);
// entityManager.Debug.CheckInternalConsistency();
//@TODO do we need to keep this index? doesn't carry any additional info
for (int i = 0; i < entitiesInMainSection.Length; ++i)
{
PublicEntityRef.Add(ref publicRefs,
new PublicEntityRef {
entityIndex = i, targetEntity = entitiesInMainSection[i]
});
}
UnityEngine.Debug.Assert(publicRefs.Length == entitiesInMainSection.Length);
// Save main section
var sectionWorld = new World("SectionWorld");
var sectionManager = sectionWorld.EntityManager;
var entityRemapping = entityManager.CreateEntityRemapArray(Allocator.TempJob);
sectionManager.MoveEntitiesFrom(entityManager, sectionQuery, entityRemapping);
AddRetainBlobAssetsEntity(sectionManager, framesToRetainBlobAssets);
// The section component is only there to break the conversion world into different sections
// We don't want to store that on the disk
//@TODO: Component should be removed but currently leads to corrupt data file. Figure out why.
//sectionManager.RemoveComponent(sectionManager.UniversalQuery, typeof(SceneSection));
var sectionFileSize = WriteEntityScene(sectionManager, sceneGUID, "0", settings, out var objectRefCount, out var objRefs);
sectionRefObjs?.Add(objRefs);
sceneSections.Add(new SceneSectionData
{
FileSize = sectionFileSize,
SceneGUID = sceneGUID,
ObjectReferenceCount = objectRefCount,
SubSectionIndex = 0,
BoundingVolume = bounds
});
entityRemapping.Dispose();
sectionWorld.Dispose();
}
{
// Index 0 is the default value of the shared component, not an actual section
for (int subSectionIndex = 0; subSectionIndex < subSectionList.Count; ++subSectionIndex)
{
var subSection = subSectionList[subSectionIndex];
if (subSection.Section == 0)
{
continue;
}
sectionQuery.SetSharedComponentFilter(subSection);
sectionBoundsQuery.SetSharedComponentFilter(subSection);
var bounds = GetBoundsAndRemove(entityManager, sectionBoundsQuery);
var entitiesInSection = sectionQuery.ToEntityArray(Allocator.TempJob);
if (entitiesInSection.Length > 0)
{
// Fetch back the external reference entity we created earlier to not disturb the mapping
var refInfoEntity = extRefInfoEntities[subSectionIndex];
entityManager.AddBuffer <ExternalEntityRef>(refInfoEntity);
var externRefs = entityManager.GetBuffer <ExternalEntityRef>(refInfoEntity);
// Store the mapping to everything in the main section
//@TODO maybe we don't need all that? is this worth worrying about?
for (int i = 0; i < entitiesInMainSection.Length; ++i)
{
ExternalEntityRef.Add(ref externRefs, new ExternalEntityRef {
entityIndex = i
});
}
var entityRemapping = entityManager.CreateEntityRemapArray(Allocator.TempJob);
// Entities will be remapped to a contiguous range in the section world, but they will
// also come with an unpredictable amount of meta entities. We have the guarantee that
// the entities in the main section won't be moved over, so there's a free range of that
// size at the end of the remapping table. So we use that range for external references.
var externEntityIndexStart = entityRemapping.Length - entitiesInMainSection.Length;
entityManager.AddComponentData(refInfoEntity,
new ExternalEntityRefInfo
{
SceneGUID = sceneGUID,
EntityIndexStart = externEntityIndexStart
});
var sectionWorld = new World("SectionWorld");
var sectionManager = sectionWorld.EntityManager;
// Insert mapping for external references, conversion world entity to virtual index in section
for (int i = 0; i < entitiesInMainSection.Length; ++i)
{
EntityRemapUtility.AddEntityRemapping(ref entityRemapping, entitiesInMainSection[i],
new Entity {
Index = i + externEntityIndexStart, Version = 1
});
}
sectionManager.MoveEntitiesFrom(entityManager, sectionQuery, entityRemapping);
AddRetainBlobAssetsEntity(sectionManager, framesToRetainBlobAssets);
// Now that all the required entities have been moved over, we can get rid of the gap between
// real entities and external references. This allows remapping during load to deal with a
// smaller remap table, containing only useful entries.
int highestEntityIndexInUse = 0;
for (int i = 0; i < externEntityIndexStart; ++i)
{
var targetIndex = entityRemapping[i].Target.Index;
if (targetIndex < externEntityIndexStart && targetIndex > highestEntityIndexInUse)
{
highestEntityIndexInUse = targetIndex;
}
}
var oldExternEntityIndexStart = externEntityIndexStart;
externEntityIndexStart = highestEntityIndexInUse + 1;
sectionManager.SetComponentData
(
EntityRemapUtility.RemapEntity(ref entityRemapping, refInfoEntity),
new ExternalEntityRefInfo
{
SceneGUID = sceneGUID,
EntityIndexStart = externEntityIndexStart
}
);
// When writing the scene, references to missing entities are set to Entity.Null by default
// (but only if they have been used, otherwise they remain untouched)
// We obviously don't want that to happen to our external references, so we add explicit mapping
// And at the same time, we put them back at the end of the effective range of real entities.
for (int i = 0; i < entitiesInMainSection.Length; ++i)
{
var src = new Entity {
Index = i + oldExternEntityIndexStart, Version = 1
};
var dst = new Entity {
Index = i + externEntityIndexStart, Version = 1
};
EntityRemapUtility.AddEntityRemapping(ref entityRemapping, src, dst);
}
// The section component is only there to break the conversion world into different sections
// We don't want to store that on the disk
//@TODO: Component should be removed but currently leads to corrupt data file. Figure out why.
//sectionManager.RemoveComponent(sectionManager.UniversalQuery, typeof(SceneSection));
var fileSize = WriteEntityScene(sectionManager, sceneGUID, subSection.Section.ToString(), settings, out var objectRefCount, out var objRefs, entityRemapping);
sectionRefObjs?.Add(objRefs);
sceneSections.Add(new SceneSectionData
{
FileSize = fileSize,
SceneGUID = sceneGUID,
ObjectReferenceCount = objectRefCount,
SubSectionIndex = subSection.Section,
BoundingVolume = bounds
});
entityRemapping.Dispose();
sectionWorld.Dispose();
}
entitiesInSection.Dispose();
}
}
{
var noSectionQuery = entityManager.CreateEntityQuery(
new EntityQueryDesc
{
None = new[] { ComponentType.ReadWrite <SceneSection>() },
Options = EntityQueryOptions.IncludePrefab | EntityQueryOptions.IncludeDisabled
}
);
if (noSectionQuery.CalculateEntityCount() != 0)
{
Debug.LogWarning($"{noSectionQuery.CalculateEntityCount()} entities in the scene '{scene.path}' had no SceneSection and as a result were not serialized at all.");
}
}
sectionQuery.Dispose();
sectionBoundsQuery.Dispose();
entitiesInMainSection.Dispose();
world.Dispose();
// Save the new header
var sceneSectionsArray = sceneSections.ToArray();
WriteHeader(sceneGUID, sceneSectionsArray, scene.name, settings.AssetImportContext);
// If we are writing assets to assets folder directly, then we need to make sure the asset database see them so they can be loaded.
if (settings.AssetImportContext == null)
{
AssetDatabase.Refresh();
}
if (disposeBlobAssetCache)
{
settings.BlobAssetStore.Dispose();
}
// Save the log of issues that happened during conversion
WriteConversionLog(sceneGUID, journalData, scene.name, settings.AssetImportContext);
return(sceneSectionsArray);
}
public static unsafe void SerializeWorld(EntityManager entityManager, BinaryWriter writer, out int[] sharedComponentsToSerialize, NativeArray <EntityRemapUtility.EntityRemapInfo> entityRemapInfos)
{
writer.Write(CurrentFileFormatVersion);
var archetypeManager = entityManager.ArchetypeManager;
Dictionary <EntityArchetype, int> archetypeToIndex;
EntityArchetype[] archetypeArray;
GetAllArchetypes(archetypeManager, out archetypeToIndex, out archetypeArray);
var typeindices = new HashSet <int>();
foreach (var archetype in archetypeArray)
{
for (int iType = 0; iType < archetype.Archetype->TypesCount; ++iType)
{
typeindices.Add(archetype.Archetype->Types[iType].TypeIndex & ~TypeManager.ChunkComponentTypeFlag);
}
}
var typeArray = typeindices.Select(index =>
{
var type = TypeManager.GetType(index);
var name = TypeManager.GetType(index).AssemblyQualifiedName;
var hash = TypeManager.GetTypeInfo(index).StableTypeHash;
return(new
{
index,
type,
name,
hash,
utf8Name = Encoding.UTF8.GetBytes(name)
});
}).OrderBy(t => t.name).ToArray();
int typeNameBufferSize = typeArray.Sum(t => t.utf8Name.Length + 1);
writer.Write(typeArray.Length);
foreach (var n in typeArray)
{
writer.Write(n.hash);
}
writer.Write(typeNameBufferSize);
foreach (var n in typeArray)
{
writer.Write(n.utf8Name);
writer.Write((byte)0);
}
var typeIndexMap = new Dictionary <int, int>();
for (int i = 0; i < typeArray.Length; ++i)
{
typeIndexMap[typeArray[i].index] = i;
}
WriteArchetypes(writer, archetypeArray, typeIndexMap);
//TODO: ensure chunks are defragged?
var bufferPatches = new NativeList <BufferPatchRecord>(128, Allocator.Temp);
var totalChunkCount = GenerateRemapInfo(entityManager, archetypeArray, entityRemapInfos);
writer.Write(totalChunkCount);
var tempChunk = (Chunk *)UnsafeUtility.Malloc(Chunk.kChunkSize, 16, Allocator.Temp);
var sharedIndexToSerialize = new Dictionary <int, int>();
for (int archetypeIndex = 0; archetypeIndex < archetypeArray.Length; ++archetypeIndex)
{
var archetype = archetypeArray[archetypeIndex].Archetype;
for (var ci = 0; ci < archetype->Chunks.Count; ++ci)
{
var chunk = archetype->Chunks.p[ci];
bufferPatches.Clear();
UnsafeUtility.MemCpy(tempChunk, chunk, Chunk.kChunkSize);
tempChunk->SharedComponentValueArray = (int *)((byte *)(tempChunk) + Chunk.GetSharedComponentOffset(archetype->NumSharedComponents));
tempChunk->metaChunkEntity = EntityRemapUtility.RemapEntity(ref entityRemapInfos, tempChunk->metaChunkEntity);
byte *tempChunkBuffer = tempChunk->Buffer;
EntityRemapUtility.PatchEntities(archetype->ScalarEntityPatches, archetype->ScalarEntityPatchCount, archetype->BufferEntityPatches, archetype->BufferEntityPatchCount, tempChunkBuffer, tempChunk->Count, ref entityRemapInfos);
// Find all buffer pointer locations and work out how much memory the deserializer must allocate on load.
for (int ti = 0; ti < archetype->TypesCount; ++ti)
{
int index = archetype->TypeMemoryOrder[ti];
if (!archetype->Types[index].IsBuffer)
{
continue;
}
int subArrayOffset = archetype->Offsets[index];
BufferHeader *header = (BufferHeader *)OffsetFromPointer(tempChunkBuffer, subArrayOffset);
int stride = archetype->SizeOfs[index];
int count = chunk->Count;
var ct = TypeManager.GetTypeInfo(archetype->Types[index].TypeIndex);
for (int bi = 0; bi < count; ++bi)
{
if (header->Pointer != null)
{
header->Pointer = null;
bufferPatches.Add(new BufferPatchRecord
{
ChunkOffset = (int)(((byte *)header) - (byte *)tempChunkBuffer),
AllocSizeBytes = ct.ElementSize * header->Capacity,
});
}
header = (BufferHeader *)OffsetFromPointer(header, stride);
}
}
ClearChunkHeaderComponents(tempChunk);
ClearUnusedChunkData(tempChunk);
tempChunk->Archetype = (Archetype *)archetypeIndex;
if (archetype->NumManagedArrays != 0)
{
throw new ArgumentException("Serialization of GameObject components is not supported for pure entity scenes");
}
for (int i = 0; i != archetype->NumSharedComponents; i++)
{
int sharedComponentIndex = tempChunk->SharedComponentValueArray[i];
int newIndex;
if (tempChunk->SharedComponentValueArray[i] != 0)
{
if (sharedIndexToSerialize.TryGetValue(sharedComponentIndex, out newIndex))
{
tempChunk->SharedComponentValueArray[i] = newIndex;
}
else
{
// 0 is reserved for null types in shared components
newIndex = sharedIndexToSerialize.Count + 1;
sharedIndexToSerialize[sharedComponentIndex] = newIndex;
tempChunk->SharedComponentValueArray[i] = newIndex;
}
}
}
writer.WriteBytes(tempChunk, Chunk.kChunkSize);
writer.Write(bufferPatches.Length);
if (bufferPatches.Length > 0)
{
writer.WriteList(bufferPatches);
// Write heap backed data for each required patch.
// TODO: PERF: Investigate static-only deserialization could manage one block and mark in pointers somehow that they are not indiviual
for (int i = 0; i < bufferPatches.Length; ++i)
{
var patch = bufferPatches[i];
// NOTE that this reads the pointer from the original, unpatched chunk.
// We have nulled out the pointer in the serialized data above.
var header = (BufferHeader *)OffsetFromPointer(chunk->Buffer, patch.ChunkOffset);
writer.WriteBytes(header->Pointer, patch.AllocSizeBytes);
}
}
}
}
bufferPatches.Dispose();
UnsafeUtility.Free(tempChunk, Allocator.Temp);
sharedComponentsToSerialize = new int[sharedIndexToSerialize.Count];
foreach (var i in sharedIndexToSerialize)
{
sharedComponentsToSerialize[i.Value - 1] = i.Key;
}
}
