public async Task AddOrUpdateAsync(SpatialSurfaceInfo spatialSurfaceInfo, SpatialSurfaceMeshOptions options)
{
var mesh = await spatialSurfaceInfo.TryComputeLatestMeshAsync(1000.0d, options);
if (mesh == null)
{
return;
}
var tempList = new Dictionary <Guid, SurfaceMesh>(_internalList);
// See if we already have this one....
var hasOne = tempList.ContainsKey(mesh.SurfaceInfo.Id);
if (hasOne)
{
// Update
var surfaceMesh = tempList[mesh.SurfaceInfo.Id];
if (surfaceMesh.UpdateTime < mesh.SurfaceInfo.UpdateTime)
{
surfaceMesh.CalculateAllVertices(mesh);
surfaceMesh.UpdateTime = mesh.SurfaceInfo.UpdateTime;
}
}
else
{
// Add new
var newMesh = new SurfaceMesh
{
Id = mesh.SurfaceInfo.Id,
UpdateTime = mesh.SurfaceInfo.UpdateTime,
CoordinateSystem = CoordinateSystem,
DirectXDevice = DirectXDevice
};
newMesh.CalculateAllVertices(mesh);
tempList[newMesh.Id] = newMesh;
}
lock (_lockObject)
_internalList = tempList;
}
public static async void surfaceModified(Guid id, SpatialSurfaceInfo info)
{
SpatialSurfaceMesh mesh = await info.TryComputeLatestMeshAsync(2);
if (mesh != null)
{
Vector3 scale = mesh.VertexPositionScale;
Matrix4x4 coordinateSystem = (Matrix4x4)mesh.CoordinateSystem.TryGetTransformTo(currentCoordinateSystem);
byte[] vertexPositions = mesh.VertexPositions.Data.ToArray();
byte[] triangleIndices = mesh.TriangleIndices.Data.ToArray();
var triangleIndicesList = new List <int>();
var vertexPositionsList = new List <Vector3>();
for (int i = 0; i < triangleIndices.Length; i += (int)mesh.TriangleIndices.Stride)
{
int j = (int)System.BitConverter.ToUInt16(triangleIndices, i);
triangleIndicesList.Add(j);
}
for (int i = 0; i < vertexPositions.Length;)
{
float x = (float)System.BitConverter.ToInt16(vertexPositions, i);
i += 2;
float y = (float)System.BitConverter.ToInt16(vertexPositions, i);
i += 2;
float z = (float)System.BitConverter.ToInt16(vertexPositions, i);
i += 2;
float w = (float)System.BitConverter.ToInt16(vertexPositions, i);
i += 2;
Vector3 position = Vector3.Transform(new Vector3(x / w, y / w, z / w) * scale, coordinateSystem);
vertexPositionsList.Add(position / Spritesheet.positionScaleFactor);
}
List <Object> message = new List <object>()
{
"surfaceModified", id, vertexPositionsList, triangleIndicesList
};
sendMessage(JsonConvert.SerializeObject(message));
}
}
private async Task GenerateMeshAsync(SpatialSurfaceInfo meshInfo, SpatialMeshObject spatialMeshObject)
{
spatialMeshObject.LastUpdated = meshInfo.UpdateTime;
// TODO Check if the spatialSurfaceMeshOptions are correct for what we need.
var spatialSurfaceMesh = await meshInfo.TryComputeLatestMeshAsync(TrianglesPerCubicMeter, spatialSurfaceMeshOptions);
await Awaiters.UnityMainThread;
var mesh = spatialMeshObject.Mesh == null ? new Mesh() : spatialMeshObject.Mesh;
mesh.name = $"Mesh_{meshInfo.Id}";
await Awaiters.BackgroundThread;
if (MeshRecalculateNormals)
{
var normalCount = (int)spatialSurfaceMesh.VertexNormals.ElementCount;
var normals = new NativeArray <VertexData>(normalCount, Allocator.None);
var vertexBuffer = DataReader.FromBuffer(spatialSurfaceMesh.VertexPositions.Data);
var normalBuffer = DataReader.FromBuffer(spatialSurfaceMesh.VertexNormals.Data);
for (int i = 0; i < normalCount; i++)
{
normals[i] = new VertexData
{
// TODO Check if spatialSurfaceMesh.VertexPositionScale needs to be accounted for.
Position = new Vector3(vertexBuffer.ReadSingle(), vertexBuffer.ReadSingle(), -vertexBuffer.ReadSingle()),
Normal = new Vector3(normalBuffer.ReadSingle(), normalBuffer.ReadSingle(), -normalBuffer.ReadSingle())
};
}
mesh.SetVertexBufferParams(normalCount, NormalsLayout);
mesh.SetVertexBufferData(normals, 0, 0, normalCount);
vertexBuffer.Dispose();
normalBuffer.Dispose();
normals.Dispose();
}
else
{
var vertexCount = (int)spatialSurfaceMesh.VertexPositions.ElementCount;
var vertices = new NativeArray <Vector3>(vertexCount, Allocator.None);
var vertexBuffer = DataReader.FromBuffer(spatialSurfaceMesh.VertexPositions.Data);
for (int i = 0; i < vertexCount; i++)
{
// TODO Check if spatialSurfaceMesh.VertexPositionScale needs to be accounted for.
vertices[i] = new Vector3(vertexBuffer.ReadSingle(), vertexBuffer.ReadSingle(), -vertexBuffer.ReadSingle());
}
mesh.SetVertexBufferParams(vertexCount, VertexLayout);
mesh.SetVertexBufferData(vertices, 0, 0, vertexCount);
vertexBuffer.Dispose();
vertices.Dispose();
}
var indicesCount = (int)spatialSurfaceMesh.TriangleIndices.ElementCount;
var indices = new NativeArray <short>(indicesCount, Allocator.None);
var indicesBuffer = DataReader.FromBuffer(spatialSurfaceMesh.TriangleIndices.Data);
for (int i = 0; i < indicesCount; i++)
{
indices[i] = indicesBuffer.ReadInt16();
}
mesh.SetIndexBufferParams(indicesCount, IndexFormat.UInt16);
mesh.SetIndexBufferData(indices, 0, 0, indicesCount);
indicesBuffer.Dispose();
indices.Dispose();
mesh.SetSubMesh(0, new SubMeshDescriptor(0, indicesCount));
mesh.Optimize();
mesh.RecalculateBounds();
if (MeshRecalculateNormals)
{
mesh.RecalculateNormals();
}
spatialMeshObject.Mesh = mesh;
await Awaiters.UnityMainThread;
}
private async void MeshInfo_Update(SpatialSurfaceInfo meshInfo, SpatialObserverStatus statusType)
{
if (statusType == SpatialObserverStatus.Removed)
{
if (SpatialMeshObjects.TryGetValue(meshInfo.Id, out var removedMeshObject))
{
RaiseMeshRemoved(removedMeshObject);
}
return;
}
var spatialMeshObject = await RequestSpatialMeshObject(meshInfo.Id);
if (meshInfo.UpdateTime.ToUniversalTime() <= spatialMeshObject.LastUpdated.ToUniversalTime())
{
// No updates
return;
}
try
{
await GenerateMeshAsync(meshInfo, spatialMeshObject);
}
catch (Exception e)
{
Debug.LogError(e);
RaiseMeshRemoved(spatialMeshObject);
return;
}
if (!SpatialMeshObjects.TryGetValue(meshInfo.Id, out var meshObject))
{
Debug.LogWarning($"Failed to find a spatial mesh object for {meshInfo.Id}!");
// Likely it was removed before data could be cooked.
return;
}
// Apply the appropriate material to the mesh.
var displayOption = MeshDisplayOption;
if (displayOption != SpatialMeshDisplayOptions.None)
{
meshObject.Collider.enabled = true;
meshObject.Renderer.enabled = displayOption == SpatialMeshDisplayOptions.Visible ||
displayOption == SpatialMeshDisplayOptions.Occlusion;
meshObject.Renderer.sharedMaterial = displayOption == SpatialMeshDisplayOptions.Visible
? MeshVisibleMaterial
: MeshOcclusionMaterial;
}
else
{
meshObject.Renderer.enabled = false;
meshObject.Collider.enabled = false;
}
if (!meshObject.GameObject.activeInHierarchy)
{
meshObject.GameObject.SetActive(true);
}
switch (statusType)
{
case SpatialObserverStatus.Added:
RaiseMeshAdded(meshObject);
break;
case SpatialObserverStatus.Updated:
RaiseMeshUpdated(meshObject);
break;
default:
throw new ArgumentOutOfRangeException($"{nameof(SpatialObserverStatus)}.{statusType} was not handled!");
}
}
async Task ProcessSurface(SpatialSurfaceInfo surface)
{
var mesh = await surface.TryComputeLatestMeshAsync(trianglesPerCubicMeter, options).AsTask().ConfigureAwait(false);
if (observer == null || mesh == null)
{
RemoveSurfaceFromCache(surface.Id);
return;
}
var bounds = mesh.SurfaceInfo.TryGetBounds(currentCoordinateSystem);
if (bounds == null)
{
RemoveSurfaceFromCache(surface.Id);
return;
}
var transform = mesh.CoordinateSystem.TryGetTransformTo(currentCoordinateSystem);
if (transform == null)
{
RemoveSurfaceFromCache(surface.Id);
return;
}
//1. TriangleIndices
var trianglesBytes = mesh.TriangleIndices.Data.ToArray();
var indeces = new short[mesh.TriangleIndices.ElementCount];
int indexOffset = 0;
for (int i = 0; i < mesh.TriangleIndices.ElementCount; i++)
{
//DirectXPixelFormat.R16UInt
var index = BitConverter.ToInt16(trianglesBytes, indexOffset);
indexOffset += 2;
indeces[i] = index;
}
var vertexCount = mesh.VertexPositions.ElementCount;
var vertexRawData = mesh.VertexPositions.Data.ToArray();
var vertexScale = mesh.VertexPositionScale;
var normalsRawData = mesh.VertexNormals.Data.ToArray();
var vertexColor = DefaultColor.ToUInt();
var vertexData = new SpatialVertex[vertexCount];
var boundsRotation = new Quaternion(-bounds.Value.Orientation.X, -bounds.Value.Orientation.Y, bounds.Value.Orientation.Z, bounds.Value.Orientation.W);
var boundsCenter = new Vector3(bounds.Value.Center.X, bounds.Value.Center.Y, -bounds.Value.Center.Z);
var boundsExtents = new Vector3(bounds.Value.Extents.X, bounds.Value.Extents.Y, bounds.Value.Extents.Z);
var transformValue = transform.Value;
Matrix4 transformUrhoMatrix;
unsafe { transformUrhoMatrix = *(Matrix4 *)(void *)&transformValue; }
//these values won't change, let's declare them as consts
const int vertexStride = 16; // (int) mesh.VertexPositions.Stride;
const int normalStride = 16; // (int) mesh.VertexNormals.Stride;
//2,3 - VertexPositions and Normals
for (int i = 0; i < vertexCount; i++)
{
var positionX = BitConverter.ToSingle(vertexRawData, i * vertexStride + 0);
var positionY = BitConverter.ToSingle(vertexRawData, i * vertexStride + 4); //4 per X,Y,Z,W (stride is 16)
var positionZ = BitConverter.ToSingle(vertexRawData, i * vertexStride + 8); //also, we don't need the W component.
var normalX = BitConverter.ToSingle(normalsRawData, i * normalStride + 0);
var normalY = BitConverter.ToSingle(normalsRawData, i * normalStride + 4);
var normalZ = BitConverter.ToSingle(normalsRawData, i * normalStride + 8);
//merge vertex+normals for Urho3D (also, change RH to LH coordinate systems)
vertexData[i].PositionX = positionX * vertexScale.X;
vertexData[i].PositionY = positionY * vertexScale.Y;
vertexData[i].PositionZ = -positionZ * vertexScale.Z;
vertexData[i].NormalX = normalX;
vertexData[i].NormalY = normalY;
vertexData[i].NormalZ = -normalZ;
//Vertex color (for VCol techniques)
vertexData[i].ColorUint = vertexColor;
}
var surfaceInfo = new SharpReality.SpatialMeshInfo
{
SurfaceId = surface.Id.ToString(),
Date = surface.UpdateTime,
VertexData = vertexData,
IndexData = indeces,
BoundsCenter = boundsCenter,
BoundsRotation = boundsRotation,
Extents = boundsExtents,
Transform = transformUrhoMatrix,
};
currentHoloApp.HandleSurfaceUpdated(surfaceInfo);
}
async Task ProcessSurface(SpatialSurfaceInfo surface, bool convertToLH)
{
var mesh = await surface.TryComputeLatestMeshAsync(trianglesPerCubicMeter, options).AsTask().ConfigureAwait(false);
if (observer == null || mesh == null)
{
RemoveSurfaceFromCache(surface.Id);
return;
}
var bounds = mesh.SurfaceInfo.TryGetBounds(currentCoordinateSystem);
if (bounds == null)
{
RemoveSurfaceFromCache(surface.Id);
return;
}
var transform = mesh.CoordinateSystem.TryGetTransformTo(currentCoordinateSystem);
if (transform == null)
{
RemoveSurfaceFromCache(surface.Id);
return;
}
// convert from RH to LH coordinate system
int rhToLh = convertToLH ? -1 : 1;
//1. TriangleIndices
var trianglesBytes = mesh.TriangleIndices.Data.ToArray();
var indeces = new short[mesh.TriangleIndices.ElementCount];
int indexOffset = 0;
for (int i = 0; i < mesh.TriangleIndices.ElementCount; i++)
{
//DirectXPixelFormat.R16UInt
var index = BitConverter.ToInt16(trianglesBytes, indexOffset);
indexOffset += 2;
indeces[i] = index;
}
var vertexCount = mesh.VertexPositions.ElementCount;
var vertexRawData = mesh.VertexPositions.Data.ToArray();
var vertexScale = mesh.VertexPositionScale;
var normalsRawData = mesh.VertexNormals.Data.ToArray();
var vertexColor = DefaultColor.ToUInt();
var vertexData = new SpatialVertex[vertexCount];
//2,3 - VertexPositions and Normals
for (int i = 0; i < vertexCount; i++)
{
//VertexPositions: DirectXPixelFormat.R16G16B16A16IntNormalized
short x = BitConverter.ToInt16(vertexRawData, i * 8 + 0);
short y = BitConverter.ToInt16(vertexRawData, i * 8 + 2);
short z = BitConverter.ToInt16(vertexRawData, i * 8 + 4);
//short to float:
float xx = (x == -32768) ? -1.0f : x * 1.0f / (32767.0f);
float yy = (y == -32768) ? -1.0f : y * 1.0f / (32767.0f);
float zz = (z == -32768) ? -1.0f : z * 1.0f / (32767.0f);
//Normals: DirectXPixelFormat.R8G8B8A8IntNormalized
var normalX = normalsRawData[i * 4 + 0];
var normalY = normalsRawData[i * 4 + 1];
var normalZ = normalsRawData[i * 4 + 2];
//merge vertex+normals for Urho3D (also, change RH to LH coordinate systems)
vertexData[i].PositionX = xx * vertexScale.X;
vertexData[i].PositionY = yy * vertexScale.Y;
vertexData[i].PositionZ = rhToLh * zz * vertexScale.Z;
vertexData[i].NormalX = normalX == 0 ? 0 : 255 / normalX;
vertexData[i].NormalY = normalY == 0 ? 0 : 255 / normalY;
vertexData[i].NormalZ = normalZ == 0 ? 0 : rhToLh * 255 / normalZ;
//Vertex color
vertexData[i].ColorUint = vertexColor;
}
var boundsRotation = new Quaternion(rhToLh * bounds.Value.Orientation.X, rhToLh * bounds.Value.Orientation.Y, bounds.Value.Orientation.Z, bounds.Value.Orientation.W);
var boundsCenter = new Vector3(bounds.Value.Center.X, bounds.Value.Center.Y, rhToLh * bounds.Value.Center.Z);
var boundsExtents = new Vector3(bounds.Value.Extents.X, bounds.Value.Extents.Y, bounds.Value.Extents.Z);
var transformValue = transform.Value;
Matrix4 transformUrhoMatrix;
unsafe { transformUrhoMatrix = *(Matrix4 *)(void *)&transformValue; }
var surfaceInfo = new HoloLens.SpatialMeshInfo
{
SurfaceId = surface.Id.ToString(),
Date = surface.UpdateTime,
VertexData = vertexData,
IndexData = indeces,
BoundsCenter = boundsCenter,
BoundsRotation = boundsRotation,
Extents = boundsExtents,
Transform = transformUrhoMatrix,
};
currentHoloApp.HandleSurfaceUpdated(surfaceInfo);
}
/// <summary>
/// Update the surface
/// </summary>
/// <param name="handler">The handler to receive the surface information</param>
/// <param name="change">The surface change</param>
/// <param name="surface">The surface to update</param>
/// <param name="trianglesPerCubicMeter">The max triangles per cubic meter</param>
/// <param name="surfaceInfo">The surface info</param>
/// <param name="surfaceOptions">The mesh options</param>
private async void UpdateSurface(OnSurfaceChangedHandler handler, SurfaceChange change, SpatialMappingSurfaceInternal surface, float trianglesPerCubicMeter, SpatialSurfaceInfo surfaceInfo, SpatialSurfaceMeshOptions surfaceOptions)
{
if (surface.IsProcessing)
{
return;
}
try
{
surface.IsProcessing = true;
// Generate the mesh from the MixedReality surface info
SpatialSurfaceMesh surfaceMesh = await surfaceInfo.TryComputeLatestMeshAsync(trianglesPerCubicMeter, surfaceOptions);
if (surfaceMesh != null && surface.UpdateTime < surfaceMesh.SurfaceInfo.UpdateTime)
{
// Update the surface mesh
surface.UpdateSurfaceMesh(surfaceMesh);
if (handler != null)
{
handler(surface.Id, surface, change, surface.UpdateTime);
}
}
}
finally
{
surface.IsProcessing = false;
}
}
async Task ProcessSurface(SpatialSurfaceInfo surface)
{
var mesh = await surface.TryComputeLatestMeshAsync(trianglesPerCubicMeter, options);
if (observer == null)
{
return;
}
var bounds = mesh.SurfaceInfo.TryGetBounds(currentCoordinateSystem);
if (bounds == null)
{
return;
}
//1. TriangleIndices
var trianglesBytes = mesh.TriangleIndices.Data.ToArray();
var indeces = new short[mesh.TriangleIndices.ElementCount];
int indexOffset = 0;
for (int i = 0; i < mesh.TriangleIndices.ElementCount; i++)
{
//DirectXPixelFormat.R16UInt
var index = BitConverter.ToInt16(trianglesBytes, indexOffset);
indexOffset += 2;
indeces[i] = index;
}
var vertexCount = mesh.VertexPositions.ElementCount;
var vertexRawData = mesh.VertexPositions.Data.ToArray();
var vertexScale = mesh.VertexPositionScale;
var normalsRawData = mesh.VertexNormals.Data.ToArray();
var vertexColor = DefaultColor.ToUInt();
SpatialVertex[] vertexData = new SpatialVertex[vertexCount];
//2,3 - VertexPositions and Normals
for (int i = 0; i < vertexCount; i++)
{
//VertexPositions: DirectXPixelFormat.R16G16B16A16IntNormalized
short x = BitConverter.ToInt16(vertexRawData, i * 8 + 0);
short y = BitConverter.ToInt16(vertexRawData, i * 8 + 2);
short z = BitConverter.ToInt16(vertexRawData, i * 8 + 4);
//short to float:
float xx = (x == -32768) ? -1.0f : x * 1.0f / (32767.0f);
float yy = (y == -32768) ? -1.0f : y * 1.0f / (32767.0f);
float zz = (z == -32768) ? -1.0f : z * 1.0f / (32767.0f);
//Normals: DirectXPixelFormat.R8G8B8A8IntNormalized
var normalX = normalsRawData[i * 4 + 0];
var normalY = normalsRawData[i * 4 + 1];
var normalZ = normalsRawData[i * 4 + 2];
//merge vertex+normals for Urho3D (also, change RH to LH coordinate systems)
vertexData[i].PositionX = xx * vertexScale.X;
vertexData[i].PositionY = yy * vertexScale.Y;
vertexData[i].PositionZ = -zz * vertexScale.Z;
vertexData[i].NormalX = normalX == 0 ? 0 : 255 / normalX;
vertexData[i].NormalY = normalY == 0 ? 0 : 255 / normalY;
vertexData[i].NormalZ = normalZ == 0 ? 0 : -255 / normalZ;
//Vertex color
vertexData[i].Color = vertexColor;
}
var boundsRotation = new Quaternion(-bounds.Value.Orientation.X, -bounds.Value.Orientation.Y, bounds.Value.Orientation.Z, bounds.Value.Orientation.W);
var boundsCenter = new Vector3(bounds.Value.Center.X, bounds.Value.Center.Y, -bounds.Value.Center.Z);
Application.InvokeOnMain(() => currentHoloApp.HandleSurfaceUpdated(surface.Id.ToString(), surface.UpdateTime, vertexData, indeces, boundsCenter, boundsRotation));
}
