private bool CollectTextureDataFromFrame(MVGraphAPI.Frame frame)
{
TextureFormat textureFormat;
ushort textureWidth, textureHeight;
UInt32 textureSizeInBytes;
if (m_textureFormat == MvxTextureFormat.DEPTH_MAP && frame.StreamContainsDataLayer(MVGraphAPI.SimpleDataLayersGuids.DEPTHMAP_TEXTURE_DATA_LAYER))
{
textureFormat = TextureFormat.R8;
MVGraphAPI.FrameTextureExtractor.GetTextureResolution(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_DEPTH, out textureWidth, out textureHeight);
textureSizeInBytes = MVGraphAPI.FrameTextureExtractor.GetTextureDataSizeInBytes(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_DEPTH) / 2;
}
else if (m_textureFormat == MvxTextureFormat.ASTC && frame.StreamContainsDataLayer(MVGraphAPI.SimpleDataLayersGuids.ASTC_TEXTURE_DATA_LAYER))
{
textureFormat = TextureFormat.ASTC_RGB_8x8;
MVGraphAPI.FrameTextureExtractor.GetTextureResolution(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_ASTC, out textureWidth, out textureHeight);
textureSizeInBytes = MVGraphAPI.FrameTextureExtractor.GetTextureDataSizeInBytes(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_ASTC);
}
else if (m_textureFormat == MvxTextureFormat.DXT1 && frame.StreamContainsDataLayer(MVGraphAPI.SimpleDataLayersGuids.DXT1_TEXTURE_DATA_LAYER))
{
textureFormat = TextureFormat.DXT1;
MVGraphAPI.FrameTextureExtractor.GetTextureResolution(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_DXT1, out textureWidth, out textureHeight);
textureSizeInBytes = MVGraphAPI.FrameTextureExtractor.GetTextureDataSizeInBytes(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_DXT1);
}
else if (m_textureFormat == MvxTextureFormat.ETC2 && frame.StreamContainsDataLayer(MVGraphAPI.SimpleDataLayersGuids.ETC2_TEXTURE_DATA_LAYER))
{
textureFormat = TextureFormat.ETC2_RGB;
MVGraphAPI.FrameTextureExtractor.GetTextureResolution(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_ETC2, out textureWidth, out textureHeight);
textureSizeInBytes = MVGraphAPI.FrameTextureExtractor.GetTextureDataSizeInBytes(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_ETC2);
}
else if (m_textureFormat == MvxTextureFormat.RGB && frame.StreamContainsDataLayer(MVGraphAPI.SimpleDataLayersGuids.RGB_TEXTURE_DATA_LAYER))
{
textureFormat = TextureFormat.RGB24;
MVGraphAPI.FrameTextureExtractor.GetTextureResolution(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_RGB, out textureWidth, out textureHeight);
textureSizeInBytes = MVGraphAPI.FrameTextureExtractor.GetTextureDataSizeInBytes(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_RGB);
}
else if (m_textureFormat == MvxTextureFormat.IR && frame.StreamContainsDataLayer(MVGraphAPI.SimpleDataLayersGuids.IR_TEXTURE_DATA_LAYER))
{
textureFormat = TextureFormat.R8;
MVGraphAPI.FrameTextureExtractor.GetTextureResolution(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_IR, out textureWidth, out textureHeight);
textureSizeInBytes = MVGraphAPI.FrameTextureExtractor.GetTextureDataSizeInBytes(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_IR);
}
else
{
return(false);
}
m_activeTextureIndex = (m_activeTextureIndex + 1) % m_textures.Length;
Texture2D newActiveTexture = m_textures[m_activeTextureIndex];
EnsureTextureProperties(ref newActiveTexture, textureFormat, textureWidth, textureHeight);
m_textures[m_activeTextureIndex] = newActiveTexture;
LoadTexture(frame, newActiveTexture, m_textureFormat, textureWidth, textureHeight, textureSizeInBytes);
return(true);
}
public static Bounds GetFrameBoundingBox(MVGraphAPI.Frame frame)
{
MVGraphAPI.FrameMeshExtractor.GetMeshData(frame).CopyBoundingBox(m_boundingBoxData);
Vector3 boundingBoxVector1 = new Vector3(m_boundingBoxData[0], m_boundingBoxData[1], m_boundingBoxData[2]);
Vector3 boundingBoxVector2 = new Vector3(m_boundingBoxData[3], m_boundingBoxData[4], m_boundingBoxData[5]);
Vector3 boundingBoxCenter = (boundingBoxVector1 + boundingBoxVector2) / 2.0f;
Vector3 boundingBoxSize = boundingBoxVector2 - boundingBoxVector1;
return(new Bounds(boundingBoxCenter, boundingBoxSize));
}
protected override void ProcessNextFrame(MVGraphAPI.Frame frame)
{
if (!frame.StreamContainsDataLayer(MVGraphAPI.SimpleDataLayersGuids.AUDIO_DATA_LAYER))
{
return;
}
lock (m_framesQueue)
m_framesQueue.Enqueue(frame);
StartAudioDataProcessingThread();
}
protected override void ProcessNextFrame(MVGraphAPI.Frame frame)
{
if (m_rawImageUI == null)
{
return;
}
if (CollectTextureDataFromFrame(frame))
{
m_rawImageUI.texture = m_textures[m_activeTextureIndex];
}
else
{
m_rawImageUI.texture = m_blackTexture;
}
}
private void ProcessAudioData()
{
while (true)
{
if (m_stopProcessingAudioData)
{
return;
}
MVGraphAPI.Frame frame = null;
lock (m_framesQueue)
{
if (m_framesQueue.Count == 0)
{
return;
}
frame = m_framesQueue.Dequeue();
}
UInt32 framePCMDataSize = MVGraphAPI.FrameAudioExtractor.GetPCMDataSize(frame);
if (framePCMDataSize == 0)
{
continue;
}
UInt32 frameChannelsCount;
UInt32 frameBitsPerSample;
UInt32 frameSampleRate;
MVGraphAPI.FrameAudioExtractor.GetAudioSamplingInfo(frame, out frameChannelsCount, out frameBitsPerSample, out frameSampleRate);
if (frameBitsPerSample != 8 && frameBitsPerSample != 16 && frameBitsPerSample != 32)
{
Debug.LogErrorFormat("Unsupported 'bits per sample' value {0}", frameBitsPerSample);
continue;
}
UInt32 frameBytesPerSample = frameBitsPerSample / 8;
byte[] frameAudioBytes = new byte[framePCMDataSize];
MVGraphAPI.FrameAudioExtractor.CopyPCMData(frame, frameAudioBytes);
MvxAudioChunk newAudioChunk = MvxAudioChunksPool.instance.AllocateAudioChunk(frameAudioBytes, frameBytesPerSample, frameChannelsCount, frameSampleRate);
lock (m_audioPlayer)
m_audioPlayer.EnqueueAudioChunk(newAudioChunk);
}
}
protected override void ProcessNextFrame(MVGraphAPI.Frame frame)
{
UpdateMeshPart(MVGraphAPI.FrameMeshExtractor.GetMeshData(frame));
}
private void OnMvxFrame(MVGraphAPI.Frame frame)
{
onNextFrameReceived.Invoke(new MVCommon.SharedRef <MVGraphAPI.Frame>(frame));
}
private unsafe void LoadTexture(MVGraphAPI.Frame frame, Texture2D targetTexture, MvxTextureFormat textureFormat, ushort width, ushort height, UInt32 sizeInBytes)
{
switch (textureFormat)
{
case MvxTextureFormat.DEPTH_MAP:
{
byte[] depthMapData = new byte[width * height * 2];
MVGraphAPI.FrameTextureExtractor.CopyTextureData(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_DEPTH, depthMapData);
byte[] convertedDepthMapData = new byte[width * height];
for (int i = 0; i < convertedDepthMapData.Length; i++)
{
ushort depthMapDataElement = (ushort)(depthMapData[2 * i] + depthMapData[2 * i + 1] * 256);
convertedDepthMapData[i] = (byte)Mathf.Clamp(depthMapDataElement / 256 * m_depthMultiplier, 0, 256);
}
fixed(byte *convertedDepthMapDataPtr = convertedDepthMapData)
targetTexture.LoadRawTextureData((IntPtr)convertedDepthMapDataPtr, (Int32)(sizeInBytes));
}
break;
case MvxTextureFormat.ASTC:
{
IntPtr textureData = MVGraphAPI.FrameTextureExtractor.GetTextureData(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_ASTC);
targetTexture.LoadRawTextureData(textureData, (Int32)sizeInBytes);
}
break;
case MvxTextureFormat.DXT1:
{
IntPtr textureData = MVGraphAPI.FrameTextureExtractor.GetTextureData(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_DXT1);
targetTexture.LoadRawTextureData(textureData, (Int32)sizeInBytes);
}
break;
case MvxTextureFormat.ETC2:
{
IntPtr textureData = MVGraphAPI.FrameTextureExtractor.GetTextureData(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_ETC2);
targetTexture.LoadRawTextureData(textureData, (Int32)sizeInBytes);
}
break;
case MvxTextureFormat.RGB:
{
IntPtr textureData = MVGraphAPI.FrameTextureExtractor.GetTextureData(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_RGB);
targetTexture.LoadRawTextureData(textureData, (Int32)sizeInBytes);
}
break;
case MvxTextureFormat.IR:
{
IntPtr textureData = MVGraphAPI.FrameTextureExtractor.GetTextureData(frame, MVGraphAPI.FrameTextureExtractor.TextureType.TT_IR);
targetTexture.LoadRawTextureData(textureData, (Int32)sizeInBytes);
}
break;
default:
return;
}
targetTexture.Apply(true, false);
}
protected void HandleNextFrame(MVGraphAPI.Frame nextFrame)
{
lastReceivedFrame = new MVCommon.SharedRef <MVGraphAPI.Frame>(nextFrame);
onNextFrameReceived.Invoke(lastReceivedFrame);
}
private void CollectTextureDataFromFrame(MVGraphAPI.Frame frame)
{
if (materialInstances == null || materialInstances.Length == 0)
{
return;
}
int textureType;
TextureFormat textureFormat;
MVGraphAPI.FrameTextureExtractor.TextureType mvxTextureType;
if (frame.StreamContainsDataLayer(MVGraphAPI.SimpleDataLayersGuids.ASTC_TEXTURE_DATA_LAYER))
{
textureType = (int)TextureTypeCodes.TTC_ASTC;
textureFormat = TextureFormat.ASTC_RGB_8x8;
mvxTextureType = MVGraphAPI.FrameTextureExtractor.TextureType.TT_ASTC;
}
else if (frame.StreamContainsDataLayer(MVGraphAPI.SimpleDataLayersGuids.DXT1_TEXTURE_DATA_LAYER))
{
textureType = (int)TextureTypeCodes.TTC_DXT1;
textureFormat = TextureFormat.DXT1;
mvxTextureType = MVGraphAPI.FrameTextureExtractor.TextureType.TT_DXT1;
}
else if (frame.StreamContainsDataLayer(MVGraphAPI.SimpleDataLayersGuids.ETC2_TEXTURE_DATA_LAYER))
{
textureType = (int)TextureTypeCodes.TTC_ETC2;
textureFormat = TextureFormat.ETC2_RGB;
mvxTextureType = MVGraphAPI.FrameTextureExtractor.TextureType.TT_ETC2;
}
else if (frame.StreamContainsDataLayer(MVGraphAPI.SimpleDataLayersGuids.NVX_TEXTURE_DATA_LAYER))
{
textureType = (int)TextureTypeCodes.TTC_NVX;
textureFormat = TextureFormat.Alpha8;
mvxTextureType = MVGraphAPI.FrameTextureExtractor.TextureType.TT_NVX;
}
else if (frame.StreamContainsDataLayer(MVGraphAPI.SimpleDataLayersGuids.RGB_TEXTURE_DATA_LAYER))
{
textureType = (int)TextureTypeCodes.TTC_RGB;
textureFormat = TextureFormat.RGB24;
mvxTextureType = MVGraphAPI.FrameTextureExtractor.TextureType.TT_RGB;
}
else
{
foreach (Material materialInstance in materialInstances)
{
materialInstance.SetTexture(TEXTURE_SHADER_PROPERTY_NAME, null);
}
return;
}
ushort textureWidth, textureHeight;
MVGraphAPI.FrameTextureExtractor.GetTextureResolution(frame, mvxTextureType, out textureWidth, out textureHeight);
UInt32 textureSizeInBytes = MVGraphAPI.FrameTextureExtractor.GetTextureDataSizeInBytes(frame, mvxTextureType);
IntPtr textureData = MVGraphAPI.FrameTextureExtractor.GetTextureData(frame, mvxTextureType);
m_activeTextureIndex = (m_activeTextureIndex + 1) % m_textures.Length;
Texture2D newActiveTexture = m_textures[m_activeTextureIndex];
EnsureTextureProperties(ref newActiveTexture, textureFormat, textureWidth, textureHeight);
m_textures[m_activeTextureIndex] = newActiveTexture;
newActiveTexture.LoadRawTextureData(textureData, (Int32)textureSizeInBytes);
newActiveTexture.Apply(true, false);
foreach (Material materialInstance in materialInstances)
{
materialInstance.SetInt(TEXTURE_TYPE_SHADER_PROPERTY_NAME, textureType);
materialInstance.SetTexture(TEXTURE_SHADER_PROPERTY_NAME, newActiveTexture);
}
}
protected override void ProcessNextFrame(MVGraphAPI.Frame frame)
{
base.ProcessNextFrame(frame);
CollectTextureDataFromFrame(frame);
}
protected abstract void ProcessNextFrame(MVGraphAPI.Frame frame);
