unsafe void OnCameraFrameReceied(ARCameraFrameEventArgs eventArgs)
{
XRCpuImage image;
if (!cameraManager.TryGetLatestImage(out image))
{
return;
}
var conversionParams = new XRCpuImage.ConversionParams
(
image,
TextureFormat.RGBA32,
XRCpuImage.Transformation.None
);
if (mTexture == null || mTexture.width != image.width || mTexture.height != image.height)
{
mTexture = new Texture2D(
conversionParams.outputDimensions.x,
conversionParams.outputDimensions.y,
conversionParams.outputFormat,
false);
}
var buffer = mTexture.GetRawTextureData <byte>();
image.Convert(conversionParams, new IntPtr(buffer.GetUnsafePtr()), buffer.Length);
mTexture.Apply();
mRenderer.material.mainTexture = mTexture;
buffer.Dispose();
image.Dispose();
}
/// <summary>
/// 获取摄像头原始数据
/// </summary>
private unsafe void ScreenRecordRawImage()
{
if (!cameraManager.TryAcquireLatestCpuImage(out XRCpuImage image))
{
return;
}
if (isRecording)
{
if (renderTexture == null)
{
renderTexture = new Texture2D(image.width, image.height, TextureFormat.BGRA32, false);
conversionParams = new XRCpuImage.ConversionParams(image, TextureFormat.BGRA32);
intPtr = new IntPtr(renderTexture.GetRawTextureData <byte>().GetUnsafePtr());
NativeAPI.StartRecordVideo(image.width, image.height, recordType);
}
var rawTextureData = renderTexture.GetRawTextureData <byte>();
try
{
image.Convert(conversionParams, intPtr, rawTextureData.Length);
}
finally
{
image.Dispose();
}
renderTexture.Apply();
NativeAPI.SendVideoData(renderTexture.GetRawTextureData(), rawTextureData.Length);
}
}
/// <summary>
/// 获取摄像头原始数据
/// </summary>
private unsafe void ScreenShotRawImage()
{
if (!cameraManager.TryAcquireLatestCpuImage(out XRCpuImage image))
{
return;
}
if (renderTexture == null)
{
renderTexture = new Texture2D(image.width, image.height, TextureFormat.BGRA32, false);
conversionParams = new XRCpuImage.ConversionParams(image, TextureFormat.BGRA32);
intPtr = new IntPtr(renderTexture.GetRawTextureData <byte>().GetUnsafePtr());
}
var rawTextureData = renderTexture.GetRawTextureData <byte>();
try
{
image.Convert(conversionParams, intPtr, rawTextureData.Length);
}
finally
{
image.Dispose();
}
renderTexture.Apply();
byte[] rawData = renderTexture.EncodeToPNG();
NativeAPI.ScreenDidShot(rawData, rawData.Length, recordType);
renderTexture = null;
}
unsafe void OnCameraFrameReceived(ARCameraFrameEventArgs eventArgs)
{
if (!arCameraManager.TryAcquireLatestCpuImage(out XRCpuImage image))
{
return;
}
var conversionParams = new XRCpuImage.ConversionParams
{
inputRect = new RectInt(0, 0, image.width, image.height),
outputDimensions = new Vector2Int(image.width, image.height),
outputFormat = TextureFormat.RGBA32,
transformation = XRCpuImage.Transformation.None
};
int imageSize = image.GetConvertedDataSize(conversionParams);
var buffer = new NativeArray <byte>(imageSize, Allocator.Temp);
image.Convert(conversionParams, new IntPtr(buffer.GetUnsafePtr()), buffer.Length);
image.Dispose();
this.m_Texture = new Texture2D(
conversionParams.outputDimensions.x,
conversionParams.outputDimensions.y,
conversionParams.outputFormat,
false
);
this.m_Texture.LoadRawTextureData(buffer);
this.m_Texture.Apply();
buffer.Dispose();
Detect();
}
unsafe void OnARCameraFrameReceived(ARCameraFrameEventArgs eventArgs)
{
//Reference:https://docs.unity3d.com/Packages/[email protected]/manual/cpu-camera-image.html
if (!CameraManager.TryAcquireLatestCpuImage(out XRCpuImage image))
{
return;
}
var conversionParams = new XRCpuImage.ConversionParams
{
inputRect = new RectInt(0, 0, image.width, image.height),
outputDimensions = new Vector2Int(image.width / 2, image.height / 2),
outputFormat = TextureFormat.RGBA32,
transformation = XRCpuImage.Transformation.MirrorY
};
int size = image.GetConvertedDataSize(conversionParams);
var buffer = new NativeArray <byte>(size, Allocator.Temp);
image.Convert(conversionParams, new IntPtr(buffer.GetUnsafePtr()), buffer.Length);
image.Dispose();
if (RGB_Texture == null)
{
RGB_Texture = new Texture2D(conversionParams.outputDimensions.x, conversionParams.outputDimensions.y, conversionParams.outputFormat, false);
}
RGB_Texture.LoadRawTextureData(buffer);
RGB_Texture.Apply();
buffer.Dispose();
}
/// <summary>
/// Convert the image with handle <paramref name="nativeHandle"/> using the provided
/// <paramref cref="conversionParams"/>.
/// </summary>
/// <param name="nativeHandle">A unique identifier for the camera image to convert.</param>
/// <param name="conversionParams">The parameters to use during the conversion.</param>
/// <param name="destinationBuffer">A buffer to write the converted image to.</param>
/// <param name="bufferLength">The number of bytes available in the buffer.</param>
/// <returns>
/// <c>true</c> if the image was converted and stored in <paramref name="destinationBuffer"/>.
/// </returns>
public override bool TryConvert(
int nativeHandle,
XRCpuImage.ConversionParams conversionParams,
IntPtr destinationBuffer,
int bufferLength)
{
return(Native.TryConvert(nativeHandle, conversionParams, destinationBuffer, bufferLength));
}
/// <summary>
/// Similar to <see cref="ConvertAsync(int, XRCpuImage.ConversionParams)"/> but takes a delegate to
/// invoke when the request is complete, rather than returning a request id.
/// </summary>
/// <remarks>
/// If the first parameter to <paramref name="callback"/> is
/// <see cref="XRCpuImage.AsyncConversionStatus.Ready"/> then the <c>dataPtr</c> parameter must be valid
/// for the duration of the invocation. The data may be destroyed immediately upon return. The
/// <paramref name="context"/> parameter must be passed back to the <paramref name="callback"/>.
/// </remarks>
/// <param name="nativeHandle">A unique identifier for the camera image to convert.</param>
/// <param name="conversionParams">The parameters to use during the conversion.</param>
/// <param name="callback">A delegate which must be invoked when the request is complete, whether the
/// conversion was successfully or not.</param>
/// <param name="context">A native pointer which must be passed back unaltered to
/// <paramref name="callback"/>.</param>
public override void ConvertAsync(
int nativeHandle,
XRCpuImage.ConversionParams conversionParams,
OnImageRequestCompleteDelegate callback,
IntPtr context)
{
Native.CreateAsyncConversionRequestWithCallback(
nativeHandle, conversionParams, callback, context);
}
public unsafe byte[] CamGetFrame()
{
XRCpuImage image;
if (m_CameraManager.TryAcquireLatestCpuImage(out image))
{
var conversionParams = new XRCpuImage.ConversionParams
{
// Get the entire image.
inputRect = new RectInt(0, 0, image.width, image.height),
outputDimensions = new Vector2Int(image.width, image.height),
// Choose RGBA format.
outputFormat = TextureFormat.RGBA32,
// Flip across the vertical axis (mirror image).
transformation = XRCpuImage.Transformation.MirrorY
};
// See how many bytes we need to store the final image.
int size = image.GetConvertedDataSize(conversionParams);
// Allocate a buffer to store the image
var buffer = new NativeArray <byte>(size, Allocator.Temp);
// Extract the image data
image.Convert(conversionParams, new IntPtr(buffer.GetUnsafePtr()), buffer.Length);
Debug.Log("buffer.Length" + buffer.Length);
// The image was converted to RGBA32 format and written into the provided buffer
// so we can dispose of the CameraImage. We must do this or it will leak resources.
image.Dispose();
// At this point, we could process the image, pass it to a computer vision algorithm, etc.
// In this example, we'll just apply it to a texture to visualize it.
// We've got the data; let's put it into a texture so we can visualize it.
Texture2D m_Texture = new Texture2D(
conversionParams.outputDimensions.x,
conversionParams.outputDimensions.y,
conversionParams.outputFormat,
false);
m_Texture.LoadRawTextureData(buffer);
m_Texture.Apply();
buffer.Dispose();
byte[] bb = m_Texture.EncodeToJPG(100);
Destroy(m_Texture);
return(bb);
}
return(null);
}
private unsafe void UpdateCameraTexture()
{
// https://docs.unity3d.com/Packages/[email protected]/manual/cpu-camera-image.html
if (CameraManager.TryAcquireLatestCpuImage(out XRCpuImage img))
{
XRCpuImage.ConversionParams conversionParams = new XRCpuImage.ConversionParams
{
// Get the entire image.
inputRect = new RectInt(0, 0, img.width, img.height),
// Downsample by 4.
outputDimensions = new Vector2Int(img.width / 4, img.height / 4),
// Choose RGBA format.
outputFormat = TextureFormat.RGBA32,
// Flip across the vertical axis (mirror image).
transformation = XRCpuImage.Transformation.MirrorY
};
// See how many bytes you need to store the final image.
int size = img.GetConvertedDataSize(conversionParams);
// Allocate a buffer to store the image.
NativeArray <byte> buffer = new NativeArray <byte>(size, Allocator.Temp);
// Extract the image data
img.Convert(conversionParams, new IntPtr(buffer.GetUnsafePtr()), buffer.Length);
// The image was converted to RGBA32 format and written into the provided buffer
// so you can dispose of the XRCpuImage. You must do this or it will leak resources.
img.Dispose();
// At this point, you can process the image, pass it to a computer vision algorithm, etc.
// In this example, you apply it to a texture to visualize it.
// You've got the data; let's put it into a texture so you can visualize it.
if (m_cameraTexture == null)
{
m_cameraTexture = new Texture2D(
conversionParams.outputDimensions.x,
conversionParams.outputDimensions.y,
conversionParams.outputFormat,
false);
}
m_cameraTexture.LoadRawTextureData(buffer);
m_cameraTexture.Apply();
// Done with your temporary data, so you can dispose it.
buffer.Dispose();
}
}
private void UpdateRawImage(ref Texture2D texture, XRCpuImage cpuImage)
{
if (texture == null || texture.width != cpuImage.width || texture.height != cpuImage.height)
{
texture = new Texture2D(cpuImage.width, cpuImage.height, TextureFormat.RGB565, false);
}
var conversionParams = new XRCpuImage.ConversionParams(cpuImage, TextureFormat.R16);
var rawTextureData = texture.GetRawTextureData <byte>();
cpuImage.Convert(conversionParams, rawTextureData);
texture.Apply();
}
private static void UpdateRawImage(Texture2D texture, XRCpuImage cpuImage)
{
var conversionParams = new XRCpuImage.ConversionParams(cpuImage, cpuImage.format.AsTextureFormat(),
XRCpuImage.Transformation.MirrorY);
var rawTextureData = texture.GetRawTextureData <byte>();
Debug.Assert(
rawTextureData.Length ==
cpuImage.GetConvertedDataSize(conversionParams.outputDimensions, conversionParams.outputFormat),
"The Texture2D is not the same size as the converted data.");
cpuImage.Convert(conversionParams, rawTextureData);
texture.Apply();
}
unsafe void convertCPUImage()
{
XRCpuImage image;
if (!cameraManager.TryAcquireLatestCpuImage(out image))
{
Debug.Log("Cant get image");
return;
}
if (float.IsNegativeInfinity(ALPHA))
{
ALPHA = (float)image.height / image.width;
imageRatio = (float)(BETA / ALPHA);
}
var conversionParams = new XRCpuImage.ConversionParams {
// Get the entire image
inputRect = new RectInt(0, 0, image.width, image.height),
// Downsample by 2
outputDimensions = new Vector2Int(image.width / 2, image.height / 2),
// Choose RGBA format
outputFormat = TextureFormat.RGBA32,
// Flip across the vertical axis (mirror image)
transformation = XRCpuImage.Transformation.MirrorY
};
int size = image.GetConvertedDataSize(conversionParams);
var buffer = new NativeArray <byte>(size, Allocator.Temp);
image.Convert(conversionParams, new IntPtr(buffer.GetUnsafePtr()), buffer.Length);
image.Dispose();
Texture2D m_Texture = new Texture2D(
conversionParams.outputDimensions.x,
conversionParams.outputDimensions.y,
conversionParams.outputFormat,
false);
m_Texture.LoadRawTextureData(buffer);
m_Texture.Apply();
buffer.Dispose();
// pass image for mediapipe
long time = new DateTimeOffset(DateTime.Now).ToUnixTimeMilliseconds();
Debug.Log("Texture loaded at: " + time.ToString());
handProcessor.addFrameTexture(m_Texture);
}
void UpdateRawImage(Texture2D texture, XRCpuImage cpuImage)
{
// For display, we need to mirror about the vertical access.
var conversionParams = new XRCpuImage.ConversionParams(cpuImage, cpuImage.format.AsTextureFormat(), XRCpuImage.Transformation.MirrorY);
// Get the Texture2D's underlying pixel buffer.
var rawTextureData = texture.GetRawTextureData <byte>();
// Make sure the destination buffer is large enough to hold the converted data (they should be the same size)
Debug.Assert(rawTextureData.Length == cpuImage.GetConvertedDataSize(conversionParams.outputDimensions, conversionParams.outputFormat),
"The Texture2D is not the same size as the converted data.");
// Perform the conversion.
cpuImage.Convert(conversionParams, rawTextureData);
// "Apply" the new pixel data to the Texture2D.
texture.Apply();
}
public static void GetPlaneDataRGB(out byte[] pixels, XRCpuImage image)
{
var conversionParams = new XRCpuImage.ConversionParams
{
inputRect = new RectInt(0, 0, image.width, image.height),
outputDimensions = new Vector2Int(image.width, image.height),
outputFormat = TextureFormat.RGB24,
transformation = XRCpuImage.Transformation.None
};
int size = image.GetConvertedDataSize(conversionParams);
pixels = new byte[size];
GCHandle bufferHandle = GCHandle.Alloc(pixels, GCHandleType.Pinned);
image.Convert(conversionParams, bufferHandle.AddrOfPinnedObject(), pixels.Length);
bufferHandle.Free();
}
unsafe void UpdateCameraImage()
{
// Attempt to get the latest camera image. If this method succeeds,
// it acquires a native resource that must be disposed (see below).
if (!m_CameraManager.TryAcquireLatestCpuImage(out XRCpuImage image))
{
return;
}
using (image)
{
// Choose an RGBA format.
// See XRCpuImage.FormatSupported for a complete list of supported formats.
var format = TextureFormat.RGBA32;
//Initialize m_CameraTexture if it's null or size of image is changed.
if (m_CameraTexture == null || m_CameraTexture.width != image.width || m_CameraTexture.height != image.height)
{
m_CameraTexture = new Texture2D(image.width, image.height, format, false);
}
// Convert the image to format, flipping the image across the Y axis.
// We can also get a sub rectangle, but we'll get the full image here.
var conversionParams = new XRCpuImage.ConversionParams(image, format, XRCpuImage.Transformation.MirrorY);
// Texture2D allows us write directly to the raw texture data
// This allows us to do the conversion in-place without making any copies.
var rawTextureData = m_CameraTexture.GetRawTextureData <byte>();
//Convert XRCpuImage into RGBA32
image.Convert(conversionParams, new IntPtr(rawTextureData.GetUnsafePtr()), rawTextureData.Length);
// Apply the updated texture data to our texture
m_CameraTexture.Apply();
// Set the RawImage's texture so we can visualize it.
m_cameraView.texture = m_CameraTexture;
}
}
private unsafe void UpdateCameraImage()
{
if (!_cameraManager.TryAcquireLatestCpuImage(out var image))
{
CloudUpdateEvent?.Invoke(GenerateDebugCloud());
return;
}
using (image)
{
if (_cameraTexture == null || !HasEqualDimensions(_cameraTexture, image))
{
_cameraTexture = new Texture2D(image.width, image.height, TextureFormat.RGBA32, false);
}
var conversionParams =
new XRCpuImage.ConversionParams(image, TextureFormat.RGBA32, XRCpuImage.Transformation.MirrorY);
var textureRaw = _cameraTexture.GetRawTextureData <byte>();
image.Convert(conversionParams, new IntPtr(textureRaw.GetUnsafePtr()), textureRaw.Length);
_cameraTexture.Apply();
_debugViews.Camera.texture = _cameraTexture;
}
}
public static extern int UnityARCore_CpuImage_CreateAsyncConversionRequest(int nativeHandle,
XRCpuImage.ConversionParams conversionParams);
public static extern bool UnityARCore_CpuImage_TryConvert(int nativeHandle,
XRCpuImage.ConversionParams conversionParams,
IntPtr buffer, int bufferLength);
/// <summary>
/// Similar to <see cref="ConvertAsync(int, XRCpuImage.ConversionParams)"/> but takes a delegate to
/// invoke when the request is complete, rather than returning a request id.
/// </summary>
/// <remarks>
/// If the first parameter to <paramref name="callback"/> is
/// <see cref="XRCpuImage.AsyncConversionStatus.Ready"/> then the <c>dataPtr</c> parameter must be valid
/// for the duration of the invocation. The data may be destroyed immediately upon return. The
/// <paramref name="context"/> parameter must be passed back to the <paramref name="callback"/>.
/// </remarks>
/// <param name="nativeHandle">A unique identifier for the camera image to convert.</param>
/// <param name="conversionParams">The parameters to use during the conversion.</param>
/// <param name="callback">A delegate which must be invoked when the request is complete, whether the
/// conversion was successfully or not.</param>
/// <param name="context">A native pointer which must be passed back unaltered to
/// <paramref name="callback"/>.</param>
public override void ConvertAsync(int nativeHandle, XRCpuImage.ConversionParams conversionParams,
XRCpuImage.Api.OnImageRequestCompleteDelegate callback, IntPtr context)
=> NativeApi.UnityARCore_CpuImage_CreateAsyncConversionRequestWithCallback(nativeHandle, conversionParams,
callback, context);
/// <summary> /// Create an asynchronous request to convert a camera image, similar to <see cref="TryConvert"/> except /// the conversion should happen on a thread other than the calling (main) thread. /// </summary> /// <param name="nativeHandle">A unique identifier for the camera image to convert.</param> /// <param name="conversionParams">The parameters to use during the conversion.</param> /// <returns>A unique identifier for this request.</returns> public override int ConvertAsync(int nativeHandle, XRCpuImage.ConversionParams conversionParams) => NativeApi.UnityARCore_CpuImage_CreateAsyncConversionRequest(nativeHandle, conversionParams);
/// <summary>
/// Convert the image with handle <paramref name="nativeHandle"/> using the provided
/// <paramref cref="conversionParams"/>.
/// </summary>
/// <param name="nativeHandle">A unique identifier for the camera image to convert.</param>
/// <param name="conversionParams">The parameters to use during the conversion.</param>
/// <param name="destinationBuffer">A buffer to write the converted image to.</param>
/// <param name="bufferLength">The number of bytes available in the buffer.</param>
/// <returns>
/// <c>true</c> if the image was converted and stored in <paramref name="destinationBuffer"/>.
/// </returns>
public override bool TryConvert(int nativeHandle, XRCpuImage.ConversionParams conversionParams,
IntPtr destinationBuffer, int bufferLength)
=> NativeApi.UnityARCore_CpuImage_TryConvert(nativeHandle, conversionParams, destinationBuffer,
bufferLength);
public static void CreateAsyncConversionRequestWithCallback(
int nativeHandle, XRCpuImage.ConversionParams conversionParams,
OnImageRequestCompleteDelegate callback, IntPtr context)
{
throw new System.NotImplementedException(k_ExceptionMsg);
}
IEnumerator ProcessImage(XRCpuImage image, Vector3 viewportScaling)
{
// Create the async conversion request.
XRCpuImage.ConversionParams conv_params = new XRCpuImage.ConversionParams
{
// Use the full image.
inputRect = new RectInt(0, 0, image.width, image.height),
// Downsample by 2.
outputDimensions = new Vector2Int(image.width, image.height),
// Color image format.
outputFormat = TextureFormat.RGBA32,
// Flip across the Y axis.
transformation = XRCpuImage.Transformation.MirrorY
};
var request = image.ConvertAsync(conv_params);
// Wait for the conversion to complete.
while (!request.status.IsDone())
{
yield return(null);
}
// Check status to see if the conversion completed successfully.
if (request.status != XRCpuImage.AsyncConversionStatus.Ready)
{
// Something went wrong.
Debug.LogErrorFormat("Request failed with status {0}", request.status);
// Dispose even if there is an error.
request.Dispose();
yield break;
}
// Image data is ready. Let's apply it to a Texture2D.
var rawData = request.GetData <byte>();
// Create a texture if necessary.
if (m_Texture == null)
{
m_Texture = new Texture2D(
request.conversionParams.outputDimensions.x,
request.conversionParams.outputDimensions.y,
request.conversionParams.outputFormat,
false);
}
// Copy the image data into the texture.
m_Texture.LoadRawTextureData(rawData);
m_Texture.Apply();
Debug.Log("TEX: " + m_Texture.height + "h " + m_Texture.width + "w");
Debug.Log("Screen: " + m_Texture.height + "h " + m_Texture.width + "w");
Mat inputMat = new Mat(image.height, image.width, CvType.CV_8UC4);
Mat outputMat = new Mat(1500, 1500, CvType.CV_8UC4);
Utils.fastTexture2DToMat(m_Texture, inputMat);
if (tex2d == null)
{
tex2d = new Texture2D(1500,
1500, conv_params.outputFormat, false);
}
Debug.Log("positionAnchor");
Debug.Log(positionAnchor);
Debug.Log("anchorRef");
Debug.Log(anchorRef);
int counter = 0;
Point[] srcPointsVec = new Point[4];
foreach (var point in anchorRef.getWorldPoints())
{
Vector3 screenPoint = mainCam.WorldToScreenPoint(point);
srcPointsVec[counter] = new Point(screenPoint.y * viewportScaling.y / 3,
100 - screenPoint.x * viewportScaling.x / 3);
counter += 1;
}
MatOfPoint2f srcPoints = new MatOfPoint2f(new[]
{
srcPointsVec[0],
srcPointsVec[1],
srcPointsVec[2],
srcPointsVec[3]
});
MatOfPoint2f dstPoints = new MatOfPoint2f(new[]
{
new Point(195 * 1.25, 0),
new Point(0, 0),
new Point(0, 280 * 1.25),
new Point(195 * 1.25, 280 * 1.25),
});
Mat H = Calib3d.findHomography(srcPoints, dstPoints);
Imgproc.warpPerspective(inputMat, outputMat, H, new Size(1500, 1500));
Utils.fastMatToTexture2D(outputMat, tex2d);
if (websocket.State == WebSocketState.Open && canProcess)
{
websocket.Send(ImageConversion.EncodeToJPG(tex2d, 50));
canProcess = false;
}
inputMat.Dispose();
inputMat = null;
outputMat.Dispose();
outputMat = null;
request.Dispose();
}
public static bool TryConvert(
int nativeHandle, XRCpuImage.ConversionParams conversionParams,
IntPtr buffer, int bufferLength)
{
throw new System.NotImplementedException(k_ExceptionMsg);
}
unsafe void OnCameraFrameReceived(ARCameraFrameEventArgs eventArgs)
{
// Attempt to get the latest camera image. If this method succeeds,
// it acquires a native resource that must be disposed (see below).
XRCpuImage image;
if (!m_CameraManager.TryAcquireLatestCpuImage(out image))
{
return;
}
var format = TextureFormat.RGBA32;
if (m_CameraTexture == null || m_CameraTexture.width != image.width || m_CameraTexture.height != image.height)
{
m_CameraTexture = new Texture2D(image.width, image.height, format, false);
}
// Convert the image to format, flipping the image across the Y axis.
// We can also get a sub rectangle, but we'll get the full image here.
var conversionParams = new XRCpuImage.ConversionParams(image, format, XRCpuImage.Transformation.MirrorY);
// Texture2D allows us write directly to the raw texture data
// This allows us to do the conversion in-place without making any copies.
var rawTextureData = m_CameraTexture.GetRawTextureData <byte>();
try
{
image.Convert(conversionParams, new IntPtr(rawTextureData.GetUnsafePtr()), rawTextureData.Length);
}
finally
{
// We must dispose of the XRCameraImage after we're finished
// with it to avoid leaking native resources.
image.Dispose();
}
// Apply the updated texture data to our texture
m_CameraTexture.Apply();
// Set the RawImage's texture so we can visualize it.
m_RawImage.material.SetTexture("_CameraTex", m_CameraTexture);
///////////////////////////////////////
Matrix4x4 cameraMatrix = eventArgs.displayMatrix ?? Matrix4x4.identity;
Vector2 affineBasisX = new Vector2(1.0f, 0.0f);
Vector2 affineBasisY = new Vector2(0.0f, 1.0f);
Vector2 affineTranslation = new Vector2(0.0f, 0.0f);
affineBasisX = new Vector2(cameraMatrix[0, 0], cameraMatrix[1, 0]);
affineBasisY = new Vector2(cameraMatrix[0, 1], cameraMatrix[1, 1]);
affineTranslation = new Vector2(cameraMatrix[2, 0], cameraMatrix[2, 1]);
// The camera display matrix includes scaling and offsets to fit the aspect ratio of the device. In most
// cases, the camera display matrix should be used directly without modification when applying depth to
// the scene because that will line up the depth image with the camera image. However, for this demo,
// we want to show the full depth image as a picture-in-picture, so we remove these scaling and offset
// factors while preserving the orientation.
affineBasisX = affineBasisX.normalized;
affineBasisY = affineBasisY.normalized;
m_DisplayRotationMatrix = Matrix4x4.identity;
m_DisplayRotationMatrix[0, 0] = affineBasisX.x;
m_DisplayRotationMatrix[0, 1] = affineBasisY.x;
m_DisplayRotationMatrix[1, 0] = affineBasisX.y;
m_DisplayRotationMatrix[1, 1] = affineBasisY.y;
m_DisplayRotationMatrix[2, 0] = Mathf.Round(affineTranslation.x);
m_DisplayRotationMatrix[2, 1] = Mathf.Round(affineTranslation.y);
// Set the matrix to the raw image material.
m_RawImage.material.SetMatrix(k_DisplayRotationPerFrameId, m_DisplayRotationMatrix);
m_RawImage.material.SetMatrix(k_InverseMatrixId, m_DisplayRotationMatrix.inverse);
}
void onCameraFrameReceived(ARCameraFrameEventArgs eventArgs)
{
if (isStart_)
{
if (nextUpdateFrameTime_ > Time.realtimeSinceStartup)
{
return;
}
//nextUpdateFrameTime_ += INTERVAL_TIME;
nextUpdateFrameTime_ = Time.realtimeSinceStartup + INTERVAL_TIME;
}
if (!cameraManager_.TryAcquireLatestCpuImage(out XRCpuImage image))
{
return;
}
if (!isStart_)
{
height_ = RESIZE_HEIGHT;
if (image.height < height_)
{
height_ = image.height;
}
width_ = height_ * image.width / image.height;
#if UNITY_ANDROID && !UNITY_EDITOR
using (AndroidJavaClass unityPlayer = new AndroidJavaClass("com.unity3d.player.UnityPlayer"))
using (AndroidJavaObject currentUnityActivity = unityPlayer.GetStatic <AndroidJavaObject>("currentActivity"))
{
multiHandMain_ = new AndroidJavaObject("online.mumeigames.mediapipe.apps.multihandtrackinggpu.MultiHandMain", currentUnityActivity, width_, height_, BothHand ? 2 : 1, 0);
}
multiHandMain_.Call("startRunningGraph");
nextUpdateFrameTime_ = Time.realtimeSinceStartup + START_DELAY_TIME;
#endif
#if UNITY_IOS && !UNITY_EDITOR
IntPtr graphName = Marshal.StringToHGlobalAnsi("handcputogpu");
multiHandSetup(graphName, width_, height_, BothHand ? 2 : 1, 0);
Marshal.FreeHGlobal(graphName);
multiHandStartRunningGraph();
nextUpdateFrameTime_ = Time.realtimeSinceStartup + START_DELAY_TIME;
#endif
focalLength_ = 0.5 / Mathf.Tan(FixedFieldOfView * Mathf.Deg2Rad * 0.5f);
hand3dInitWithValues(focalLength_, focalLength_, 0.5, 0.5);
resetHandValues();
isStart_ = true;
image.Dispose();
return;
}
var conversionParams = new XRCpuImage.ConversionParams
{
inputRect = new RectInt(0, 0, image.width, image.height),
outputDimensions = new Vector2Int(width_, height_),
#if UNITY_IOS && !UNITY_EDITOR
outputFormat = TextureFormat.BGRA32,
#else
outputFormat = TextureFormat.ARGB32,
#endif
transformation = XRCpuImage.Transformation.None
};
unsafe
{
int size = image.GetConvertedDataSize(conversionParams);
var buffer = new NativeArray <byte>(size, Allocator.Temp);
image.Convert(conversionParams, new IntPtr(buffer.GetUnsafePtr()), buffer.Length);
image.Dispose();
byte[] frameImage = new byte[size];
buffer.CopyTo(frameImage);
#if UNITY_ANDROID && !UNITY_EDITOR
sbyte[] frameImageSigned = Array.ConvertAll(frameImage, b => unchecked ((sbyte)b));
multiHandMain_.Call("setFrame", frameImageSigned);
#endif
#if UNITY_IOS && !UNITY_EDITOR
IntPtr frameIntPtr = Marshal.AllocHGlobal(frameImage.Length * Marshal.SizeOf <byte>());
Marshal.Copy(frameImage, 0, frameIntPtr, frameImage.Length);
multiHandSetFrame(frameIntPtr, frameImage.Length);
Marshal.FreeHGlobal(frameIntPtr);
#endif
buffer.Dispose();
}
}
public static int CreateAsyncConversionRequest(
int nativeHandle, XRCpuImage.ConversionParams conversionParams)
{
throw new System.NotImplementedException(k_ExceptionMsg);
}
public static extern void UnityARCore_CpuImage_CreateAsyncConversionRequestWithCallback(
int nativeHandle, XRCpuImage.ConversionParams conversionParams,
XRCpuImage.Api.OnImageRequestCompleteDelegate callback, IntPtr context);
unsafe void UpdateCameraImage()
{
// Attempt to get the latest camera image. If this method succeeds,
// it acquires a native resource that must be disposed (see below).
if (!m_CameraManager.TryAcquireLatestCpuImage(out XRCpuImage image))
{
return;
}
// Choose an RGBA format.
// See XRCpuImage.FormatSupported for a complete list of supported formats.
var format = TextureFormat.RGBA32;
//Create new RenderTexture with correct width and height
if (m_CameraTexture == null || m_CameraTexture.width != image.width || m_CameraTexture.height != image.height)
{
m_CameraTexture = new Texture2D(image.width, image.height, format, false);
}
// Convert the image to format, flipping the image across the Y axis.
// We can also get a sub rectangle, but we'll get the full image here.
var conversionParams = new XRCpuImage.ConversionParams(image, format, XRCpuImage.Transformation.MirrorY);
// Texture2D allows us write directly to the raw texture data
// This allows us to do the conversion in-place without making any copies.
var rawTextureData = m_CameraTexture.GetRawTextureData <byte>();
try
{
image.Convert(conversionParams, new IntPtr(rawTextureData.GetUnsafePtr()), rawTextureData.Length);
}
finally
{
// We must dispose of the XRCpuImage after we're finished
// with it to avoid leaking native resources.
image.Dispose();
}
// Apply the updated texture data to our texture
m_CameraTexture.Apply();
// Set the RawImage's texture so we can visualize it.
if (Input.touchCount > 0 && Input.touches[0].phase == TouchPhase.Ended)
{
if (canSpawn)
{
canSpawn = false;
nextSpawn = Time.time + timeToWait;
cols = m_CameraTexture.GetPixels((m_CameraTexture.width / 2) - 80, (m_CameraTexture.height / 2) - 80, 160, 160);
Color col = Color.black;
for (int i = 0; i < cols.Length; i++)
{
col += cols[i];
}
col /= cols.Length;
ConvertColor(col);
}
}
}
/// <summary>
/// Requests the localization.
/// </summary>
public unsafe void RequestLocalization()
{
XRCameraIntrinsics intr;
ARCameraManager cameraManager = m_Sdk.cameraManager;
var cameraSubsystem = cameraManager.subsystem;
if (cameraSubsystem != null && cameraSubsystem.TryGetIntrinsics(out intr) && cameraManager.TryAcquireLatestCpuImage(out XRCpuImage image))
{
loaderText.text = "Localizing...";
loaderPanel.SetActive(true);
LocalizationRequest lr = new LocalizationRequest();
lr.cloud_Ids = cloudMaps;
lr.width = image.width;
lr.height = image.height;
lr.channel = 3;
lr.Camera_fx = intr.focalLength.x;
lr.Camera_fy = intr.focalLength.y;
lr.Camera_px = intr.principalPoint.x;
lr.Camera_py = intr.principalPoint.y;
lr.version = m_Sdk.arwaysdkversion;
Vector3 camPos = ARCamera.transform.position;
Quaternion camRot = ARCamera.transform.rotation;
var format = TextureFormat.RGB24;
if (m_Texture == null || m_Texture.width != image.width || m_Texture.height != image.height)
{
m_Texture = new Texture2D(image.width, image.height, format, false);
}
// Convert the image to format, flipping the image across the Y axis.
// We can also get a sub rectangle, but we'll get the full image here.
var conversionParams = new XRCpuImage.ConversionParams(image, format, XRCpuImage.Transformation.MirrorX);
// Texture2D allows us write directly to the raw texture data
// This allows us to do the conversion in-place without making any copies.
var rawTextureData = m_Texture.GetRawTextureData <byte>();
try
{
image.Convert(conversionParams, new IntPtr(rawTextureData.GetUnsafePtr()), rawTextureData.Length);
}
finally
{
// We must dispose of the XRCameraImage after we're finished
// with it to avoid leaking native resources.
image.Dispose();
}
// Apply the updated texture data to our texture
m_Texture.Apply();
byte[] _bytesjpg = m_Texture.EncodeToJPG();
lr.image = Convert.ToBase64String(_bytesjpg);
lr.timestamp = image.timestamp;
//show requeset counts..
loc_attempts_txt.GetComponent <TMP_Text>().enabled = true;
string output = JsonUtility.ToJson(lr);
StartCoroutine(sendCameraImages(output, camPos, camRot));
}
}
