private Color GetColourAt(CameraImageBytes cim, int cx, int cy, out bool foundColour)
{
Color colour = new Color(0, 0, 0);
foundColour = false;
if ((cx >= 0) && (cy >= 0) && (cx < cim.Width) && (cy < cim.Height))
{
byte[] buf = new byte[1];
byte y;
byte u;
byte v;
Marshal.Copy(new IntPtr(cim.Y.ToInt64() + cy * cim.YRowStride + cx * 1), buf, 0, 1);
y = buf[0];
//UV planes are at quarter resolution, make sure to keep in mind when editing values
Marshal.Copy(new IntPtr(cim.U.ToInt64() + (cy / 2) * cim.UVRowStride + (cx / 2) * cim.UVPixelStride), buf, 0, 1);
u = buf[0];
Marshal.Copy(new IntPtr(cim.V.ToInt64() + (cy / 2) * cim.UVRowStride + (cx / 2) * cim.UVPixelStride), buf, 0, 1);
v = buf[0];
colour = YUV2RGB(y, u, v);
foundColour = true;
}
return(colour);
}
void OnImageAvailableC(int width, int height, CameraImageBytes image, int bufferSize)
{
byte[] bufferY = new byte[bufferSize];
byte[] bufferU = new byte[bufferSize];
byte[] bufferV = new byte[bufferSize];
Marshal.Copy(image.Y, bufferY, 0, bufferSize);
Marshal.Copy(image.U, bufferU, 0, bufferSize);
Marshal.Copy(image.V, bufferV, 0, bufferSize);
int bufferSizeYUV = width * height * 3 / 2;
byte[] bufferYUV = new byte[bufferSizeYUV];
Marshal.Copy(image.Y, bufferYUV, 0, bufferSizeYUV);
Texture2D m_TextureRender = new Texture2D(width, height, TextureFormat.RGBA32, false, false);
Color c = new Color();
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
float Y = bufferY[y * width + x];
float U = bufferU[y * width + x];
float V = bufferV[y * width + x];
/*
* c.r = Y; // + 1.4075f * (V - 128);
* c.g = 0f; //Y - 0.3455f * (U - 128) - (0.7169f * (V - 128));
* c.b = 0f; // Y + 1.7790f * (U - 128);*/
/*float Yvalue = bufferYUV[y * width + x];
* float Uvalue = bufferYUV[(y / 2) * (width / 2) + x / 2 + (width * height)];
* float Vvalue = bufferYUV[(y / 2) * (width / 2) + x / 2 + (width * height) + (width * height) / 4];*/
c.r = Y + (float)(1.370705 * (V - 128.0f));
c.g = Y - (float)(0.698001 * (V - 128.0f)) - (float)(0.337633 * (U - 128.0f));
c.b = Y + (float)(1.732446 * (U - 128.0f));
c.r /= 255.0f;
c.g /= 255.0f;
c.b /= 255.0f;
MinMaxColor(ref c);
c.a = 1.0f;
m_TextureRender.SetPixel(width - 1 - x, y, c);
}
}
string fn = System.DateTime.Now.ToString("yyyy-MM-dd-HH-mm-ss") + "_photo_bw.jpg";
string path = Application.persistentDataPath + "/" + fn;
Utils.Toast("Image " + width + "x" + height + " " + fn);
File.WriteAllBytes(path, m_TextureRender.EncodeToJPG());
}
/// <summary>
/// This is the standard Unity function called at every frame, in this particular case it acquire an image if neccesary, and enqueque inside
/// </summary>
public void Update()
{
if (nextFrameId == long.MinValue)
{
return;
}
// Method 1 (some problem on Xiaomi, resolve on 1.8 version)
using (CameraImageBytes image = Frame.CameraImage.AcquireCameraImageBytes())
{
if (!image.IsAvailable)
{
return;
}
Debug.Log("Acquired frame nextFrameId: " + nextFrameId);
byte[] yBuffer = new byte[image.Width * image.Height];
byte[] vuBuffer = new byte[image.Width * (image.Height / 2)];
Marshal.Copy(image.Y, yBuffer, 0, yBuffer.Length);
Marshal.Copy(image.V, vuBuffer, 0, vuBuffer.Length);
Vector3 position = trsmatrix.MultiplyPoint3x4(Frame.Pose.position);
Quaternion rotation = Frame.Pose.rotation * Quaternion.LookRotation(
trsmatrix.GetColumn(2), trsmatrix.GetColumn(1));
Quaternion rotationNoEdit = new Quaternion(rotation.x, rotation.y, rotation.z, rotation.w);
Vector3 positionOri = cameraTransform.position;
Quaternion rotationOri = cameraTransform.rotation;
FrameNode frame = new FrameNode
{
frameId = nextFrameId,
imageHeight = image.Height,
imageWidth = image.Width,
focalLength = Frame.CameraImage.ImageIntrinsics.FocalLength,
principalPoint = Frame.CameraImage.ImageIntrinsics.PrincipalPoint,
position = position,
positionOri = positionOri,
rotation = rotation,
rotationOri = rotationOri,
vuBuffer = vuBuffer,
yBuffer = yBuffer
};
sendingSocket.EnquequeFrame(ref frame);
frameAcquired.Add(nextFrameId);
nextFrameId = long.MinValue;
}
}
/* public Texture2D CameraToTexture() {
* // Create the object for the result - this has to be done before the
* // using {} clause.
* Texture2D result;
*
* // Use using to make sure that C# disposes of the CameraImageBytes afterwards
* using (CameraImageBytes camBytes = Frame.CameraImage.AcquireCameraImageBytes()) {
*
* // If acquiring failed, return null
* if (!camBytes.IsAvailable) {
* Debug.LogWarning("camBytes not available");
* return null;
* }
*
* // To save a YUV_420_888 image, you need 1.5*pixelCount bytes.
* // I will explain later, why.
*
* byte[] YUVimage = new byte[(int)(camBytes.Width * camBytes.Height * 1.5f)];
*
* // As CameraImageBytes keep the Y, U and V data in three separate
* // arrays, we need to put them in a single array. This is done using
* // native pointers, which are considered unsafe in C#.
* unsafe {
* for (int i = 0; i < camBytes.Width * camBytes.Height; i++) {
* YUVimage[i] = *((byte*)camBytes.Y.ToPointer() + (i * sizeof(byte)));
* }
*
* for (int i = 0; i < camBytes.Width * camBytes.Height / 4; i++) {
* YUVimage[(camBytes.Width * camBytes.Height) + 2 * i] = *((byte*)camBytes.U.ToPointer() + (i * camBytes.UVPixelStride * sizeof(byte)));
* YUVimage[(camBytes.Width * camBytes.Height) + 2 * i + 1] = *((byte*)camBytes.V.ToPointer() + (i * camBytes.UVPixelStride * sizeof(byte)));
* }
* }
*
* // Create the output byte array. RGB is three channels, therefore
* // we need 3 times the pixel count
* byte[] RGBimage = new byte[camBytes.Width * camBytes.Height * 3];
*
* // GCHandles help us "pin" the arrays in the memory, so that we can
* // pass them to the C++ code.
* GCHandle YUVhandle = GCHandle.Alloc(YUVimage, GCHandleType.Pinned);
* GCHandle RGBhandle = GCHandle.Alloc(RGBimage, GCHandleType.Pinned);
*
* // Call the C++ function that we created.
* int k = ConvertYUV2RGBA(YUVhandle.AddrOfPinnedObject(), RGBhandle.AddrOfPinnedObject(), camBytes.Width, camBytes.Height);
*
* // If OpenCV conversion failed, return null
* if (k != 0) {
* Debug.LogWarning("Color conversion - k != 0");
* return null;
* }
*
* // Create a new texture object
* result = new Texture2D(camBytes.Width, camBytes.Height, TextureFormat.RGB24, false);
*
* // Load the RGB array to the texture, send it to GPU
* result.LoadRawTextureData(RGBimage);
* result.Apply();
*
* // Save the texture as an PNG file. End the using {} clause to
* // dispose of the CameraImageBytes.
* File.WriteAllBytes(Application.persistentDataPath + "/tex.png", result.EncodeToPNG());
* }
*
* // Return the texture.
* return result;
* }*/
void takePhotoGrayscale640x480()
{
Utils.Log(_CameraIntrinsicsToString(Frame.CameraImage.ImageIntrinsics, "ff"));
using (CameraImageBytes image = Frame.CameraImage.AcquireCameraImageBytes()) {
if (!image.IsAvailable)
{
Utils.Toast("not available");
return;
}
OnImageAvailableBW(image.Width, image.Height, image.YRowStride, image.Y, image.Width * image.Height); //works but sometime camera not availble and only in B&W
//OnImageAvailableC(image.Width, image.Height, image, image.Width * image.Height);
}
void OnImageAvailableBW(int width, int height, int rowStride, IntPtr pixelBuffer, int bufferSize)
{
byte[] bufferY = new byte[bufferSize];
System.Runtime.InteropServices.Marshal.Copy(pixelBuffer, bufferY, 0, bufferSize);
Texture2D m_TextureRender = new Texture2D(width, height, TextureFormat.RGBA32, false, false);
Color c = new Color();
for (int y = 0; y < height; y++)
{
for (int x = 0; x < width; x++)
{
float Y = bufferY[y * width + x];
c.r = Y;
c.g = Y;
c.b = Y;
c.r /= 255.0f;
c.g /= 255.0f;
c.b /= 255.0f;
MinMaxColor(ref c);
c.a = 1.0f;
m_TextureRender.SetPixel(width - 1 - x, y, c);
}
}
string fn = System.DateTime.Now.ToString("yyyy-MM-dd-HH-mm-ss") + "_photo_bw.jpg";
string path = Application.persistentDataPath + "/" + fn;
Utils.Toast("Image " + width + "x" + height + " " + fn);
File.WriteAllBytes(path, m_TextureRender.EncodeToJPG());
}
}
/// <summary>
/// Captures a YUV420888 image from the devices CPU, using ARCores CameraImageBytes class
/// Image is then converted to RGB using OpenCV+ A free plugin based on OpenCV Sharp
/// Image is then rotated and flipped to align with the viewport.
/// Adds images to a List of Mat Arrays, which can be converted to JPG using Unity Textures at a later date
/// NOTE: No Unity classes are used in this function to ensure that it is thread safe (Unity itself is not threadsafe).
/// </summary>
public static void GetCameraImage()
{
// Use using to make sure that C# disposes of the CameraImageBytes afterwards
using (CameraImageBytes camBytes = Frame.CameraImage.AcquireCameraImageBytes())
{
// If acquiring failed, return null
if (!camBytes.IsAvailable)
{
return;
}
// To save a YUV_420_888 image, you need 1.5*pixelCount bytes.
byte[] YUVimage = new byte[(int)(camBytes.Width * camBytes.Height * 1.5f)];
// As CameraImageBytes keep the Y, U and V data in three separate
// arrays, we need to put them in a single array. This is done using
// native pointers, which are considered unsafe in C#.
unsafe
{
for (int i = 0; i < camBytes.Width * camBytes.Height; i++)
{
YUVimage[i] = *((byte *)camBytes.Y.ToPointer() + (i * sizeof(byte)));
}
for (int i = 0; i < camBytes.Width * camBytes.Height / 4; i++)
{
YUVimage[(camBytes.Width * camBytes.Height) + 2 * i] = *((byte *)camBytes.U.ToPointer() + (i * camBytes.UVPixelStride * sizeof(byte)));
YUVimage[(camBytes.Width * camBytes.Height) + 2 * i + 1] = *((byte *)camBytes.V.ToPointer() + (i * camBytes.UVPixelStride * sizeof(byte)));
}
}
// GCHandles help us "pin" the arrays in the memory, so that we can
// pass them to the C++ code.
GCHandle pinnedArray = GCHandle.Alloc(YUVimage, GCHandleType.Pinned);
IntPtr pointerYUV = pinnedArray.AddrOfPinnedObject();
Mat input = new Mat(camBytes.Height + camBytes.Height / 2, camBytes.Width, MatType.CV_8UC1, pointerYUV);
Mat output = new Mat(camBytes.Height, camBytes.Width, MatType.CV_8UC3);
Cv2.CvtColor(input, output, ColorConversionCodes.YUV2BGR_NV12);
// FLIP AND TRANPOSE TO VERTICAL
Cv2.Transpose(output, output);
Cv2.Flip(output, output, FlipMode.Y);
Mesh3DController.AllData.Add(output);
pinnedArray.Free();
Mesh3DController.ErrorString = "Thread Sucess!";
}
}
public static void GetCameraImage()
{
Texture2D result;
// Use using to make sure that C# disposes of the CameraImageBytes afterwards
using (CameraImageBytes camBytes = Frame.CameraImage.AcquireCameraImageBytes())
{
// If acquiring failed, return null
if (!camBytes.IsAvailable)
{
Debug.LogWarning("camBytes not available");
return;
}
// To save a YUV_420_888 image, you need 1.5*pixelCount bytes.
// I will explain later, why.
byte[] YUVimage = new byte[(int)(camBytes.Width * camBytes.Height * 1.5f)];
// As CameraImageBytes keep the Y, U and V data in three separate
// arrays, we need to put them in a single array. This is done using
// native pointers, which are considered unsafe in C#.
unsafe
{
for (int i = 0; i < camBytes.Width * camBytes.Height; i++)
{
YUVimage[i] = *((byte*)camBytes.Y.ToPointer() + (i * sizeof(byte)));
}
for (int i = 0; i < camBytes.Width * camBytes.Height / 4; i++)
{
YUVimage[(camBytes.Width * camBytes.Height) + 2 * i] = *((byte*)camBytes.U.ToPointer() + (i * camBytes.UVPixelStride * sizeof(byte)));
YUVimage[(camBytes.Width * camBytes.Height) + 2 * i + 1] = *((byte*)camBytes.V.ToPointer() + (i * camBytes.UVPixelStride * sizeof(byte)));
}
}
// Create the output byte array. RGB is three channels, therefore
// we need 3 times the pixel count
byte[] RGBimage = new byte[camBytes.Width * camBytes.Height * 3];
// GCHandles help us "pin" the arrays in the memory, so that we can
// pass them to the C++ code.
GCHandle pinnedArray = GCHandle.Alloc(YUVimage, GCHandleType.Pinned);
IntPtr pointer = pinnedArray.AddrOfPinnedObject();
Mat input = new Mat(camBytes.Height + camBytes.Height / 2, camBytes.Width, MatType.CV_8UC1, pointer);
Mat output = new Mat(camBytes.Height, camBytes.Width, MatType.CV_8UC3);
Cv2.CvtColor(input, output, ColorConversionCodes.YUV2BGR_NV12 );// YUV2RGB_NV12);
Cv2.Transpose(output, output);
Cv2.Flip(output, output, FlipMode.Y);
result = Unity.MatToTexture(output);
result.Apply();
byte[] im = result.EncodeToJPG(100);
AllData.Add(im);
Destroy(result);
pinnedArray.Free();
}
}
public override Texture2D captureCurrentFrame()
{
Texture2D texture = null;
using (CameraImageBytes image = Frame.CameraImage.AcquireCameraImageBytes())
{
texture = new Texture2D(image.Width, image.Height, TextureFormat.RGBA32, false, false);
byte[] imageBinary = new byte[image.Width * image.Height * 4];
int bufferSize = image.YRowStride * image.Height;
System.Runtime.InteropServices.Marshal.Copy(image.Y, imageBinary, 0, bufferSize);
texture.LoadRawTextureData(imageBinary);
texture.Apply();
}
return(texture);
}
// Update is called once per frame
void Update()
{
if (Frame.PointCloud.IsUpdatedThisFrame)
{
if (logger != null)
{
logger.text = "Have Points";
}
if (AddVoxels)
{
CameraImageBytes cim = Frame.CameraImage.AcquireCameraImageBytes();
for (int i = 0; i < Frame.PointCloud.PointCount; i++)
{
Color colour = new Color(0, 0, 0);
bool foundColour = false;
PointCloudPoint p = Frame.PointCloud.GetPointAsStruct(i);
Vector3 cameraCoordinates = arCamera.WorldToViewportPoint(p);
if (cim.IsAvailable)
{
var uvQuad = Frame.CameraImage.DisplayUvCoords;
int cx = (int)(cameraCoordinates.x * cim.Width);
int cy = (int)((1.0f - cameraCoordinates.y) * cim.Height);
colour = GetColourAt(cim, cx, cy, out foundColour);
}
if (foundColour)
{
tree.addPoint(p, colour);
}
}
cim.Release();
}
tree.renderOctTree(voxelParent);
}
else
{
if (logger != null)
{
logger.text = "No Points";
}
}
}
/**
* Sends a snapshot of the current camera image and convert it into a JPG image
* to be processed by a OCR service.
* The response can arrive at any time. The default timeout time is 10 seconds
* The average response time from Google Cloud Vision is 1 second
*/
private void InitiateOCRDetection()
{
image = Frame.CameraImage.AcquireCameraImageBytes();
// Detection needs to continue until enough cpu resources have been freed
// If it fails, the user has to start the whole process again
if (!image.IsAvailable)
{
_SystemStatePresenter.DisplayUserMessage("Couldn't access camera image! Please try again.");
}
/**
* The camera image is split into its brightness(Y) channel and the meta data needed for calculations
* This data is needed to create a Texture2D that can be converted into a JPG
*/
else
{
_TextDetection.DetectText(image.Width, image.Height, image.Y, image.YRowStride);
}
image.Release();
}
/// <summary>
/// The Unity Update() method.
/// </summary>
public void Update()
{
if (Input.GetKey(KeyCode.Escape))
{
Application.Quit();
}
_QuitOnConnectionErrors();
_UpdateFrameRate();
// Change the CPU resolution checkbox visibility.
LowResConfigToggle.gameObject.SetActive(EdgeDetectionBackgroundImage.enabled);
HighResConfigToggle.gameObject.SetActive(EdgeDetectionBackgroundImage.enabled);
m_ImageTextureToggleText.text = EdgeDetectionBackgroundImage.enabled ?
"Switch to GPU Texture" : "Switch to CPU Image";
if (!Session.Status.IsValid())
{
return;
}
using (CameraImageBytes image = Frame.CameraImage.AcquireCameraImageBytes())
{
if (!image.IsAvailable)
{
return;
}
_OnImageAvailable(image.Width, image.Height, image.YRowStride, image.Y, 0);
}
var cameraIntrinsics = EdgeDetectionBackgroundImage.enabled
? Frame.CameraImage.ImageIntrinsics : Frame.CameraImage.TextureIntrinsics;
string intrinsicsType =
EdgeDetectionBackgroundImage.enabled ? "CPU Image" : "GPU Texture";
CameraIntrinsicsOutput.text =
_CameraIntrinsicsToString(cameraIntrinsics, intrinsicsType);
}
private string takeSnapshot2()
{
string fn = System.DateTime.Now.ToString("yyyy-MM-dd-HH-mm-ss") + "_screenshot.jpg";
string pathSnap = Application.persistentDataPath + "/" + fn;
//ScreenCapture.CaptureScreenshot(fn);
Utils.Toast("Photo " + pathSnap);
CameraImageBytes image = Frame.CameraImage.AcquireCameraImageBytes(); //using
int width = image.Width;
int height = image.Height;
byte[] m_EdgeImage = new byte[image.Width * image.Height * 4];
//System.Runtime.InteropServices.Marshal.Copy(image.Y, m_EdgeImage, 0, image.Width * image.Height);
Texture2D tx = new Texture2D(width, height, TextureFormat.R8, false, false);
byte[] rgbaBuffer = new byte[width * height * 4];
System.Runtime.InteropServices.Marshal.Copy(image.Y, rgbaBuffer, 0, width * height * 4);
// Copy the red channel to the grayscale image buffer, since the shader only reads the red channel.
for (int i = 0; i < width * height; i++)
{
m_EdgeImage[i] = rgbaBuffer[i * 4 + 0];
}
tx.LoadRawTextureData(m_EdgeImage);
tx.Apply();
var encodedJpg = tx.EncodeToJPG();
File.WriteAllBytes(pathSnap, encodedJpg);
return(fn);
}
private static IEnumerator UpdateLoop(string adbPath)
{
Debug.Log("Entering UpdateLoop");
var renderEventFunc = NativeApi.GetRenderEventFunc();
var shouldConvertToBgra = SystemInfo.graphicsDeviceType == GraphicsDeviceType.Direct3D11;
var loggedAspectRatioWarning = false;
// Waits until the end of the first frame until capturing the screen size,
// because it might be incorrect when first querying it.
yield return(k_WaitForEndOfFrame);
var currentWidth = 0;
var currentHeight = 0;
var needToStartActivity = true;
var prevFrameLandscape = false;
RenderTexture screenTexture = null;
RenderTexture targetTexture = null;
RenderTexture bgrTexture = null;
// Begins update loop. The coroutine will cease when the
// ARCoreSession component it's called from is destroyed.
for (; ;)
{
yield return(k_WaitForEndOfFrame);
var curFrameLandscape = Screen.width > Screen.height;
if (prevFrameLandscape != curFrameLandscape)
{
needToStartActivity = true;
}
prevFrameLandscape = curFrameLandscape;
if (needToStartActivity)
{
Debug.Log("Need to start InstantPreview activity");
string activityName = curFrameLandscape ? "InstantPreviewLandscapeActivity" :
"InstantPreviewActivity";
string output;
string errors;
ShellHelper.RunCommand(adbPath,
"shell am start -S -n com.google.ar.core.instantpreview/." + activityName,
out output, out errors);
needToStartActivity = false;
}
// Creates a target texture to capture the preview window onto.
// Some video encoders prefer the dimensions to be a multiple of 16.
var targetWidth = RoundUpToNearestMultipleOf16(Screen.width);
var targetHeight = RoundUpToNearestMultipleOf16(Screen.height);
if (targetWidth != currentWidth || targetHeight != currentHeight)
{
screenTexture = new RenderTexture(targetWidth, targetHeight, 0);
targetTexture = screenTexture;
if (shouldConvertToBgra)
{
bgrTexture = new RenderTexture(screenTexture.width, screenTexture.height, 0, RenderTextureFormat.BGRA32);
targetTexture = bgrTexture;
}
currentWidth = targetWidth;
currentHeight = targetHeight;
Debug.Log("Created target texture is was needed");
}
Debug.Log("Native Api update");
NativeApi.Update();
Debug.Log("InstantPreviewInput update");
InstantPreviewInput.Update();
AddInstantPreviewTrackedPoseDriverWhenNeeded();
Graphics.Blit(null, screenTexture);
if (shouldConvertToBgra)
{
Graphics.Blit(screenTexture, bgrTexture);
}
var cameraTexture = Frame.CameraImage.Texture;
if (cameraTexture == null)
{
Debug.Log("cameraTexture is null");
Debug.Log("Trying to acquire camera image bytes.");
CameraImageBytes cameraBytes = Frame.CameraImage.AcquireCameraImageBytes();
if (cameraBytes.IsAvailable)
{
Debug.Log("Acquire cpu was successful. Retrying to assign cameraTexture");
cameraTexture = Frame.CameraImage.Texture;
if (cameraTexture == null)
{
Debug.Log("cameraTexture STILL NULL");
}
}
else
{
Debug.Log("Acquire failed");
}
}
if (!loggedAspectRatioWarning && cameraTexture != null)
{
var sourceWidth = cameraTexture.width;
var sourceHeight = cameraTexture.height;
var sourceAspectRatio = (float)sourceWidth / sourceHeight;
var destinationWidth = Screen.width;
var destinationHeight = Screen.height;
var destinationAspectRatio = (float)destinationWidth / destinationHeight;
if (Mathf.Abs(sourceAspectRatio - destinationAspectRatio) >
k_MaxTolerableAspectRatioDifference)
{
Debug.LogWarningFormat(k_MismatchedAspectRatioWarningFormatString, sourceWidth,
sourceHeight);
loggedAspectRatioWarning = true;
}
}
NativeApi.SendFrame(targetTexture.GetNativeTexturePtr());
GL.IssuePluginEvent(renderEventFunc, 1);
}
}
