/// <summary>
/// 照片拍摄完成，获取拍摄的照片，调用Custom Vision API,对图片进行分析
/// </summary>
/// <param name="result">拍照的结果</param>
/// <param name="photoCaptureFrame">拍摄的图片</param>
private void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (result.success)
{
audioSource.Stop();
audioSource.clip = captureAudioClip;
audioSource.Play();
ModelManager.Instance.SetPhotoImageActive(true);
ModelManager.Instance.SetWaitingCanvas(false);
//ModelManager.Instance.SetTipText("正在处理中..."); //changed by yimei
ModelManager.Instance.SetTipText("正在处理中...\r\nIn processing...");
if (ConfigurationManager.Instance.GetMode() == CurrentMode.EdittingMode)
{
ToolManager.Instance.ShowMenu();
currentStatus = CurrentStatus.EdittingPhoto;
}
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
imageBufferList = FlipVertical(imageBufferList, cameraParameters.cameraResolutionWidth, cameraParameters.cameraResolutionHeight, 4);
Texture2D targetTexture = CreateTexture(imageBufferList, cameraParameters.cameraResolutionWidth, cameraParameters.cameraResolutionHeight);
Sprite sprite = Sprite.Create(targetTexture, new Rect(0, 0, targetTexture.width, targetTexture.height), new Vector2(0.5f, 0.5f));
ModelManager.Instance.SetPhotoImage(sprite);
if (ConfigurationManager.Instance.GetMode() == CurrentMode.SimpleMode)
{
StartCoroutine(PostToCustomVisionAPI(targetTexture));
}
else
{
ModelManager.Instance.PlayAnimation("ShowAnimation");
}
}
else
{
audioSource.Stop();
audioSource.clip = failedAudioClip;
audioSource.Play();
currentStatus = CurrentStatus.Ready;
//ModelManager.Instance.SetTipText("点击进行拍照"); //changed by yimei
ModelManager.Instance.SetTipText("点击进行拍照\r\nAirtap for taking picture");
ModelManager.Instance.PlayAnimation("IdleAnimation");
}
photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
}
void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (result.success)
{
List <byte> imageBufferList = new List <byte>();
// Copy the raw IMFMediaBuffer data into our empty byte list.
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
// In this example, we captured the image using the BGRA32 format.
// So our stride will be 4 since we have a byte for each rgba channel.
// The raw image data will also be flipped so we access our pixel data
// in the reverse order.
int stride = 4;
float denominator = 1.0f / 255.0f;
List <Color> colorArray = new List <Color>();
Debug.Log(imageBufferList.Count);
for (int i = imageBufferList.Count - 1; i - 3 >= 0; i -= stride)
{
float a = (int)(imageBufferList[i - 0]) * denominator;
float r = (int)(imageBufferList[i - 1]) * denominator;
float g = (int)(imageBufferList[i - 2]) * denominator;
float b = (int)(imageBufferList[i - 3]) * denominator;
colorArray.Add(new Color(r, g, b, a));
}
targetTexture_.SetPixels(colorArray.ToArray());
targetTexture_.Apply();
if (_debug)
{
if (quadRenderer_ == null)
{
GameObject p = GameObject.CreatePrimitive(PrimitiveType.Quad);
quadRenderer_ = p.GetComponent <Renderer>() as Renderer;
Debug.Log(quadRenderer_);
Debug.Log(Shader.Find("Unlit/Texture"));
quadRenderer_.material = new Material(Shader.Find("Unlit/Texture"));
p.transform.parent = this.transform;
p.transform.localPosition = new Vector3(0.0f, 0.0f, 1.0f);
}
quadRenderer_.material.SetTexture("_MainTex", targetTexture_);
}
}
photoCaptureObject_.StopPhotoModeAsync(OnStoppedPhotoMode);
// Take another photo
}
/// <summary>
/// 照相完毕后调用
/// </summary>
/// <param name="result">结果</param>
/// <param name="photoCaptureFrame">帧</param>
private void OnProcessFrame(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
Debug.Log("OnProcessFrame");
if (result.success)
{
List <byte> imageBufferList = new List <byte>();
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
photoCaptureFrame.TryGetCameraToWorldMatrix(out martrix_camera_to_world);
photoCaptureFrame.TryGetProjectionMatrix(out martrix_projection);
//photoCaptureFrame.Dispose();
#if WINDOWS_UWP
SendData(imageBufferList.ToArray());
//ShowHoloGrams = !ShowHoloGrams;
#endif
photo_capture.TakePhotoAsync(OnProcessFrame);
}
}
private void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (result.success)
{
// ComputerVision APIに送るimageBufferListにメモリ上の画像をコピーする
List <byte> imageBufferList = new List <byte>();
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
//ここはデバッグ用 送信画像の出力。どんな画像が取れたのか確認したい場合に使用。邪魔ならphotoPanelごと消してもよい。
Texture2D debugTexture = new Texture2D(100, 100);
debugTexture.LoadImage(imageBufferList.ToArray());
photoPanel.texture = debugTexture;
StartCoroutine(PostToVisionAPI(imageBufferList.ToArray()));
}
photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
}
public void Print(PhotoCaptureFrame captured)
{
if (mSetupped == false)
{
throw new System.Exception("Not setuped yet.");
}
List <byte> buf = new List <byte>();
captured.CopyRawImageDataIntoBuffer(buf);
var b = Serializer.Serialize(
new Pair <string, byte []>().Set("てすと", buf.ToArray()));
Print(b.ToArray());
}
//void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
//{
// if (result.success)
// {
// // Create our Texture2D for use and set the correct resolution
// Resolution cameraResolution = PhotoCapture.SupportedResolutions.OrderByDescending((res) => res.width * res.height).First();
// Texture2D targetTexture = new Texture2D(cameraResolution.width, cameraResolution.height);
// // Copy the raw image data into our target texture
// photoCaptureFrame.UploadImageDataToTexture(targetTexture);
// // Do as we wish with the texture such as apply it to a material, etc.
// _outputMaterial.mainTexture = targetTexture;
// }
// // Clean up
// photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
//}
void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (result.success)
{
List <byte> imageBufferList = new List <byte>();
// Copy the raw IMFMediaBuffer data into our empty byte list.
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
//_visionApiTest.QueryVisionApi("http://www.technewscentral.co.uk/wp-content/uploads/2015/07/sony-tablets1-hands2-lg.jpg"); //tablet
//_visionApiTest.QueryVisionApi("https://pixabay.com/static/uploads/photo/2012/04/01/18/38/television-23936_960_720.png"); //tv
//_visionApiTest.QueryVisionApi("https://i5.walmartimages.com/dfw/4ff9c6c9-9356/k2-_c59a878c-3d51-4807-aabd-84b0410de921.v1.jpg"); //phone
//_visionApiTest.QueryVisionApi("http://core0.staticworld.net/images/article/2015/02/hp-spectre-x360_beauty-100570598-orig.jpg"); //laptop
_visionApiTest.QueryVisionApi(imageBufferList.ToArray());
}
photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
}
void OnProcessFrame(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
UnityEngine.Debug.Log("++OnProcessFrame");
if (result.success)
{
if (!Const.LOAD_IMAGES)
{
List <byte> imageBufferList = new List <byte>();
// Copy the raw IMFMediaBuffer data into our empty byte list.
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
photoCaptureFrame.TryGetCameraToWorldMatrix(out _cameraToWorldMatrix);
photoCaptureFrame.TryGetProjectionMatrix(out _projectionMatrix);
//UnityEngine.Debug.Log(cameraToWorldMatrix);
photoCaptureFrame.Dispose();
_imageDataRaw = imageBufferList.ToArray();
_frameReadyFlag = true;
}
else
{
/*
* _indexImageFile = (int)(frameID % _imageFiles.LongCount());
* using (IRandomAccessStreamWithContentType stream = await _imageFiles[_indexImageFile].OpenReadAsync())
* {
* imageData = new byte[stream.Size];
* using (DataReader reader = new DataReader(stream))
* {
* await reader.LoadAsync((uint)stream.Size);
* reader.ReadBytes(imageData);
* }
* }
*/
}
}
if (Const.HOLO_CAPTURE)
{
_photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoModeHOLO);
}
else
{
_isCapturing = false;
}
}
private void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
Debug.Log(photoCaptureFrame.dataLength);
Debug.Log(photoCaptureFrame.hasLocationData);
var rgb = new List <byte>();
photoCaptureFrame.CopyRawImageDataIntoBuffer(rgb);
//var bytes = File.ReadAllBytes(@"C:\Users\asd14\Pictures\hololens.jpg");
//var stream = new MemoryStream(bytes);
var bytes = rgb.ToArray();
UploadFile(bytes, "https://hologate.tav.cc/image");
StartCoroutine("DoRequest");
}
private void OnPhotoCapturedToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame frame)
{
if (result.success)
{
Debug.Log("Captured image to memory. Sending data to requester.");
Matrix4x4 calibrationMatrix;
frame.TryGetProjectionMatrix(out calibrationMatrix);
List <byte> imageData = new List <byte>();
frame.CopyRawImageDataIntoBuffer(imageData);
_currentState.Value = State.READY_TO_TAKE_PHOTO;
_currentRequester.ReceiveTakenPictureAsBytes(imageData, _cameraParameters.cameraResolutionWidth, _cameraParameters.cameraResolutionHeight, calibrationMatrix);
}
else
{
Debug.LogError("Failed to capture image to memory.");
_currentState.Value = State.READY_TO_TAKE_PHOTO;
}
}
/// <summary>
/// Processes the received frame, converts the image to grayscale if requested, and invokes the next photo request.
/// </summary>
private void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (_stopped?.Task != null)
{
return;
}
if (result.resultType == PhotoCapture.CaptureResultType.UnknownError)
{
return;
}
if (photoCaptureFrame == null)
{
return;
}
Size size = new Size(FrameWidth, (double)FrameHeight * 3 / 2); // Luminance (grayscale) of the NV12 format requires image height, chrominance is stored in half resolution. <see href="https://docs.microsoft.com/en-us/windows/win32/medfound/recommended-8-bit-yuv-formats-for-video-rendering#nv12"/>.
_image = new Mat(size, CvType.CV_8UC1);
List <byte> imageBuffer = new List <byte>();
photoCaptureFrame?.CopyRawImageDataIntoBuffer(imageBuffer);
MatUtils.copyToMat(imageBuffer.ToArray(), _image);
if (_format == ColorFormat.Grayscale)
{
Imgproc.cvtColor(_image, _image, Imgproc.COLOR_YUV2GRAY_NV12);
}
Matrix4x4 cameraToWorldMatrix = Matrix4x4.identity;
photoCaptureFrame?.TryGetCameraToWorldMatrix(out cameraToWorldMatrix);
CameraExtrinsic extrinsic = new CameraExtrinsic(cameraToWorldMatrix);
Matrix4x4 projectionMatrix = Matrix4x4.identity;
photoCaptureFrame?.TryGetProjectionMatrix(out projectionMatrix);
CameraIntrinsic intrinsic = new CameraIntrinsic(projectionMatrix);
CameraFrame cameraFrame = new CameraFrame(_image, intrinsic, extrinsic, FrameWidth, FrameHeight, FrameCount++, _format);
FrameArrivedEventArgs args = new FrameArrivedEventArgs(cameraFrame);
FrameArrived?.Invoke(this, args);
_photoCaptureObject?.TakePhotoAsync(OnCapturedPhotoToMemory);
}
void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
var photoBuffer = new List <byte>();
if (photoCaptureFrame.pixelFormat == CapturePixelFormat.JPEG)
{
photoCaptureFrame.CopyRawImageDataIntoBuffer(photoBuffer);
}
else
{
photoBuffer = ConvertAndShowOnDebugPane(photoCaptureFrame);
}
Messenger.Instance.Broadcast(
new PhotoCaptureMessage(photoBuffer, _cameraResolution, CopyCameraTransForm()));
// Deactivate our camera
_photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
}
private void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
// Copy the raw image data into the target texture
lock (imageBuffer)
{
imageBuffer.Clear();
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBuffer);
byte[] vs = imageBuffer.ToArray();
tex.LoadRawTextureData(vs);
tex.Apply();
rawImage.texture = tex;
//IMPORTANT: Dispose the capture frame, or the app will crash after a while with access violation
photoCaptureFrame.Dispose();
firstScan = false;
}
// Deactivate the camera
photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
}
void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
// Copy the raw image data into the target texture
photoCaptureFrame.UploadImageDataToTexture(targetTexture);
photoCaptureFrame.CopyRawImageDataIntoBuffer(buffer);
Color32[] pix = targetTexture.GetPixels32();
int with = targetTexture.width;
int hight = targetTexture.height;
int index = MaxRedColor.MaxRedColors(pix);
int lines = (int)(index + 1) / 640;
int stakes = index - (lines * 640);
if (Variabless.First_x == -1)
{
Variabless.First_x = stakes;
Variabless.First_y = lines;
print(stakes);
}
else if (Variabless.Second_x == -1)
{
Variabless.Second_x = stakes;
Variabless.Second_y = lines;
print(stakes);
}
else if (Variabless.Third_x == -1)
{
Variabless.Third_x = stakes;
Variabless.Third_y = lines;
print(stakes);
}
else if (Variabless.Four_x == -1)
{
Variabless.Four_x = stakes;
Variabless.Four_y = lines;
print(stakes);
}
// Deactivate the camera
photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
}
void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (result.success)
{
DebugDisplay.Instance.Log("OnCapturedPhotoToMemory Copy Started ");
imageBufferList = new List <byte>();
// Copy the raw IMFMediaBuffer data into our empty byte list.
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
DebugDisplay.Instance.Log("OnCapturedPhotoToMemory " + imageBufferList.Count);
}
else
{
DebugDisplay.Instance.Log("Failed to save Photo to memory");
photoReadyCallBack(false, imageBufferList);
}
photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
}
void OnPhotoCapturedCopyToBytes(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
Matrix4x4 cameraToWorldMatrix;
photoCaptureFrame.TryGetCameraToWorldMatrix(out cameraToWorldMatrix);
Matrix4x4 projectionMatrix;
photoCaptureFrame.TryGetProjectionMatrix(out projectionMatrix);
List <byte> capturedImg = new List <byte>();
photoCaptureFrame.CopyRawImageDataIntoBuffer(capturedImg);
photoCaptureFrame.Dispose();
_takePhotoActionCopyToBytes?.Invoke(cameraToWorldMatrix, projectionMatrix, capturedImg, _cameraParameters.cameraResolutionHeight, _cameraParameters.cameraResolutionWidth);
CanTakePhoto = false;
_takePhotoActionCopyToBytes = null;
}
void OnCapturePhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
Matrix4x4 cameraToWorldMatrix;
List <byte> buffer = new List <byte>();
photoCaptureFrame.CopyRawImageDataIntoBuffer(buffer);
//Check if we can receive the position where the photo was taken
if (!photoCaptureFrame.TryGetCameraToWorldMatrix(out cameraToWorldMatrix))
{
cameraToWorldMatrix = Matrix4x4.identity;
}
if (photoReady != null)
{
photoReady(id, buffer.ToArray(), cameraToWorldMatrix, cameraResolution);
}
// stop the photo mode
photoCapture.StopPhotoModeAsync(OnPhotoModeStopped);
}
void OnCapturedPhotoToMemoryAsync(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (result.success)
{
//将photoCaptureFrame转为List<byte>,再转为byte[].
imageBufferList = new List <byte>();
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
//将拍摄内容保存到imageBufferArray中
imageBufferArray = imageBufferList.ToArray();
photoCaptureFrame.TryGetCameraToWorldMatrix(out cameraToWorldMatrix);
worldToCameraMatrix = cameraToWorldMatrix.inverse;
photoCaptureFrame.TryGetProjectionMatrix(out projectionMatrix);
UnityEngine.Debug.LogFormat(@"The value of cameraToWorld Matrix: {0}{1}{2}{3} ", cameraToWorldMatrix.GetRow(0), cameraToWorldMatrix.GetRow(1), cameraToWorldMatrix.GetRow(2), cameraToWorldMatrix.GetRow(3));
UnityEngine.Debug.Log("Captured Photo To Memory Succeed! ");
}
photoCaptureObj.StopPhotoModeAsync(OnStoppedPhotoMode);
}
private void onCapturedPhotoToMemoryCallback(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (!result.success)
{
Debug.LogError("Failed to take photo");
onPhotoCaptured.OnNext(null);
photoCapture.StopPhotoModeAsync(onPhotoModeStoppedCallback);
return;
}
var buffer = new List <byte>();
photoCaptureFrame.CopyRawImageDataIntoBuffer(buffer);
// カメラの向きをワールド座標に変換するためのパラメータ保持
Matrix4x4 cameraToWorldMatrix;
photoCaptureFrame.TryGetCameraToWorldMatrix(out cameraToWorldMatrix);
//var cameraRotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
Matrix4x4 projectionMatrix;
photoCaptureFrame.TryGetProjectionMatrix(Camera.main.nearClipPlane, Camera.main.farClipPlane, out projectionMatrix);
var pixelToCameraMatrix = projectionMatrix.inverse;
Plan = new ShootingPlan
{
Resolution = Resolution,
ShootingLocation = Camera.main.transform.position,
CameraToWorld = cameraToWorldMatrix,
PixelToCamera = pixelToCameraMatrix,
};
photoCapture.StopPhotoModeAsync(onPhotoModeStoppedCallback);
var value = buffer.ToArray();
onPhotoCaptured.OnNext(value);
}
void OnPhotoCaptured(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
//Matrix4x4 cameraToWorldMatrix;
//photoCaptureFrame.TryGetCameraToWorldMatrix(out cameraToWorldMatrix);
//Matrix4x4 worldToCameraMatrix = cameraToWorldMatrix.inverse;
//Matrix4x4 projectionMatrix;
//photoCaptureFrame.TryGetProjectionMatrix(out projectionMatrix);
//var texture = new Texture2D(this.cameraParameters.cameraResolutionWidth, this.cameraParameters.cameraResolutionHeight, TextureFormat.ARGB32, false);
//photoCaptureFrame.UploadImageDataToTexture(texture);
List <byte> byteses = new List <byte>();
photoCaptureFrame.CopyRawImageDataIntoBuffer(byteses);
TakeImageAction?.Invoke(byteses);
//texture.wrapMode = TextureWrapMode.Clamp;
photoCaptureFrame.Dispose();
//texture.Compress(true);//ここでの圧縮はDXTフォーマットに圧縮するということ。
Resources.UnloadUnusedAssets();
isCapturingPhoto = false;
}
void Publish_to_ROS(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (result.success)
{
memory = photoCaptureFrame;
//画像をコピーする
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
byte[] ROS_data = new byte[imageBufferList.Count];
Resolution cameraResolution = PhotoCapture.SupportedResolutions.OrderByDescending((res) => res.width * res.height).First();
Texture2D targetTexture = new Texture2D(cameraResolution.width, cameraResolution.height);
photoCaptureFrame.UploadImageDataToTexture(targetTexture);
ROS_data = targetTexture.EncodeToJPG(qualityLevel);
GameObject.Destroy(targetTexture);
Publish_to_ROS(ROS_data);
}
}
void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (result.success)
{
List <byte> imageBufferList = new List <byte>();
Debug.Log("OnCapturedPhotoToMemory Copy Started");
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
Debug.Log("OnCapturedPhotoToMemory " + imageBufferList.Count);
//Execute OCR Coroutine
ExecuteMCSComputerVisionOCR(imageBufferList, "ocr");
}
else
{
Debug.Log("Failed to save Photo to memory");
}
photoCapture.StopPhotoModeAsync(OnStoppedPhotoMode);
}
void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (result.success)
{
List <byte> imageBufferList = new List <byte>();
// Copy the raw IMFMediaBuffer data into our empty byte list.
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
// In this example, we captured the image using the BGRA32 format.
// So our stride will be 4 since we have a byte for each rgba channel.
// The raw image data will also be flipped so we access our pixel data
// in the reverse order.
int stride = 4;
float denominator = 1.0f / 255.0f;
List <Color> colorArray = new List <Color>();
for (int i = imageBufferList.Count - 1; i >= 0; i -= stride)
{
float a = (int)(imageBufferList[i - 0]) * denominator;
float r = (int)(imageBufferList[i - 1]) * denominator;
float g = (int)(imageBufferList[i - 2]) * denominator;
float b = (int)(imageBufferList[i - 3]) * denominator;
colorArray.Add(new Color(r, g, b, a));
}
// Now we could do something with the array such as texture.SetPixels() or run image processing on the list
//Debug.Log(String.Format("imageBufferList={0}", imageBufferList.Count));
CreateMesh(ref colorArray, photoCaptureFrame);
// save ?
if (CaptureCount % 1 == 0)
{
SaveFile(PointAll, ColorAll);
Debug.Log(String.Format("SaveFile={0}", CaptureCount));
}
}
photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
}
void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (result.success)
{
List <byte> imageBufferList = new List <byte>();
// Copy the raw IMFMediaBuffer data into our empty byte list.
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
var cameraToWorldMatrix = new Matrix4x4();
photoCaptureFrame.TryGetCameraToWorldMatrix(out cameraToWorldMatrix);
cameraPosition = cameraToWorldMatrix.MultiplyPoint3x4(new Vector3(0, 0, -1));
cameraRotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
Matrix4x4 projectionMatrix;
photoCaptureFrame.TryGetProjectionMatrix(Camera.main.nearClipPlane, Camera.main.farClipPlane, out projectionMatrix);
Matrix4x4 pixelToCameraMatrix = projectionMatrix.inverse;
status.GetComponent <TextMesh>().text = "Processing";
StartCoroutine(PostToFaceAPI(imageBufferList.ToArray(), cameraToWorldMatrix, pixelToCameraMatrix));
}
photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
}
void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
List <byte> imageBufferList = new List <byte>();
byte[] imageArray;
if (photoCaptureFrame.pixelFormat == CapturePixelFormat.JPEG)
{
// Copy the raw IMFMediaBuffer data into our empty byte list.
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
imageArray = imageBufferList.ToArray();
}
else
{
// Copy the raw image data into our target texture
imageArray = ConvertAndShowOnDebugPane(photoCaptureFrame);
}
string data = System.Convert.ToBase64String(imageArray);
string pictureID = System.Guid.NewGuid().ToString();
Log("sending picture via MQTT");
Log("ID: " + pictureID);
mqttHelper.Publish(outboundTopic, "{\"ID\":\"" + pictureID + "\",\"image\":\"" + data + "\"}");
}
void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (result.success)
{
Debug.Log("Success. Changing things now!");
List <byte> imageBufferList = new List <byte>();
// Copy the raw IMFMediaBuffer data into our empty byte list.
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
// In this example, we captured the image using the BGRA32 format.
// So our stride will be 4 since we have a byte for each rgba channel.
// The raw image data will also be flipped so we access our pixel data
// in the reverse order.
int stride = 4;
float denominator = 1.0f / 255.0f;
List <Color> colorArray = new List <Color>();
for (int i = imageBufferList.Count - 1; i >= 0; i -= stride)
{
float a = (int)(imageBufferList[i - 0]) * denominator;
float r = (int)(imageBufferList[i - 1]) * denominator;
float g = (int)(imageBufferList[i - 2]) * denominator;
float b = (int)(imageBufferList[i - 3]) * denominator;
colorArray.Add(new Color(r, g, b, a));
}
// Now we could do something with the array such as texture.SetPixels() or run image processing on the list
Debug.Log("Color Array" + r.ToString());
int counter = 0;
r = 0; g = 0; b = 0;
for (int w = (width / 2) - 50; w < ((width) / 2) + 50; w++)
{
for (int hi = (height / 2) - 50; hi < ((height) / 2) + 50; hi++)
{
int index = w * height + hi;
counter++;
r += colorArray[index].r;
g += colorArray[index].g;
b += colorArray[index].b;
}
}
r /= counter;
g /= counter;
b /= counter;
// Now we could do something with the array such as texture.SetPixels() or run image processing on the list
//r /= counter;
//g /= counter;
//b /= counter;
Color rgbColor;
rgbColor = new Color(r, g, b, 1);
//Color.RGBToHSV(rgbColor, out h, out s, out v);
//Color32[] pix = targetTexture.GetPixels32();
//for (int i = 0; i < pix.Length; i++)
//{
// pix[i].r = (byte)(r * 255);
// pix[i].g = (byte)(g * 255);
// pix[i].b = (byte)(b * 255);
//}
Debug.Log("Current RGB = (" + r.ToString() + ", " + g.ToString() + ", " + b.ToString() + ")");
//targetTexture.SetPixels32(pix);
//targetTexture.Apply();
//GetComponent<MeshRenderer>().material.mainTexture = targetTexture;
}
else
{
Debug.Log("Failed!! and F****d ");
}
photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
}
void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
#if !UNITY_EDITOR
if (result.success)
{
List <byte> imageBufferList = new List <byte>();
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
Resolution cameraResolution = PhotoCapture.SupportedResolutions.OrderByDescending((res) => res.width * res.height).First();
int imageWidth = cameraResolution.width;
int imageHeight = cameraResolution.height;
ZXing.BarcodeReader qrReader = new ZXing.BarcodeReader();
var qrResult = qrReader.Decode(imageBufferList.ToArray(), imageWidth, imageHeight, ZXing.BitmapFormat.RGBA32);
if (qrReader == null)
{
Debug.Log("error: BarcodeReader.Decode");
return;
}
Debug.Log(qrResult.Text);
Matrix4x4 projectionMat;
if (!photoCaptureFrame.TryGetProjectionMatrix(out projectionMat))
{
Debug.Log("error: PhotoCaptureFrame.TryGetProjectionMatrix");
return;
}
Matrix4x4 cameraToWorldMat;
if (!photoCaptureFrame.TryGetCameraToWorldMatrix(out cameraToWorldMat))
{
Debug.Log("error: PhotoCaptureFrame.TryGetCameraToWorldMatrix");
return;
}
if (qrResult.ResultPoints.Length < 3)
{
Debug.Log("error: too few ResultPoints");
return;
}
Vector3[] points = new Vector3[3];
for (int i = 0; i < 3; ++i)
{
var pixelPos = new Vector2(qrResult.ResultPoints[i].X, qrResult.ResultPoints[i].Y);
var imagePosZeroToOne = new Vector2(pixelPos.x / imageWidth, 1 - (pixelPos.y / imageHeight));
var imagePosProjected = (imagePosZeroToOne * 2) - new Vector2(1, 1); // -1 to 1 space
var cameraSpacePos = UnProjectVector(projectionMat, new Vector3(imagePosProjected.x, imagePosProjected.y, 1));
var worldSpaceRayPoint1 = cameraToWorldMat.MultiplyPoint(Vector3.zero); // camera location in world space
var worldSpaceRayPoint2 = cameraToWorldMat.MultiplyPoint(cameraSpacePos); // ray point in world space
RaycastHit hit;
if (!Physics.Raycast(worldSpaceRayPoint1, worldSpaceRayPoint2 - worldSpaceRayPoint1, out hit, 5, 1 << 31))
{
Debug.Log("error: Physics.Raycast failed");
return;
}
points[i] = hit.point;
}
var worldTopLeft = points[1];
var worldTopRight = points[2];
var worldBottomLeft = points[0];
var bottomToTop = worldTopLeft - worldBottomLeft;
var leftToRight = worldTopRight - worldTopLeft;
qrcodePlane.transform.forward = bottomToTop;
qrcodePlane.transform.position = worldBottomLeft + (bottomToTop + leftToRight) * 0.5f;
plane.transform.localScale = new Vector3(leftToRight.magnitude, 1, bottomToTop.magnitude);
//photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
}
#endif
}
private bool OnCaptureCompleted(PhotoCaptureFrame frame)
{
if (this.isProcessingImage)
{
return(true);
}
this.isProcessingImage = true;
Matrix4x4 cameraToWorldMatrix;
frame.TryGetCameraToWorldMatrix(out cameraToWorldMatrix);
cameraToWorldMatrix = cameraToWorldMatrix * Matrix4x4.Scale(new Vector3(1, 1, -1));
// Update the preview image with the last image processed
UpdatePreviewTexture(cameraImageBuffer);
var res = PVCamManager.Instance.PhotoCapCamResolution;
var w = res.width;
var h = res.height;
if (posterLocationHandler == null)
{
posterLocationHandler = new PosterLocationHandler(w, h);
}
Matrix4x4 cameraClipToWorld;
Vector3 cameraPos;
var camProj = TuneProjectionMatrix(frame);
GetCameraClipTransformAndWorldPosition(cameraToWorldMatrix, camProj,
out cameraClipToWorld, out cameraPos);
posterLocationHandler.UpdateCamera(cameraToWorldMatrix, camProj, cameraPos);
DoOnSeperateThread(() =>
{
// Allocate the buffers to hold the image data
if ((cameraImageListBuffer == null) || (cameraImageListBuffer.Count != frame.dataLength))
{
cameraImageListBuffer = new List <byte>(frame.dataLength);
}
if ((cameraImageBuffer == null) || (cameraImageBuffer.Length != frame.dataLength))
{
cameraImageBuffer = new byte[frame.dataLength];
cameraImageGCHandle = GCHandle.Alloc(cameraImageBuffer, GCHandleType.Pinned);
cameraImagePtrToData = cameraImageGCHandle.AddrOfPinnedObject();
}
frame.CopyRawImageDataIntoBuffer(cameraImageListBuffer);
// Copy the image data from the list into the byte array
cameraImageListBuffer.CopyTo(cameraImageBuffer);
PVCamManager.Instance.SignalFinishedWithFrame();
// Find the poster:
this.FindPoster(w, h);
this.isProcessingImage = false;
});
return(false);
}
public CaptureFrameInstance(PhotoCaptureFrame managedFrame)
{
// build up image struct, copy it to unmanaged memory
image_u8 temp = new image_u8();
temp.width = WebcamSystem.instance.cameraResolution.width;
temp.height = WebcamSystem.instance.cameraResolution.height;
// TASK: implement stride optimization in cases where shape is unoptimized
// https://docs.microsoft.com/en-us/windows/win32/medfound/image-stride
temp.stride = WebcamSystem.instance.cameraResolution.width;
temp.buf = Marshal.AllocHGlobal(temp.height * temp.stride);
_bufPtr = temp.buf; // stored for easy deallocation later
// Obtain managed BGRA image bytes, allocate space for
// one-fourth-sized grayscale image bytes (transformedImage)
List <byte> managedBuffer = new List <byte>();
managedFrame.CopyRawImageDataIntoBuffer(managedBuffer);
byte[] transformedImg = new byte[temp.width * temp.height];
// Do the transformation from BGRA to grayscale, using hardware acceleration if possible
int kernelHandle = WebcamSystem.instance.BGRAtoGrayscaleShader.FindKernel("ProcessImage");
uint groupSize;
WebcamSystem.instance.BGRAtoGrayscaleShader.GetKernelThreadGroupSizes(kernelHandle, out groupSize, out _, out _);
if ((temp.width * temp.height) % groupSize == 0) // TASK: implement stride optimization
{
lock (bufLock)
{
if (_inputBuffer == null)
{
_inputBuffer = new ComputeBuffer(temp.width * temp.height, sizeof(uint)); // bgra values; 4 bytes per pixel
}
if (_outputBuffer == null)
{
_outputBuffer = new ComputeBuffer(temp.width * temp.height / 4, sizeof(uint)); // grayscale values; 1 byte per pixel
}
_inputBuffer.SetData(managedBuffer.ToArray());
WebcamSystem.instance.BGRAtoGrayscaleShader.SetBuffer(kernelHandle, "In", _inputBuffer);
WebcamSystem.instance.BGRAtoGrayscaleShader.SetBuffer(kernelHandle, "Out", _outputBuffer);
// each group processes groupSize "clumps" of four pixels
int threadGroupCount = temp.width * temp.height / (int)groupSize / 4;
WebcamSystem.instance.BGRAtoGrayscaleShader.Dispatch(kernelHandle, threadGroupCount, 1, 1);
_outputBuffer.GetData(transformedImg);
}
}
else
{
Debug.LogError("Unusual resolution used-- cannot hardware accelerate! Defaulting to iterative approach...");
ProcessImage(managedBuffer.ToArray(), temp.width, temp.height, transformedImg);
}
// Copy the processed image to unmanaged memory
Marshal.Copy(transformedImg, 0, temp.buf, transformedImg.Length);
// Allocate the unmanaged image struct
unmanagedFrame = Marshal.AllocHGlobal(Marshal.SizeOf(temp));
Marshal.StructureToPtr <image_u8>(temp, unmanagedFrame, false);
}
void OnCapturedPhotoToMemoryBYTE(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (result.success)
{
List<byte> imageBufferList = new List<byte>();
// Copy the raw IMFMediaBuffer data into our empty byte list.
photoCaptureFrame.CopyRawImageDataIntoBuffer(imageBufferList);
// In this example, we captured the image using the BGRA32 format.
// So our stride will be 4 since we have a byte for each rgba channel.
// The raw image data will also be flipped so we access our pixel data
// in the reverse order.
int stride = 4;
float denominator = 1.0f / 255.0f;
List<Color> colorArray = new List<Color>();
for (int i = imageBufferList.Count - 1; i >= 0; i -= stride)
{
float a = (int)(imageBufferList[i - 0]) * denominator;
float r = (int)(imageBufferList[i - 1]) * denominator;
float g = (int)(imageBufferList[i - 2]) * denominator;
float b = (int)(imageBufferList[i - 3]) * denominator;
colorArray.Add(new Color(r, g, b, a));
}
colorArray.Reverse();
// Converting the List<byte> to a bute array
byte[] imageBufferArray = imageBufferList.ToArray();
// Now we could do something with the array such as texture.SetPixels() or run image processing on the list
//#if !UNITY_EDITOR
// create a barcode reader instance
BarcodeReader barcodeReader = new BarcodeReader { AutoRotate = true };
// Here use the Decoder class to decode the Qr code
//Decoder.DecodeFromByteArray(imageBufferArray, cameraWidth, cameraHeight);
try
{
// Locatable camera from the HoloLens use BGRA32 format in the MRCManager class
Result TextResult = barcodeReader.Decode(imageBufferArray,w,h, RGBLuminanceSource.BitmapFormat.BGRA32);
//If QR code detected
if (TextResult != null)
{
Debug.Log("Result decoding: " + TextResult.Text);
Textd.text = "Result decoding: " + TextResult.Text;
//
// Do what you want with the result here
//
}
//If QR code not detected
else
{
Textd.text = "No QR code detected";
Debug.Log("No QR code detected");
}
camTexture.Play();
}
//If error while decoding
catch(System.Exception e)
{
Debug.Log("Exception:"+e);
}
//#endif
}
photoCaptureObject.StopPhotoModeAsync(OnStoppedPhotoMode);
}