void OnFrameSampleAcquired(VideoCaptureSample sample)
{
//When copying the bytes out of the buffer, you must supply a byte[] that is appropriately sized.
//You can reuse this byte[] until you need to resize it (for whatever reason).
if (_latestImageBytes == null || _latestImageBytes.Length < sample.dataLength)
{
_latestImageBytes = new byte[sample.dataLength];
}
sample.CopyRawImageDataIntoBuffer(_latestImageBytes);
sample.Dispose();
_videoPanelUI.SetBytes(_latestImageBytes);
}
void OnFrameSampleAcquired(VideoCaptureSample sample)
{
//When copying the bytes out of the buffer, you must supply a byte[] that is appropriately sized.
//You can reuse this byte[] until you need to resize it (for whatever reason).
if (_latestImageBytes == null || _latestImageBytes.Length < sample.dataLength)
{
_latestImageBytes = new byte[sample.dataLength];
}
sample.CopyRawImageDataIntoBuffer(_latestImageBytes);
//If you need to get the cameraToWorld matrix for purposes of compositing you can do it like this
float[] cameraToWorldMatrix;
if (sample.TryGetCameraToWorldMatrix(out cameraToWorldMatrix) == false)
{
return;
}
//If you need to get the projection matrix for purposes of compositing you can do it like this
float[] projectionMatrix;
if (sample.TryGetProjectionMatrix(out projectionMatrix) == false)
{
return;
}
sample.Dispose();
//This is where we actually use the image data
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
_videoPanelUI.SetBytes(_latestImageBytes);
//text.text = _resolution.width + "x"+ _resolution.height+"+"+ _latestImageBytes.Length;
RGBUpdated = true;
}, false);
}
void OnFrameSampleAcquired(VideoCaptureSample sample)
{
//When copying the bytes out of the buffer, you must supply a byte[] that is appropriately sized.
//You can reuse this byte[] until you need to resize it (for whatever reason).
if (_latestImageBytes == null || _latestImageBytes.Length < sample.dataLength)
{
_latestImageBytes = new byte[sample.dataLength];
}
// edit the color of each frame
sample.CopyRawImageDataIntoBuffer(_latestImageBytes);
//If you need to get the cameraToWorld matrix for purposes of compositing you can do it like this
float[] cameraToWorldMatrix;
if (sample.TryGetCameraToWorldMatrix(out cameraToWorldMatrix) == false)
{
return;
}
//If you need to get the projection matrix for purposes of compositing you can do it like this
float[] projectionMatrix;
if (sample.TryGetProjectionMatrix(out projectionMatrix) == false)
{
return;
}
sample.Dispose();
//This is where we actually use the image data
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
int f = 10000;
int PixelSize = 4;
int width = _resolution.width;
int height = _resolution.height;
int Line = width * PixelSize;
// get normal image from mirror image
for (int i = 0; i < height; ++i)
{
for (int j = 0; j + 4 < Line / 2; j += 4)
{
Swap <byte>(ref _latestImageBytes[Line * i + j], ref _latestImageBytes[Line * i + Line - j - 4]);
Swap <byte>(ref _latestImageBytes[Line * i + j + 1], ref _latestImageBytes[Line * i + Line - j - 3]);
Swap <byte>(ref _latestImageBytes[Line * i + j + 2], ref _latestImageBytes[Line * i + Line - j - 2]);
Swap <byte>(ref _latestImageBytes[Line * i + j + 3], ref _latestImageBytes[Line * i + Line - j - 1]);
// reduce the alpha
//_latestImageBytes[Line * i + j + 3] = _latestImageBytes[Line * i + Line - j - 1] = 100;
}
}
_videoPanelUI.SetBytes(_latestImageBytes);
}, true);
}
// Everything above here is boilerplate from the VideoPanelApp.cs project
void OnFrameSampleAcquired(VideoCaptureSample sample)
{
//When copying the bytes out of the buffer, you must supply a byte[] that is appropriately sized.
//You can reuse this byte[] until you need to resize it (for whatever reason).
if (_latestImageBytes == null || _latestImageBytes.Length < sample.dataLength)
{
_latestImageBytes = new byte[sample.dataLength];
}
sample.CopyRawImageDataIntoBuffer(_latestImageBytes);
//If you need to get the cameraToWorld matrix for purposes of compositing you can do it like this
float[] cameraToWorldMatrixAsFloat;
if (sample.TryGetCameraToWorldMatrix(out cameraToWorldMatrixAsFloat) == false)
{
return;
}
//If you need to get the projection matrix for purposes of compositing you can do it like this
float[] projectionMatrixAsFloat;
if (sample.TryGetProjectionMatrix(out projectionMatrixAsFloat) == false)
{
return;
}
// Right now we pass things across the pipe as a float array then convert them back into UnityEngine.Matrix using a utility method
Matrix4x4 cameraToWorldMatrix = LocatableCameraUtils.ConvertFloatArrayToMatrix4x4(cameraToWorldMatrixAsFloat);
Matrix4x4 projectionMatrix = LocatableCameraUtils.ConvertFloatArrayToMatrix4x4(projectionMatrixAsFloat);
//This is where we actually use the image data
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
_videoPanelUI.SetBytes(_latestImageBytes);
Vector3 inverseNormal = LocatableCameraUtils.GetNormalOfPose(cameraToWorldMatrix);
Vector3 imagePosition = cameraToWorldMatrix.MultiplyPoint(Vector3.zero);
// Throw out an indicator in the composite space 2 meters in front of us using the corresponding view matrices
float distanceToMarker = 2f;
Vector3 pointOnFaceBoxPlane = imagePosition - inverseNormal * distanceToMarker;
Plane surfacePlane = new Plane(inverseNormal, pointOnFaceBoxPlane);
Vector2 targetPoint = new Vector2(_resolution.width * 0.5f, _resolution.height * 0.5f);
Vector3 mdPoint = LocatableCameraUtils.PixelCoordToWorldCoord(cameraToWorldMatrix, projectionMatrix, _resolution, targetPoint, surfacePlane);
_targetIndicator.SetPosition(mdPoint);
_targetIndicator.SetText("P");
_videoPanelUI.gameObject.SetActive(false);
}, false);
}
void OnFrameSampleAcquired(VideoCaptureSample sample)
{
//When copying the bytes out of the buffer, you must supply a byte[] that is appropriately sized.
//You can reuse this byte[] until you need to resize it (for whatever reason).
if (_latestImageBytes == null || _latestImageBytes.Length < sample.dataLength)
{
_latestImageBytes = new byte[sample.dataLength];
}
sample.CopyRawImageDataIntoBuffer(_latestImageBytes);
sample.Dispose();
//This is where we actually use the image data
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
_videoPanelUI.SetBytes(_latestImageBytes);
}, false);
}
void OnFrameSampleAcquired(VideoCaptureSample sample)
{
lock (_mainThreadActions)
{
if (_mainThreadActions.Count > 2)
{
return;
}
}
//When copying the bytes out of the buffer, you must supply a byte[] that is appropriately sized.
//You can reuse this byte[] until you need to resize it (for whatever reason).
if (_latestImageBytes == null || _latestImageBytes.Length < sample.dataLength)
{
_latestImageBytes = new byte[sample.dataLength];
}
sample.CopyRawImageDataIntoBuffer(_latestImageBytes);
//If you need to get the cameraToWorld matrix for purposes of compositing you can do it like this
float[] cameraToWorldMatrix;
if (sample.TryGetCameraToWorldMatrix(out cameraToWorldMatrix) == false)
{
return;
}
//If you need to get the projection matrix for purposes of compositing you can do it like this
float[] projectionMatrix;
if (sample.TryGetProjectionMatrix(out projectionMatrix) == false)
{
return;
}
sample.Dispose();
//Debug.Log("Got frame: " + sample.FrameWidth + "x" + sample.FrameHeight + " | " + sample.pixelFormat);
Enqueue(() => _videoPanelUI.SetBytes(_latestImageBytes));
}
// Update is called once per frame
void Update()
{
byte[] imgData = ProcessFrame(640, 480);
videoPanelUI.SetBytes(imgData);
FreeMemory();
}
void OnFrameSampleAcquired(VideoCaptureSample sample)
{
m_NumFrames++;
//When copying the bytes out of the buffer, you must supply a byte[] that is appropriately sized.
//You can reuse this byte[] until you need to resize it (for whatever reason).
if (_latestImageBytes == null || _latestImageBytes.Length < sample.dataLength)
{
_latestImageBytes = new byte[sample.dataLength];
}
sample.CopyRawImageDataIntoBuffer(_latestImageBytes);
//If you need to get the cameraToWorld matrix for purposes of compositing you can do it like this
float[] cameraToWorldMatrixAsFloat;
if (sample.TryGetCameraToWorldMatrix(out cameraToWorldMatrixAsFloat) == false)
{
return;
}
//If you need to get the projection matrix for purposes of compositing you can do it like this
float[] projectionMatrixAsFloat;
if (sample.TryGetProjectionMatrix(out projectionMatrixAsFloat) == false)
{
return;
}
// Right now we pass things across the pipe as a float array then convert them back into UnityEngine.Matrix using a utility method
Matrix4x4 cameraToWorldMatrix = LocatableCameraUtils.ConvertFloatArrayToMatrix4x4(cameraToWorldMatrixAsFloat);
Matrix4x4 projectionMatrix = LocatableCameraUtils.ConvertFloatArrayToMatrix4x4(projectionMatrixAsFloat);
//This is where we actually use the image data
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
int numFrames = m_NumFrames;
_videoPanelUI.SetBytes(_latestImageBytes);
Texture2D tex = _videoPanelUI.rawImage.texture as Texture2D;
byte[] jpgBytes = tex.EncodeToJPG();
/*
* Vector3 inverseNormal = LocatableCameraUtils.GetNormalOfPose(cameraToWorldMatrix);
* Vector3 imagePosition = cameraToWorldMatrix.MultiplyPoint(Vector3.zero);
*
* // Throw out an indicator in the composite space 2 meters in front of us using the corresponding view matrices
* float distanceToMarker = 2f;
* Vector3 pointOnFaceBoxPlane = imagePosition - inverseNormal * distanceToMarker;
* Plane surfacePlane = new Plane(inverseNormal, pointOnFaceBoxPlane);
*
* Vector2 targetPoint = new Vector2(_resolution.width * 0.5f, _resolution.height * 0.5f);
* Vector3 mdPoint = LocatableCameraUtils.PixelCoordToWorldCoord(cameraToWorldMatrix, projectionMatrix, _resolution, targetPoint, surfacePlane);
*/
//string infoText = String.Format("Position: {0}\t{1}\t{2}\nRotation: {3}\t{4}\t{5}", position.x, position.y, position.z, eulerAngles.x, eulerAngles.y, eulerAngles.z);
//Debug.Log(infoText);
float timestamp = Time.time - m_RecordingStartTime;
PoseData pose = new PoseData();
pose.timestamp = timestamp;
pose.position = cameraToWorldMatrix.MultiplyPoint(Vector3.zero);
Quaternion rotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
pose.eulerAngles = rotation.eulerAngles;
if (PlacingAxisMarkers && AxisPrefab != null)
{
Instantiate(AxisPrefab, pose.position, rotation);
}
m_CurrentPoses.Add(pose);
SaveFrame(jpgBytes, numFrames);
if (m_CurrentRecordingState == RecordingState.Recording)
{
this._videoCapture.RequestNextFrameSample(OnFrameSampleAcquired);
}
}, false);
}
void OnFrameSampleAcquired(VideoCaptureSample sample)
{
//When copying the bytes out of the buffer, you must supply a byte[] that is appropriately sized.
//You can reuse this byte[] until you need to resize it (for whatever reason).
if (_latestImageBytes == null || _latestImageBytes.Length < sample.dataLength)
{
_latestImageBytes = new byte[sample.dataLength];
}
sample.CopyRawImageDataIntoBuffer(_latestImageBytes);
//If you need to get the cameraToWorld matrix for purposes of compositing you can do it like this
float[] cameraToWorldMatrixAsFloat;
if (sample.TryGetCameraToWorldMatrix(out cameraToWorldMatrixAsFloat) == false)
{
return;
}
//If you need to get the projection matrix for purposes of compositing you can do it like this
float[] projectionMatrixAsFloat;
if (sample.TryGetProjectionMatrix(out projectionMatrixAsFloat) == false)
{
return;
}
// Right now we pass things across the pipe as a float array then convert them back into UnityEngine.Matrix using a utility method
Matrix4x4 cameraToWorldMatrix = LocatableCameraUtils.ConvertFloatArrayToMatrix4x4(cameraToWorldMatrixAsFloat);
Matrix4x4 projectionMatrix = LocatableCameraUtils.ConvertFloatArrayToMatrix4x4(projectionMatrixAsFloat);
//This is where we actually use the image data
UnityEngine.WSA.Application.InvokeOnAppThread(() =>
{
Vector3 cameraWorldPosition = cameraToWorldMatrix.MultiplyPoint(Vector3.zero);
Quaternion cameraWorldRotation = Quaternion.LookRotation(-cameraToWorldMatrix.GetColumn(2), cameraToWorldMatrix.GetColumn(1));
_videoPanelUI.SetBytes(_latestImageBytes);
Texture2D tex = _videoPanelUI.rawImage.texture as Texture2D;
Color32[] c = tex.GetPixels32();
IntPtr imageHandle = getImageHandle(c);
newImage(imageHandle);
// do any detection here...
int chessX = topDownCameraCalibrationData["chess_x"].AsInt;
int chessY = topDownCameraCalibrationData["chess_y"].AsInt;
float chessSquareMeters = topDownCameraCalibrationData["chess_square_size_meters"].AsFloat;
double cam_mtx_fx = HoloCam_fx;
double cam_mtx_fy = HoloCam_fy;
double cam_mtx_cx = HoloCam_cx;
double cam_mtx_cy = HoloCam_cy;
double dist_k1 = HoloCam_k1;
double dist_k2 = HoloCam_k2;
double dist_p1 = HoloCam_p1;
double dist_p2 = HoloCam_p2;
double dist_k3 = HoloCam_k3;
bool gotValidPose = findExtrinsics(chessX, chessY, chessSquareMeters, cam_mtx_fx, cam_mtx_fy, cam_mtx_cx, cam_mtx_cy, dist_k1, dist_k2, dist_p1, dist_p2, dist_k3);
if (gotValidPose)
{
Debug.Log("Got valid pose!");
List <double> rvec = new List <double>();
rvec.Add(GetRvec0());
rvec.Add(GetRvec1());
rvec.Add(GetRvec2());
List <double> tvec = new List <double>();
tvec.Add(GetTvec0());
tvec.Add(GetTvec1());
tvec.Add(GetTvec2());
Debug.Log("rvec between HoloLens camera and chessboard is " + rvec[0] + " " + rvec[1] + " " + rvec[2]);
Debug.Log("tvec between HoloLens camera and chessboard is " + tvec[0] + " " + tvec[1] + " " + tvec[2]);
// NOTE: At this point, we should STOP the HoloLens frame processing by:
// 1: setting _processingCameraFrames to FALSE
// 2: calling StopCameraProcessing()
// the data we have available is:
// --- rvec and tvec (the pose data between the HoloLens camera and the chessboard)
// --- topDownCameraCalibrationData ... the received data from the top-down camera app, which includes:
// ------ topDownCameraCalibrationData["rvec"] ... the pose between the top-down camera and the chessboard
// ------ topDownCameraCalibrationData["tvec"]
// ------ topDownCameraCalibrationData["fx"], fy, cx, cy (the top-down camera matrix)
// ------ topDownCameraCalibrationData["k1"], k2, p1, p2, k3 (the top-down camera distortion coefficients)
// --- cameraWorldPosition and cameraWorldRotation (the pose data in Unity's coord system between the HoloLens camera and the world)
// StopCameraProcessing();
// _processingCameraFrames = false;
}
//// Fetch the processed image and render
imageHandle = getProcessedImage();
Marshal.Copy(imageHandle, processedImageData, 0, _resolution.width * _resolution.height * 4);
tex.LoadRawTextureData(processedImageData);
tex.Apply();
if (_processingCameraFrames)
{
this._videoCapture.RequestNextFrameSample(OnFrameSampleAcquired);
}
}, false);
}