/// <summary>
/// Processes the received frame, converts the image to grayscale if requested, and invokes the next photo request.
/// </summary>
private void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (_stopped?.Task != null)
{
return;
}
if (result.resultType == PhotoCapture.CaptureResultType.UnknownError)
{
return;
}
if (photoCaptureFrame == null)
{
return;
}
Size size = new Size(FrameWidth, (double)FrameHeight * 3 / 2); // Luminance (grayscale) of the NV12 format requires image height, chrominance is stored in half resolution. <see href="https://docs.microsoft.com/en-us/windows/win32/medfound/recommended-8-bit-yuv-formats-for-video-rendering#nv12"/>.
_image = new Mat(size, CvType.CV_8UC1);
List <byte> imageBuffer = new List <byte>();
photoCaptureFrame?.CopyRawImageDataIntoBuffer(imageBuffer);
MatUtils.copyToMat(imageBuffer.ToArray(), _image);
if (_format == ColorFormat.Grayscale)
{
Imgproc.cvtColor(_image, _image, Imgproc.COLOR_YUV2GRAY_NV12);
}
Matrix4x4 cameraToWorldMatrix = Matrix4x4.identity;
photoCaptureFrame?.TryGetCameraToWorldMatrix(out cameraToWorldMatrix);
CameraExtrinsic extrinsic = new CameraExtrinsic(cameraToWorldMatrix);
Matrix4x4 projectionMatrix = Matrix4x4.identity;
photoCaptureFrame?.TryGetProjectionMatrix(out projectionMatrix);
CameraIntrinsic intrinsic = new CameraIntrinsic(projectionMatrix);
CameraFrame cameraFrame = new CameraFrame(_image, intrinsic, extrinsic, FrameWidth, FrameHeight, FrameCount++, _format);
FrameArrivedEventArgs args = new FrameArrivedEventArgs(cameraFrame);
FrameArrived?.Invoke(this, args);
_photoCaptureObject?.TakePhotoAsync(OnCapturedPhotoToMemory);
}
/// <summary>
/// Invoked on each received video frame. Extracts the image according to the <see cref="ColorFormat"/> and invokes the <see cref="FrameArrived"/> event containing a <see cref="CameraFrame"/>.
/// </summary>
private unsafe void OnFrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args)
{
if (sender == null)
{
throw new ArgumentNullException(nameof(sender));
}
if (args == null)
{
throw new ArgumentNullException(nameof(args));
}
using (MediaFrameReference frame = sender.TryAcquireLatestFrame())
{
if (frame == null)
{
return;
}
SoftwareBitmap originalSoftwareBitmap = frame.VideoMediaFrame?.SoftwareBitmap;
if (originalSoftwareBitmap == null)
{
_logger.LogWarning("Received frame without image.");
return;
}
CameraExtrinsic extrinsic = new CameraExtrinsic(frame.CoordinateSystem, WorldOrigin);
CameraIntrinsic intrinsic = new CameraIntrinsic(frame.VideoMediaFrame.CameraIntrinsics);
using (var input = originalSoftwareBitmap.LockBuffer(BitmapBufferAccessMode.Read))
using (var inputReference = input.CreateReference())
{
byte *inputBytes;
uint inputCapacity;
((IMemoryBufferByteAccess)inputReference).GetBuffer(out inputBytes, out inputCapacity);
MatUtils.copyToMat((IntPtr)inputBytes, _bitmap);
int thisFrameCount = Interlocked.Increment(ref FrameCount);
// TODO: Check out of using block
CameraFrame cameraFrame = new CameraFrame(_bitmap, intrinsic, extrinsic, FrameWidth, FrameHeight, (uint)thisFrameCount, _format);
FrameArrivedEventArgs eventArgs = new FrameArrivedEventArgs(cameraFrame);
FrameArrived?.Invoke(this, eventArgs);
}
originalSoftwareBitmap?.Dispose();
}
}
public void Update()
{
if (!_available)
{
return;
}
if (!_cameraTexture.didUpdateThisFrame)
{
return;
}
Color32[] pixels32 = _cameraTexture.GetPixels32();
Utils.setDebugMode(true);
Mat argbMat = new Mat(_targetVideoHeight, _targetVideoWidth, CvType.CV_8UC4);
MatUtils.copyToMat(pixels32, argbMat);
if (argbMat.empty())
{
return;
}
// workaround obs cam: drop frame if grey / empty.
double[] values = argbMat.get(0, 0);
if (values[0] == 128 && values[1] == 129 && values[2] == 127 && values[3] == 255)
{
return;
}
Mat yuvMat = new Mat(_targetVideoHeight * 2 / 3, _targetVideoWidth, CvType.CV_8UC1);
Imgproc.cvtColor(argbMat, yuvMat, Imgproc.COLOR_BGRA2YUV_I420);
Mat submat = yuvMat.submat(0, _targetVideoHeight, 0, _targetVideoWidth);
Core.flip(submat, submat, 0);
Utils.setDebugMode(false);
CameraIntrinsic intrinsic = new CameraIntrinsic();
CameraExtrinsic extrinsic = new CameraExtrinsic(Matrix4x4.identity);
CameraFrame frame = new CameraFrame(submat, intrinsic, extrinsic, _targetVideoWidth, _targetVideoHeight, frameCount++, ColorFormat.Unknown);
FrameArrivedEventArgs args = new FrameArrivedEventArgs(frame);
FrameArrived?.Invoke(this, args);
}
public CameraFrame([NotNull] Mat mat, [NotNull] CameraIntrinsic intrinsic, [NotNull] CameraExtrinsic extrinsic, int width, int height, uint frameCount, ColorFormat format)
{
if (mat == null)
{
throw new ArgumentNullException(nameof(mat));
}
if (intrinsic == null)
{
throw new ArgumentNullException(nameof(intrinsic));
}
if (extrinsic == null)
{
throw new ArgumentNullException(nameof(extrinsic));
}
Mat = mat;
Intrinsic = intrinsic;
Extrinsic = extrinsic;
Width = width;
Height = height;
FrameCount = frameCount;
Format = format;
}
public CameraExtrinsic(CameraExtrinsic extrinsic)
{
viewFromWorld = extrinsic.viewFromWorld;
}