public TrackedObject([NotNull] Rect2d rect, [NotNull] string label, [NotNull] CameraIntrinsic intrinsic, [NotNull] CameraExtrinsic extrinsic, int frameHeigth)
{
if (rect == null)
{
throw new ArgumentNullException(nameof(rect));
}
if (label == null)
{
throw new ArgumentNullException(nameof(label));
}
if (intrinsic == null)
{
throw new ArgumentNullException(nameof(intrinsic));
}
if (extrinsic == null)
{
throw new ArgumentNullException(nameof(extrinsic));
}
if (frameHeigth == 0)
{
throw new ArgumentOutOfRangeException(nameof(frameHeigth));
}
Label = label;
Rect = rect;
Intrinsic = intrinsic;
Extrinsic = extrinsic;
FrameHeight = frameHeigth;
}
/// <summary>
/// Copy Constructor (shallow). On Windows UWP, values of the original camera intrinsics are used.
/// </summary>
public CameraIntrinsic([NotNull] CameraIntrinsic intrinsic)
{
if (intrinsic == null)
{
throw new ArgumentNullException(nameof(intrinsic));
}
#if ENABLE_WINMD_SUPPORT
WindowsCameraIntrinsics = intrinsic.WindowsCameraIntrinsics;
if (WindowsCameraIntrinsics == null)
{
throw new NullReferenceException(nameof(WindowsCameraIntrinsics));
}
FocalLength = new Vector2(WindowsCameraIntrinsics.FocalLength.X, WindowsCameraIntrinsics.FocalLength.Y);
ImageWidth = WindowsCameraIntrinsics.ImageWidth;
ImageHeight = WindowsCameraIntrinsics.ImageHeight;
PrincipalPoint = new Vector2(WindowsCameraIntrinsics.PrincipalPoint.X, WindowsCameraIntrinsics.PrincipalPoint.Y);
RadialDistortion = new Vector3(WindowsCameraIntrinsics.RadialDistortion.X, WindowsCameraIntrinsics.RadialDistortion.Y, WindowsCameraIntrinsics.RadialDistortion.Z);
TangentialDistortion = new Vector2(WindowsCameraIntrinsics.TangentialDistortion.X, WindowsCameraIntrinsics.TangentialDistortion.Y);
UndistortedProjectionTransform = WindowsCameraIntrinsics.UndistortedProjectionTransform.ToUnity();
#else
FocalLength = intrinsic.FocalLength;
ImageWidth = intrinsic.ImageWidth;
ImageHeight = intrinsic.ImageHeight;
PrincipalPoint = intrinsic.PrincipalPoint;
RadialDistortion = intrinsic.RadialDistortion;
TangentialDistortion = intrinsic.TangentialDistortion;
UndistortedProjectionTransform = intrinsic.UndistortedProjectionTransform;
#endif
}
public TrackedObject([NotNull] DetectedObject detectedObject)
{
if (detectedObject == null)
{
throw new ArgumentNullException(nameof(detectedObject));
}
Rect = detectedObject.Rect;
Label = detectedObject.Label;
CameraFrame frame = detectedObject.Frame;
Intrinsic = frame.Intrinsic;
Extrinsic = frame.Extrinsic;
FrameHeight = frame.Height;
}
public TrackedObject([NotNull] TrackedObject trackedObject)
{
if (trackedObject == null)
{
throw new ArgumentNullException(nameof(trackedObject));
}
if (trackedObject.FrameHeight == 0)
{
throw new ArgumentOutOfRangeException(nameof(trackedObject.FrameHeight));
}
Rect = trackedObject.Rect.clone();
Label = string.Copy(trackedObject.Label);
Intrinsic = new CameraIntrinsic(trackedObject.Intrinsic);
Extrinsic = new CameraExtrinsic(trackedObject.Extrinsic);
FrameHeight = trackedObject.FrameHeight;
}
/// <summary>
/// Processes the received frame, converts the image to grayscale if requested, and invokes the next photo request.
/// </summary>
private void OnCapturedPhotoToMemory(PhotoCapture.PhotoCaptureResult result, PhotoCaptureFrame photoCaptureFrame)
{
if (_stopped?.Task != null)
{
return;
}
if (result.resultType == PhotoCapture.CaptureResultType.UnknownError)
{
return;
}
if (photoCaptureFrame == null)
{
return;
}
Size size = new Size(FrameWidth, (double)FrameHeight * 3 / 2); // Luminance (grayscale) of the NV12 format requires image height, chrominance is stored in half resolution. <see href="https://docs.microsoft.com/en-us/windows/win32/medfound/recommended-8-bit-yuv-formats-for-video-rendering#nv12"/>.
_image = new Mat(size, CvType.CV_8UC1);
List <byte> imageBuffer = new List <byte>();
photoCaptureFrame?.CopyRawImageDataIntoBuffer(imageBuffer);
MatUtils.copyToMat(imageBuffer.ToArray(), _image);
if (_format == ColorFormat.Grayscale)
{
Imgproc.cvtColor(_image, _image, Imgproc.COLOR_YUV2GRAY_NV12);
}
Matrix4x4 cameraToWorldMatrix = Matrix4x4.identity;
photoCaptureFrame?.TryGetCameraToWorldMatrix(out cameraToWorldMatrix);
CameraExtrinsic extrinsic = new CameraExtrinsic(cameraToWorldMatrix);
Matrix4x4 projectionMatrix = Matrix4x4.identity;
photoCaptureFrame?.TryGetProjectionMatrix(out projectionMatrix);
CameraIntrinsic intrinsic = new CameraIntrinsic(projectionMatrix);
CameraFrame cameraFrame = new CameraFrame(_image, intrinsic, extrinsic, FrameWidth, FrameHeight, FrameCount++, _format);
FrameArrivedEventArgs args = new FrameArrivedEventArgs(cameraFrame);
FrameArrived?.Invoke(this, args);
_photoCaptureObject?.TakePhotoAsync(OnCapturedPhotoToMemory);
}
/// <summary>
/// Invoked on each received video frame. Extracts the image according to the <see cref="ColorFormat"/> and invokes the <see cref="FrameArrived"/> event containing a <see cref="CameraFrame"/>.
/// </summary>
private unsafe void OnFrameArrived(MediaFrameReader sender, MediaFrameArrivedEventArgs args)
{
if (sender == null)
{
throw new ArgumentNullException(nameof(sender));
}
if (args == null)
{
throw new ArgumentNullException(nameof(args));
}
using (MediaFrameReference frame = sender.TryAcquireLatestFrame())
{
if (frame == null)
{
return;
}
SoftwareBitmap originalSoftwareBitmap = frame.VideoMediaFrame?.SoftwareBitmap;
if (originalSoftwareBitmap == null)
{
_logger.LogWarning("Received frame without image.");
return;
}
CameraExtrinsic extrinsic = new CameraExtrinsic(frame.CoordinateSystem, WorldOrigin);
CameraIntrinsic intrinsic = new CameraIntrinsic(frame.VideoMediaFrame.CameraIntrinsics);
using (var input = originalSoftwareBitmap.LockBuffer(BitmapBufferAccessMode.Read))
using (var inputReference = input.CreateReference())
{
byte *inputBytes;
uint inputCapacity;
((IMemoryBufferByteAccess)inputReference).GetBuffer(out inputBytes, out inputCapacity);
MatUtils.copyToMat((IntPtr)inputBytes, _bitmap);
int thisFrameCount = Interlocked.Increment(ref FrameCount);
// TODO: Check out of using block
CameraFrame cameraFrame = new CameraFrame(_bitmap, intrinsic, extrinsic, FrameWidth, FrameHeight, (uint)thisFrameCount, _format);
FrameArrivedEventArgs eventArgs = new FrameArrivedEventArgs(cameraFrame);
FrameArrived?.Invoke(this, eventArgs);
}
originalSoftwareBitmap?.Dispose();
}
}
public void Update()
{
if (!_available)
{
return;
}
if (!_cameraTexture.didUpdateThisFrame)
{
return;
}
Color32[] pixels32 = _cameraTexture.GetPixels32();
Utils.setDebugMode(true);
Mat argbMat = new Mat(_targetVideoHeight, _targetVideoWidth, CvType.CV_8UC4);
MatUtils.copyToMat(pixels32, argbMat);
if (argbMat.empty())
{
return;
}
// workaround obs cam: drop frame if grey / empty.
double[] values = argbMat.get(0, 0);
if (values[0] == 128 && values[1] == 129 && values[2] == 127 && values[3] == 255)
{
return;
}
Mat yuvMat = new Mat(_targetVideoHeight * 2 / 3, _targetVideoWidth, CvType.CV_8UC1);
Imgproc.cvtColor(argbMat, yuvMat, Imgproc.COLOR_BGRA2YUV_I420);
Mat submat = yuvMat.submat(0, _targetVideoHeight, 0, _targetVideoWidth);
Core.flip(submat, submat, 0);
Utils.setDebugMode(false);
CameraIntrinsic intrinsic = new CameraIntrinsic();
CameraExtrinsic extrinsic = new CameraExtrinsic(Matrix4x4.identity);
CameraFrame frame = new CameraFrame(submat, intrinsic, extrinsic, _targetVideoWidth, _targetVideoHeight, frameCount++, ColorFormat.Unknown);
FrameArrivedEventArgs args = new FrameArrivedEventArgs(frame);
FrameArrived?.Invoke(this, args);
}
public CameraFrame([NotNull] Mat mat, [NotNull] CameraIntrinsic intrinsic, [NotNull] CameraExtrinsic extrinsic, int width, int height, uint frameCount, ColorFormat format)
{
if (mat == null)
{
throw new ArgumentNullException(nameof(mat));
}
if (intrinsic == null)
{
throw new ArgumentNullException(nameof(intrinsic));
}
if (extrinsic == null)
{
throw new ArgumentNullException(nameof(extrinsic));
}
Mat = mat;
Intrinsic = intrinsic;
Extrinsic = extrinsic;
Width = width;
Height = height;
FrameCount = frameCount;
Format = format;
}
