public static CVPixelBuffer ToCVPixelBuffer(this UIImage self)
{
var attrs = new CVPixelBufferAttributes();
attrs.CGImageCompatibility = true;
attrs.CGBitmapContextCompatibility = true;
var cgImg = self.CGImage;
var pb = new CVPixelBuffer(cgImg.Width, cgImg.Height, CVPixelFormatType.CV32ARGB, attrs);
pb.Lock(CVPixelBufferLock.None);
var pData = pb.BaseAddress;
var colorSpace = CGColorSpace.CreateDeviceRGB();
var ctxt = new CGBitmapContext(pData, cgImg.Width, cgImg.Height, 8, pb.BytesPerRow, colorSpace, CGImageAlphaInfo.NoneSkipFirst);
ctxt.TranslateCTM(0, cgImg.Height);
ctxt.ScaleCTM(1.0f, -1.0f);
UIGraphics.PushContext(ctxt);
self.Draw(new CGRect(0, 0, cgImg.Width, cgImg.Height));
UIGraphics.PopContext();
pb.Unlock(CVPixelBufferLock.None);
return(pb);
}
public static NSImage ToNSImage(this CMSampleBuffer sampleBuffer)
{
NSImage image;
using (CVPixelBuffer pixelBuffer = sampleBuffer.GetImageBuffer() as CVPixelBuffer)
{
// Lock the base address
pixelBuffer.Lock(CVPixelBufferLock.ReadOnly);
using (CIImage cIImage = new CIImage(pixelBuffer))
{
image = null;
AutoResetEvent e = new AutoResetEvent(false);
Xamarin.Forms.Device.BeginInvokeOnMainThread(delegate
{
NSCIImageRep rep = new NSCIImageRep(cIImage);
image = new NSImage(rep.Size);
image.AddRepresentation(rep);
e.Set();
});
e.WaitOne();
}
pixelBuffer.Unlock(CVPixelBufferLock.ReadOnly);
}
return(image);
}
//private static MCvFont font = new MCvFont(Emgu.CV.CvEnum.FONT.CV_FONT_HERSHEY_PLAIN, 1.0, 1.0);
UIImage ImageFromSampleBuffer(CMSampleBuffer sampleBuffer)
{
// Get the CoreVideo image
using (CVPixelBuffer pixelBuffer = sampleBuffer.GetImageBuffer() as CVPixelBuffer)
{
// Lock the base address
pixelBuffer.Lock(CVPixelBufferLock.ReadOnly);
// Get the number of bytes per row for the pixel buffer
IntPtr baseAddress = pixelBuffer.BaseAddress;
int bytesPerRow = (int)pixelBuffer.BytesPerRow;
int width = (int)pixelBuffer.Width;
int height = (int)pixelBuffer.Height;
using (Image <Bgra, byte> bgra = new Image <Bgra, byte>(width, height, bytesPerRow, baseAddress))
using (Image <Bgr, byte> bgr = bgra.Convert <Bgr, byte>())
using (Image <Bgr, byte> bgr2 = bgr.Rotate(90, new Bgr(0, 0, 0)))
{
bgr2.Draw(
string.Format("{0} x {1}", width, height),
new Point(20, 20),
FontFace.HersheySimplex,
1.0,
new Bgr(255, 0, 0));
//CvInvoke.cvCvtColor(bgr2, bgra, Emgu.CV.CvEnum.COLOR_CONVERSION.CV_BGR2BGRA);
UIImage result = bgr2.ToUIImage();
pixelBuffer.Unlock(CVPixelBufferLock.ReadOnly);
return(result);
}
}
}
public void ToCGImageTest()
{
TestRuntime.AssertSystemVersion(PlatformName.iOS, 9, 0, throwIfOtherPlatform: false);
TestRuntime.AssertSystemVersion(PlatformName.MacOSX, 10, 11, throwIfOtherPlatform: false);
TestRuntime.AssertSystemVersion(PlatformName.TvOS, 10, 2, throwIfOtherPlatform: false);
#if MONOMAC
var originalImage = new NSImage(NSBundle.MainBundle.PathForResource("Xam", "png", "CoreImage"));
#else
var originalImage = UIImage.FromBundle("CoreImage/Xam.png");
#endif
var originalCGImage = originalImage.CGImage;
var pxbuffer = new CVPixelBuffer(originalCGImage.Width, originalCGImage.Height, CVPixelFormatType.CV32ARGB,
new CVPixelBufferAttributes {
CGImageCompatibility = true, CGBitmapContextCompatibility = true
});
#if !XAMCORE_3_0
pxbuffer.Lock(CVOptionFlags.None);
#else
pxbuffer.Lock(CVPixelBufferLock.None);
#endif
using (var colorSpace = CGColorSpace.CreateDeviceRGB())
using (var ctx = new CGBitmapContext(pxbuffer.BaseAddress, originalCGImage.Width, originalCGImage.Height, 8,
4 * originalCGImage.Width, colorSpace, CGBitmapFlags.NoneSkipLast)) {
ctx.RotateCTM(0);
ctx.DrawImage(new CGRect(0, 0, originalCGImage.Width, originalCGImage.Height), originalCGImage);
#if !XAMCORE_3_0
pxbuffer.Unlock(CVOptionFlags.None);
#else
pxbuffer.Unlock(CVPixelBufferLock.None);
#endif
}
Assert.NotNull(pxbuffer, "VTUtilitiesTests.ToCGImageTest pxbuffer should not be null");
CGImage newImage;
var newImageStatus = pxbuffer.ToCGImage(out newImage);
Assert.That(newImageStatus == VTStatus.Ok, "VTUtilitiesTests.ToCGImageTest must be ok");
Assert.NotNull(newImage, "VTUtilitiesTests.ToCGImageTest pxbuffer should not be newImage");
Assert.AreEqual(originalCGImage.Width, newImage.Width, "VTUtilitiesTests.ToCGImageTest");
Assert.AreEqual(originalCGImage.Height, newImage.Height, "VTUtilitiesTests.ToCGImageTest");
var retainCount = CFGetRetainCount(newImage.Handle);
Assert.That(retainCount, Is.EqualTo(1), "RetainCount");
}
NSImage ImageFromSampleBuffer(CMSampleBuffer sampleBuffer)
#endif
{
#if __IOS__
UIImage image;
#else
NSImage image;
#endif
Random r = new Random();
int flag = r.Next();
// Get the CoreVideo image
using (CVPixelBuffer pixelBuffer = sampleBuffer.GetImageBuffer() as CVPixelBuffer)
{
// Lock the base address
pixelBuffer.Lock(CVPixelBufferLock.ReadOnly);
using (CIImage cIImage = new CIImage(pixelBuffer))
{
#if __IOS__
image = new UIImage(cIImage);
#else
image = null;
AutoResetEvent e = new AutoResetEvent(false);
//e.WaitOne();
//Semaphore s = new Semaphore(1, 1);
//s.WaitOne();
BeginInvokeOnMainThread(delegate
{
NSCIImageRep rep = new NSCIImageRep(cIImage);
Debug.WriteLine(String.Format("({2}) NSCIImageRep: {0}x{1}", rep.Size.Width, rep.Size.Height, flag));
image = new NSImage(rep.Size);
image.AddRepresentation(rep);
//s.Release();
//Monitor.Exit(this);
Debug.WriteLine(String.Format("({2}) NSImage: {0}x{1}", image.Size.Width, image.Size.Height, flag));
e.Set();
});
e.WaitOne();
//Monitor.Enter(this);
//Monitor.Exit(this);
//s.WaitOne();
//s.Release();
#endif
}
pixelBuffer.Unlock(CVPixelBufferLock.ReadOnly);
}
Debug.WriteLine(String.Format("({2}) Received NSImage: {0}x{1}", image.Size.Width, image.Size.Height, flag));
return(image);
}
public static UIImage ToUIImage(this CMSampleBuffer sampleBuffer)
{
UIImage image;
using (CVPixelBuffer pixelBuffer = sampleBuffer.GetImageBuffer() as CVPixelBuffer)
{
// Lock the base address
pixelBuffer.Lock(CVPixelBufferLock.ReadOnly);
using (CIImage cIImage = new CIImage(pixelBuffer))
{
image = new UIImage(cIImage);
}
pixelBuffer.Unlock(CVPixelBufferLock.ReadOnly);
}
//Debug.WriteLine(String.Format("({2}) Received NSImage: {0}x{1}", image.Size.Width, image.Size.Height, flag));
return(image);
}
private byte[] CVPixelBufferToByte(CVPixelBuffer pixelBuffer)
{
pixelBuffer.Lock(CVPixelBufferLock.None);
CVPixelFormatType ft = pixelBuffer.PixelFormatType;
var baseAddress = pixelBuffer.BaseAddress;
var bytesPerRow = (int)pixelBuffer.BytesPerRow;
var width = (int)pixelBuffer.Width;
var height = (int)pixelBuffer.Height;
var lenght = bytesPerRow * height;
byte[] managedArray = new byte[lenght];
Marshal.Copy(baseAddress, managedArray, 0, lenght);
pixelBuffer.Unlock(CVPixelBufferLock.None);
return(managedArray);
}
unsafe void UpdateBackground(ARFrame frame)
{
using (var img = frame.CapturedImage)
{
using (var ciImage = CIImage.FromImageBuffer(img))
{
using (var rotatedImage = ciImage.CreateByApplyingOrientation(ImageIO.CGImagePropertyOrientation.Right))
{
var size = new CGSize(rotatedImage.Extent.Size);
using (var rotatedBuff = new CVPixelBuffer((nint)size.Width,
(nint)size.Height, img.PixelFormatType))
{
var _coreImageContext = new CIContext(null);
rotatedBuff.Lock(CVPixelBufferLock.None);
_coreImageContext.Render(rotatedImage, rotatedBuff);
var yPtr = rotatedBuff.BaseAddress;
var uvPtr = rotatedBuff.GetBaseAddress(1);
if (yPtr == IntPtr.Zero || uvPtr == IntPtr.Zero)
{
return;
}
int wY = (int)rotatedBuff.Width;
int hY = (int)rotatedBuff.Height;
int wUv = (int)rotatedBuff.GetWidthOfPlane(1);
int hUv = (int)rotatedBuff.GetHeightOfPlane(1);
cameraYtexture.SetData(0, 0, 0, wY, hY, (void *)yPtr);
cameraUVtexture.SetData(0, 0, 0, wUv, hUv, (void *)uvPtr);
rotatedBuff.Unlock(CVPixelBufferLock.None);
_coreImageContext.Dispose();
}
}
}
}
}
public static CVPixelBuffer ToPixelBuffer(this UIImage self)
{
var width = self.Size.Width;
var height = self.Size.Height;
var attrs = new CVPixelBufferAttributes();
attrs.CGBitmapContextCompatibility = true;
attrs.CGImageCompatibility = true;
var resultPixelBuffer = new CVPixelBuffer((int)(width),
(int)(height),
CVPixelFormatType.CV32ARGB,
attrs);
resultPixelBuffer.Lock(CVPixelBufferLock.None);
var pixelData = resultPixelBuffer.GetBaseAddress(0);
var rgbColorSpace = CGColorSpace.CreateDeviceRGB();
var context = new CGBitmapContext(data: pixelData,
width: (int)(width),
height: (int)(height),
bitsPerComponent: 8,
bytesPerRow: resultPixelBuffer.GetBytesPerRowOfPlane(0),
colorSpace: rgbColorSpace,
bitmapInfo: CGImageAlphaInfo.NoneSkipFirst);
context.TranslateCTM(tx: 0, ty: height);
context.ScaleCTM(sx: 1.0f, sy: -1.0f);
UIGraphics.PushContext(context);
self.Draw(new CGRect(x: 0, y: 0, width: width, height: height));
UIGraphics.PopContext();
resultPixelBuffer.Unlock(CVPixelBufferLock.None);
return(resultPixelBuffer);
}
public override void DidOutputSampleBuffer(AVCaptureOutput captureOutput, CMSampleBuffer sampleBuffer, AVCaptureConnection connection)
{
if (ready)
{
ready = false;
CVPixelBuffer cVPixelBuffer = (CVPixelBuffer)sampleBuffer.GetImageBuffer();
cVPixelBuffer.Lock(CVPixelBufferLock.ReadOnly);
nint dataSize = cVPixelBuffer.DataSize;
width = cVPixelBuffer.Width;
height = cVPixelBuffer.Height;
IntPtr baseAddress = cVPixelBuffer.BaseAddress;
bpr = cVPixelBuffer.BytesPerRow;
cVPixelBuffer.Unlock(CVPixelBufferLock.ReadOnly);
buffer = NSData.FromBytes(baseAddress, (nuint)dataSize);
cVPixelBuffer.Dispose();
queue.DispatchAsync(ReadTask);
}
sampleBuffer.Dispose();
}
public void ToCGImageTest()
{
if (!TestRuntime.CheckSystemAndSDKVersion(9, 0))
{
Assert.Ignore("Ignoring VideoToolbox.VTUtilitiesTests: Requires iOS9+");
}
#if MONOMAC
var originalImage = new NSImage(NSBundle.MainBundle.PathForResource("Xam", "png", "CoreImage"));
#else
var originalImage = UIImage.FromBundle("CoreImage/Xam.png");
#endif
var originalCGImage = originalImage.CGImage;
var pxbuffer = new CVPixelBuffer(originalCGImage.Width, originalCGImage.Height, CVPixelFormatType.CV32ARGB,
new CVPixelBufferAttributes {
CGImageCompatibility = true, CGBitmapContextCompatibility = true
});
pxbuffer.Lock(CVOptionFlags.None);
using (var colorSpace = CGColorSpace.CreateDeviceRGB())
using (var ctx = new CGBitmapContext(pxbuffer.BaseAddress, originalCGImage.Width, originalCGImage.Height, 8,
4 * originalCGImage.Width, colorSpace, CGBitmapFlags.NoneSkipLast)) {
ctx.RotateCTM(0);
ctx.DrawImage(new RectangleF(0, 0, originalCGImage.Width, originalCGImage.Height), originalCGImage);
pxbuffer.Unlock(CVOptionFlags.None);
}
Assert.NotNull(pxbuffer, "VTUtilitiesTests.ToCGImageTest pxbuffer should not be null");
CGImage newImage;
var newImageStatus = pxbuffer.ToCGImage(out newImage);
Assert.That(newImageStatus == VTStatus.Ok, "VTUtilitiesTests.ToCGImageTest must be ok");
Assert.NotNull(newImage, "VTUtilitiesTests.ToCGImageTest pxbuffer should not be newImage");
Assert.AreEqual(originalCGImage.Width, newImage.Width, "VTUtilitiesTests.ToCGImageTest");
Assert.AreEqual(originalCGImage.Height, newImage.Height, "VTUtilitiesTests.ToCGImageTest");
var retainCount = CFGetRetainCount(newImage.Handle);
Assert.That(retainCount, Is.EqualTo(1), "RetainCount");
}
public override void DidOutputSampleBuffer(AVCaptureOutput captureOutput, CMSampleBuffer sampleBuffer, AVCaptureConnection connection)
{
if (ready)
{
ready = false;
pixelBuffer = (CVPixelBuffer)sampleBuffer.GetImageBuffer();
pixelBuffer.Lock(CVPixelBufferLock.None);
width = pixelBuffer.Width;
height = pixelBuffer.Height;
bytesPerRow = pixelBuffer.BytesPerRow;
context = new CGBitmapContext(pixelBuffer.BaseAddress, width, height, 8, bytesPerRow, colorSpace, CGImageAlphaInfo.PremultipliedFirst);
cgImage = context.ToImage();
uiImage = new UIImage(cgImage);
pixelBuffer.Unlock(CVPixelBufferLock.None);
pixelBuffer.Dispose();
queue.DispatchAsync(ReadTask);
}
sampleBuffer.Dispose();
}
/// <summary>
/// for debugging...
/// </summary>
public UIImage ImageBufferToUIImage(CVPixelBuffer imageBuffer)
{
//imageBuffer
imageBuffer.Lock(CVPixelBufferLock.None);
var baseAddress = imageBuffer.BaseAddress;
var bytesPerRow = imageBuffer.BytesPerRow;
var width = imageBuffer.Width;
var height = imageBuffer.Height;
var colorSpace = CGColorSpace.CreateDeviceRGB();
var bitmapInfo = (uint)CGImageAlphaInfo.NoneSkipFirst | (uint)CGBitmapFlags.ByteOrder32Little;
var context = new CGBitmapContext(baseAddress, width, height, 8, bytesPerRow, colorSpace, (CGImageAlphaInfo)bitmapInfo);
var quartzImage = context?.ToImage();
imageBuffer.Unlock(CVPixelBufferLock.None);
var image = new UIImage(quartzImage, 1.0f, UIImageOrientation.Right);
// HACK: save as file
//var documentsDirectory = Environment.GetFolderPath
// (Environment.SpecialFolder.Personal);
//string jpgFilename = System.IO.Path.Combine(documentsDirectory, "Photo.jpg"); // hardcoded filename, overwritten each time
//NSData imgData = image.AsJPEG();
//NSError err = null;
//if (imgData.Save(jpgFilename, false, out err))
//{
// Console.WriteLine("saved as " + jpgFilename);
//}
//else
//{
// Console.WriteLine("NOT saved as " + jpgFilename + " because" + err.LocalizedDescription);
//}
return(image);
}
/// <summary>
/// This is an expensive call. Used by preview thumbnail.
/// </summary>
/// <returns>The (processed) video frame, as a UIImage.</returns>
/// <param name="imageBuffer">The (processed) video frame.</param>
public static UIImage ImageBufferToUIImage(CVPixelBuffer imageBuffer)
{
imageBuffer.Lock(CVPixelBufferLock.None);
var baseAddress = imageBuffer.BaseAddress;
var bytesPerRow = imageBuffer.BytesPerRow;
var width = imageBuffer.Width;
var height = imageBuffer.Height;
var colorSpace = CGColorSpace.CreateDeviceRGB();
var bitmapInfo = (uint)CGImageAlphaInfo.NoneSkipFirst | (uint)CGBitmapFlags.ByteOrder32Little;
using (var context = new CGBitmapContext(baseAddress, width, height, 8, bytesPerRow, colorSpace, (CGImageAlphaInfo)bitmapInfo))
{
var quartzImage = context?.ToImage();
imageBuffer.Unlock(CVPixelBufferLock.None);
var image = new UIImage(quartzImage, 1.0f, UIImageOrientation.Up);
return(image);
}
}
