public void captureSetup(iVideoTrack videoTrack, int decodedBuffersCount, sDecodedVideoSize decodedSize) { this.decodedSize = decodedSize; if (pendingFrames.capacity != decodedBuffersCount) { pendingFrames = new PendingFrames(decodedBuffersCount); } // Set decoded format. Pi4 Linux failed to implement V4L2 stateful decoder setup workflow, instead computing everything manually, from parsed SPS. sPixelFormatMP sdf = computeDecodedFormat(ref decodedSize); device.setDataFormat(eBufferType.VideoCaptureMPlane, ref sdf); // Apparently, Pi4 hardware or drivers is unable to process S_SELECTION request and crop the video. Cropping it later while rendering NV12 into RGB. /* sSelection selection = default; * selection.type = eBufferType.VideoCaptureMPlane; * selection.target = eSelectionTarget.Compose; * selection.flags = eSelectionFlags.LesserOrEqual; * selection.rect = decodedSize.cropRect; * device.file.call( eControlCode.S_SELECTION, ref selection ); * device.file.call( eControlCode.G_SELECTION, ref selection ); * CRect selectedRect = selection.rect; * if( selectedRect == decodedSize.cropRect ) * Logger.logVerbose( "Video cropping: decoded size {0}, cropped to {1}", decodedSize.size, selectedRect ); * else * Logger.logInfo( "Video cropping: decoded size {0}, asked to crop to {1}, GPU driver replaced with {2}", decodedSize.size, decodedSize.cropRect, selectedRect ); */ sdf = device.getDataFormat(eBufferType.VideoCaptureMPlane); decoded = new DecodedQueue(device, decodedBuffersCount); // decoded.exportTextures( renderDev, device, ref sdf ); // Start streaming of the output queue device.startStreaming(eBufferType.VideoCaptureMPlane); }
sPixelFormatMP iVideoTrack.getDecodedFormat() { sPixelFormatMP res = new sPixelFormatMP(); res.size = videoParams.decodedSize.size; res.pixelFormat = ePixelFormat.NV12; res.field = eField.Progressive; videoParams.setColorAttributes(ref res); return(res); }
public Nv12Texture exportNv12(iGlesRenderDevice gles, VideoDevice device, ref sPixelFormatMP pixelFormat, ref ColorFormat colorFormat) { device.file.call(eControlCode.QUERYBUF, ref buffer); sDmaBuffer dma = exportOutputBuffer(device, ref pixelFormat); ITexture texture = gles.importNv12Texture(ref dma, ref colorFormat); // Logger.logVerbose( "Exported NV12 texture: {0}", dma ); return(new Nv12Texture(texture)); }
/// <summary>Export all buffers from V4L2, import them into GLES in Diligent Engine</summary> public void exportTextures(IRenderDevice renderDevice, VideoDevice device, ref sPixelFormatMP pixelFormat, ref ColorFormat color) { iGlesRenderDevice gles = ComLightCast.cast <iGlesRenderDevice>(renderDevice); textures = new Nv12Texture[buffers.Length]; for (int i = 0; i < buffers.Length; i++) { textures[i] = buffers[i].exportNv12(gles, device, ref pixelFormat, ref color); } }
public EncodedQueue(VideoDevice device, int encodedBuffersCount, ref sPixelFormatMP encodedFormat) : base(allocateBuffers(device, encodedBuffersCount), device.file) { bufferCapacity = encodedFormat.getPlaneFormat(0).sizeImage; // Create encoded buffers, this does the memory mapping for (int i = 0; i < buffersCount; i++) { var eb = new EncodedBuffer(device.file, i); buffers[i] = eb; } }
public override sPixelFormatMP getDecodedFormat() { sPixelFormatMP res = new sPixelFormatMP(); res.size = sizePixels; res.pixelFormat = ePixelFormat.NV12; res.field = eField.Progressive; res.colorSpace = eColorSpace.BT709; res.encoding = (byte)eYCbCrEncoding.BT709; res.quantization = eQuantization.FullRange; res.transferFunction = eTransferFunction.BT_709; return(res); }
public sDmaBuffer exportOutputBuffer(VideoDevice device, ref sPixelFormatMP pixelFormat) { sExportBuffer eb = device.exportOutputBuffer(bufferIndex); sPlanePixelFormat planeFormat = pixelFormat.getPlaneFormat(0); sDmaBuffer dma = new sDmaBuffer() { fd = eb.fd, offset = 0, stride = planeFormat.bytesPerLine, imageSize = planeFormat.bytesPerLine * pixelFormat.size.cy, sizePixels = pixelFormat.size, bufferIndex = bufferIndex }; return(dma); }
public override sPixelFormatMP getEncodedFormat() { sPixelFormatMP res = new sPixelFormatMP(); res.size = sizePixels; res.pixelFormat = ePixelFormat.H264; res.field = eField.Progressive; // res.sizeImage = bitRate.decodingBufferSize; res.colorSpace = eColorSpace.BT709; //if( !extended ) // return res; res.encoding = (byte)eYCbCrEncoding.BT709; res.quantization = eQuantization.LimitedRange; res.transferFunction = eTransferFunction.BT_709; return(res); }
public void initialize(iVideoTrack videoTrack, int encodedBuffersCount) { // dbgPrintEncodedFormats(); // Determine format of the encoded video sPixelFormatMP encodedFormat = videoTrack.getEncodedFormat(); // Set format of the first and the only plane of the compressed video. encodedFormat.numPlanes = 1; encodedFormat.setPlaneFormat(0, new sPlanePixelFormat() { sizeImage = EncodedQueue.encodedVideoBufferSize(videoTrack) }); // 4.5.1.5. Initialization // 1. Set the coded format on OUTPUT via VIDIOC_S_FMT() sStreamDataFormat sdf = new sStreamDataFormat() { bufferType = eBufferType.VideoOutputMPlane, pix_mp = encodedFormat }; device.file.call(eControlCode.S_FMT, ref sdf); // Logger.logVerbose( "eControlCode.S_FMT completed OK for encoded format: {0}", sdf.pix_mp ); // 2 Allocate source (bytestream) buffers via VIDIOC_REQBUFS() on OUTPUT encoded = new EncodedQueue(device, encodedBuffersCount, ref encodedFormat); // 3 Start streaming on the OUTPUT queue via VIDIOC_STREAMON() device.startStreaming(eBufferType.VideoOutputMPlane); // Continue queuing/dequeuing bytestream buffers to/from the OUTPUT queue via VIDIOC_QBUF() and VIDIOC_DQBUF(). // The buffers will be processed and returned to the client in order, until required metadata to configure the CAPTURE queue are found. // This is indicated by the decoder sending a V4L2_EVENT_SOURCE_CHANGE event with changes set to V4L2_EVENT_SRC_CH_RESOLUTION. eventSubscription = new EventSubscription(device); // Linux kernel on Pi4 appears to be too old and does not implement that spec. The event never arrives, while the encoded buffers are stuck in the Queued state. // For this reason, we have to deal with dynamic resolution changes instead :-( // WaitForResolution.wait( device, encoded, reader, waitForResolutionTimeout ); }
static sPixelFormatMP computeDecodedFormat(ref sDecodedVideoSize decodedSize) { if (decodedSize.chromaFormat != eChromaFormat.c420) { throw new NotImplementedException("So far, the library only supports 4:2:0 chroma sampling"); } // Apparently, the hardware decoder of the Pi4 can't crop video. Not a huge deal, will crop while rendering NV12 into RGB. // You would expect you need to pass decodedSize.size here, but no, Linux only plays the video when cropped size is passed there. // The size of the output buffers actually created by that Linux ain't cropped. Crazy stuff. CSize px = decodedSize.cropRect.size; // Set stride to be a multiple of 4 bytes, GLES requirement on Pi4 int stride = (px.cx + 3) & (~3); sPixelFormatMP pmp = new sPixelFormatMP() { size = px, pixelFormat = ePixelFormat.NV12, field = eField.Progressive, colorSpace = eColorSpace.BT709, numPlanes = 2, encoding = (byte)eYCbCrEncoding.BT709, quantization = eQuantization.FullRange, transferFunction = eTransferFunction.BT_709, }; pmp.setPlaneFormat(0, new sPlanePixelFormat() { sizeImage = px.cy * stride, bytesPerLine = stride }); pmp.setPlaneFormat(1, new sPlanePixelFormat() { sizeImage = px.cy * stride / 2, bytesPerLine = stride }); return(pmp); }
internal abstract void setColorAttributes(ref sPixelFormatMP pixFormat);
internal override void setColorAttributes(ref sPixelFormatMP pixFormat) { throw new NotImplementedException(); }
public VideoTextures exportTextures(iGlesRenderDevice gles, VideoDevice device, ref sPixelFormatMP pixelFormat) { sExportBuffer eb = new sExportBuffer() { type = eBufferType.VideoCaptureMPlane, index = bufferIndex, plane = 0, flags = eFileFlags.O_RDONLY | eFileFlags.O_CLOEXEC }; device.file.call(eControlCode.EXPBUF, ref eb); sPlanePixelFormat planeFormat = pixelFormat.getPlaneFormat(0); sDmaBuffer dma = new sDmaBuffer() { fd = eb.fd, offset = 0, stride = planeFormat.bytesPerLine, imageSize = planeFormat.bytesPerLine * pixelFormat.size.cy, sizePixels = pixelFormat.size, bufferIndex = bufferIndex }; ITexture luma = gles.importLumaTexture(ref dma); Logger.logVerbose("Exported luma texture: {0}", dma); // I asked V4L2 for 2 planes, however QUERYBUF returned a single plane, with the complete NV12 image in it. // No big deal, we have EGL_DMA_BUF_PLANE0_OFFSET_EXT for that; that's where the sDmaBuffer.offset field goes. dma.offset = dma.imageSize; dma.sizePixels = chromaSize(pixelFormat.size); dma.imageSize = dma.stride * dma.sizePixels.cy; ITexture chroma = gles.importChromaTexture(ref dma); Logger.logVerbose("Exported chroma texture: {0}", dma); return(new VideoTextures(luma, chroma)); }