public void captureSetup(iVideoTrack videoTrack, int decodedBuffersCount, sDecodedVideoSize decodedSize) { this.decodedSize = decodedSize; if (pendingFrames.capacity != decodedBuffersCount) { pendingFrames = new PendingFrames(decodedBuffersCount); } // Set decoded format. Pi4 Linux failed to implement V4L2 stateful decoder setup workflow, instead computing everything manually, from parsed SPS. sPixelFormatMP sdf = computeDecodedFormat(ref decodedSize); device.setDataFormat(eBufferType.VideoCaptureMPlane, ref sdf); // Apparently, Pi4 hardware or drivers is unable to process S_SELECTION request and crop the video. Cropping it later while rendering NV12 into RGB. /* sSelection selection = default; * selection.type = eBufferType.VideoCaptureMPlane; * selection.target = eSelectionTarget.Compose; * selection.flags = eSelectionFlags.LesserOrEqual; * selection.rect = decodedSize.cropRect; * device.file.call( eControlCode.S_SELECTION, ref selection ); * device.file.call( eControlCode.G_SELECTION, ref selection ); * CRect selectedRect = selection.rect; * if( selectedRect == decodedSize.cropRect ) * Logger.logVerbose( "Video cropping: decoded size {0}, cropped to {1}", decodedSize.size, selectedRect ); * else * Logger.logInfo( "Video cropping: decoded size {0}, asked to crop to {1}, GPU driver replaced with {2}", decodedSize.size, decodedSize.cropRect, selectedRect ); */ sdf = device.getDataFormat(eBufferType.VideoCaptureMPlane); decoded = new DecodedQueue(device, decodedBuffersCount); // decoded.exportTextures( renderDev, device, ref sdf ); // Start streaming of the output queue device.startStreaming(eBufferType.VideoCaptureMPlane); }
static sPixelFormatMP computeDecodedFormat(ref sDecodedVideoSize decodedSize) { if (decodedSize.chromaFormat != eChromaFormat.c420) { throw new NotImplementedException("So far, the library only supports 4:2:0 chroma sampling"); } // Apparently, the hardware decoder of the Pi4 can't crop video. Not a huge deal, will crop while rendering NV12 into RGB. // You would expect you need to pass decodedSize.size here, but no, Linux only plays the video when cropped size is passed there. // The size of the output buffers actually created by that Linux ain't cropped. Crazy stuff. CSize px = decodedSize.cropRect.size; // Set stride to be a multiple of 4 bytes, GLES requirement on Pi4 int stride = (px.cx + 3) & (~3); sPixelFormatMP pmp = new sPixelFormatMP() { size = px, pixelFormat = ePixelFormat.NV12, field = eField.Progressive, colorSpace = eColorSpace.BT709, numPlanes = 2, encoding = (byte)eYCbCrEncoding.BT709, quantization = eQuantization.FullRange, transferFunction = eTransferFunction.BT_709, }; pmp.setPlaneFormat(0, new sPlanePixelFormat() { sizeImage = px.cy * stride, bytesPerLine = stride }); pmp.setPlaneFormat(1, new sPlanePixelFormat() { sizeImage = px.cy * stride / 2, bytesPerLine = stride }); return(pmp); }
protected abstract void initRendering(IRenderDevice device, Nv12Texture[] textures, ref sDecodedVideoSize videoSize);
/// <summary>Load an mp4 media file; this runs on a background thread from the thread pool.</summary> void loadMediaImpl(string url, TaskCompletionSource <bool> completionSource) { try { // Deal with paths starting from "~/", transform that into user's home folder if (url.StartsWith("~/")) { string rel = url.Substring(2); string home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile); url = Path.Combine(home, rel); } // Parse the complete MP4, except the largest `mdat` box which has the actual payload of the video. file = OpenMedia.openMedia(File.OpenRead(url)); if (!shutdownHandle) { shutdownHandle = EventHandle.create(); } // Initialize the video if (null == file.videoTrack) { throw new ApplicationException("The file doesn’t have a video track"); } decoder?.Dispose(); decoder = new StatefulVideoDecoder(h264.device, shutdownHandle); int videoEncodedBuffers = encodedBuffersCount; int audioEncodedBuffers = this.audioEncodedBuffers; decoder.initialize(file.videoTrack, videoEncodedBuffers); decodedSize = file.videoTrack.decodedSize; decoder.captureSetup(file.videoTrack, decodedBuffersCount, decodedSize); // Initialize the audio audioPlayer?.Dispose(); audioPlayer = null; if (null != file.audioTrack) { try { audioPlayer = Audio.Player.create(file.audioTrack, audioEncodedBuffers, shutdownHandle); } catch (Exception ex) { // Logger.logError( ex.ToString() ); ex.logError("Error initializing audio"); } } else { Logger.logWarning("The file doesn’t have an audio track"); } // Initialize presentation clock source if (null != audioPlayer) { // Use audio player for the source of presentation time if (displayRefresh.HasValue) { presentationClock = new Clocks.AudioWithTimer(decoder, audioPlayer, displayRefresh.Value); } else { presentationClock = new Clocks.Audio(decoder, audioPlayer); } } else { // Use eClock.Monotonic OS clock for presentation time presentationClock = new Clocks.Video(decoder); } m_state = eState.Prepared; if (m_autoPlay) { play(); } completionSource.SetResult(true); } catch (Exception ex) { completionSource.SetException(ex); } }