public void captureSetup(iVideoTrack videoTrack, int decodedBuffersCount, sDecodedVideoSize decodedSize)
{
this.decodedSize = decodedSize;
if (pendingFrames.capacity != decodedBuffersCount)
{
pendingFrames = new PendingFrames(decodedBuffersCount);
}
// Set decoded format. Pi4 Linux failed to implement V4L2 stateful decoder setup workflow, instead computing everything manually, from parsed SPS.
sPixelFormatMP sdf = computeDecodedFormat(ref decodedSize);
device.setDataFormat(eBufferType.VideoCaptureMPlane, ref sdf);
// Apparently, Pi4 hardware or drivers is unable to process S_SELECTION request and crop the video. Cropping it later while rendering NV12 into RGB.
/* sSelection selection = default;
* selection.type = eBufferType.VideoCaptureMPlane;
* selection.target = eSelectionTarget.Compose;
* selection.flags = eSelectionFlags.LesserOrEqual;
* selection.rect = decodedSize.cropRect;
* device.file.call( eControlCode.S_SELECTION, ref selection );
* device.file.call( eControlCode.G_SELECTION, ref selection );
* CRect selectedRect = selection.rect;
* if( selectedRect == decodedSize.cropRect )
* Logger.logVerbose( "Video cropping: decoded size {0}, cropped to {1}", decodedSize.size, selectedRect );
* else
* Logger.logInfo( "Video cropping: decoded size {0}, asked to crop to {1}, GPU driver replaced with {2}", decodedSize.size, decodedSize.cropRect, selectedRect ); */
sdf = device.getDataFormat(eBufferType.VideoCaptureMPlane);
decoded = new DecodedQueue(device, decodedBuffersCount);
// decoded.exportTextures( renderDev, device, ref sdf );
// Start streaming of the output queue
device.startStreaming(eBufferType.VideoCaptureMPlane);
}
static sPixelFormatMP computeDecodedFormat(ref sDecodedVideoSize decodedSize)
{
if (decodedSize.chromaFormat != eChromaFormat.c420)
{
throw new NotImplementedException("So far, the library only supports 4:2:0 chroma sampling");
}
// Apparently, the hardware decoder of the Pi4 can't crop video. Not a huge deal, will crop while rendering NV12 into RGB.
// You would expect you need to pass decodedSize.size here, but no, Linux only plays the video when cropped size is passed there.
// The size of the output buffers actually created by that Linux ain't cropped. Crazy stuff.
CSize px = decodedSize.cropRect.size;
// Set stride to be a multiple of 4 bytes, GLES requirement on Pi4
int stride = (px.cx + 3) & (~3);
sPixelFormatMP pmp = new sPixelFormatMP()
{
size = px,
pixelFormat = ePixelFormat.NV12,
field = eField.Progressive,
colorSpace = eColorSpace.BT709,
numPlanes = 2,
encoding = (byte)eYCbCrEncoding.BT709,
quantization = eQuantization.FullRange,
transferFunction = eTransferFunction.BT_709,
};
pmp.setPlaneFormat(0, new sPlanePixelFormat()
{
sizeImage = px.cy * stride, bytesPerLine = stride
});
pmp.setPlaneFormat(1, new sPlanePixelFormat()
{
sizeImage = px.cy * stride / 2, bytesPerLine = stride
});
return(pmp);
}
protected abstract void initRendering(IRenderDevice device, Nv12Texture[] textures, ref sDecodedVideoSize videoSize);
/// <summary>Load an mp4 media file; this runs on a background thread from the thread pool.</summary>
void loadMediaImpl(string url, TaskCompletionSource <bool> completionSource)
{
try
{
// Deal with paths starting from "~/", transform that into user's home folder
if (url.StartsWith("~/"))
{
string rel = url.Substring(2);
string home = Environment.GetFolderPath(Environment.SpecialFolder.UserProfile);
url = Path.Combine(home, rel);
}
// Parse the complete MP4, except the largest `mdat` box which has the actual payload of the video.
file = OpenMedia.openMedia(File.OpenRead(url));
if (!shutdownHandle)
{
shutdownHandle = EventHandle.create();
}
// Initialize the video
if (null == file.videoTrack)
{
throw new ApplicationException("The file doesn’t have a video track");
}
decoder?.Dispose();
decoder = new StatefulVideoDecoder(h264.device, shutdownHandle);
int videoEncodedBuffers = encodedBuffersCount;
int audioEncodedBuffers = this.audioEncodedBuffers;
decoder.initialize(file.videoTrack, videoEncodedBuffers);
decodedSize = file.videoTrack.decodedSize;
decoder.captureSetup(file.videoTrack, decodedBuffersCount, decodedSize);
// Initialize the audio
audioPlayer?.Dispose();
audioPlayer = null;
if (null != file.audioTrack)
{
try
{
audioPlayer = Audio.Player.create(file.audioTrack, audioEncodedBuffers, shutdownHandle);
}
catch (Exception ex)
{
// Logger.logError( ex.ToString() );
ex.logError("Error initializing audio");
}
}
else
{
Logger.logWarning("The file doesn’t have an audio track");
}
// Initialize presentation clock source
if (null != audioPlayer)
{
// Use audio player for the source of presentation time
if (displayRefresh.HasValue)
{
presentationClock = new Clocks.AudioWithTimer(decoder, audioPlayer, displayRefresh.Value);
}
else
{
presentationClock = new Clocks.Audio(decoder, audioPlayer);
}
}
else
{
// Use eClock.Monotonic OS clock for presentation time
presentationClock = new Clocks.Video(decoder);
}
m_state = eState.Prepared;
if (m_autoPlay)
{
play();
}
completionSource.SetResult(true);
}
catch (Exception ex)
{
completionSource.SetException(ex);
}
}
