static CRect computeMargins(ref sDecodedVideoSize videoSize)
{
return(new CRect(videoSize.cropRect.left,
videoSize.cropRect.top,
videoSize.size.cx - videoSize.cropRect.right,
videoSize.size.cy - videoSize.cropRect.bottom));
}
/// <summary>Valid range of UV coordinates; rendering them [ 0 .. 1 ] would produce cropping borders, we don’t want them.</summary>
/// <remarks>Returns substitution values for HLSL/GLSL, e.g. ( "0.0, 0.0", "1.0, 0.97523452345123235356452363" )</remarks>
public static (string, string) videoUvCroppedRect(ref sDecodedVideoSize videoSize)
{
double x = rel(videoSize.cropRect.left, videoSize.size.cx);
double y = rel(videoSize.cropRect.top, videoSize.size.cy);
string minUv = Utils.printFloat2(x, y);
x = rel(videoSize.cropRect.right, videoSize.size.cx);
y = rel(videoSize.cropRect.bottom, videoSize.size.cy);
string maxUv = Utils.printFloat2(x, y);
return(minUv, maxUv);
}
/// <summary>NV12 texture rectangle in clip space units.
/// NV12 texture is slightly larger than the video 'coz Pi4 h264 decoder ain't smart enough to crop the damn footage on their side.</summary>
static RectD nv12Rectangle(ref RectD video, ref sDecodedVideoSize videoSize)
{
double cx = videoSize.cropRect.size.cx * 0.5;
double cy = videoSize.cropRect.size.cy * 0.5;
CRect margins = computeMargins(ref videoSize);
RectD inflationRel = new RectD(
(cx + margins.left) / cx,
(cy + margins.top) / cy,
(cx + margins.right) / cx,
(cy + margins.bottom) / cy
);
return(video * inflationRel);
}
public RenderBase(IRenderDevice device, CSize renderTargetSize, SwapChainFormats formats, Vector4 borderColor, sDecodedVideoSize videoSize)
{
this.videoSize = videoSize;
// Create vertex buffer
vertexBuffer = createVideoVertexBuffer(device, renderTargetSize, ref videoSize);
// Create pipeline state
var pso = new PipelineStateDesc(false);
pso.GraphicsPipeline.DepthStencilDesc.DepthEnable = false;
pso.GraphicsPipeline.PrimitiveTopology = PrimitiveTopology.TriangleList;
pso.GraphicsPipeline.NumRenderTargets = 1;
pso.GraphicsPipeline.setRTVFormat(0, formats.color);
pso.GraphicsPipeline.DSVFormat = formats.depth;
pso.ResourceLayout.DefaultVariableType = ShaderResourceVariableType.Static;
var compiler = device.GetShaderFactory();
iStorageFolder assets = StorageFolder.embeddedResources(System.Reflection.Assembly.GetExecutingAssembly(), resourceFolder);
using (var psf = device.CreatePipelineStateFactory())
{
psf.setName("Video PSO");
setupVideoInputLayout(psf);
using (var vs = compiler.compileHlslFile(assets, "VideoVS.hlsl", ShaderType.Vertex))
psf.graphicsVertexShader(vs);
(string uvMin, string uvMax) = videoUvCroppedRect(ref videoSize);
string colorString = Utils.printFloat4(borderColor);
using (var ps = compilePixelShader(compiler, assets, uvMin, uvMax, colorString))
psf.graphicsPixelShader(ps);
psf.layoutVariable(ShaderType.Pixel, ShaderResourceVariableType.Dynamic, varTexture);
var sampler = new SamplerDesc(false)
{
MipFilter = FilterType.Point,
};
psf.layoutStaticSampler(ShaderType.Pixel, ref sampler, varTexture);
psf.apply(ref pso);
pipelineState = device.CreatePipelineState(ref pso);
}
// Create resource binding and cache the variable, we gonna need both on every frame rendered
binding = pipelineState.CreateShaderResourceBinding(true);
textureVariable = binding.GetVariableByName(ShaderType.Pixel, varTexture);
}
/// <summary>Cropped video rectangle in clip space units</summary>
static RectD videoRectangle(CSize pxRenderTarget, ref sDecodedVideoSize videoSize)
{
CSize pxVideo = videoSize.cropRect.size;
if (pxVideo.cx * pxRenderTarget.cy >= pxVideo.cy * pxRenderTarget.cx)
{
// scale X to fit, center vertically
double h = mulDiv(pxVideo.cy, pxRenderTarget.cx, pxVideo.cx, pxRenderTarget.cy);
return(new RectD(-1, -h, 1, h));
}
else
{
// scale Y to fit, center horizontally
double w = mulDiv(pxVideo.cx, pxRenderTarget.cy, pxVideo.cy, pxRenderTarget.cx);
return(new RectD(-w, -1, w, 1));
}
}
static void produceVertices(Span <sVideoVertex> span, CSize pxRenderTarget, ref sDecodedVideoSize videoSize)
{
// Non-trivial amount of arithmetics, hopefully with 64-bit floats the numerical precision won't be too bad as it's pretty critical here.
// Ideally, need to solve symbolically and copy-paste the solution from Maple solver.
RectD rc = videoRectangle(pxRenderTarget, ref videoSize);
rc = nv12Rectangle(ref rc, ref videoSize);
rc = textureCoordinates(ref rc);
// Produce the 3 vertices.
// Positions are set so the triangle covers complete render target.
// Texture coordinates are set to position video rectangle within the render target.
span[0].position = new Vector2(-1, 1);
span[0].texCoords = new Vector2((float)rc.left, (float)rc.top);
span[1].position = new Vector2(3, 1);
span[1].texCoords = new Vector2((float)(rc.right * 2 - rc.left), (float)rc.top);
span[2].position = new Vector2(-1, -3);
span[2].texCoords = new Vector2((float)rc.left, (float)(rc.bottom * 2 - rc.top));
}
internal RepresentationFormat(ref BitReader reader, eChromaFormat chromaFormat)
{
CSize size = default;
size.cx = reader.readInt(16); // pic_width_vps_in_luma_samples
size.cy = reader.readInt(16); // pic_height_vps_in_luma_samples
bool chroma_and_bit_depth_vps_present_flag = reader.readBit();
if (chroma_and_bit_depth_vps_present_flag)
{
int chroma_format_vps_idc = reader.readInt(2);
if (chroma_format_vps_idc == 3)
{
bool separate_colour_plane_vps_flag = reader.readBit();
}
bit_depth_vps_luma = (byte)(reader.readInt(4) + 8);
bit_depth_vps_chroma = (byte)(reader.readInt(4) + 8);
}
else
{
bit_depth_vps_luma = bit_depth_vps_chroma = 8;
}
bool conformance_window_vps_flag = reader.readBit();
if (conformance_window_vps_flag)
{
CRect rc = default;
rc.left = (int)reader.unsignedGolomb(); // conf_win_vps_left_offset
rc.right = size.cx - (int)reader.unsignedGolomb(); // conf_win_vps_right_offset
rc.top = (int)reader.unsignedGolomb(); // conf_win_vps_top_offset
rc.bottom = size.cy - (int)reader.unsignedGolomb(); // conf_win_vps_bottom_offset
decodedVideoSize = new sDecodedVideoSize(size, rc, chromaFormat);
}
else
{
decodedVideoSize = new sDecodedVideoSize(size, new CRect(default, size), chromaFormat);
internal VideoParams264(TrackEntry videoTrack)
{
// File.WriteAllBytes( @"C:\Temp\2remove\mkv\videoPrivateData.bin", videoTrack.codecPrivate );
ReadOnlySpan <byte> codecPrivate = videoTrack.codecPrivate.AsSpan();
int cbHeader = Marshal.SizeOf <NativeStruct>();
NativeStruct ns = codecPrivate.Slice(0, cbHeader).cast <NativeStruct>()[0];
profile = ns.profileCode;
profileCompatibility = ns.profileCompatibility;
levelCode = ns.levelCode;
int offset = cbHeader;
sps = ContainerUtils.copyBlobs(ns.numOfSequenceParameterSets, codecPrivate, ref offset);
// File.WriteAllBytes( @"C:\Temp\2remove\mkv\sps.bin", sps[ 0 ] );
int ppsCount = codecPrivate[offset++];
pps = ContainerUtils.copyBlobs(ppsCount, codecPrivate, ref offset);
ReadOnlySpan <byte> spsBlob = sps[0].AsSpan();
if (MiscUtils.getNaluType(spsBlob[0]) != eNaluType.SPS)
{
throw new ApplicationException("The SPS is invalid, wrong NALU type");
}
spsBlob = spsBlob.Slice(1);
BitReader spsReader = new BitReader(spsBlob);
parsedSps = new SequenceParameterSet(ref spsReader);
chromaFormat = parsedSps.chromaFormat;
bitDepthLuma = parsedSps.bitDepthLuma;
bitDepthChroma = parsedSps.bitDepthChroma;
m_decodedSize = new sDecodedVideoSize(parsedSps.decodedSize, parsedSps.cropRectangle, chromaFormat);
}
public Nv12State(IRenderDevice device, TextureFormat format, Nv12Texture[] textures, ref sDecodedVideoSize videoSize)
{
// Create the pipeline state
var pso = new PipelineStateDesc(false);
pso.GraphicsPipeline.DepthStencilDesc.DepthEnable = false;
pso.GraphicsPipeline.RasterizerDesc.CullMode = CullMode.None;
pso.GraphicsPipeline.PrimitiveTopology = PrimitiveTopology.TriangleList;
pso.GraphicsPipeline.NumRenderTargets = 1;
pso.GraphicsPipeline.setRTVFormat(0, format);
pso.ResourceLayout.DefaultVariableType = ShaderResourceVariableType.Static;
var compiler = device.GetShaderFactory();
iStorageFolder assets = StorageFolder.embeddedResources(System.Reflection.Assembly.GetExecutingAssembly(), resourceFolder);
using (var psf = device.CreatePipelineStateFactory())
{
psf.setName("Video PSO");
using (var vs = compiler.compileHlslFile(assets, "VideoVS.hlsl", ShaderType.Vertex))
psf.graphicsVertexShader(vs);
using (var ps = compilePixelShader(compiler, assets))
psf.graphicsPixelShader(ps);
psf.layoutVariable(ShaderType.Pixel, ShaderResourceVariableType.Dynamic, varTexture);
var sampler = samplerDesc();
psf.layoutStaticSampler(ShaderType.Pixel, ref sampler, varTexture);
psf.apply(ref pso);
pipelineState = device.CreatePipelineState(ref pso);
}
// Create resource binding and cache the variable
binding = pipelineState.CreateShaderResourceBinding(true);
textureVariable = binding.GetVariableByName(ShaderType.Pixel, varTexture);
// Copy views of the textures into array
sourceTextures = new ITextureView[textures.Length];
for (int i = 0; i < textures.Length; i++)
{
sourceTextures[i] = textures[i].view;
}
// Create GL viewport structure with weird values for cropping the video
viewport = new Viewport(false)
{
TopLeftX = -videoSize.cropRect.left,
// TopLeftY = videoSize.cropRect.top,
// OpenGL uses opposite Y direction there.
TopLeftY = videoSize.cropRect.bottom - videoSize.size.cy,
Width = videoSize.size.cx,
Height = videoSize.size.cy,
};
}
/// <summary>Create immutable VB with the full-screen triangle with cropping-included texture coordinates</summary>
public static IBuffer createVideoVertexBuffer(IRenderDevice device, CSize renderTargetSize, ref sDecodedVideoSize videoSize)
{
Span <sVideoVertex> data = stackalloc sVideoVertex[3];
produceVertices(data, renderTargetSize, ref videoSize);
BufferDesc desc = new BufferDesc(false)
{
uiSizeInBytes = 16 * 3,
BindFlags = BindFlags.VertexBuffer,
Usage = Usage.Static,
};
ReadOnlySpan <sVideoVertex> readOnly = data;
return(device.CreateBuffer(desc, readOnly, "Video VB"));
}
protected override void initRendering(IRenderDevice device, Nv12Texture[] textures, ref sDecodedVideoSize videoSize)
{
state = new Nv12State(device, rgbFormat, textures, ref videoSize);
}
