public static int ProcessAudioFrame(Decoder decoder, MediaFrame mFrame, AVFrame *frame)
{
/* References
*
* https://www.programmersought.com/article/70255648018/
* https://ffmpeg.org/doxygen/2.4/resampling__audio_8c_source.html
*
* Notes
*
* Currently output sample rate = input sample rate which means in case we change that we should review that actually works properly
* If frame->nb_samples are known initially or by decoding on frame we can avoid av_samples_alloc_array_and_samples here and transfer it to SetupAudio
*/
int ret = 0;
try
{
int dst_nb_samples;
if (decoder.m_max_dst_nb_samples == -1)
{
if (decoder.m_dst_data != null && (IntPtr)(*decoder.m_dst_data) != IntPtr.Zero)
{
av_freep(&decoder.m_dst_data[0]); decoder.m_dst_data = null;
}
decoder.m_max_dst_nb_samples = (int)av_rescale_rnd(frame->nb_samples, decoder.codecCtx->sample_rate, decoder.codecCtx->sample_rate, AVRounding.AV_ROUND_UP);
fixed(byte ***dst_data = &decoder.m_dst_data)
fixed(int *dst_linesize = &decoder.m_dst_linesize)
ret = av_samples_alloc_array_and_samples(dst_data, dst_linesize, Decoder.AOutChannels, decoder.m_max_dst_nb_samples, Decoder.AOutSampleFormat, 0);
}
fixed(int *dst_linesize = &decoder.m_dst_linesize)
{
dst_nb_samples = (int)av_rescale_rnd(swr_get_delay(decoder.swrCtx, decoder.codecCtx->sample_rate) + frame->nb_samples, decoder.codecCtx->sample_rate, decoder.codecCtx->sample_rate, AVRounding.AV_ROUND_UP);
if (dst_nb_samples > decoder.m_max_dst_nb_samples)
{
av_freep(&decoder.m_dst_data[0]);
ret = av_samples_alloc(decoder.m_dst_data, dst_linesize, Decoder.AOutChannels, (int)dst_nb_samples, Decoder.AOutSampleFormat, 0);
}
ret = swr_convert(decoder.swrCtx, decoder.m_dst_data, dst_nb_samples, (byte **)&frame->data, frame->nb_samples);
if (ret < 0)
{
return(ret);
}
int dst_data_len = av_samples_get_buffer_size(dst_linesize, Decoder.AOutChannels, ret, Decoder.AOutSampleFormat, 1);
mFrame.audioData = new byte[dst_data_len]; Marshal.Copy((IntPtr)(*decoder.m_dst_data), mFrame.audioData, 0, mFrame.audioData.Length); //Marshal.FreeHGlobal((IntPtr)(*m_dst_data));
}
} catch (Exception e) { ret = -1; Log("Error[" + (ret).ToString("D4") + "], Func: ProcessAudioFrame(), Msg: " + e.Message + " - " + e.StackTrace); }
return(ret);
}
public void ReSync(long ms = -1)
{
if (status == Status.None)
{
return;
}
Pause();
decoder.Flush();
if (ms == -1)
{
MediaFrame vFrame = null;
do
{
decCtx.vDecoder.frames.TryPeek(out vFrame);
if (vFrame != null)
{
break;
}
else
{
Thread.Sleep(5);
}
} while (vFrame == null || forcePause);
if (forcePause)
{
return;
}
Seek(vFrame.timestamp / 10000);
}
else
{
Seek(ms);
}
if (decCtx.status == Status.Playing)
{
decoder.decodeARE.Set();
}
}
public static int ProcessSubsFrame(Decoder decoder, MediaFrame mFrame, AVSubtitle *sub)
{
int ret = 0;
try
{
string line = "";
byte[] buffer;
AVSubtitleRect **rects = sub->rects;
AVSubtitleRect * cur = rects[0];
switch (cur->type)
{
case AVSubtitleType.SUBTITLE_ASS:
buffer = new byte[1024];
line = Utils.BytePtrToStringUTF8(cur->ass);
break;
case AVSubtitleType.SUBTITLE_TEXT:
buffer = new byte[1024];
line = Utils.BytePtrToStringUTF8(cur->ass);
break;
case AVSubtitleType.SUBTITLE_BITMAP:
Log("Subtitles BITMAP -> Not Implemented yet");
return(-1);
}
mFrame.text = SSAtoSubStyles(line, out List <SubStyle> subStyles);
mFrame.subStyles = subStyles;
mFrame.duration = (int)(sub->end_display_time - sub->start_display_time);
//Log("SUBS ......... " + Utils.TicksToTime(mFrame.timestamp));
} catch (Exception e) { ret = -1; Log("Error[" + (ret).ToString("D4") + "], Func: ProcessSubsFrame(), Msg: " + e.Message + " - " + e.StackTrace); }
return(ret);
}
public void Decode()
{
//int xf = 0;
AVPacket *pkt;
while (!stopThread)
{
if (status != Status.Ended)
{
status = Status.Paused;
}
decodeARE.Reset();
decodeARE.WaitOne();
if (stopThread)
{
stopThread = false; break;
}
forcePause = false;
status = Status.Playing;
bool shouldStop = false;
int allowedErrors = decCtx.cfg.decoder.MaxErrors;
int ret = -1;
Log("Started");
// Wait for demuxer to come up
if (demuxer.status == Status.Paused)
{
demuxer.demuxARE.Set();
while (!demuxer.isPlaying && demuxer.status != Status.Ended && !forcePause && decCtx.isPlaying)
{
Thread.Sleep(1);
}
}
while (!stopThread)
{
// No Packets || Max Frames Brakes
if (packets.Count == 0 ||
(type == MediaType.Audio && frames.Count > decCtx.cfg.decoder.MaxAudioFrames) ||
(type == MediaType.Video && frames.Count > decCtx.cfg.decoder.MaxVideoFrames) ||
(type == MediaType.Subs && frames.Count > decCtx.cfg.decoder.MaxSubsFrames))
{
shouldStop = false;
//isWaiting = true;
do
{
if (!decCtx.isPlaying || forcePause) // Proper Pause
{
Log("Pausing"); shouldStop = true; break;
}
else if (packets.Count == 0 && demuxer.status == Status.Ended) // Drain
{
Log("Draining"); break;
}
//else if (packets.Count == 0 && (!demuxer.isPlaying || demuxer.isWaiting)) // No reason to run
else if (packets.Count == 0 && (!demuxer.isPlaying || ((!isEmbedded || type == MediaType.Video) && demuxer.isWaiting))) // No reason to run
{
Log("Exhausted " + isPlaying); shouldStop = true; break;
}
Thread.Sleep(10);
} while (packets.Count == 0 ||
(type == MediaType.Audio && frames.Count > decCtx.cfg.decoder.MaxAudioFrames) ||
(type == MediaType.Video && frames.Count > decCtx.cfg.decoder.MaxVideoFrames) ||
(type == MediaType.Subs && frames.Count > decCtx.cfg.decoder.MaxSubsFrames));
//isWaiting = false;
if (shouldStop)
{
break;
}
}
if (!decCtx.isPlaying || forcePause)
{
break;
}
if (packets.Count == 0 && demuxer.status == Status.Ended)
{
if (type == MediaType.Video)
{
// Check case pause while draining
Log("Draining...");
pkt = null;
}
else
{
status = Status.Ended;
Log("EOF");
break;
}
}
else
{
packets.TryDequeue(out IntPtr pktPtr);
pkt = (AVPacket *)pktPtr;
if (type == MediaType.Subs)
{
MediaFrame mFrame = new MediaFrame();
mFrame.pts = pkt->pts;
mFrame.timestamp = (long)((mFrame.pts * info.Timebase)) + decCtx.cfg.audio.LatencyTicks + decCtx.cfg.subs.DelayTicks;
//Log(Utils.TicksToTime((long)(mFrame.pts * demuxer.streams[st->index].timebase)) + " | pts -> " + mFrame.pts);
//xf++;
if (mFrame.pts == AV_NOPTS_VALUE)
{
av_packet_free(&pkt);
continue;
}
int gotFrame = 0;
AVSubtitle sub = new AVSubtitle();
// drain mode todo
// pkt->data set to NULL && pkt->size = 0 until it stops returning subtitles
ret = avcodec_decode_subtitle2(codecCtx, &sub, &gotFrame, pkt);
if (ret < 0)
{
allowedErrors--;
Log($"[ERROR-2] {Utils.FFmpeg.ErrorCodeToMsg(ret)} ({ret})");
if (allowedErrors == 0)
{
Log("[ERROR-0] Too many errors!");
break;
}
continue;
}
if (gotFrame < 1 || sub.num_rects < 1)
{
continue;
}
MediaFrame.ProcessSubsFrame(this, mFrame, &sub);
frames.Enqueue(mFrame);
avsubtitle_free(&sub);
av_packet_free(&pkt);
continue;
}
}
ret = avcodec_send_packet(codecCtx, pkt);
if (ret != 0 && ret != AVERROR(EAGAIN))
{
if (ret == AVERROR_EOF)
{
status = Status.Ended;
Log("EOF");
break;
}
else
//if (ret == AVERROR_INVALIDDATA) // We also get Error number -16976906 occurred
{
allowedErrors--;
Log($"[ERROR-2] {Utils.FFmpeg.ErrorCodeToMsg(ret)} ({ret})");
if (allowedErrors == 0)
{
Log("[ERROR-0] Too many errors!");
break;
}
continue;
}
}
av_packet_free(&pkt);
while (true)
{
ret = avcodec_receive_frame(codecCtx, frame);
if (ret == 0)
{
MediaFrame mFrame = new MediaFrame();
mFrame.pts = frame->best_effort_timestamp == AV_NOPTS_VALUE ? frame->pts : frame->best_effort_timestamp;
if (mFrame.pts == AV_NOPTS_VALUE)
{
av_frame_unref(frame);
continue;
}
//Log(Utils.TicksToTime((long)(mFrame.pts * demuxer.streams[st->index].Timebase)) + " | pts -> " + mFrame.pts);
if (type == MediaType.Video)
{
if (hwAccelSuccess && frame->hw_frames_ctx == null)
{
Log("HW Acceleration Failed 2");
hwAccelSuccess = false;
decCtx.renderer.FrameResized();
}
mFrame.timestamp = ((long)(mFrame.pts * info.Timebase) - info.StartTime) + decCtx.cfg.audio.LatencyTicks;
if (MediaFrame.ProcessVideoFrame(this, mFrame, frame) != 0)
{
mFrame = null;
}
}
else // Audio
{
mFrame.timestamp = ((long)(mFrame.pts * info.Timebase) - info.StartTime) + decCtx.cfg.audio.DelayTicks + (info.StartTime - demuxer.decCtx.vDecoder.info.StartTime);
if (MediaFrame.ProcessAudioFrame(this, mFrame, frame) < 0)
{
mFrame = null;
}
}
if (mFrame != null)
{
frames.Enqueue(mFrame);
//xf++;
}
av_frame_unref(frame);
continue;
}
av_frame_unref(frame);
break;
}
if (ret == AVERROR_EOF)
{
status = Status.Ended;
Log("EOF");
if (type == MediaType.Video && decCtx.aDecoder.status != Status.Playing)
{
Log("EOF All"); decCtx.status = Status.Ended;
}
else if (type == MediaType.Audio && decCtx.vDecoder.status != Status.Playing)
{
Log("EOF All"); decCtx.status = Status.Ended;
}
break;
}
if (ret != AVERROR(EAGAIN))
{
Log($"[ERROR-3] {Utils.FFmpeg.ErrorCodeToMsg(ret)} ({ret})"); break;
}
}
Log($"Done {(allowedErrors == decCtx.cfg.decoder.MaxErrors ? "" : $"[Errors: {decCtx.cfg.decoder.MaxErrors - allowedErrors}]")}");
}
}
public static int ProcessVideoFrame(Decoder decoder, MediaFrame mFrame, AVFrame *frame)
{
int ret = 0;
try
{
// Hardware Frame (NV12|P010) | CopySubresourceRegion FFmpeg Texture Array -> Device Texture[1] (NV12|P010) / SRV (RX_RXGX) -> PixelShader (Y_UV)
if (decoder.hwAccelSuccess)
{
decoder.textureFFmpeg = new Texture2D((IntPtr)frame->data.ToArray()[0]);
decoder.textDesc.Format = decoder.textureFFmpeg.Description.Format;
mFrame.textures = new Texture2D[1];
mFrame.textures[0] = new Texture2D(decoder.decCtx.renderer.device, decoder.textDesc);
decoder.decCtx.renderer.device.ImmediateContext.CopySubresourceRegion(decoder.textureFFmpeg, (int)frame->data.ToArray()[1], new ResourceRegion(0, 0, 0, mFrame.textures[0].Description.Width, mFrame.textures[0].Description.Height, 1), mFrame.textures[0], 0);
return(ret);
}
// Software Frame (8-bit YUV) | YUV byte* -> Device Texture[3] (RX) / SRV (RX_RX_RX) -> PixelShader (Y_U_V)
else if (decoder.info.PixelFormatType == PixelFormatType.Software_Handled)
{
mFrame.textures = new Texture2D[3];
// YUV Planar [Y0 ...] [U0 ...] [V0 ....]
if (decoder.info.IsPlanar)
{
DataBox db = new DataBox();
db.DataPointer = (IntPtr)frame->data.ToArray()[0];
db.RowPitch = frame->linesize.ToArray()[0];
mFrame.textures[0] = new Texture2D(decoder.decCtx.renderer.device, decoder.textDesc, new DataBox[] { db });
db = new DataBox();
db.DataPointer = (IntPtr)frame->data.ToArray()[1];
db.RowPitch = frame->linesize.ToArray()[1];
mFrame.textures[1] = new Texture2D(decoder.decCtx.renderer.device, decoder.textDescUV, new DataBox[] { db });
db = new DataBox();
db.DataPointer = (IntPtr)frame->data.ToArray()[2];
db.RowPitch = frame->linesize.ToArray()[2];
mFrame.textures[2] = new Texture2D(decoder.decCtx.renderer.device, decoder.textDescUV, new DataBox[] { db });
}
// YUV Packed ([Y0U0Y1V0] ....)
else
{
DataStream dsY = new DataStream(decoder.textDesc.Width * decoder.textDesc.Height, true, true);
DataStream dsU = new DataStream(decoder.textDescUV.Width * decoder.textDescUV.Height, true, true);
DataStream dsV = new DataStream(decoder.textDescUV.Width * decoder.textDescUV.Height, true, true);
DataBox dbY = new DataBox();
DataBox dbU = new DataBox();
DataBox dbV = new DataBox();
dbY.DataPointer = dsY.DataPointer;
dbU.DataPointer = dsU.DataPointer;
dbV.DataPointer = dsV.DataPointer;
dbY.RowPitch = decoder.textDesc.Width;
dbU.RowPitch = decoder.textDescUV.Width;
dbV.RowPitch = decoder.textDescUV.Width;
long totalSize = frame->linesize.ToArray()[0] * decoder.textDesc.Height;
byte *dataPtr = frame->data.ToArray()[0];
AVComponentDescriptor[] comps = decoder.info.PixelFormatDesc->comp.ToArray();
for (int i = 0; i < totalSize; i += decoder.info.Comp0Step)
{
dsY.WriteByte(*(dataPtr + i));
}
for (int i = 1; i < totalSize; i += decoder.info.Comp1Step)
{
dsU.WriteByte(*(dataPtr + i));
}
for (int i = 3; i < totalSize; i += decoder.info.Comp2Step)
{
dsV.WriteByte(*(dataPtr + i));
}
mFrame.textures[0] = new Texture2D(decoder.decCtx.renderer.device, decoder.textDesc, new DataBox[] { dbY });
mFrame.textures[1] = new Texture2D(decoder.decCtx.renderer.device, decoder.textDescUV, new DataBox[] { dbU });
mFrame.textures[2] = new Texture2D(decoder.decCtx.renderer.device, decoder.textDescUV, new DataBox[] { dbV });
Utilities.Dispose(ref dsY); Utilities.Dispose(ref dsU); Utilities.Dispose(ref dsV);
}
}
// Software Frame (OTHER/sws_scale) | X byte* -> Sws_Scale RGBA -> Device Texture[1] (RGBA) / SRV (RGBA) -> PixelShader (RGBA)
else
{
if (decoder.swsCtx == null)
{
decoder.textDesc.Format = SharpDX.DXGI.Format.R8G8B8A8_UNorm;
decoder.outData = new byte_ptrArray4();
decoder.outLineSize = new int_array4();
decoder.outBufferSize = av_image_get_buffer_size(Decoder.VOutPixelFormat, decoder.codecCtx->width, decoder.codecCtx->height, 1);
Marshal.FreeHGlobal(decoder.outBufferPtr);
decoder.outBufferPtr = Marshal.AllocHGlobal(decoder.outBufferSize);
av_image_fill_arrays(ref decoder.outData, ref decoder.outLineSize, (byte *)decoder.outBufferPtr, Decoder.VOutPixelFormat, decoder.codecCtx->width, decoder.codecCtx->height, 1);
int vSwsOptFlags = decoder.decCtx.cfg.video.SwsHighQuality ? DecoderContext.SCALING_HQ : DecoderContext.SCALING_LQ;
decoder.swsCtx = sws_getContext(decoder.codecCtx->coded_width, decoder.codecCtx->coded_height, decoder.codecCtx->pix_fmt, decoder.codecCtx->width, decoder.codecCtx->height, Decoder.VOutPixelFormat, vSwsOptFlags, null, null, null);
if (decoder.swsCtx == null)
{
Log($"[ProcessVideoFrame|RGB] [ERROR-1] Failed to allocate SwsContext"); return(ret);
}
}
sws_scale(decoder.swsCtx, frame->data, frame->linesize, 0, frame->height, decoder.outData, decoder.outLineSize);
DataBox db = new DataBox();
db.DataPointer = (IntPtr)decoder.outData.ToArray()[0];
db.RowPitch = decoder.outLineSize[0];
mFrame.textures = new Texture2D[1];
mFrame.textures[0] = new Texture2D(decoder.decCtx.renderer.device, decoder.textDesc, new DataBox[] { db });
}
return(ret);
} catch (Exception e) { ret = -1; Log("Error[" + (ret).ToString("D4") + "], Func: ProcessVideoFrame(), Msg: " + e.Message + " - " + e.StackTrace); }
return(ret);
}
public long GetVideoFrame()
{
int ret;
long firstTs = -1;
while (interrupt != 1)
{
AVPacket *pkt = av_packet_alloc();
ret = av_read_frame(demuxer.fmtCtx, pkt);
if (ret != 0)
{
return(-1);
}
if (!demuxer.enabledStreams.Contains(pkt->stream_index))
{
av_packet_free(&pkt);
continue;
}
switch (demuxer.fmtCtx->streams[pkt->stream_index]->codecpar->codec_type)
{
case AVMEDIA_TYPE_AUDIO:
aDecoder.packets.Enqueue((IntPtr)pkt);
break;
case AVMEDIA_TYPE_VIDEO:
ret = avcodec_send_packet(vDecoder.codecCtx, pkt);
av_packet_free(&pkt);
if (ret != 0)
{
return(-1);
}
while (interrupt != 1)
{
AVFrame *frame = av_frame_alloc();
ret = avcodec_receive_frame(vDecoder.codecCtx, frame);
if (ret == 0)
{
MediaFrame mFrame = new MediaFrame();
mFrame.pts = frame->best_effort_timestamp == AV_NOPTS_VALUE ? frame->pts : frame->best_effort_timestamp;
mFrame.timestamp = ((long)(mFrame.pts * vDecoder.info.Timebase) - vDecoder.info.StartTime) + cfg.audio.LatencyTicks;
if (mFrame.pts == AV_NOPTS_VALUE || frame->pict_type != AVPictureType.AV_PICTURE_TYPE_I)
{
if (frame->pict_type != AVPictureType.AV_PICTURE_TYPE_I)
{
Log($"Invalid Seek to Keyframe, skip... {frame->pict_type} | {frame->key_frame.ToString()}");
}
av_frame_free(&frame);
continue;
}
if (firstTs == -1)
{
if (vDecoder.hwAccelSuccess && frame->hw_frames_ctx == null)
{
Log("HW Acceleration Failed 2");
vDecoder.hwAccelSuccess = false;
renderer.FrameResized();
}
if (vDecoder.hwAccelSuccess && frame->hw_frames_ctx == null)
{
vDecoder.hwAccelSuccess = false;
}
firstTs = mFrame.timestamp;
}
if (MediaFrame.ProcessVideoFrame(vDecoder, mFrame, frame) != 0)
{
mFrame = null;
}
if (mFrame != null)
{
vDecoder.frames.Enqueue(mFrame);
}
//Log(Utils.TicksToTime((long)(mFrame.pts * avs.streams[video.st->index].timebase)));
av_frame_free(&frame);
continue;
}
av_frame_free(&frame);
break;
}
break;
case AVMEDIA_TYPE_SUBTITLE:
sDecoder.packets.Enqueue((IntPtr)pkt);
break;
default:
av_packet_free(&pkt);
break;
}
if (firstTs != -1)
{
break;
}
}
return(firstTs);
}