public static int ProcessAudioFrame(Decoder decoder, MediaFrame mFrame, AVFrame *frame)
{
/* References
*
* https://www.programmersought.com/article/70255648018/
* https://ffmpeg.org/doxygen/2.4/resampling__audio_8c_source.html
*
* Notes
*
* Currently output sample rate = input sample rate which means in case we change that we should review that actually works properly
* If frame->nb_samples are known initially or by decoding on frame we can avoid av_samples_alloc_array_and_samples here and transfer it to SetupAudio
*/
int ret = 0;
try
{
int dst_nb_samples;
if (decoder.m_max_dst_nb_samples == -1)
{
if (decoder.m_dst_data != null && (IntPtr)(*decoder.m_dst_data) != IntPtr.Zero)
{
av_freep(&decoder.m_dst_data[0]); decoder.m_dst_data = null;
}
decoder.m_max_dst_nb_samples = (int)av_rescale_rnd(frame->nb_samples, decoder.opt.audio.SampleRate, decoder.codecCtx->sample_rate, AVRounding.AV_ROUND_UP);
fixed(byte ***dst_data = &decoder.m_dst_data)
fixed(int *dst_linesize = &decoder.m_dst_linesize)
ret = av_samples_alloc_array_and_samples(dst_data, dst_linesize, decoder.opt.audio.Channels, decoder.m_max_dst_nb_samples, decoder.opt.audio.SampleFormat, 0);
}
fixed(int *dst_linesize = &decoder.m_dst_linesize)
{
dst_nb_samples = (int)av_rescale_rnd(swr_get_delay(decoder.swrCtx, decoder.codecCtx->sample_rate) + frame->nb_samples, decoder.opt.audio.SampleRate, decoder.codecCtx->sample_rate, AVRounding.AV_ROUND_UP);
if (dst_nb_samples > decoder.m_max_dst_nb_samples)
{
av_freep(&decoder.m_dst_data[0]);
ret = av_samples_alloc(decoder.m_dst_data, dst_linesize, decoder.opt.audio.Channels, (int)dst_nb_samples, decoder.opt.audio.SampleFormat, 0);
}
ret = swr_convert(decoder.swrCtx, decoder.m_dst_data, dst_nb_samples, (byte **)&frame->data, frame->nb_samples);
if (ret < 0)
{
return(ret);
}
int dst_data_len = av_samples_get_buffer_size(dst_linesize, decoder.opt.audio.Channels, ret, decoder.opt.audio.SampleFormat, 1);
mFrame.audioData = new byte[dst_data_len]; Marshal.Copy((IntPtr)(*decoder.m_dst_data), mFrame.audioData, 0, mFrame.audioData.Length); //Marshal.FreeHGlobal((IntPtr)(*m_dst_data));
}
} catch (Exception e) { ret = -1; Log("Error[" + (ret).ToString("D4") + "], Func: ProcessAudioFrame(), Msg: " + e.Message + " - " + e.StackTrace); }
return(ret);
}
public void ReSync(long ms = -1)
{
if (status == Status.NOTSET)
{
return;
}
Pause();
decoder.Flush();
if (ms == -1)
{
MediaFrame vFrame = null;
do
{
decCtx.vDecoder.frames.TryPeek(out vFrame);
if (vFrame != null)
{
break;
}
else
{
Thread.Sleep(5);
}
} while (vFrame == null || forcePause);
if (forcePause)
{
return;
}
Seek(vFrame.timestamp / 10000);
}
else
{
Seek(ms);
}
if (decCtx.status == Status.PLAY)
{
decoder.decodeARE.Set();
}
}
public static int ProcessSubsFrame(Decoder decoder, MediaFrame mFrame, AVSubtitle *sub)
{
int ret = 0;
try
{
string line = "";
byte[] buffer;
AVSubtitleRect **rects = sub->rects;
AVSubtitleRect * cur = rects[0];
switch (cur->type)
{
case AVSubtitleType.SUBTITLE_ASS:
buffer = new byte[1024];
line = Utils.BytePtrToStringUTF8(cur->ass);
break;
case AVSubtitleType.SUBTITLE_TEXT:
buffer = new byte[1024];
line = Utils.BytePtrToStringUTF8(cur->ass);
break;
case AVSubtitleType.SUBTITLE_BITMAP:
Log("Subtitles BITMAP -> Not Implemented yet");
return(-1);
}
mFrame.text = Subtitles.SSAtoSubStyles(line, out List <OSDMessage.SubStyle> subStyles);
mFrame.subStyles = subStyles;
mFrame.duration = (int)(sub->end_display_time - sub->start_display_time);
//Log("SUBS ......... " + Utils.TicksToTime(mFrame.timestamp));
} catch (Exception e) { ret = -1; Log("Error[" + (ret).ToString("D4") + "], Func: ProcessSubsFrame(), Msg: " + e.Message + " - " + e.StackTrace); }
return(ret);
}
public long GetVideoFrame()
{
int ret;
long firstTs = -1;
while (interrupt != 1)
{
AVPacket *pkt = av_packet_alloc();
ret = av_read_frame(demuxer.fmtCtx, pkt);
if (ret != 0)
{
return(-1);
}
if (!demuxer.enabledStreams.Contains(pkt->stream_index))
{
av_packet_free(&pkt);
continue;
}
switch (demuxer.fmtCtx->streams[pkt->stream_index]->codecpar->codec_type)
{
case AVMEDIA_TYPE_AUDIO:
aDecoder.packets.Enqueue((IntPtr)pkt);
break;
case AVMEDIA_TYPE_VIDEO:
lock (device)
ret = avcodec_send_packet(vDecoder.codecCtx, pkt);
av_packet_free(&pkt);
if (ret != 0)
{
return(-1);
}
while (interrupt != 1)
{
AVFrame *frame = av_frame_alloc();
lock (device)
ret = avcodec_receive_frame(vDecoder.codecCtx, frame);
if (ret == 0)
{
MediaFrame mFrame = new MediaFrame();
mFrame.pts = frame->best_effort_timestamp == AV_NOPTS_VALUE ? frame->pts : frame->best_effort_timestamp;
mFrame.timestamp = ((long)(mFrame.pts * vDecoder.info.Timebase) - demuxer.streams[vDecoder.st->index].StartTime) + opt.audio.LatencyTicks;
if (mFrame.pts == AV_NOPTS_VALUE)
{
av_frame_free(&frame);
continue;
}
if (firstTs == -1)
{
if (vDecoder.hwAccelSuccess && frame->hw_frames_ctx == null)
{
vDecoder.hwAccelSuccess = false;
}
firstTs = mFrame.timestamp;
}
if (MediaFrame.ProcessVideoFrame(vDecoder, mFrame, frame) != 0)
{
mFrame = null;
}
if (mFrame != null)
{
vDecoder.frames.Enqueue(mFrame);
}
//Log(Utils.TicksToTime((long)(mFrame.pts * avs.streams[video.st->index].timebase)));
av_frame_free(&frame);
continue;
}
av_frame_free(&frame);
break;
}
break;
case AVMEDIA_TYPE_SUBTITLE:
sDecoder.packets.Enqueue((IntPtr)pkt);
break;
default:
av_packet_free(&pkt);
break;
}
if (firstTs != -1)
{
break;
}
}
return(firstTs);
}
public void Decode()
{
//int xf = 0;
AVPacket *pkt;
while (true)
{
if (status != Status.END)
{
status = Status.READY;
}
decodeARE.Reset();
decodeARE.WaitOne();
status = Status.PLAY;
forcePause = false;
bool shouldStop = false;
int allowedErrors = decCtx.opt.demuxer.MaxErrors;
int ret = -1;
Log("Started");
// Wait for demuxer to come up
if (demuxer.status == Status.READY)
{
demuxer.demuxARE.Set();
while (!demuxer.isPlaying && demuxer.status != Status.END)
{
Thread.Sleep(1);
}
}
while (true)
{
// No Packets || Max Frames Brakes
if (packets.Count == 0 ||
(type == Type.Audio && frames.Count > decCtx.opt.audio.MaxDecodedFrames) ||
(type == Type.Video && frames.Count > decCtx.opt.video.MaxDecodedFrames) ||
(type == Type.Subs && frames.Count > decCtx.opt.subs.MaxDecodedFrames))
{
shouldStop = false;
//isWaiting = true;
do
{
if (!decCtx.isPlaying || forcePause) // Proper Pause
{
Log("Pausing"); shouldStop = true; break;
}
else if (packets.Count == 0 && demuxer.status == Status.END) // Drain
{
Log("Draining"); break;
}
//else if (packets.Count == 0 && (!demuxer.isPlaying || demuxer.isWaiting)) // No reason to run
else if (packets.Count == 0 && (!demuxer.isPlaying || ((!isEmbedded || type == Type.Video) && demuxer.isWaiting))) // No reason to run
{
Log("Exhausted " + isPlaying); shouldStop = true; break;
}
Thread.Sleep(10);
} while (packets.Count == 0 ||
(type == Type.Audio && frames.Count > decCtx.opt.audio.MaxDecodedFrames) ||
(type == Type.Video && frames.Count > decCtx.opt.video.MaxDecodedFrames) ||
(type == Type.Subs && frames.Count > decCtx.opt.subs.MaxDecodedFrames));
//isWaiting = false;
if (shouldStop)
{
break;
}
}
if (packets.Count == 0 && demuxer.status == Status.END)
{
if (type == Type.Video)
{
// Check case pause while draining
Log("Draining...");
pkt = null;
}
else
{
status = Status.END;
Log("EOF");
break;
}
}
else
{
packets.TryDequeue(out IntPtr pktPtr);
pkt = (AVPacket *)pktPtr;
if (type == Type.Subs)
{
MediaFrame mFrame = new MediaFrame();
mFrame.pts = pkt->pts;
mFrame.timestamp = (long)((mFrame.pts * info.Timebase)) + opt.audio.LatencyTicks + opt.subs.DelayTicks;
//Log(Utils.TicksToTime((long)(mFrame.pts * demuxer.streams[st->index].timebase)) + " | pts -> " + mFrame.pts);
//xf++;
if (mFrame.pts == AV_NOPTS_VALUE)
{
av_packet_free(&pkt);
continue;
}
int gotFrame = 0;
AVSubtitle sub = new AVSubtitle();
// drain mode todo
// pkt->data set to NULL && pkt->size = 0 until it stops returning subtitles
ret = avcodec_decode_subtitle2(codecCtx, &sub, &gotFrame, pkt);
if (ret < 0)
{
allowedErrors--;
Log($"[ERROR-2] {Utils.ErrorCodeToMsg(ret)} ({ret})");
if (allowedErrors == 0)
{
Log("[ERROR-0] Too many errors!");
break;
}
continue;
}
if (gotFrame < 1 || sub.num_rects < 1)
{
continue;
}
MediaFrame.ProcessSubsFrame(this, mFrame, &sub);
frames.Enqueue(mFrame);
avsubtitle_free(&sub);
av_packet_free(&pkt);
continue;
}
}
lock (demuxer.decCtx.device)
ret = avcodec_send_packet(codecCtx, pkt);
if (ret != 0 && ret != AVERROR(EAGAIN))
{
if (ret == AVERROR_EOF)
{
status = Status.END;
Log("EOF");
break;
}
else
//if (ret == AVERROR_INVALIDDATA) // We also get Error number -16976906 occurred
{
allowedErrors--;
Log($"[ERROR-2] {Utils.ErrorCodeToMsg(ret)} ({ret})");
if (allowedErrors == 0)
{
Log("[ERROR-0] Too many errors!");
break;
}
continue;
}
}
av_packet_free(&pkt);
while (true)
{
lock (demuxer.decCtx.device)
ret = avcodec_receive_frame(codecCtx, frame);
if (ret == 0)
{
MediaFrame mFrame = new MediaFrame();
mFrame.pts = frame->best_effort_timestamp == AV_NOPTS_VALUE ? frame->pts : frame->best_effort_timestamp;
if (mFrame.pts == AV_NOPTS_VALUE)
{
av_frame_unref(frame);
continue;
}
//Log(Utils.TicksToTime((long)(mFrame.pts * demuxer.streams[st->index].Timebase)) + " | pts -> " + mFrame.pts);
if (type == Type.Video)
{
if (hwAccelSuccess && frame->hw_frames_ctx == null)
{
hwAccelSuccess = false;
}
mFrame.timestamp = ((long)(mFrame.pts * info.Timebase) - demuxer.streams[st->index].StartTime) + opt.audio.LatencyTicks;
if (MediaFrame.ProcessVideoFrame(this, mFrame, frame) != 0)
{
mFrame = null;
}
}
else // Audio
{
mFrame.timestamp = ((long)(mFrame.pts * info.Timebase) - demuxer.streams[st->index].StartTime) + opt.audio.DelayTicks + (demuxer.streams[st->index].StartTime - demuxer.decCtx.vDecoder.info.StartTime);
if (MediaFrame.ProcessAudioFrame(this, mFrame, frame) < 0)
{
mFrame = null;
}
}
if (mFrame != null)
{
frames.Enqueue(mFrame);
//xf++;
}
av_frame_unref(frame);
continue;
}
av_frame_unref(frame);
break;
}
if (ret == AVERROR_EOF)
{
status = Status.END;
Log("EOF");
if (type == Type.Video && decCtx.aDecoder.status != Status.PLAY)
{
Log("EOF All"); decCtx.status = Status.END;
}
else if (type == Type.Audio && decCtx.vDecoder.status != Status.PLAY)
{
Log("EOF All"); decCtx.status = Status.END;
}
break;
}
if (ret != AVERROR(EAGAIN))
{
Log($"[ERROR-3] {Utils.ErrorCodeToMsg(ret)} ({ret})"); break;
}
}
Log($"Done {(allowedErrors == decCtx.opt.demuxer.MaxErrors ? "" : $"[Errors: {decCtx.opt.demuxer.MaxErrors - allowedErrors}]")}");
}
}
public static int ProcessVideoFrame(Decoder decoder, MediaFrame mFrame, AVFrame *frame)
{
int ret = 0;
try
{
// Hardware Frame (NV12|P010) | CopySubresourceRegion FFmpeg -> textureHW -> VideoProcessBlt RGBA
if (decoder.hwAccelSuccess)
{
decoder.textureFFmpeg = new Texture2D((IntPtr)frame->data.ToArray()[0]);
decoder.textDescHW.Format = decoder.textureFFmpeg.Description.Format;
mFrame.textureHW = new Texture2D(decoder.decCtx.device, decoder.textDescHW);
lock (decoder.decCtx.device)
decoder.decCtx.device.ImmediateContext.CopySubresourceRegion(decoder.textureFFmpeg, (int)frame->data.ToArray()[1], new ResourceRegion(0, 0, 0, mFrame.textureHW.Description.Width, mFrame.textureHW.Description.Height, 1), mFrame.textureHW, 0);
return(ret);
}
// Software Frame (YUV420P) | YUV byte* -> Device YUV (srv R8 * 3) -> PixelShader YUV->RGBA
else if (frame->format == (int)AVPixelFormat.AV_PIX_FMT_YUV420P)
{
decoder.textDescYUV.Width = decoder.codecCtx->width;
decoder.textDescYUV.Height = decoder.codecCtx->height;
DataStream dsY = new DataStream(frame->linesize.ToArray()[0] * decoder.codecCtx->height, true, true);
DataStream dsU = new DataStream(frame->linesize.ToArray()[1] * decoder.codecCtx->height / 2, true, true);
DataStream dsV = new DataStream(frame->linesize.ToArray()[2] * decoder.codecCtx->height / 2, true, true);
DataBox dbY = new DataBox();
DataBox dbU = new DataBox();
DataBox dbV = new DataBox();
dbY.DataPointer = dsY.DataPointer;
dbU.DataPointer = dsU.DataPointer;
dbV.DataPointer = dsV.DataPointer;
dbY.RowPitch = frame->linesize.ToArray()[0];
dbU.RowPitch = frame->linesize.ToArray()[1];
dbV.RowPitch = frame->linesize.ToArray()[2];
dsY.WriteRange((IntPtr)frame->data.ToArray()[0], dsY.Length);
dsU.WriteRange((IntPtr)frame->data.ToArray()[1], dsU.Length);
dsV.WriteRange((IntPtr)frame->data.ToArray()[2], dsV.Length);
mFrame.textureY = new Texture2D(decoder.decCtx.device, decoder.textDescYUV, new DataBox[] { dbY });
decoder.textDescYUV.Width = decoder.codecCtx->width / 2;
decoder.textDescYUV.Height = decoder.codecCtx->height / 2;
mFrame.textureU = new Texture2D(decoder.decCtx.device, decoder.textDescYUV, new DataBox[] { dbU });
mFrame.textureV = new Texture2D(decoder.decCtx.device, decoder.textDescYUV, new DataBox[] { dbV });
Utilities.Dispose(ref dsY);
Utilities.Dispose(ref dsU);
Utilities.Dispose(ref dsV);
}
// Software Frame (OTHER/sws_scale) | X byte* -> Sws_Scale RGBA -> Device RGA
else if (!decoder.hwAccelSuccess)
{
if (decoder.swsCtx == null)
{
decoder.outData = new byte_ptrArray4();
decoder.outLineSize = new int_array4();
decoder.outBufferSize = av_image_get_buffer_size(decoder.opt.video.PixelFormat, decoder.codecCtx->width, decoder.codecCtx->height, 1);
Marshal.FreeHGlobal(decoder.outBufferPtr);
decoder.outBufferPtr = Marshal.AllocHGlobal(decoder.outBufferSize);
av_image_fill_arrays(ref decoder.outData, ref decoder.outLineSize, (byte *)decoder.outBufferPtr, decoder.opt.video.PixelFormat, decoder.codecCtx->width, decoder.codecCtx->height, 1);
int vSwsOptFlags = decoder.opt.video.SwsHighQuality ? DecoderContext.SCALING_HQ : DecoderContext.SCALING_LQ;
decoder.swsCtx = sws_getContext(decoder.codecCtx->coded_width, decoder.codecCtx->coded_height, decoder.codecCtx->pix_fmt, decoder.codecCtx->width, decoder.codecCtx->height, decoder.opt.video.PixelFormat, vSwsOptFlags, null, null, null);
if (decoder.swsCtx == null)
{
Log($"[ProcessVideoFrame|RGB] [ERROR-1] Failed to allocate SwsContext"); return(ret);
}
}
sws_scale(decoder.swsCtx, frame->data, frame->linesize, 0, frame->height, decoder.outData, decoder.outLineSize);
DataStream ds = new DataStream(decoder.outLineSize[0] * decoder.codecCtx->height, true, true);
DataBox db = new DataBox();
db.DataPointer = ds.DataPointer;
db.RowPitch = decoder.outLineSize[0];
ds.WriteRange((IntPtr)decoder.outData.ToArray()[0], ds.Length);
mFrame.textureRGB = new Texture2D(decoder.decCtx.device, decoder.textDescRGB, new DataBox[] { db });
Utilities.Dispose(ref ds);
}
return(ret);
} catch (Exception e) { ret = -1; Log("Error[" + (ret).ToString("D4") + "], Func: ProcessVideoFrame(), Msg: " + e.Message + " - " + e.StackTrace); }
return(ret);
}
