private static Mat AudioFrameToMat(AudioFrame frame)
{
int planar = ffmpeg.av_sample_fmt_is_planar((AVSampleFormat)frame.AVFrame.format);
int planes = planar != 0 ? frame.AVFrame.channels : 1;
int block_align = ffmpeg.av_get_bytes_per_sample((AVSampleFormat)frame.AVFrame.format) * (planar != 0 ? 1 : frame.AVFrame.channels);
int stride = frame.AVFrame.nb_samples * block_align;
int channels = planar != 0 ? 1 : frame.AVFrame.channels;
MatType dstType;
switch ((AVSampleFormat)frame.AVFrame.format)
{
case AVSampleFormat.AV_SAMPLE_FMT_U8:
case AVSampleFormat.AV_SAMPLE_FMT_U8P:
dstType = MatType.CV_8UC(channels);
break;
case AVSampleFormat.AV_SAMPLE_FMT_S16:
case AVSampleFormat.AV_SAMPLE_FMT_S16P:
dstType = MatType.CV_16SC(channels);
break;
case AVSampleFormat.AV_SAMPLE_FMT_S32:
case AVSampleFormat.AV_SAMPLE_FMT_S32P:
dstType = MatType.CV_32SC(channels);
break;
case AVSampleFormat.AV_SAMPLE_FMT_FLT:
case AVSampleFormat.AV_SAMPLE_FMT_FLTP:
dstType = MatType.CV_32FC(channels);
break;
case AVSampleFormat.AV_SAMPLE_FMT_DBL:
case AVSampleFormat.AV_SAMPLE_FMT_DBLP:
// opencv not have 64S, use 64F
case AVSampleFormat.AV_SAMPLE_FMT_S64:
case AVSampleFormat.AV_SAMPLE_FMT_S64P:
dstType = MatType.CV_64FC(channels);
break;
default:
throw new FFmpegException(FFmpegException.NotSupportFormat);
}
Mat mat = new Mat(planes, frame.AVFrame.nb_samples, dstType);
for (int i = 0; i < planes; i++)
{
FFmpegHelper.CopyMemory(mat.Data + i * stride, frame.Data[i], stride);
}
return(mat);
}
private static AudioFrame MatToAudioFrame(Mat mat, AVSampleFormat srctFormat, int sampleRate)
{
int channels = mat.NumberOfChannels > 1 ? mat.NumberOfChannels : mat.Height;
AudioFrame frame = new AudioFrame(channels, mat.Width, srctFormat, sampleRate);
bool isPlanar = ffmpeg.av_sample_fmt_is_planar(srctFormat) > 0;
int stride = mat.Step;
for (int i = 0; i < (isPlanar ? channels : 1); i++)
{
FFmpegHelper.CopyMemory(frame.Data[i], mat.DataPointer + i * stride, stride);
}
return(frame);
}
private static Mat AudioFrameToMat(AudioFrame frame)
{
DepthType dstType;
switch ((AVSampleFormat)frame.AVFrame.format)
{
case AVSampleFormat.AV_SAMPLE_FMT_U8:
case AVSampleFormat.AV_SAMPLE_FMT_U8P:
dstType = DepthType.Cv8U;
break;
case AVSampleFormat.AV_SAMPLE_FMT_S16:
case AVSampleFormat.AV_SAMPLE_FMT_S16P:
dstType = DepthType.Cv16S;
break;
case AVSampleFormat.AV_SAMPLE_FMT_S32:
case AVSampleFormat.AV_SAMPLE_FMT_S32P:
dstType = DepthType.Cv32S;
break;
case AVSampleFormat.AV_SAMPLE_FMT_FLT:
case AVSampleFormat.AV_SAMPLE_FMT_FLTP:
dstType = DepthType.Cv32F;
break;
case AVSampleFormat.AV_SAMPLE_FMT_DBL:
case AVSampleFormat.AV_SAMPLE_FMT_DBLP:
// emgucv not have S64, use 64F
case AVSampleFormat.AV_SAMPLE_FMT_S64:
case AVSampleFormat.AV_SAMPLE_FMT_S64P:
dstType = DepthType.Cv64F;
break;
default:
throw new FFmpegException(FFmpegException.NotSupportFormat);
}
int planar = ffmpeg.av_sample_fmt_is_planar((AVSampleFormat)frame.AVFrame.format);
int planes = planar != 0 ? frame.AVFrame.channels : 1;
int block_align = ffmpeg.av_get_bytes_per_sample((AVSampleFormat)frame.AVFrame.format) * (planar != 0 ? 1 : frame.AVFrame.channels);
int stride = frame.AVFrame.nb_samples * block_align;
Mat mat = new Mat(planes, frame.AVFrame.nb_samples, dstType, (planar != 0 ? 1 : frame.AVFrame.channels));
for (int i = 0; i < planes; i++)
{
FFmpegHelper.CopyMemory(frame.Data[i], mat.DataPointer + i * stride, stride);
}
return(mat);
}
/// <summary>
/// create a audio frame by codec's parames
/// </summary>
/// <param name="codec"></param>
/// <returns></returns>
public static AudioFrame CreateFrameByCodec(MediaCodec codec)
{
if (codec.Type != AVMediaType.AVMEDIA_TYPE_AUDIO)
{
throw new FFmpegException(FFmpegException.CodecTypeError);
}
AudioFrame audioFrame = new AudioFrame(codec.AVCodecContext.channels, codec.AVCodecContext.frame_size, codec.AVCodecContext.sample_fmt, codec.AVCodecContext.sample_rate);
if (codec.AVCodecContext.channel_layout > 0)
{
audioFrame.pFrame->channel_layout = codec.AVCodecContext.channel_layout;
}
return(audioFrame);
}
/// <summary>
/// Convert to audio frame to <paramref name="dstFotmat"/>
/// <para><see cref="DepthType"/> to <see cref="AVSampleFormat"/> mapping table.
/// if <see cref="Mat.NumberOfChannels"/> > 1, use packet format, otherwise planar</para>
/// <list type="table" >
/// <item>
/// <term><see cref="DepthType.Cv8U"/></term>
/// <description1><see cref="AVSampleFormat.AV_SAMPLE_FMT_U8"/>/<see cref="AVSampleFormat.AV_SAMPLE_FMT_U8P"/></description1>
/// </item>
/// <item>
/// <term><see cref="DepthType.Cv16S"/></term>
/// <description1><see cref="AVSampleFormat.AV_SAMPLE_FMT_S16"/>/<see cref="AVSampleFormat.AV_SAMPLE_FMT_S16P"/></description1>
/// </item>
/// <item>
/// <term><see cref="DepthType.Cv32S"/></term>
/// <description1><see cref="AVSampleFormat.AV_SAMPLE_FMT_S32"/>/<see cref="AVSampleFormat.AV_SAMPLE_FMT_S32P"/></description1>
/// </item>
/// <item>
/// <term><see cref="DepthType.Cv32F"/></term>
/// <description1><see cref="AVSampleFormat.AV_SAMPLE_FMT_FLT"/>/<see cref="AVSampleFormat.AV_SAMPLE_FMT_FLTP"/></description1>
/// </item>
/// <item>
/// <term><see cref="DepthType.Cv64F"/></term>
/// <description1><see cref="AVSampleFormat.AV_SAMPLE_FMT_DBL"/>/<see cref="AVSampleFormat.AV_SAMPLE_FMT_DBLP"/></description1>
/// </item>
/// <item>
/// <term><see cref="DepthType.Cv64F"/></term>
/// <description1><see cref="AVSampleFormat.AV_SAMPLE_FMT_S64"/>/<see cref="AVSampleFormat.AV_SAMPLE_FMT_S64P"/></description1>
/// </item>
/// <item>NOTE: Emgucv not supported int64, mapping Cv64F to int64,
/// so set Mat with int64 if <paramref name="dstFotmat"/> is <see cref="AVSampleFormat.AV_SAMPLE_FMT_S64"/> or <see cref="AVSampleFormat.AV_SAMPLE_FMT_S64P"/>
/// </item>
/// </list>
/// </summary>
/// <param name="mat"></param>
/// <param name="dstFotmat">Default is auto format by <see cref="Mat.Depth"/> and <see cref="Mat.NumberOfChannels"/> use mapping table</param>
/// <param name="dstSampleRate">Mat not have sample rate, set value here or later</param>
/// <returns></returns>
public static AudioFrame ToAudioFrame(this Mat mat, AVSampleFormat dstFotmat = AVSampleFormat.AV_SAMPLE_FMT_NONE, int dstSampleRate = 0)
{
AVSampleFormat srcformat;
switch (mat.Depth)
{
case DepthType.Default:
case DepthType.Cv8U:
case DepthType.Cv8S:
srcformat = mat.NumberOfChannels > 1 ? AVSampleFormat.AV_SAMPLE_FMT_U8 : AVSampleFormat.AV_SAMPLE_FMT_U8P;
break;
case DepthType.Cv16U:
case DepthType.Cv16S:
srcformat = mat.NumberOfChannels > 1 ? AVSampleFormat.AV_SAMPLE_FMT_S16 : AVSampleFormat.AV_SAMPLE_FMT_S16P;
break;
case DepthType.Cv32S:
srcformat = mat.NumberOfChannels > 1 ? AVSampleFormat.AV_SAMPLE_FMT_S32 : AVSampleFormat.AV_SAMPLE_FMT_S32P;
break;
case DepthType.Cv32F:
srcformat = mat.NumberOfChannels > 1 ? AVSampleFormat.AV_SAMPLE_FMT_FLT : AVSampleFormat.AV_SAMPLE_FMT_FLTP;
break;
case DepthType.Cv64F:
srcformat = mat.NumberOfChannels > 1 ? AVSampleFormat.AV_SAMPLE_FMT_DBL : AVSampleFormat.AV_SAMPLE_FMT_DBLP;
break;
default:
throw new FFmpegException(FFmpegException.NotSupportFormat);
}
if (dstFotmat != AVSampleFormat.AV_SAMPLE_FMT_NONE && dstFotmat != srcformat)
{
// converter must need set sample rate
using (SampleConverter converter = new SampleConverter(dstFotmat, mat.NumberOfChannels > 1 ? mat.NumberOfChannels : mat.Height, mat.Width, Math.Min(1, dstSampleRate)))
{
AudioFrame frame = converter.ConvertFrame(MatToAudioFrame(mat, srcformat, Math.Min(1, dstSampleRate)), out int a, out int b);
unsafe
{
// set real sample rate after convert
((AVFrame *)frame)->sample_rate = dstSampleRate;
}
}
}
return(MatToAudioFrame(mat, srcformat, dstSampleRate));
}
/// <summary>
/// convert current frame to packet frame.
/// if current frame is planer return new packet frame
/// else return current frame.
/// </summary>
/// <returns></returns>
public AudioFrame ToPacket()
{
unsafe
{
if (ffmpeg.av_sample_fmt_is_planar((AVSampleFormat)pFrame->format) <= 0)
{
return(this);
}
AVSampleFormat outFormat = (AVSampleFormat)pFrame->format;
if (outFormat == AVSampleFormat.AV_SAMPLE_FMT_NB || outFormat == AVSampleFormat.AV_SAMPLE_FMT_NONE)
{
throw new FFmpegException(FFmpegException.NotSupportFormat);
}
switch ((AVSampleFormat)pFrame->format)
{
case AVSampleFormat.AV_SAMPLE_FMT_U8P:
outFormat = AVSampleFormat.AV_SAMPLE_FMT_U8;
break;
case AVSampleFormat.AV_SAMPLE_FMT_S16P:
outFormat = AVSampleFormat.AV_SAMPLE_FMT_S16;
break;
case AVSampleFormat.AV_SAMPLE_FMT_S32P:
outFormat = AVSampleFormat.AV_SAMPLE_FMT_S32;
break;
case AVSampleFormat.AV_SAMPLE_FMT_FLTP:
outFormat = AVSampleFormat.AV_SAMPLE_FMT_FLT;
break;
case AVSampleFormat.AV_SAMPLE_FMT_DBLP:
outFormat = AVSampleFormat.AV_SAMPLE_FMT_DBL;
break;
case AVSampleFormat.AV_SAMPLE_FMT_S64P:
outFormat = AVSampleFormat.AV_SAMPLE_FMT_S64;
break;
}
AudioFrame outFrame = new AudioFrame(pFrame->channels, pFrame->nb_samples, outFormat, pFrame->sample_rate);
outFrame.pFrame->channel_layout = pFrame->channel_layout;
using (SampleConverter converter = new SampleConverter(outFrame))
{
return(converter.ConvertFrame(this, out int _, out int __));
}
}
}
/// <summary>
/// A full copy.
/// </summary>
/// <returns></returns>
public override MediaFrame Copy()
{
AudioFrame dstFrame = new AudioFrame();
AVFrame * dst = dstFrame;
dst->format = pFrame->format;
dst->channel_layout = pFrame->channel_layout;
dst->channels = pFrame->channels;
dst->nb_samples = pFrame->nb_samples;
dst->sample_rate = pFrame->sample_rate;
if (ffmpeg.av_frame_is_writable(pFrame) != 0)
{
ffmpeg.av_frame_get_buffer(dst, 0).ThrowIfError();
ffmpeg.av_frame_copy(dst, pFrame).ThrowIfError();
}
ffmpeg.av_frame_copy_props(dst, pFrame).ThrowIfError();
return(dstFrame);
}
