/// <summary>
/// Called when MediaCodec wants a new frame to decode
/// </summary>
/// <param name="codec">Codec.</param>
/// <param name="index">Index.</param>
public override void OnInputBufferAvailable(MediaCodec codec, int index)
{
if (_videoQueue.Size < 1)
{
// FIXME: Is it proper to enqueue an empty
// buffer like this?
codec.QueueInputBuffer(index, 0, 0, 0, MediaCodecBufferFlags.None);
return;
}
var data = _videoQueue.Back();
_videoQueue.PopBack();
if (data != null)
{
// Get pre-allocated buffer from MediaCodec
Java.Nio.ByteBuffer buffer = codec.GetInputBuffer(index);
// Stuff in our raw framedata
buffer.Put(data);
// Tell the decoder to process the frame
codec.QueueInputBuffer(index, 0, data.Length, 0, MediaCodecBufferFlags.None);
}
}
public void encode(byte[] data, int offset, int size)
{
if (!running)
{
return;
}
if (size == 0)
{
return;
}
int buffer_index = -1;
lock (availableBuffers)
{
if (availableBuffers.Count > 0)
{
buffer_index = availableBuffers[0];
availableBuffers.RemoveAt(0);
}
}
if (buffer_index > -1)
{
var ib = audioEncoder.GetInputBuffer(buffer_index);
ib.Clear();
ib.Put(data);
audioEncoder.QueueInputBuffer(buffer_index, offset, size, 0, 0);
}
return;
}
public override void OnInputBufferAvailable(MediaCodec codec, int index)
{
H264Frame frame;
bool success = _videoFrameQueue.TryDequeue(out frame);
if (!success)
{
codec.QueueInputBuffer(index, 0, 0, 0, 0);
return;
}
Java.Nio.ByteBuffer buffer = codec.GetInputBuffer(index);
buffer.Put(frame.RawData);
// tell the decoder to process the frame
codec.QueueInputBuffer(index, 0, frame.RawData.Length, 0, 0);
}
private void OnInputBufferAvailableInner(MediaCodec mc, int inputBufferId)
{
byte[] encoded_data = null;
while (((encoded_data = mCallbackObj.getEncodedFrameData()) == null) && isClosed == false)
{
Thread.Sleep(500);
}
if (isClosed)
{
return;
}
Console.WriteLine("OnInputBufferAvailable: got encoded data!");
if (encoded_data != null)
{
int sampleSize = encoded_data.Length;
if (sampleSize > 0)
{
ByteBuffer inputBuffer = mDecoder.GetInputBuffer(inputBufferId);
inputBuffer.Put(encoded_data);
if (frameCounter == 0)
{
Console.WriteLine("feed a frame contains SSP and PSP");
mDecoder.QueueInputBuffer(inputBufferId, 0, sampleSize, 0, MediaCodec.BufferFlagCodecConfig);
}
else
{
Console.WriteLine("QueueInputBuffer inputIndex=" + inputBufferId.ToString());
mDecoder.QueueInputBuffer(inputBufferId, 0, sampleSize, frameCounter * 1000 /* 1FPS */, 0);
}
}
else
{
Console.WriteLine("QueueInputBuffer set MediaCodec.BufferFlagEndOfStream");
mDecoder.QueueInputBuffer(inputBufferId, 0, 0, 0, MediaCodec.BufferFlagEndOfStream);
}
frameCounter++;
}
}
public void encodeAndSend(int num_bytes)
{
if (num_bytes > 0)
{
int buffer_index = -1;
lock (availableBuffers)
{
if (availableBuffers.Count > 0)
{
buffer_index = availableBuffers[0];
availableBuffers.RemoveAt(0);
}
}
if (buffer_index > -1)
{
var ib = audioEncoder.GetInputBuffer(buffer_index);
ib.Clear();
ib.Put(buffer);
audioEncoder.QueueInputBuffer(buffer_index, 0, num_bytes, 0, 0);
}
}
byte[] data_to_send = null;
lock (outputBuffers)
{
int total_size = 0;
foreach (var buf in outputBuffers)
{
total_size += buf.Length;
}
if (total_size >= 400)
{
data_to_send = new byte[total_size];
int data_written = 0;
foreach (var buf in outputBuffers)
{
Array.Copy(buf, 0, data_to_send, data_written, buf.Length);
data_written += buf.Length;
}
outputBuffers.Clear();
}
}
if (data_to_send != null)
{
OnSoundDataReceived(data_to_send);
}
}
private void decode(int buffer_index, byte[] data)
{
if (!running)
{
return;
}
try
{
var ib = audioDecoder.GetInputBuffer(buffer_index);
ib.Clear();
ib.Put(data);
audioDecoder.QueueInputBuffer(buffer_index, 0, data.Length, 0, 0);
}
catch (Exception e)
{
Logging.error("Exception occured in audio decoder: " + e);
}
}
private Thread GetDecoderThread()
{
var decoderThread = new Thread(() =>
{
try
{
while (!_disposed)
{
int inputBufferIndex = _mediaCodec.DequeueInputBuffer(50000);
if (inputBufferIndex >= 0)
{
ByteBuffer inputBuffer = _mediaCodec.GetInputBuffer(inputBufferIndex);
if (inputBuffer != null)
{
byte[] sample = null;
do
{
sample = _audioQueue.Size < 1 ? null : _audioQueue.Back();
} while (sample == null && !_disposed);
_audioQueue.PopBack();
if (sample != null)
{
inputBuffer.Put(sample, 0, sample.Length);
_mediaCodec.QueueInputBuffer(inputBufferIndex, 0, sample.Length, 0, MediaCodecBufferFlags.None);
}
}
}
}
}
catch (ThreadInterruptedException)
{
// Ignore Thread got interrupted from outside
}
});
decoderThread.Daemon = true;
decoderThread.Priority = Thread.MaxPriority;
return(decoderThread);
}
private bool WriteAudio()
{
int index = audioEncoder.DequeueInputBuffer(-1);
if (index >= 0)
{
ByteBuffer[] inputBuffers = audioEncoder.GetInputBuffers();
ByteBuffer buffer = inputBuffers[index];
var len = Utils.Clamp(audioData.Length - audioDataIdx, 0, buffer.Remaining());
buffer.Clear();
buffer.Put(audioData, audioDataIdx, len);
long presentationTime = (audioDataIdx * SecondsToMicroSeconds) / (44100 * 2);
audioDataIdx += len;
var done = audioDataIdx == audioData.Length;
audioEncoder.QueueInputBuffer(index, 0, len, presentationTime, done ? MediaCodecBufferFlags.EndOfStream : MediaCodecBufferFlags.None);
}
return(audioDataIdx < audioData.Length);
}
private void ExtractMedia()
{
if (MediaDecoder == null)
{
throw new InvalidOperationException("The Media Codec Extractor has not been initialized");
}
if (!isInitialized)
{
throw new InvalidOperationException("The Media Codec has not been initialized for a media");
}
var bufferInfo = new MediaCodec.BufferInfo();
var waitDefaultTime = TimeSpan.FromMilliseconds(10);
MediaDecoder.Start();
while (true)
{
var waitTime = waitDefaultTime; // time to wait at the end of the loop iteration
//Process the commands
if (ProcessCommandsAndUpdateCurrentState())
{
waitTime = TimeSpan.Zero;
}
// terminate the thread on disposal
if (currentState == SchedulerAsyncCommandEnum.Dispose)
{
return;
}
//=================================================================================================
//Extract video inputs
if (!inputExtractionDone)
{
int inputBufIndex = MediaDecoder.DequeueInputBuffer(0);
if (inputBufIndex >= 0)
{
waitTime = TimeSpan.Zero;
var inputBuffer = MediaDecoder.GetInputBuffer(inputBufIndex);
// Read the sample data into the ByteBuffer. This neither respects nor updates inputBuf's position, limit, etc.
int chunkSize = mediaExtractor.ReadSampleData(inputBuffer, 0);
if (chunkSize > 0)
{
if (mediaExtractor.SampleTrackIndex != mediaTrackIndex)
{
throw new Exception($"Got media sample from track {mediaExtractor.SampleTrackIndex}, track expected {mediaTrackIndex}");
}
MediaDecoder.QueueInputBuffer(inputBufIndex, 0, chunkSize, mediaExtractor.SampleTime, 0);
mediaExtractor.Advance();
}
else // End of stream -- send empty frame with EOS flag set.
{
MediaDecoder.QueueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodecBufferFlags.EndOfStream);
inputExtractionDone = true;
}
}
else
{
//do nothing: the input buffer queue is full (we need to output them first)
}
}
//=================================================================================================
// Process the output buffers
if (ShouldProcessDequeueOutput(ref waitTime))
{
int indexOutput = MediaDecoder.DequeueOutputBuffer(bufferInfo, 0);
switch (indexOutput)
{
case (int)MediaCodecInfoState.TryAgainLater: // decoder not ready yet (haven't processed input yet)
case (int)MediaCodecInfoState.OutputBuffersChanged: //deprecated: we just ignore it
break;
case (int)MediaCodecInfoState.OutputFormatChanged:
Logger.Verbose("decoder output format changed: " + MediaDecoder.OutputFormat.ToString());
break;
default: // the index of the output buffer
if (indexOutput < 0)
{
Logger.Warning("unexpected index from decoder.dequeueOutputBuffer: " + indexOutput);
isEOF = true;
break;
}
if ((bufferInfo.Flags & MediaCodecBufferFlags.EndOfStream) != 0)
{
isEOF = true;
MediaDecoder.ReleaseOutputBuffer(indexOutput, false);
break;
}
MediaCurrentTime = TimeSpanExtensions.FromMicroSeconds(bufferInfo.PresentationTimeUs);
ProcessOutputBuffer(bufferInfo, indexOutput);
break;
}
}
if (waitTime > TimeSpan.Zero)
{
// sleep required time to avoid active looping
// Note: do not sleep more than 'waitDefaultTime' to continue processing play commands
Utilities.Sleep(TimeSpanExtensions.Min(waitDefaultTime, waitTime));
}
}
}
private bool ExtractSomeAudioData(out bool endOfFile)
{
endOfFile = extractionOutputDone;
if (endOfFile)
{
return(false);
}
var hasExtractedData = false;
int TimeoutUs = 20000;
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
if (!extractionInputDone)
{
int inputBufIndex = audioMediaDecoder.DequeueInputBuffer(TimeoutUs);
if (inputBufIndex >= 0)
{
Java.Nio.ByteBuffer inputBuffer = audioMediaDecoder.GetInputBuffer(inputBufIndex);
//Read the sample data into the ByteBuffer. This neither respects nor updates inputBuf's position, limit, etc.
int chunkSize = audioMediaExtractor.ReadSampleData(inputBuffer, 0);
if (chunkSize < 0)
{
//End of stream: send empty frame with EOS flag set
audioMediaDecoder.QueueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodecBufferFlags.EndOfStream);
extractionInputDone = true;
//Logger.Verbose("sent input EOS");
}
else
{
if (audioMediaExtractor.SampleTrackIndex != trackIndexAudio)
{
Logger.Warning(string.Format("got audio sample from track {0}, expected {1}", audioMediaExtractor.SampleTrackIndex, trackIndexAudio));
}
var presentationTimeMicroSeconds = audioMediaExtractor.SampleTime;
audioMediaDecoder.QueueInputBuffer(inputBufIndex, 0, chunkSize, presentationTimeMicroSeconds, 0);
audioMediaExtractor.Advance();
}
}
else
{
//do nothing: the input buffer queue is full (we need to output them first)
//continue;
}
}
int decoderStatus = audioMediaDecoder.DequeueOutputBuffer(info, TimeoutUs);
switch (decoderStatus)
{
case (int)MediaCodecInfoState.TryAgainLater:
{
Logger.Verbose("no output from decoder available");
break;
}
case (int)MediaCodecInfoState.OutputFormatChanged:
{
MediaFormat newFormat = audioMediaDecoder.OutputFormat;
string newFormatStr = newFormat.ToString();
Logger.Verbose("audio decoder output format changed: " + newFormatStr);
break;
}
case (int)MediaCodecInfoState.OutputBuffersChanged:
{
//deprecated: we just ignore it
break;
}
default:
{
if (decoderStatus < 0)
{
throw new InvalidOperationException(string.Format("unexpected result from audio decoder.DequeueOutputBuffer: {0}", decoderStatus));
}
if ((info.Flags & MediaCodecBufferFlags.EndOfStream) != 0)
{
Logger.Verbose("audio: output EOS");
extractionOutputDone = true;
}
if (info.Size > 0)
{
hasExtractedData = true;
var buffer = audioMediaDecoder.GetOutputBuffer(decoderStatus);
var presentationTime = TimeSpanExtensions.FromMicroSeconds(info.PresentationTimeUs);
if (StorageBuffer.CountDataBytes + info.Size <= StorageBuffer.Data.Length)
{
buffer.Get(StorageBuffer.Data, StorageBuffer.CountDataBytes, info.Size); // Read the buffer all at once
buffer.Clear(); // MUST DO!!! OTHERWISE THE NEXT TIME YOU GET THIS SAME BUFFER BAD THINGS WILL HAPPEN
buffer.Position(0);
if (StorageBuffer.CountDataBytes == 0)
{
StorageBuffer.PresentationTime = presentationTime;
}
StorageBuffer.CountDataBytes += info.Size;
}
else
{
Logger.Error("The storage buffer has reached full capacity. Current data will be dropped");
}
}
audioMediaDecoder.ReleaseOutputBuffer(decoderStatus, false);
break;
}
}
endOfFile = extractionOutputDone;
return(hasExtractedData);
}
/**
* Work loop.
*/
private void doExtract(MediaExtractor extractor, int trackIndex, MediaCodec decoder, CodecOutputSurface outputSurface)
{
Stopwatch stopWatch = new Stopwatch();
const int TIMEOUT_USEC = 10000;
ByteBuffer [] decoderInputBuffers = decoder.GetInputBuffers();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int inputChunk = 0;
int decodeCount = 0;
var frameTimestamps = new List <long>();
bool outputDone = false;
bool inputDone = false;
//speed vs accuracy tradeoffs https://stackoverflow.com/questions/34132444/google-mobile-vision-poor-facedetector-performance-without-camerasource
//reducing bitmap resolution helps the most and thats ok because i'm not using them after
var detector = new FaceDetector.Builder(Application.Context)
.SetTrackingEnabled(true) //tracking enables false makes it much slow wtf?!?!
.SetClassificationType(ClassificationType.All)
.SetProminentFaceOnly(true) // no diff
//.SetMinFaceSize((float)0.1) //small performance gain when removed
.SetMode(FaceDetectionMode.Fast) // tiny small performance gain
.Build();
while (!outputDone)
{
stopWatch.Start();
// Feed more data to the decoder.
if (!inputDone)
{
int inputBufIndex = decoder.DequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0)
{
ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
// Read the sample data into the ByteBuffer. This neither respects nor
// updates inputBuf's position, limit, etc.
int chunkSize = extractor.ReadSampleData(inputBuf, 0);
if (chunkSize < 0)
{
// End of stream -- send empty frame with EOS flag set.
decoder.QueueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodec.BufferFlagEndOfStream);
inputDone = true;
//if (VERBOSE) Log.d(TAG, "sent input EOS");
}
else
{
if (extractor.SampleTrackIndex != trackIndex)
{
//Log.w(TAG, "WEIRD: got sample from track " + extractor.getSampleTrackIndex() + ", expected " + trackIndex);
}
frameTimestamps.Add(extractor.SampleTime); //might need to play with offset here to get right sync from decoder
decoder.QueueInputBuffer(inputBufIndex, 0, chunkSize, extractor.SampleTime, 0 /*flags*/);
//if (VERBOSE) {
// Log.d(TAG, "submitted frame " + inputChunk + " to dec, size=" +
// chunkSize);
//}
inputChunk++;
extractor.Advance();
}
}
else
{
//if (VERBOSE) Log.d(TAG, "input buffer not available");
}
}
if (!outputDone)
{
int decoderStatus = decoder.DequeueOutputBuffer(info, TIMEOUT_USEC);
if (decoderStatus == (int)MediaCodecInfoState.TryAgainLater)
{
// no output available yet
//if (VERBOSE) Log.d(TAG, "no output from decoder available");
}
else if (decoderStatus == (int)MediaCodecInfoState.OutputBuffersChanged)
{
// not important for us, since we're using Surface
//if (VERBOSE) Log.d(TAG, "decoder output buffers changed");
}
else if (decoderStatus == (int)MediaCodecInfoState.OutputFormatChanged)
{
//MediaFormat newFormat = decoder.OutputFormat;
//if (VERBOSE) Log.d(TAG, "decoder output format changed: " + newFormat);
}
else if (decoderStatus < 0)
{
//fail("unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus);
throw new InvalidOperationException();
}
else
{
//if (VERBOSE) Log.d(TAG, "surface decoder given buffer " + decoderStatus + " (size=" + info.size + ")");
if ((info.Flags & MediaCodecBufferFlags.EndOfStream) != 0)
{
//if (VERBOSE) Log.d(TAG, "output EOS");
outputDone = true;
}
bool doRender = (info.Size != 0);
//could not get this working!!!
// As soon as we call releaseOutputBuffer, the buffer will be forwarded
// to SurfaceTexture to convert to a texture. The API doesn't guarantee
// that the texture will be available before the call returns, so we
// need to wait for the onFrameAvailable callback to fire.
decoder.ReleaseOutputBuffer(decoderStatus, doRender);
if (doRender)
{
//outputSurface.awaitNewImage(); //could not get callback to work and even so do not want to wait 2.5 seconds for each frame, might need to revist
outputSurface.mTextureRender.checkGlError("before updateTexImage");
outputSurface.mSurfaceTexture.UpdateTexImage();
outputSurface.drawImage(true);
//Log.Info("innerSTOPWATCH_begin!!!!:", stopWatch.ElapsedMilliseconds.ToString());
//can't call face detector this way its too slow or maybe there is a busy loop???
//_FaceFetchDataTasks.Add(Task.Run(() => CreateFaceframes(detector, outputSurface.GetFramebitmap(), decodeCount, frameTimestamps[decodeCount])));
//if (decodeCount % 2 ==0) //doesn't help that much and messes with rating algo
CreateFaceframes(detector, outputSurface.GetFramebitmap(), frameTimestamps[decodeCount]);
//Log.Info("innerSTOPWATCH_end!!!!:", stopWatch.ElapsedMilliseconds.ToString());
decodeCount++;
}
}
}
}
stopWatch.Stop();
Log.Info("inner STOPWATCH!!!!:", string.Format("numberofframes = {0}, totaltime = {1}", decodeCount, stopWatch.ElapsedMilliseconds));
detector.Release();
}
public void decode(byte[] array)
{
if (bConfigured == false)
{
Init();
}
var nalType = array[4] & 0x1f;
//Console.WriteLine("nal:" + nalType);
if (nalType == 7)
{
//sps = array.ToArray();
if (array.Length != sps.Length)
{
stop();
sps = array.ToArray();
Init();
}
return;
}
if (nalType == 8)
{
//pps = array.ToArray();
return;
}
if (bConfigured == false)
{
return;
}
//Make sure keyframe is first.
if (nalType == 5)
{
bWaitForKeyframe = false;
//pps = array.ToArray();
//return;
}
if (bWaitForKeyframe)
{
return;
}
if (bConfigured)
{
try
{
ByteBuffer[] inputBuffers = codec.GetInputBuffers();
ByteBuffer[] outputBuffers = codec.GetOutputBuffers();
int dequeueInputBuffer = codec.DequeueInputBuffer(-1L);
if (dequeueInputBuffer >= 0)
{
//Send data to decoder.
ByteBuffer byteBuffer = inputBuffers[dequeueInputBuffer];
byteBuffer.Clear();
byteBuffer.Put(array);
codec.QueueInputBuffer(dequeueInputBuffer, 0, array.Length, 0L, 0);
}
//Show decoded frame
MediaCodec.BufferInfo BufferInfo = new MediaCodec.BufferInfo();
int i = codec.DequeueOutputBuffer(BufferInfo, 0L);
while (i >= 0)
{
/*if (picSurface == null)//Only if not using display surface.
* {
* ByteBuffer byteBuffer2 = outputBuffers[i];
* if (buffer == null || buffer.Length != BufferInfo.Size)
* {
* buffer = new byte[BufferInfo.Size];
* }
* byteBuffer2.Get(buffer);
* //do something with raw frame in buffer.
* }*/
codec.ReleaseOutputBuffer(i, true);
codec.SetVideoScalingMode(VideoScalingMode.ScaleToFit);
i = codec.DequeueOutputBuffer(BufferInfo, 0L);
}
}
catch (Exception ex)
{
MainActivity.getActivity().notifyUser("VideoDecoder decode exception " + ex.Message, false);
//attempt to recover.
//codec.Release();
//codec = null;
//bConfigured = false;
stop();
}
}
return;// ret;
}
public void decode(byte[] array)
{
if (bConfigured == false)
{
Init();
}
var nalType = array[4] & 0x1f;
if (nalType == 7)
{
//sps = array.ToArray();
if (array.Length != sps.Length)
{
stop();
sps = array.ToArray();
Init();
}
return;
}
if (nalType == 8)
{
//pps = array.ToArray();
return;
}
if (bConfigured == false)
{
return;
}
if (bConfigured)
{
try
{
ByteBuffer[] inputBuffers = codec.GetInputBuffers();
ByteBuffer[] outputBuffers = codec.GetOutputBuffers();
int dequeueInputBuffer = codec.DequeueInputBuffer(-1L);
if (dequeueInputBuffer >= 0)
{
//Send data to decoder.
ByteBuffer byteBuffer = inputBuffers[dequeueInputBuffer];
byteBuffer.Clear();
byteBuffer.Put(array);
codec.QueueInputBuffer(dequeueInputBuffer, 0, array.Length, 0L, 0);
}
//Show decoded frame
MediaCodec.BufferInfo BufferInfo = new MediaCodec.BufferInfo();
int i = codec.DequeueOutputBuffer(BufferInfo, 0L);
while (i >= 0)
{
/*if (picSurface == null)//Only if not using display surface.
{
ByteBuffer byteBuffer2 = outputBuffers[i];
if (buffer == null || buffer.Length != BufferInfo.Size)
{
buffer = new byte[BufferInfo.Size];
}
byteBuffer2.Get(buffer);
//do something with raw frame in buffer.
}*/
codec.ReleaseOutputBuffer(i, true);
codec.SetVideoScalingMode(VideoScalingMode.ScaleToFit);
i = codec.DequeueOutputBuffer(BufferInfo, 0L);
}
}
catch (Exception ex)
{
//attempt to recover.
stop();
}
}
return;// ret;
}
private void RtspClient_FrameReceived(object sender, RtspClientSharp.RawFrames.RawFrame e)
{
if (rtspCancel.Token.IsCancellationRequested)
{
return;
}
//Console.WriteLine("Got Frame " + e.ToString());
switch (e)
{
case RawH264IFrame h264IFrame:
{
if (callbacks.buffers.Count == 0)
{
return;
}
var index = callbacks.buffers.Pop();
var buffer = codec.GetInputBuffer(index);
buffer.Clear();
buffer.Put(h264IFrame.SpsPpsSegment.Array);
codec.QueueInputBuffer(index, 0, h264IFrame.SpsPpsSegment.Count, 0, MediaCodecBufferFlags.CodecConfig);
if (callbacks.buffers.Count == 0)
{
return;
}
index = callbacks.buffers.Pop();
buffer = codec.GetInputBuffer(index);
buffer.Clear();
buffer.Put(h264IFrame.FrameSegment.Array);
codec.QueueInputBuffer(index, 0, h264IFrame.FrameSegment.Count, 0, MediaCodecBufferFlags.None);
iframestart = false;
break;
}
case RawH264PFrame h264PFrame:
{
if (iframestart)
{
return;
}
if (callbacks.buffers.Count == 0)
{
return;
}
var index = callbacks.buffers.Pop();
var buffer = codec.GetInputBuffer(index);
buffer.Clear();
buffer.Put(h264PFrame.FrameSegment.Array);
codec.QueueInputBuffer(index, 0, h264PFrame.FrameSegment.Count, 0, 0);
break;
}
case RawJpegFrame jpegFrame:
case RawAACFrame aacFrame:
case RawG711AFrame g711AFrame:
case RawG711UFrame g711UFrame:
case RawPCMFrame pcmFrame:
case RawG726Frame g726Frame:
break;
}
}
/**
* Edits a stream of video data.
*/
private void editVideoData(VideoChunks inputData, MediaCodec decoder,
OutputSurface outputSurface, InputSurface inputSurface, MediaCodec encoder,
VideoChunks outputData)
{
const int TIMEOUT_USEC = 10000;
ByteBuffer[] decoderInputBuffers = decoder.GetInputBuffers();
ByteBuffer[] encoderOutputBuffers = encoder.GetOutputBuffers();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int inputChunk = 0;
int outputCount = 0;
bool outputDone = false;
bool inputDone = false;
bool decoderDone = false;
while (!outputDone)
{
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "edit loop");
}
// Feed more data to the decoder.
if (!inputDone)
{
int inputBufIndex = decoder.DequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0)
{
if (inputChunk == inputData.NumChunks)
{
// End of stream -- send empty frame with EOS flag set.
decoder.QueueInputBuffer(inputBufIndex, 0, 0, 0L,
MediaCodecBufferFlags.EndOfStream);
inputDone = true;
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "sent input EOS (with zero-length frame)");
}
}
else
{
// Copy a chunk of input to the decoder. The first chunk should have
// the BUFFER_FLAG_CODEC_CONFIG flag set.
ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
inputBuf.Clear();
inputData.getChunkData(inputChunk, inputBuf);
int flags = inputData.getChunkFlags(inputChunk);
long time = inputData.getChunkTime(inputChunk);
decoder.QueueInputBuffer(inputBufIndex, 0, inputBuf.Position(),
time, (MediaCodecBufferFlags)flags); // TODO: Not sure if it's MediaCodecBufferFlags, verify.
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "submitted frame " + inputChunk + " to dec, size=" +
inputBuf.Position() + " flags=" + flags);
}
inputChunk++;
}
}
else
{
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "input buffer not available");
}
}
}
// Assume output is available. Loop until both assumptions are false.
bool decoderOutputAvailable = !decoderDone;
bool encoderOutputAvailable = true;
while (decoderOutputAvailable || encoderOutputAvailable)
{
// Start by draining any pending output from the encoder. It's important to
// do this before we try to stuff any more data in.
int encoderStatus = encoder.DequeueOutputBuffer(info, TIMEOUT_USEC);
if (encoderStatus == (int)MediaCodecInfoState.TryAgainLater)
{
// no output available yet
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "no output from encoder available");
}
encoderOutputAvailable = false;
}
else if (encoderStatus == (int)MediaCodecInfoState.OutputBuffersChanged)
{
encoderOutputBuffers = encoder.GetOutputBuffers();
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "encoder output buffers changed");
}
}
else if (encoderStatus == (int)MediaCodecInfoState.OutputFormatChanged)
{
MediaFormat newFormat = encoder.OutputFormat;
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "encoder output format changed: " + newFormat);
}
}
else if (encoderStatus < 0)
{
fail("unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus);
}
else // encoderStatus >= 0
{
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null)
{
fail("encoderOutputBuffer " + encoderStatus + " was null");
}
// Write the data to the output "file".
if (info.Size != 0)
{
encodedData.Position(info.Offset);
encodedData.Limit(info.Offset + info.Size);
outputData.addChunk(encodedData, (int)info.Flags, info.PresentationTimeUs);
outputCount++;
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "encoder output " + info.Size + " bytes");
}
}
outputDone = (info.Flags & MediaCodec.BufferFlagEndOfStream) != 0;
encoder.ReleaseOutputBuffer(encoderStatus, false);
}
if (encoderStatus != (int)MediaCodec.InfoTryAgainLater)
{
// Continue attempts to drain output.
continue;
}
// Encoder is drained, check to see if we've got a new frame of output from
// the decoder. (The output is going to a Surface, rather than a ByteBuffer,
// but we still get information through BufferInfo.)
if (!decoderDone)
{
int decoderStatus = decoder.DequeueOutputBuffer(info, TIMEOUT_USEC);
if (decoderStatus == (int)MediaCodec.InfoTryAgainLater)
{
// no output available yet
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "no output from decoder available");
}
decoderOutputAvailable = false;
}
else if (decoderStatus == (int)MediaCodec.InfoOutputBuffersChanged)
{
//decoderOutputBuffers = decoder.getOutputBuffers();
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "decoder output buffers changed (we don't care)");
}
}
else if (decoderStatus == (int)MediaCodec.InfoOutputFormatChanged)
{
// expected before first buffer of data
MediaFormat newFormat = decoder.OutputFormat;
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "decoder output format changed: " + newFormat);
}
}
else if (decoderStatus < 0)
{
fail("unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus);
}
else // decoderStatus >= 0
{
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "surface decoder given buffer " + decoderStatus + " (size=" + info.Size + "(");
}
// The ByteBuffers are null references, but we still get a nonzero
// size for the decoded data.
bool doRender = (info.Size != 0);
// As soon as we call releaseOutputBuffer, the buffer will be forwarded
// to SurfaceTexture to convert to a texture. The API doesn't
// guarantee that the texture will be available before the call
// returns, so we need to wait for the onFrameAvailable callback to
// fire. If we don't wait, we risk rendering from the previous frame.
decoder.ReleaseOutputBuffer(decoderStatus, doRender);
if (doRender)
{
// This waits for the image and renders it after it arrives.
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "awaiting frame");
}
outputSurface.AwaitNewImage();
outputSurface.DrawImage();
// Send it to the encoder.
inputSurface.SetPresentationTime(info.PresentationTimeUs * 1000);
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "swapBuffers");
}
inputSurface.SwapBuffers();
}
if ((info.Flags & MediaCodec.BufferFlagEndOfStream) != 0)
{
// forward decoder EOS to encoder
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "signaling input EOS");
}
if (WORK_AROUND_BUGS)
{
// Bail early, possibly dropping a frame.
return;
}
else
{
encoder.SignalEndOfInputStream();
}
}
}
}
}
}
if (inputChunk != outputCount)
{
throw new RuntimeException("frame lost: " + inputChunk + " in, " +
outputCount + " out");
}
}
public async void rtspClientStart()
{
rtspCancel = new CancellationTokenSource();
var url = "rtsp://192.168.0.10:8554/H264Video";
String now = DateTime.Now.ToString("yyyyMMdd_HHmmss");
MemoryStream fs_vps = null; // used to write the video
MemoryStream fs_v = null; // used to write the video
MemoryStream fs_a = null; // used to write the audio
h264 = false;
h265 = false;
bool spsdone = false;
RTSPClient c = new RTSPClient();
// The SPS/PPS comes from the SDP data
// or it is the first SPS/PPS from the H264 video stream
c.Received_SPS_PPS += (byte[] sps, byte[] pps) => {
h264 = true;
if (fs_vps == null)
{
String filename = "rtsp_capture_" + now + ".264";
fs_vps = new MemoryStream();
}
if (fs_vps != null)
{
fs_vps.SetLength(0);
fs_vps.Write(new byte[] { 0x00, 0x00, 0x00, 0x01 }, 0, 4); // Write Start Code
fs_vps.Write(sps, 0, sps.Length);
fs_vps.Write(new byte[] { 0x00, 0x00, 0x00, 0x01 }, 0, 4); // Write Start Code
fs_vps.Write(pps, 0, pps.Length);
}
};
c.Received_VPS_SPS_PPS += (byte[] vps, byte[] sps, byte[] pps) => {
h265 = true;
if (fs_vps == null)
{
String filename = "rtsp_capture_" + now + ".265";
fs_vps = new MemoryStream();
}
if (fs_vps != null)
{
fs_vps.SetLength(0);
fs_vps.Write(new byte[] { 0x00, 0x00, 0x00, 0x01 }, 0, 4); // Write Start Code
fs_vps.Write(vps, 0, vps.Length); // Video parameter set
fs_vps.Write(new byte[] { 0x00, 0x00, 0x00, 0x01 }, 0, 4); // Write Start Code
fs_vps.Write(sps, 0, sps.Length); // Sequence Parameter Set
fs_vps.Write(new byte[] { 0x00, 0x00, 0x00, 0x01 }, 0, 4); // Write Start Code
fs_vps.Write(pps, 0, pps.Length); // Picture Parameter Set
}
};
// Video NALs. May also include the SPS and PPS in-band for H264
c.Received_NALs += (List <byte[]> nal_units) =>
{
foreach (byte[] nal_unit in nal_units)
{
// Output some H264 stream information
if (h264 && nal_unit.Length > 0)
{
int nal_ref_idc = (nal_unit[0] >> 5) & 0x03;
int nal_unit_type = nal_unit[0] & 0x1F;
String description = "";
if (nal_unit_type == 1)
{
description = "NON IDR NAL";
}
else if (nal_unit_type == 5)
{
description = "IDR NAL";
}
else if (nal_unit_type == 6)
{
description = "SEI NAL";
}
else if (nal_unit_type == 7)
{
description = "SPS NAL";
}
else if (nal_unit_type == 8)
{
description = "PPS NAL";
}
else if (nal_unit_type == 9)
{
description = "ACCESS UNIT DELIMITER NAL";
}
else
{
description = "OTHER NAL";
}
//Console.WriteLine("NAL Ref = " + nal_ref_idc + " NAL Type = " + nal_unit_type + " " + description);
}
// Output some H265 stream information
if (h265 && nal_unit.Length > 0)
{
int nal_unit_type = (nal_unit[0] >> 1) & 0x3F;
String description = "";
if (nal_unit_type == 1)
{
description = "NON IDR NAL";
}
else if (nal_unit_type == 19)
{
description = "IDR NAL";
}
else if (nal_unit_type == 32)
{
description = "VPS NAL";
}
else if (nal_unit_type == 33)
{
description = "SPS NAL";
}
else if (nal_unit_type == 34)
{
description = "PPS NAL";
}
else if (nal_unit_type == 39)
{
description = "SEI NAL";
}
else
{
description = "OTHER NAL";
}
//Console.WriteLine("NAL Type = " + nal_unit_type + " " + description);
}
// we need sps... first
if (!h264 && !h265)
{
return;
}
if (!spsdone)
{
if (callbacks == null || callbacks.buffers.Count == 0)
{
return;
}
var index = callbacks.buffers.Pop();
var buffer = codec.GetInputBuffer(index);
buffer.Clear();
buffer.Put(fs_vps.ToArray());
codec.QueueInputBuffer(index, 0, (int)fs_vps.Length, 0, MediaCodecBufferFlags.CodecConfig);
spsdone = true;
fs_v = new MemoryStream();
}
if (fs_v != null)
{
fs_v.Write(new byte[] { 0x00, 0x00, 0x00, 0x01 }, 0, 4); // Write Start Code
fs_v.Write(nal_unit, 0, nal_unit.Length); // Write NAL
}
if (callbacks == null || fs_v == null || callbacks.buffers.Count == 0)
{
return;
}
try
{
var index = callbacks.buffers.Pop();
var buffer = codec.GetInputBuffer(index);
buffer.Clear();
buffer.Put(fs_v.ToArray());
codec.QueueInputBuffer(index, 0, (int)fs_v.Length, 0, MediaCodecBufferFlags.None);
fs_v.SetLength(0);
}
catch
{
}
}
};
// seperate and stay running
Task.Run(() =>
{
while (rtspCancel != null && true)
{
try
{
if (rtspCancel.Token.IsCancellationRequested)
{
return;
}
c.Connect(url, RTSPClient.RTP_TRANSPORT.UDP);
var lastrtp = 0;
int cnt = 0;
while (!c.StreamingFinished())
{
rtsprunning = true;
Thread.Sleep(500);
// existing
if (rtspCancel.Token.IsCancellationRequested)
{
c.Stop();
return;
}
// no rtp in .5 sec
if (lastrtp == c.rtp_count && cnt++ > 5)
{
c.Stop();
rtspCancel = null;
return;
}
lastrtp = c.rtp_count;
}
rtsprunning = false;
}
catch
{
}
}
});
}
private void ResampleVideo(MediaExtractor extractor, MediaCodec decoder, SamplerClip clip)
{
ByteBuffer[] decoderInputBuffers = decoder.GetInputBuffers();
ByteBuffer[] encoderOutputBuffers = mEncoder.GetOutputBuffers();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int inputChunk = 0;
int outputCount = 0;
long endTime = clip.getEndTime();
if (endTime == -1)
{
endTime = clip.getVideoDuration();
}
bool outputDoneNextTimeWeCheck = false;
bool outputDone = false;
bool inputDone = false;
bool decoderDone = false;
while (!outputDone)
{
// Feed more data to the decoder.
if (!inputDone)
{
int inputBufIndex = decoder.DequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0)
{
if (extractor.SampleTime / 1000 >= endTime)
{
// End of stream -- send empty frame with EOS flag set.
decoder.QueueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodecBufferFlags.EndOfStream);
inputDone = true;
}
else
{
// Copy a chunk of input to the decoder. The first chunk should have
// the BUFFER_FLAG_CODEC_CONFIG flag set.
ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
inputBuf.Clear();
int sampleSize = extractor.ReadSampleData(inputBuf, 0);
if (sampleSize < 0)
{
decoder.QueueInputBuffer(inputBufIndex, 0, 0, 0, MediaCodecBufferFlags.EndOfStream);
}
else
{
decoder.QueueInputBuffer(inputBufIndex, 0, sampleSize, extractor.SampleTime, 0);
extractor.Advance();
}
inputChunk++;
}
}
}
// Assume output is available. Loop until both assumptions are false.
bool decoderOutputAvailable = !decoderDone;
bool encoderOutputAvailable = true;
while (decoderOutputAvailable || encoderOutputAvailable)
{
// Start by draining any pending output from the encoder. It's important to
// do this before we try to stuff any more data in.
int encoderStatus = mEncoder.DequeueOutputBuffer(info, TIMEOUT_USEC);
if (encoderStatus == (int)MediaCodecInfoState.TryAgainLater)
{
encoderOutputAvailable = false;
}
else if (encoderStatus == (int)MediaCodecInfoState.OutputBuffersChanged)
{
encoderOutputBuffers = mEncoder.GetOutputBuffers();
}
else if (encoderStatus == (int)MediaCodecInfoState.OutputFormatChanged)
{
MediaFormat newFormat = mEncoder.OutputFormat;
mTrackIndex = mMuxer.AddTrack(newFormat);
mMuxer.Start();
mMuxerStarted = true;
}
else if (encoderStatus < 0)
{
// fail( "unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus );
}
else
{ // encoderStatus >= 0
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null)
{
// fail( "encoderOutputBuffer " + encoderStatus + " was null" );
}
// Write the data to the output "file".
if (info.Size != 0)
{
encodedData.Position(info.Offset);
encodedData.Limit(info.Offset + info.Size);
outputCount++;
mMuxer.WriteSampleData(mTrackIndex, encodedData, info);
}
outputDone = (info.Flags & MediaCodecBufferFlags.EndOfStream) != 0;
mEncoder.ReleaseOutputBuffer(encoderStatus, false);
}
if (outputDoneNextTimeWeCheck)
{
outputDone = true;
}
if (encoderStatus != (int)MediaCodecInfoState.TryAgainLater)
{
// Continue attempts to drain output.
continue;
}
// Encoder is drained, check to see if we've got a new frame of output from
// the decoder. (The output is going to a Surface, rather than a ByteBuffer,
// but we still get information through BufferInfo.)
if (!decoderDone)
{
int decoderStatus = decoder.DequeueOutputBuffer(info, TIMEOUT_USEC);
if (decoderStatus == (int)MediaCodecInfoState.TryAgainLater)
{
decoderOutputAvailable = false;
}
else if (decoderStatus == (int)MediaCodecInfoState.OutputBuffersChanged)
{
// decoderOutputBuffers = decoder.GetOutputBuffers();
}
else if (decoderStatus == (int)MediaCodecInfoState.OutputFormatChanged)
{
// expected before first buffer of data
MediaFormat newFormat = decoder.OutputFormat;
}
else if (decoderStatus < 0)
{
// fail( "unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus );
}
else
{ // decoderStatus >= 0
// The ByteBuffers are null references, but we still get a nonzero
// size for the decoded data.
bool doRender = (info.Size != 0);
// As soon as we call ReleaseOutputBuffer, the buffer will be forwarded
// to SurfaceTexture to convert to a texture. The API doesn't
// guarantee that the texture will be available before the call
// returns, so we need to wait for the onFrameAvailable callback to
// fire. If we don't wait, we risk rendering from the previous frame.
decoder.ReleaseOutputBuffer(decoderStatus, doRender);
if (doRender)
{
mOutputSurface.AwaitNewImage(true);
mOutputSurface.DrawImage();
// Send it to the encoder.
long nSecs = info.PresentationTimeUs * 1000;
if (clip.getStartTime() != -1)
{
nSecs = (info.PresentationTimeUs - (clip.getStartTime() * 1000)) * 1000;
}
nSecs = Java.Lang.Math.Max(0, nSecs);
mEncoderPresentationTimeUs += (nSecs - mLastSampleTime);
mLastSampleTime = nSecs;
mInputSurface.SetPresentationTime(mEncoderPresentationTimeUs);
mInputSurface.SwapBuffers();
}
if ((info.Flags & MediaCodecBufferFlags.EndOfStream) != 0)
{
// mEncoder.signalEndOfInputStream();
outputDoneNextTimeWeCheck = true;
}
}
}
}
}
if (inputChunk != outputCount)
{
// throw new RuntimeException( "frame lost: " + inputChunk + " in, " + outputCount + " out" );
}
}
/**
* Checks the video data.
*
* @return the number of bad frames
*/
private int checkVideoData(VideoChunks inputData, MediaCodec decoder, OutputSurface surface)
{
const int TIMEOUT_USEC = 1000;
ByteBuffer[] decoderInputBuffers = decoder.GetInputBuffers();
ByteBuffer[] decoderOutputBuffers = decoder.GetOutputBuffers();
MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
int inputChunk = 0;
int checkIndex = 0;
int badFrames = 0;
bool outputDone = false;
bool inputDone = false;
while (!outputDone)
{
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "check loop");
}
// Feed more data to the decoder.
if (!inputDone)
{
int inputBufIndex = decoder.DequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0)
{
if (inputChunk == inputData.NumChunks)
{
// End of stream -- send empty frame with EOS flag set.
decoder.QueueInputBuffer(inputBufIndex, 0, 0, 0L,
MediaCodec.BufferFlagEndOfStream);
inputDone = true;
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "sent input EOS");
}
}
else
{
// Copy a chunk of input to the decoder. The first chunk should have
// the BUFFER_FLAG_CODEC_CONFIG flag set.
ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex];
inputBuf.Clear();
inputData.getChunkData(inputChunk, inputBuf);
int flags = inputData.getChunkFlags(inputChunk);
long time = inputData.getChunkTime(inputChunk);
decoder.QueueInputBuffer(inputBufIndex, 0, inputBuf.Position(),
time, (MediaCodecBufferFlags)flags);
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "submitted frame " + inputChunk + " to dec, size=" +
inputBuf.Position() + " flags=" + flags);
}
inputChunk++;
}
}
else
{
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "input buffer not available");
}
}
}
if (!outputDone)
{
int decoderStatus = decoder.DequeueOutputBuffer(info, TIMEOUT_USEC);
if (decoderStatus == (int)MediaCodec.InfoTryAgainLater)
{
// no output available yet
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "no output from decoder available");
}
}
else if (decoderStatus == (int)MediaCodec.InfoOutputBuffersChanged)
{
decoderOutputBuffers = decoder.GetOutputBuffers();
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "decoder output buffers changed");
}
}
else if (decoderStatus == (int)MediaCodec.InfoOutputFormatChanged)
{
MediaFormat newFormat = decoder.OutputFormat;
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "decoder output format changed: " + newFormat);
}
}
else if (decoderStatus < 0)
{
fail("unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus);
}
else // decoderStatus >= 0
{
ByteBuffer decodedData = decoderOutputBuffers[decoderStatus];
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "surface decoder given buffer " + decoderStatus +
" (size=" + info.Size + ")");
}
if ((info.Flags & MediaCodec.BufferFlagEndOfStream) != 0)
{
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "output EOS");
}
outputDone = true;
}
bool doRender = (info.Size != 0);
// As soon as we call releaseOutputBuffer, the buffer will be forwarded
// to SurfaceTexture to convert to a texture. The API doesn't guarantee
// that the texture will be available before the call returns, so we
// need to wait for the onFrameAvailable callback to fire.
decoder.ReleaseOutputBuffer(decoderStatus, doRender);
if (doRender)
{
if (AppSettings.Logging.SendToConsole)
{
Log.Debug(TAG, "awaiting frame " + checkIndex);
}
assertEquals("Wrong time stamp", computePresentationTime(checkIndex),
info.PresentationTimeUs);
surface.AwaitNewImage();
surface.DrawImage();
if (!checkSurfaceFrame(checkIndex++))
{
badFrames++;
}
}
}
}
}
return(badFrames);
}
/**
* Tries to obtain the SPS and the PPS for the encoder.
*/
private long searchSPSandPPS()
{
ByteBuffer[] inputBuffers = mEncoder.GetInputBuffers();
ByteBuffer[] outputBuffers = mEncoder.GetOutputBuffers();
BufferInfo info = new BufferInfo();
byte[] csd = new byte[128];
int len = 0, p = 4, q = 4;
long elapsed = 0, now = timestamp();
while (elapsed < 3000000 && (mSPS == null || mPPS == null))
{
// Some encoders won't give us the SPS and PPS unless they receive something to encode first...
int bufferIndex = mEncoder.DequeueInputBuffer(1000000 / FRAMERATE);
if (bufferIndex >= 0)
{
check(inputBuffers[bufferIndex].Capacity() >= mData.Length, "The input buffer is not big enough.");
inputBuffers[bufferIndex].Clear();
inputBuffers[bufferIndex].Put(mData, 0, mData.Length);
mEncoder.QueueInputBuffer(bufferIndex, 0, mData.Length, timestamp(), 0);
}
else
{
if (VERBOSE)
{
Log.e(TAG, "No buffer available !");
}
}
// We are looking for the SPS and the PPS here. As always, Android is very inconsistent, I have observed that some
// encoders will give those parameters through the MediaFormat object (that is the normal behaviour).
// But some other will not, in that case we try to find a NAL unit of type 7 or 8 in the byte stream outputed by the encoder...
int index = mEncoder.DequeueOutputBuffer(info, 1000000 / FRAMERATE);
if (index == (int)MediaCodecInfoState.OutputFormatChanged)
{
// The PPS and PPS shoud be there
MediaFormat format = mEncoder.OutputFormat;
ByteBuffer spsb = format.GetByteBuffer("csd-0");
ByteBuffer ppsb = format.GetByteBuffer("csd-1");
mSPS = new byte[spsb.Capacity() - 4];
spsb.Position(4);
spsb.Get(mSPS, 0, mSPS.Length);
mPPS = new byte[ppsb.Capacity() - 4];
ppsb.Position(4);
ppsb.Get(mPPS, 0, mPPS.Length);
break;
}
else if (index == (int)MediaCodecInfoState.OutputBuffersChanged)
{
outputBuffers = mEncoder.GetOutputBuffers();
}
else if (index >= 0)
{
len = info.Size;
if (len < 128)
{
outputBuffers[index].Get(csd, 0, len);
if (len > 0 && csd[0] == 0 && csd[1] == 0 && csd[2] == 0 && csd[3] == 1)
{
// Parses the SPS and PPS, they could be in two different packets and in a different order
//depending on the phone so we don't make any assumption about that
while (p < len)
{
while (!(csd[p + 0] == 0 && csd[p + 1] == 0 && csd[p + 2] == 0 && csd[p + 3] == 1) && p + 3 < len)
{
p++;
}
if (p + 3 >= len)
{
p = len;
}
if ((csd[q] & 0x1F) == 7)
{
mSPS = new byte[p - q];
JavaSystem.Arraycopy(csd, q, mSPS, 0, p - q);
}
else
{
mPPS = new byte[p - q];
JavaSystem.Arraycopy(csd, q, mPPS, 0, p - q);
}
p += 4;
q = p;
}
}
}
mEncoder.ReleaseOutputBuffer(index, false);
}
elapsed = timestamp() - now;
}
check(mPPS != null && mSPS != null, "Could not determine the SPS & PPS.");
mB64PPS = Base64.EncodeToString(mPPS, 0, mPPS.Length, Base64Flags.NoWrap);
mB64SPS = Base64.EncodeToString(mSPS, 0, mSPS.Length, Base64Flags.NoWrap);
return(elapsed);
}
private void EncodeMux()
{
int TIMEOUT_USEC = 10000;
ByteBuffer[] encoderInputBuffers = _Encoder.GetInputBuffers();
bool inputDone = false;
int frameIndex = 0;
try
{
while (true)
{
if (!inputDone)
{
int inputBufIndex = _Encoder.DequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0)
{
long ptsUsec = computePresentationTime(frameIndex);
if (frameIndex == _ByteBuffers.Count)
{
// Send an empty frame with the end-of-stream flag set. If we set EOS on a frame with data, that frame data will be ignored, and the output will be short one frame.
_Encoder.QueueInputBuffer(inputBufIndex, 0, 0, ptsUsec, MediaCodec.BufferFlagEndOfStream);
inputDone = true;
Log.Info(TAG, "sent input EOS (with zero-length frame)");
}
else
{
Log.Info(TAG, string.Format("Adding _ByteBuffers image index {0} to encoder", frameIndex));
ByteBuffer inputBuf = encoderInputBuffers[inputBufIndex];
var imagedata = _ByteBuffers[frameIndex];
int chunkSize = 0;
if (imagedata == null)
{
Log.Warn(TAG, string.Format("Adding _ByteBuffers image index {0} to encoder", frameIndex));
}
else
{
//old way don't need to do this anymore.
//Bitmap b = GetBitmap(imagedata);
//byte[] yuv = new byte[b.Width * b.Height * 3 / 2];
//int[] argb = new int[b.Width * b.Height];
//b.GetPixels(argb, 0, b.Width, 0, 0, b.Width, b.Height);
//encodeYUV420SP(yuv, argb, b.Width, b.Height);
//b.Recycle();
//old way don't need to do this anymore?
//int[] argb = new int[imagedata.Width * imagedata.Height];
//imagedata.GetPixels(argb, 0, imagedata.Width, 0, 0, imagedata.Width, imagedata.Height);
//byte[] yuv = new byte[imagedata.Width * imagedata.Height * 3 / 2];
//encodeYUV420SP(yuv, argb, imagedata.Width, imagedata.Height);
//YuvImage yuv = GetYUVImage(imagedata);
//byte[] decomB = Utils.DecompressFast(imagedata);
//var yuv = new YuvImage(decomB, _CameraColorFormat, _Width, _Height, null);
//Bitmap b = BitmapFactory.DecodeByteArray(imagedata, 0, imagedata.Length);
//byte[] yuv = new byte[b.Width * b.Height * 3 / 2];
//int[] argb = new int[b.Width * b.Height];
//b.GetPixels(argb, 0, b.Width, 0, 0, b.Width, b.Height);
//encodeYUV420SP(yuv, argb, b.Width, b.Height);
Bitmap b = BitmapFactory.DecodeByteArray(imagedata, 0, imagedata.Length);
byte[] yuv = new byte[b.Width * b.Height * 3 / 2];
int[] argb = new int[b.Width * b.Height];
b.GetPixels(argb, 0, b.Width, 0, 0, b.Width, b.Height);
encodeYUV420SP(yuv, argb, b.Width, b.Height);
var yuvimage = new YuvImage(yuv, _CameraColorFormat, _Width, _Height, null);
var yuvarray = yuvimage.GetYuvData();
colorcorrection(ref yuvarray, b.Width, b.Height); //method for fixing common color matching issues see below for comments
inputBuf.Put(yuvarray);
chunkSize = yuvarray.Length;
//yuv = null;
//GC.Collect(); //essential to fix memory leak from new YuvImage allocation above
b.Recycle();
}
// the buffer should be sized to hold one full frame
inputBuf.Clear();
_Encoder.QueueInputBuffer(inputBufIndex, 0, chunkSize, ptsUsec, 0);
frameIndex++;
}
}
else
{
// either all in use, or we timed out during initial setup
Log.Warn(TAG, "input buffer not available");
}
}
ByteBuffer[] encoderOutputBuffers = _Encoder.GetOutputBuffers();
var mBufferInfo = new MediaCodec.BufferInfo();
int encoderStatus = _Encoder.DequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
if (encoderStatus == (int)MediaCodecInfoState.TryAgainLater)
{
Log.Info(TAG, "no output available, spinning to await EOS");
}
else if (encoderStatus == (int)MediaCodecInfoState.OutputBuffersChanged)
{
// not expected for an encoder
Log.Warn(TAG, "not expected OutputBuffersChanged happened");
encoderOutputBuffers = _Encoder.GetOutputBuffers();
}
else if (encoderStatus == (int)MediaCodecInfoState.OutputFormatChanged)
{
// should happen before receiving buffers, and should only happen once
if (_MuxerStarted)
{
Log.Error(TAG, "format changed twice and should never happen");
throw new RuntimeException("format changed twice");
}
MediaFormat newFormat = _Encoder.OutputFormat;
Log.Info(TAG, "format changed and starting MUX");
_TrackIndex = _Muxer.AddTrack(newFormat);
_Muxer.Start();
_MuxerStarted = true;
}
else if (encoderStatus < 0)
{
Log.Warn(TAG, "unexpected but lets ignore");
// let's ignore it
}
else
{
ByteBuffer encodedData = encoderOutputBuffers[encoderStatus];
if (encodedData == null)
{
Log.Error(TAG, string.Format("encoderOutputBuffer {0} was null!!", encoderStatus));
throw new RuntimeException(string.Format("encoderOutputBuffer {0} was null!!", encoderStatus));
}
if ((mBufferInfo.Flags & MediaCodecBufferFlags.CodecConfig) != 0)
{
// The codec config data was pulled out and fed to the muxer when we got
// the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
mBufferInfo.Size = 0;
}
if (mBufferInfo.Size != 0)
{
if (!_MuxerStarted)
{
Log.Error(TAG, "muxer hasnt started!!");
throw new RuntimeException("muxer hasnt started");
}
// adjust the ByteBuffer values to match BufferInfo (not needed?) old
//encodedData.Position(mBufferInfo.Offset);
//encodedData.Limit(mBufferInfo.Offset + this.mBufferInfo.Size);
_Muxer.WriteSampleData(_TrackIndex, encodedData, mBufferInfo);
Log.Info(TAG, string.Format("{0} bytes to muxer", mBufferInfo.Size));
}
_Encoder.ReleaseOutputBuffer(encoderStatus, false);
if ((mBufferInfo.Flags & MediaCodecBufferFlags.EndOfStream) != 0)
{
Log.Info(TAG, "End of Stream Reached!!");
break;
}
}
}
}
catch (Exception e)
{
Log.Error(TAG, "Decode or Muxer failed", e, e.Message);
throw;
}
}
public void Decode(MediaCodec _Decoder, MediaExtractor extractor)
{
Stopwatch s = new Stopwatch();
s.Start();
int TIMEOUT_USEC = 10000;
ByteBuffer[] encoderInputBuffers = _Decoder.GetInputBuffers();
ByteBuffer[] outputBuffers = _Decoder.GetOutputBuffers();
var mBufferInfo = new MediaCodec.BufferInfo();
bool inputDone = false;
var index = 0;
try
{
while (true)
{
if (!inputDone)
{
int inputBufIndex = _Decoder.DequeueInputBuffer(TIMEOUT_USEC);
if (inputBufIndex >= 0)
{
ByteBuffer buffer = encoderInputBuffers[inputBufIndex];
//long ptsUsec = computePresentationTime(frameIndex);
int sampleSize = extractor.ReadSampleData(buffer, 0);
if (sampleSize < 0)
{
// Send an empty frame with the end-of-stream flag set. If we set EOS on a frame with data, that frame data will be ignored, and the output will be short one frame.
_Decoder.QueueInputBuffer(inputBufIndex, 0, 0, 0, MediaCodec.BufferFlagEndOfStream);
inputDone = true;
Log.Info(TAG, "sent input EOS (with zero-length frame)");
}
else
{
Log.Info(TAG, "adding encoded video to decoder input ");
_Decoder.QueueInputBuffer(inputBufIndex, 0, sampleSize, extractor.SampleTime, 0);
extractor.Advance();
}
}
else
{
// either all in use, or we timed out during initial setup
Log.Warn(TAG, "input buffer not available");
}
}
//ByteBuffer[] encoderOutputBuffers = _Decoder.GetOutputBuffers();
int encoderStatus = _Decoder.DequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC);
if (encoderStatus == (int)MediaCodecInfoState.TryAgainLater)
{
Log.Info(TAG, "no output available, spinning to await EOS");
}
else if (encoderStatus == (int)MediaCodecInfoState.OutputBuffersChanged)
{
// not expected for an encoder
Log.Warn(TAG, "not expected OutputBuffersChanged happened");
outputBuffers = _Decoder.GetOutputBuffers();
}
else if (encoderStatus == (int)MediaCodecInfoState.OutputFormatChanged)
{
// should happen before receiving buffers, and should only happen once
//if (_MuxerStarted)
//{
// Log.Error(TAG, "format changed twice and should never happen");
// throw new RuntimeException("format changed twice");
//}
//MediaFormat newFormat = _Decoder.OutputFormat;
//Log.Info(TAG, "format changed and starting MUX");
//_TrackIndex = _Muxer.AddTrack(newFormat);
//_Muxer.Start();
//_MuxerStarted = true;
}
else if (encoderStatus < 0)
{
Log.Warn(TAG, "unexpected but lets ignore");
// let's ignore it
}
else
{
ByteBuffer encodedData = outputBuffers[encoderStatus];
if (encodedData == null)
{
Log.Error(TAG, string.Format("encoderOutputBuffer {0} was null!!", encoderStatus));
throw new RuntimeException(string.Format("encoderOutputBuffer {0} was null!!", encoderStatus));
}
if ((mBufferInfo.Flags & MediaCodecBufferFlags.CodecConfig) != 0)
{
// The codec config data was pulled out and fed to the muxer when we got
// the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it.
mBufferInfo.Size = 0;
}
if (mBufferInfo.Size != 0)
{
//if (!_MuxerStarted)
//{
// Log.Error(TAG, "muxer hasnt started!!");
// throw new RuntimeException("muxer hasnt started");
//}
// adjust the ByteBuffer values to match BufferInfo (not needed?) old
//encodedData.Position(mBufferInfo.Offset);
//encodedData.Limit(mBufferInfo.Offset + this.mBufferInfo.Size);
try
{
//byte[] dst = new byte[outputBuffers[encoderStatus].Capacity()];
//outputBuffers[encoderStatus].Get(dst);
//ByteBuffer buffer = outputBuffers[encoderStatus];
//byte[] ba = new byte[encodedData.Remaining()];
//encodedData.Get(ba);
//ByteBuffer buffer = outputBuffers[encoderStatus];
//buffer.Position(mBufferInfo.Offset);
//buffer.Limit(mBufferInfo.Offset + mBufferInfo.Size);
//byte[] ba = new byte[buffer.Remaining()];
//buffer.Get(ba);
//if (index < 10)
//{
YuvImage yuv = Utils.GetYUVImage(encodedData, _CameraColorFormat, _Width, _Height);
//var imagedata = yuv.GetYuvData();
//Utils.swapNV21_NV12(ref imagedata, _Width, _Height);
//Image might need to be corrected later
//Bitmap b = Utils.GetBitmap(yuv, _Width, _Height);
//Bitmap bmp = BitmapFactory.DecodeByteArray(ba, 0, ba.Length);// this return null
//var createfilepath = new File(_downloadsfilesdir, DateTime.Now.Ticks + ".png").AbsolutePath;
//using (FileStream bos = new FileStream(createfilepath, FileMode.CreateNew))
//{
// b.Compress(Bitmap.CompressFormat.Png, 100, bos);
//}
//b.Recycle();
//}
index++;
//writeFrameToSDCard(dst, i, dst.length);
//i++;
}
catch (Exception e)
{
//Log("iDecodeActivity", "Error while creating bitmap with: ");
}
_Decoder.ReleaseOutputBuffer(encoderStatus, false);
}
if ((mBufferInfo.Flags & MediaCodecBufferFlags.EndOfStream) != 0)
{
Log.Info(TAG, "End of Stream Reached!!");
break;
}
}
}
s.Stop();
Log.Info("inner STOPWATCH!!!!:", string.Format("numberofframes = {0}, totaltime = {1}", index, s.ElapsedMilliseconds));
}
catch (Exception e)
{
Log.Error(TAG, "Decode or Muxer failed", e, e.Message);
throw;
}
}