public int Read(byte[] buffer, int offset, int length) { int min = 0; try { if (mBuffer == null) { while (!Thread.Interrupted() && !mClosed) { mIndex = mMediaCodec.DequeueOutputBuffer(mBufferInfo, 500000); if (mIndex >= 0) { //Log.d(TAG,"Index: "+mIndex+" Time: "+mBufferInfo.presentationTimeUs+" size: "+mBufferInfo.size); mBuffer = mBuffers[mIndex]; mBuffer.Position(0); break; } else if (mIndex == (int)MediaCodec.InfoOutputBuffersChanged) { mBuffers = mMediaCodec.GetOutputBuffers(); } else if (mIndex == (int)MediaCodec.InfoOutputFormatChanged) { mMediaFormat = mMediaCodec.GetOutputFormat(mIndex); Log.i(TAG, mMediaFormat.ToString()); } else if (mIndex == (int)MediaCodec.InfoTryAgainLater) { Log.v(TAG, "No buffer available..."); //return 0; } else { Log.e(TAG, "Message: " + mIndex); //return 0; } } } if (mClosed) { throw new IOException("This InputStream was closed"); } min = length < mBufferInfo.Size - mBuffer.Position() ? length : mBufferInfo.Size - mBuffer.Position(); mBuffer.Get(buffer, offset, min); if (mBuffer.Position() >= mBufferInfo.Size) { mMediaCodec.ReleaseOutputBuffer(mIndex, false); mBuffer = null; } } catch (RuntimeException e) { e.PrintStackTrace(); } return(min); }
private void flushMediaCodec(MediaCodec mc) { int index = 0; BufferInfo info = new BufferInfo(); while (index != (int)MediaCodec.InfoTryAgainLater) { index = mc.DequeueOutputBuffer(info, 1000000 / FRAMERATE); if (index >= 0) { mc.ReleaseOutputBuffer(index, false); } } }
private Thread GetEncoderThread() { var encoderThread = new Thread(() => { MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); try { while (!_disposed) { // Try to get an available pcm audio frame int outIndex = _mediaCodec.DequeueOutputBuffer(info, 50000); if (outIndex >= 0) { int lastIndex = outIndex; // Get the last available output buffer while ((outIndex = this._mediaCodec.DequeueOutputBuffer(info, 0)) >= 0) { this._mediaCodec.ReleaseOutputBuffer(lastIndex, false); lastIndex = outIndex; } ByteBuffer outputBuffer = _mediaCodec.GetOutputBuffer(lastIndex); _audioTrack.Write(outputBuffer, outputBuffer.Limit(), WriteMode.NonBlocking); _mediaCodec.ReleaseOutputBuffer(lastIndex, false); } } } catch (ThreadInterruptedException) { // Ignore Thread got interrupted from outside } }); encoderThread.Daemon = true; encoderThread.Priority = Thread.MaxPriority; return(encoderThread); }
/** * Work loop. */ private void doExtract(MediaExtractor extractor, int trackIndex, MediaCodec decoder, CodecOutputSurface outputSurface) { Stopwatch stopWatch = new Stopwatch(); const int TIMEOUT_USEC = 10000; ByteBuffer [] decoderInputBuffers = decoder.GetInputBuffers(); MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); int inputChunk = 0; int decodeCount = 0; var frameTimestamps = new List <long>(); bool outputDone = false; bool inputDone = false; //speed vs accuracy tradeoffs https://stackoverflow.com/questions/34132444/google-mobile-vision-poor-facedetector-performance-without-camerasource //reducing bitmap resolution helps the most and thats ok because i'm not using them after var detector = new FaceDetector.Builder(Application.Context) .SetTrackingEnabled(true) //tracking enables false makes it much slow wtf?!?! .SetClassificationType(ClassificationType.All) .SetProminentFaceOnly(true) // no diff //.SetMinFaceSize((float)0.1) //small performance gain when removed .SetMode(FaceDetectionMode.Fast) // tiny small performance gain .Build(); while (!outputDone) { stopWatch.Start(); // Feed more data to the decoder. if (!inputDone) { int inputBufIndex = decoder.DequeueInputBuffer(TIMEOUT_USEC); if (inputBufIndex >= 0) { ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex]; // Read the sample data into the ByteBuffer. This neither respects nor // updates inputBuf's position, limit, etc. int chunkSize = extractor.ReadSampleData(inputBuf, 0); if (chunkSize < 0) { // End of stream -- send empty frame with EOS flag set. decoder.QueueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodec.BufferFlagEndOfStream); inputDone = true; //if (VERBOSE) Log.d(TAG, "sent input EOS"); } else { if (extractor.SampleTrackIndex != trackIndex) { //Log.w(TAG, "WEIRD: got sample from track " + extractor.getSampleTrackIndex() + ", expected " + trackIndex); } frameTimestamps.Add(extractor.SampleTime); //might need to play with offset here to get right sync from decoder decoder.QueueInputBuffer(inputBufIndex, 0, chunkSize, extractor.SampleTime, 0 /*flags*/); //if (VERBOSE) { // Log.d(TAG, "submitted frame " + inputChunk + " to dec, size=" + // chunkSize); //} inputChunk++; extractor.Advance(); } } else { //if (VERBOSE) Log.d(TAG, "input buffer not available"); } } if (!outputDone) { int decoderStatus = decoder.DequeueOutputBuffer(info, TIMEOUT_USEC); if (decoderStatus == (int)MediaCodecInfoState.TryAgainLater) { // no output available yet //if (VERBOSE) Log.d(TAG, "no output from decoder available"); } else if (decoderStatus == (int)MediaCodecInfoState.OutputBuffersChanged) { // not important for us, since we're using Surface //if (VERBOSE) Log.d(TAG, "decoder output buffers changed"); } else if (decoderStatus == (int)MediaCodecInfoState.OutputFormatChanged) { //MediaFormat newFormat = decoder.OutputFormat; //if (VERBOSE) Log.d(TAG, "decoder output format changed: " + newFormat); } else if (decoderStatus < 0) { //fail("unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus); throw new InvalidOperationException(); } else { //if (VERBOSE) Log.d(TAG, "surface decoder given buffer " + decoderStatus + " (size=" + info.size + ")"); if ((info.Flags & MediaCodecBufferFlags.EndOfStream) != 0) { //if (VERBOSE) Log.d(TAG, "output EOS"); outputDone = true; } bool doRender = (info.Size != 0); //could not get this working!!! // As soon as we call releaseOutputBuffer, the buffer will be forwarded // to SurfaceTexture to convert to a texture. The API doesn't guarantee // that the texture will be available before the call returns, so we // need to wait for the onFrameAvailable callback to fire. decoder.ReleaseOutputBuffer(decoderStatus, doRender); if (doRender) { //outputSurface.awaitNewImage(); //could not get callback to work and even so do not want to wait 2.5 seconds for each frame, might need to revist outputSurface.mTextureRender.checkGlError("before updateTexImage"); outputSurface.mSurfaceTexture.UpdateTexImage(); outputSurface.drawImage(true); //Log.Info("innerSTOPWATCH_begin!!!!:", stopWatch.ElapsedMilliseconds.ToString()); //can't call face detector this way its too slow or maybe there is a busy loop??? //_FaceFetchDataTasks.Add(Task.Run(() => CreateFaceframes(detector, outputSurface.GetFramebitmap(), decodeCount, frameTimestamps[decodeCount]))); //if (decodeCount % 2 ==0) //doesn't help that much and messes with rating algo CreateFaceframes(detector, outputSurface.GetFramebitmap(), frameTimestamps[decodeCount]); //Log.Info("innerSTOPWATCH_end!!!!:", stopWatch.ElapsedMilliseconds.ToString()); decodeCount++; } } } } stopWatch.Stop(); Log.Info("inner STOPWATCH!!!!:", string.Format("numberofframes = {0}, totaltime = {1}", decodeCount, stopWatch.ElapsedMilliseconds)); detector.Release(); }
public void decode(byte[] array) { if (bConfigured == false) { Init(); } var nalType = array[4] & 0x1f; //Console.WriteLine("nal:" + nalType); if (nalType == 7) { //sps = array.ToArray(); if (array.Length != sps.Length) { stop(); sps = array.ToArray(); Init(); } return; } if (nalType == 8) { //pps = array.ToArray(); return; } if (bConfigured == false) { return; } //Make sure keyframe is first. if (nalType == 5) { bWaitForKeyframe = false; //pps = array.ToArray(); //return; } if (bWaitForKeyframe) { return; } if (bConfigured) { try { ByteBuffer[] inputBuffers = codec.GetInputBuffers(); ByteBuffer[] outputBuffers = codec.GetOutputBuffers(); int dequeueInputBuffer = codec.DequeueInputBuffer(-1L); if (dequeueInputBuffer >= 0) { //Send data to decoder. ByteBuffer byteBuffer = inputBuffers[dequeueInputBuffer]; byteBuffer.Clear(); byteBuffer.Put(array); codec.QueueInputBuffer(dequeueInputBuffer, 0, array.Length, 0L, 0); } //Show decoded frame MediaCodec.BufferInfo BufferInfo = new MediaCodec.BufferInfo(); int i = codec.DequeueOutputBuffer(BufferInfo, 0L); while (i >= 0) { /*if (picSurface == null)//Only if not using display surface. * { * ByteBuffer byteBuffer2 = outputBuffers[i]; * if (buffer == null || buffer.Length != BufferInfo.Size) * { * buffer = new byte[BufferInfo.Size]; * } * byteBuffer2.Get(buffer); * //do something with raw frame in buffer. * }*/ codec.ReleaseOutputBuffer(i, true); codec.SetVideoScalingMode(VideoScalingMode.ScaleToFit); i = codec.DequeueOutputBuffer(BufferInfo, 0L); } } catch (Exception ex) { MainActivity.getActivity().notifyUser("VideoDecoder decode exception " + ex.Message, false); //attempt to recover. //codec.Release(); //codec = null; //bConfigured = false; stop(); } } return;// ret; }
public void decode(byte[] array) { if (bConfigured == false) { Init(); } var nalType = array[4] & 0x1f; if (nalType == 7) { //sps = array.ToArray(); if (array.Length != sps.Length) { stop(); sps = array.ToArray(); Init(); } return; } if (nalType == 8) { //pps = array.ToArray(); return; } if (bConfigured == false) { return; } if (bConfigured) { try { ByteBuffer[] inputBuffers = codec.GetInputBuffers(); ByteBuffer[] outputBuffers = codec.GetOutputBuffers(); int dequeueInputBuffer = codec.DequeueInputBuffer(-1L); if (dequeueInputBuffer >= 0) { //Send data to decoder. ByteBuffer byteBuffer = inputBuffers[dequeueInputBuffer]; byteBuffer.Clear(); byteBuffer.Put(array); codec.QueueInputBuffer(dequeueInputBuffer, 0, array.Length, 0L, 0); } //Show decoded frame MediaCodec.BufferInfo BufferInfo = new MediaCodec.BufferInfo(); int i = codec.DequeueOutputBuffer(BufferInfo, 0L); while (i >= 0) { /*if (picSurface == null)//Only if not using display surface. { ByteBuffer byteBuffer2 = outputBuffers[i]; if (buffer == null || buffer.Length != BufferInfo.Size) { buffer = new byte[BufferInfo.Size]; } byteBuffer2.Get(buffer); //do something with raw frame in buffer. }*/ codec.ReleaseOutputBuffer(i, true); codec.SetVideoScalingMode(VideoScalingMode.ScaleToFit); i = codec.DequeueOutputBuffer(BufferInfo, 0L); } } catch (Exception ex) { //attempt to recover. stop(); } } return;// ret; }
private void DrainEncoder(MediaCodec encoder, MediaCodec.BufferInfo bufferInfo, int trackIndex, bool endOfStream) { Debug.WriteLine($"DrainEncoder {endOfStream})"); const int TIMEOUT_USEC = 10000; if (endOfStream) { Debug.WriteLine("Sending EOS to encoder"); encoder.SignalEndOfInputStream(); } ByteBuffer[] encoderOutputBuffers = encoder.GetOutputBuffers(); while (true) { int encoderStatus = encoder.DequeueOutputBuffer(bufferInfo, TIMEOUT_USEC); if (encoderStatus == (int)MediaCodecInfoState.TryAgainLater) { // no output available yet if (!endOfStream) { break; // out of while } else { Debug.WriteLine("No output available, spinning to await EOS"); } } else if (encoderStatus == (int)MediaCodecInfoState.OutputBuffersChanged) { encoderOutputBuffers = encoder.GetOutputBuffers(); } else if (encoderStatus == (int)MediaCodecInfoState.OutputFormatChanged) { Debug.Assert(!muxerStarted); MediaFormat newFormat = encoder.OutputFormat; Debug.WriteLine($"Encoder output format changed: {newFormat}"); var isVideo = encoder == videoEncoder; if (isVideo) { videoTrackIndex = muxer.AddTrack(newFormat); trackIndex = videoTrackIndex; } else { audioTrackIndex = muxer.AddTrack(newFormat); trackIndex = audioTrackIndex; } // now that we have the Magic Goodies, start the muxer if (videoTrackIndex >= 0 && audioTrackIndex >= 0) { muxer.Start(); muxerStarted = true; muxerStartEvent.Set(); } } else if (encoderStatus < 0) { Debug.WriteLine($"Unexpected result from encoder.dequeueOutputBuffer: {encoderStatus}"); } else { ByteBuffer encodedData = encoderOutputBuffers[encoderStatus]; Debug.Assert(encodedData != null); if ((bufferInfo.Flags & MediaCodecBufferFlags.CodecConfig) != 0) { Debug.WriteLine("Ignoring BUFFER_FLAG_CODEC_CONFIG"); bufferInfo.Size = 0; } if (bufferInfo.Size != 0) { muxerStartEvent.WaitOne(); encodedData.Position(bufferInfo.Offset); encodedData.Limit(bufferInfo.Offset + bufferInfo.Size); muxer.WriteSampleData(trackIndex, encodedData, bufferInfo); Debug.WriteLine($"Sent {bufferInfo.Size} bytes to muxer"); } encoder.ReleaseOutputBuffer(encoderStatus, false); if ((bufferInfo.Flags & MediaCodecBufferFlags.EndOfStream) != 0) { if (!endOfStream) { Debug.WriteLine("Reached end of stream unexpectedly"); } else { Debug.WriteLine("End of stream reached"); } break; } } } }
/** * Extracts all pending data from the encoder and forwards it to the muxer. * <p> * If endOfStream is not set, this returns when there is no more data to drain. If it * is set, we send EOS to the encoder, and then iterate until we see EOS on the output. * Calling this with endOfStream set should be done once, right before stopping the muxer. * <p> * We're just using the muxer to get a .mp4 file (instead of a raw H.264 stream). We're * not recording audio. */ private void drainEncoder(bool endOfStream) { int TIMEOUT_USEC = 10000; if (VERBOSE) { Log.Debug(TAG, "drainEncoder(" + endOfStream + ")"); } if (endOfStream) { if (VERBOSE) { Log.Debug(TAG, "sending EOS to encoder"); } mEncoder.SignalEndOfInputStream(); } ByteBuffer[] encoderOutputBuffers = mEncoder.GetOutputBuffers(); while (true) { int encoderStatus = mEncoder.DequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC); if (encoderStatus == (int)MediaCodec.InfoTryAgainLater) { // no output available yet if (!endOfStream) { break; // out of while } else { if (VERBOSE) { Log.Debug(TAG, "no output available, spinning to await EOS"); } } } else if (encoderStatus == (int)MediaCodec.InfoOutputBuffersChanged) { // not expected for an encoder encoderOutputBuffers = mEncoder.GetOutputBuffers(); } else if (encoderStatus == (int)MediaCodec.InfoOutputFormatChanged) { // should happen before receiving buffers, and should only happen once if (mMuxerStarted) { throw new RuntimeException("format changed twice"); } MediaFormat newFormat = mEncoder.OutputFormat; Log.Debug(TAG, "encoder output format changed: " + newFormat); // now that we have the Magic Goodies, start the muxer mTrackIndex = mMuxer.AddTrack(newFormat); mMuxer.Start(); mMuxerStarted = true; } else if (encoderStatus < 0) { Log.Warn(TAG, "unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus); // let's ignore it } else { ByteBuffer encodedData = encoderOutputBuffers[encoderStatus]; if (encodedData == null) { throw new RuntimeException("encoderOutputBuffer " + encoderStatus + " was null"); } if ((mBufferInfo.Flags & MediaCodec.BufferFlagCodecConfig) != 0) { // The codec config data was pulled out and fed to the muxer when we got // the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it. if (VERBOSE) { Log.Debug(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG"); } mBufferInfo.Size = 0; } if (mBufferInfo.Size != 0) { if (!mMuxerStarted) { throw new RuntimeException("muxer hasn't started"); } // adjust the ByteBuffer values to match BufferInfo (not needed?) encodedData.Position(mBufferInfo.Offset); encodedData.Limit(mBufferInfo.Offset + mBufferInfo.Size); mMuxer.WriteSampleData(mTrackIndex, encodedData, mBufferInfo); if (VERBOSE) { Log.Debug(TAG, "sent " + mBufferInfo.Size + " bytes to muxer"); } } mEncoder.ReleaseOutputBuffer(encoderStatus, false); if ((mBufferInfo.Flags & MediaCodec.BufferFlagEndOfStream) != 0) { if (!endOfStream) { Log.Warn(TAG, "reached end of stream unexpectedly"); } else { if (VERBOSE) { Log.Debug(TAG, "end of stream reached"); } } break; // out of while } } } }
/** * Generates video frames, feeds them into the encoder, and writes the output to the * VideoChunks instance. */ private void generateVideoData(MediaCodec encoder, InputSurface inputSurface, VideoChunks output) { ByteBuffer[] encoderOutputBuffers = encoder.GetOutputBuffers(); MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); int generateIndex = 0; int outputCount = 0; // Loop until the output side is done. bool inputDone = false; bool outputDone = false; while (!outputDone) { if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "gen loop"); } // If we're not done submitting frames, generate a new one and submit it. The // eglSwapBuffers call will block if the input is full. if (!inputDone) { if (generateIndex == NUM_FRAMES) { // Send an empty frame with the end-of-stream flag set. if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "signaling input EOS"); } if (WORK_AROUND_BUGS) { // Might drop a frame, but at least we won't crash mediaserver. try { Thread.Sleep(500); } catch (InterruptedException ie) {} outputDone = true; } else { encoder.SignalEndOfInputStream(); } inputDone = true; } else { generateSurfaceFrame(generateIndex); inputSurface.SetPresentationTime(computePresentationTime(generateIndex) * 1000); if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "inputSurface swapBuffers"); } inputSurface.SwapBuffers(); } generateIndex++; } // Check for output from the encoder. If there's no output yet, we either need to // provide more input, or we need to wait for the encoder to work its magic. We // can't actually tell which is the case, so if we can't get an output buffer right // away we loop around and see if it wants more input. // // If we do find output, drain it all before supplying more input. while (true) { int encoderStatus = encoder.DequeueOutputBuffer(info, TIMEOUT_USEC); if (encoderStatus == (int)MediaCodecInfoState.TryAgainLater) { // no output available yet if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "no output from encoder available"); } break; // out of while } else if (encoderStatus == (int)MediaCodecInfoState.OutputBuffersChanged) { // not expected for an encoder encoderOutputBuffers = encoder.GetOutputBuffers(); if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "encoder output buffers changed"); } } else if (encoderStatus == (int)MediaCodecInfoState.OutputFormatChanged) { // not expected for an encoder MediaFormat newFormat = encoder.OutputFormat; if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "encoder output format changed: " + newFormat); } } else if (encoderStatus < 0) { fail("unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus); } else // encoderStatus >= 0 { ByteBuffer encodedData = encoderOutputBuffers[encoderStatus]; if (encodedData == null) { fail("encoderOutputBuffer " + encoderStatus + " was null"); } // Codec config flag must be set iff this is the first chunk of output. This // may not hold for all codecs, but it appears to be the case for video/avc. assertTrue((info.Flags & MediaCodec.BufferFlagCodecConfig) != 0 || outputCount != 0); if (info.Size != 0) { // Adjust the ByteBuffer values to match BufferInfo. encodedData.Position(info.Offset); encodedData.Limit(info.Offset + info.Size); output.addChunk(encodedData, (int)info.Flags, info.PresentationTimeUs); outputCount++; } encoder.ReleaseOutputBuffer(encoderStatus, false); if ((info.Flags & MediaCodec.BufferFlagEndOfStream) != 0) { outputDone = true; break; // out of while } } } } assertEquals("Frame count", NUM_FRAMES + 1, outputCount); }
/** * Extracts all pending data from the encoder and forwards it to the muxer. * <p> * If endOfStream is not set, this returns when there is no more data to drain. If it * is set, we send EOS to the encoder, and then iterate until we see EOS on the output. * Calling this with endOfStream set should be done once, right before stopping the muxer. * <p> * We're just using the muxer to get a .mp4 file (instead of a raw H.264 stream). We're * not recording audio. */ private void D(bool es) { if (es) { if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "sending EOS to encoder"); } mEncoder.SignalEndOfInputStream(); this.Progress.Invoke(new EncoderMinArgs(_bitsEncodedSoFar, _estimatedTotalSize, true)); } ByteBuffer[] encoderOutputBuffers = mEncoder.GetOutputBuffers(); while (true) { int encoderStatus = mEncoder.DequeueOutputBuffer(_bfi, TIMEOUT_USEC); if (encoderStatus == (int)MediaCodec.InfoTryAgainLater) { // no output available yet if (!es) { break; // out of while } else { if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "no output available, spinning to await EOS"); } } } else if (encoderStatus == (int)MediaCodec.InfoOutputBuffersChanged) { // not expected for an encoder encoderOutputBuffers = mEncoder.GetOutputBuffers(); } else if (encoderStatus == (int)MediaCodec.InfoOutputFormatChanged) { // should happen before receiving buffers, and should only happen once if (MuxerStarted) { throw new RuntimeException("format changed twice"); } MediaFormat newFormat = mEncoder.OutputFormat; if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "encoder output format changed: " + newFormat); } mTrackIndex = _muxer.AddTrack(newFormat); LatestAudioTrackIndex = _muxer.AddTrack(LatestAudioInputFormat); // @TODO No processing on this yet _muxer.Start(); MuxerStarted = true; } else if (encoderStatus < 0) { Log.Warn(TAG, "unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus); // let's ignore it } else { ByteBuffer encoderBuffers = encoderOutputBuffers[encoderStatus]; if (encoderBuffers == null) { throw new RuntimeException("encoderOutputBuffer " + encoderStatus + " was null"); } if ((_bfi.Flags & MediaCodec.BufferFlagCodecConfig) != 0) { // The codec config data was pulled out and fed to the muxer when we got // the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it. if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "ignoring BUFFER_FLAG_CODEC_CONFIG"); } _bfi.Size = 0; } if (_bfi.Size != 0) { if (!MuxerStarted) { throw new RuntimeException("muxer hasn't started"); } // adjust the ByteBuffer values to match BufferInfo encoderBuffers.Position(_bfi.Offset); encoderBuffers.Limit(_bfi.Offset + _bfi.Size); _muxer.WriteSampleData(mTrackIndex, encoderBuffers, _bfi); EncodedBits(_bfi.Size); #if DEBUG if (AppSettings.Logging.SendToConsole) { System.Console.WriteLine($"Media player @ " + $"{_mediaPlayer.Timestamp.AnchorMediaTimeUs} us while sT @ " + $"{_outputSurface.WeakSurfaceTexture.Timestamp} & output buffer info @ {_bitsEncodedSoFar}"); } #endif if (_firstKnownBuffer == 0) { _firstKnownBuffer = (_bfi.PresentationTimeUs - GetMicroSecondsFromByteCount(_bfi.Size)); // calculate the first known buffer and use it to offset/align other tracks if (InputUriToEncode != null) { this.StartAudioEncoder(_firstKnownBuffer, null, InputUriToEncode); } else { this.StartAudioEncoder(_firstKnownBuffer, LatestInputPath, null); } System.Console.WriteLine($"started draining @ {_bfi.PresentationTimeUs}"); } //we don't want to flood the system with EventArgs so only send once every 120 frames if (_frameCount >= 120) { _frameCount = 0; Notify(_bitsEncodedSoFar, _estimatedTotalSize); } } mEncoder.ReleaseOutputBuffer(encoderStatus, false); if ((_bfi.Flags & MediaCodec.BufferFlagEndOfStream) != 0) { if (!es) { Log.Warn(TAG, "reached end of stream unexpectedly"); } else { if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "end of stream reached"); } } this.Progress.Invoke(new EncoderMinArgs(_bitsEncodedSoFar, _estimatedTotalSize, true, false)); break; // out of while } } } }
private void EncodeMux() { int TIMEOUT_USEC = 10000; ByteBuffer[] encoderInputBuffers = _Encoder.GetInputBuffers(); bool inputDone = false; int frameIndex = 0; try { while (true) { if (!inputDone) { int inputBufIndex = _Encoder.DequeueInputBuffer(TIMEOUT_USEC); if (inputBufIndex >= 0) { long ptsUsec = computePresentationTime(frameIndex); if (frameIndex == _ByteBuffers.Count) { // Send an empty frame with the end-of-stream flag set. If we set EOS on a frame with data, that frame data will be ignored, and the output will be short one frame. _Encoder.QueueInputBuffer(inputBufIndex, 0, 0, ptsUsec, MediaCodec.BufferFlagEndOfStream); inputDone = true; Log.Info(TAG, "sent input EOS (with zero-length frame)"); } else { Log.Info(TAG, string.Format("Adding _ByteBuffers image index {0} to encoder", frameIndex)); ByteBuffer inputBuf = encoderInputBuffers[inputBufIndex]; var imagedata = _ByteBuffers[frameIndex]; int chunkSize = 0; if (imagedata == null) { Log.Warn(TAG, string.Format("Adding _ByteBuffers image index {0} to encoder", frameIndex)); } else { //old way don't need to do this anymore. //Bitmap b = GetBitmap(imagedata); //byte[] yuv = new byte[b.Width * b.Height * 3 / 2]; //int[] argb = new int[b.Width * b.Height]; //b.GetPixels(argb, 0, b.Width, 0, 0, b.Width, b.Height); //encodeYUV420SP(yuv, argb, b.Width, b.Height); //b.Recycle(); //old way don't need to do this anymore? //int[] argb = new int[imagedata.Width * imagedata.Height]; //imagedata.GetPixels(argb, 0, imagedata.Width, 0, 0, imagedata.Width, imagedata.Height); //byte[] yuv = new byte[imagedata.Width * imagedata.Height * 3 / 2]; //encodeYUV420SP(yuv, argb, imagedata.Width, imagedata.Height); //YuvImage yuv = GetYUVImage(imagedata); //byte[] decomB = Utils.DecompressFast(imagedata); //var yuv = new YuvImage(decomB, _CameraColorFormat, _Width, _Height, null); //Bitmap b = BitmapFactory.DecodeByteArray(imagedata, 0, imagedata.Length); //byte[] yuv = new byte[b.Width * b.Height * 3 / 2]; //int[] argb = new int[b.Width * b.Height]; //b.GetPixels(argb, 0, b.Width, 0, 0, b.Width, b.Height); //encodeYUV420SP(yuv, argb, b.Width, b.Height); Bitmap b = BitmapFactory.DecodeByteArray(imagedata, 0, imagedata.Length); byte[] yuv = new byte[b.Width * b.Height * 3 / 2]; int[] argb = new int[b.Width * b.Height]; b.GetPixels(argb, 0, b.Width, 0, 0, b.Width, b.Height); encodeYUV420SP(yuv, argb, b.Width, b.Height); var yuvimage = new YuvImage(yuv, _CameraColorFormat, _Width, _Height, null); var yuvarray = yuvimage.GetYuvData(); colorcorrection(ref yuvarray, b.Width, b.Height); //method for fixing common color matching issues see below for comments inputBuf.Put(yuvarray); chunkSize = yuvarray.Length; //yuv = null; //GC.Collect(); //essential to fix memory leak from new YuvImage allocation above b.Recycle(); } // the buffer should be sized to hold one full frame inputBuf.Clear(); _Encoder.QueueInputBuffer(inputBufIndex, 0, chunkSize, ptsUsec, 0); frameIndex++; } } else { // either all in use, or we timed out during initial setup Log.Warn(TAG, "input buffer not available"); } } ByteBuffer[] encoderOutputBuffers = _Encoder.GetOutputBuffers(); var mBufferInfo = new MediaCodec.BufferInfo(); int encoderStatus = _Encoder.DequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC); if (encoderStatus == (int)MediaCodecInfoState.TryAgainLater) { Log.Info(TAG, "no output available, spinning to await EOS"); } else if (encoderStatus == (int)MediaCodecInfoState.OutputBuffersChanged) { // not expected for an encoder Log.Warn(TAG, "not expected OutputBuffersChanged happened"); encoderOutputBuffers = _Encoder.GetOutputBuffers(); } else if (encoderStatus == (int)MediaCodecInfoState.OutputFormatChanged) { // should happen before receiving buffers, and should only happen once if (_MuxerStarted) { Log.Error(TAG, "format changed twice and should never happen"); throw new RuntimeException("format changed twice"); } MediaFormat newFormat = _Encoder.OutputFormat; Log.Info(TAG, "format changed and starting MUX"); _TrackIndex = _Muxer.AddTrack(newFormat); _Muxer.Start(); _MuxerStarted = true; } else if (encoderStatus < 0) { Log.Warn(TAG, "unexpected but lets ignore"); // let's ignore it } else { ByteBuffer encodedData = encoderOutputBuffers[encoderStatus]; if (encodedData == null) { Log.Error(TAG, string.Format("encoderOutputBuffer {0} was null!!", encoderStatus)); throw new RuntimeException(string.Format("encoderOutputBuffer {0} was null!!", encoderStatus)); } if ((mBufferInfo.Flags & MediaCodecBufferFlags.CodecConfig) != 0) { // The codec config data was pulled out and fed to the muxer when we got // the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it. mBufferInfo.Size = 0; } if (mBufferInfo.Size != 0) { if (!_MuxerStarted) { Log.Error(TAG, "muxer hasnt started!!"); throw new RuntimeException("muxer hasnt started"); } // adjust the ByteBuffer values to match BufferInfo (not needed?) old //encodedData.Position(mBufferInfo.Offset); //encodedData.Limit(mBufferInfo.Offset + this.mBufferInfo.Size); _Muxer.WriteSampleData(_TrackIndex, encodedData, mBufferInfo); Log.Info(TAG, string.Format("{0} bytes to muxer", mBufferInfo.Size)); } _Encoder.ReleaseOutputBuffer(encoderStatus, false); if ((mBufferInfo.Flags & MediaCodecBufferFlags.EndOfStream) != 0) { Log.Info(TAG, "End of Stream Reached!!"); break; } } } } catch (Exception e) { Log.Error(TAG, "Decode or Muxer failed", e, e.Message); throw; } }
private void ResampleVideo(MediaExtractor extractor, MediaCodec decoder, SamplerClip clip) { ByteBuffer[] decoderInputBuffers = decoder.GetInputBuffers(); ByteBuffer[] encoderOutputBuffers = mEncoder.GetOutputBuffers(); MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); int inputChunk = 0; int outputCount = 0; long endTime = clip.getEndTime(); if (endTime == -1) { endTime = clip.getVideoDuration(); } bool outputDoneNextTimeWeCheck = false; bool outputDone = false; bool inputDone = false; bool decoderDone = false; while (!outputDone) { // Feed more data to the decoder. if (!inputDone) { int inputBufIndex = decoder.DequeueInputBuffer(TIMEOUT_USEC); if (inputBufIndex >= 0) { if (extractor.SampleTime / 1000 >= endTime) { // End of stream -- send empty frame with EOS flag set. decoder.QueueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodecBufferFlags.EndOfStream); inputDone = true; } else { // Copy a chunk of input to the decoder. The first chunk should have // the BUFFER_FLAG_CODEC_CONFIG flag set. ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex]; inputBuf.Clear(); int sampleSize = extractor.ReadSampleData(inputBuf, 0); if (sampleSize < 0) { decoder.QueueInputBuffer(inputBufIndex, 0, 0, 0, MediaCodecBufferFlags.EndOfStream); } else { decoder.QueueInputBuffer(inputBufIndex, 0, sampleSize, extractor.SampleTime, 0); extractor.Advance(); } inputChunk++; } } } // Assume output is available. Loop until both assumptions are false. bool decoderOutputAvailable = !decoderDone; bool encoderOutputAvailable = true; while (decoderOutputAvailable || encoderOutputAvailable) { // Start by draining any pending output from the encoder. It's important to // do this before we try to stuff any more data in. int encoderStatus = mEncoder.DequeueOutputBuffer(info, TIMEOUT_USEC); if (encoderStatus == (int)MediaCodecInfoState.TryAgainLater) { encoderOutputAvailable = false; } else if (encoderStatus == (int)MediaCodecInfoState.OutputBuffersChanged) { encoderOutputBuffers = mEncoder.GetOutputBuffers(); } else if (encoderStatus == (int)MediaCodecInfoState.OutputFormatChanged) { MediaFormat newFormat = mEncoder.OutputFormat; mTrackIndex = mMuxer.AddTrack(newFormat); mMuxer.Start(); mMuxerStarted = true; } else if (encoderStatus < 0) { // fail( "unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus ); } else { // encoderStatus >= 0 ByteBuffer encodedData = encoderOutputBuffers[encoderStatus]; if (encodedData == null) { // fail( "encoderOutputBuffer " + encoderStatus + " was null" ); } // Write the data to the output "file". if (info.Size != 0) { encodedData.Position(info.Offset); encodedData.Limit(info.Offset + info.Size); outputCount++; mMuxer.WriteSampleData(mTrackIndex, encodedData, info); } outputDone = (info.Flags & MediaCodecBufferFlags.EndOfStream) != 0; mEncoder.ReleaseOutputBuffer(encoderStatus, false); } if (outputDoneNextTimeWeCheck) { outputDone = true; } if (encoderStatus != (int)MediaCodecInfoState.TryAgainLater) { // Continue attempts to drain output. continue; } // Encoder is drained, check to see if we've got a new frame of output from // the decoder. (The output is going to a Surface, rather than a ByteBuffer, // but we still get information through BufferInfo.) if (!decoderDone) { int decoderStatus = decoder.DequeueOutputBuffer(info, TIMEOUT_USEC); if (decoderStatus == (int)MediaCodecInfoState.TryAgainLater) { decoderOutputAvailable = false; } else if (decoderStatus == (int)MediaCodecInfoState.OutputBuffersChanged) { // decoderOutputBuffers = decoder.GetOutputBuffers(); } else if (decoderStatus == (int)MediaCodecInfoState.OutputFormatChanged) { // expected before first buffer of data MediaFormat newFormat = decoder.OutputFormat; } else if (decoderStatus < 0) { // fail( "unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus ); } else { // decoderStatus >= 0 // The ByteBuffers are null references, but we still get a nonzero // size for the decoded data. bool doRender = (info.Size != 0); // As soon as we call ReleaseOutputBuffer, the buffer will be forwarded // to SurfaceTexture to convert to a texture. The API doesn't // guarantee that the texture will be available before the call // returns, so we need to wait for the onFrameAvailable callback to // fire. If we don't wait, we risk rendering from the previous frame. decoder.ReleaseOutputBuffer(decoderStatus, doRender); if (doRender) { mOutputSurface.AwaitNewImage(true); mOutputSurface.DrawImage(); // Send it to the encoder. long nSecs = info.PresentationTimeUs * 1000; if (clip.getStartTime() != -1) { nSecs = (info.PresentationTimeUs - (clip.getStartTime() * 1000)) * 1000; } nSecs = Java.Lang.Math.Max(0, nSecs); mEncoderPresentationTimeUs += (nSecs - mLastSampleTime); mLastSampleTime = nSecs; mInputSurface.SetPresentationTime(mEncoderPresentationTimeUs); mInputSurface.SwapBuffers(); } if ((info.Flags & MediaCodecBufferFlags.EndOfStream) != 0) { // mEncoder.signalEndOfInputStream(); outputDoneNextTimeWeCheck = true; } } } } } if (inputChunk != outputCount) { // throw new RuntimeException( "frame lost: " + inputChunk + " in, " + outputCount + " out" ); } }
/** * Tries to obtain the SPS and the PPS for the encoder. */ private long searchSPSandPPS() { ByteBuffer[] inputBuffers = mEncoder.GetInputBuffers(); ByteBuffer[] outputBuffers = mEncoder.GetOutputBuffers(); BufferInfo info = new BufferInfo(); byte[] csd = new byte[128]; int len = 0, p = 4, q = 4; long elapsed = 0, now = timestamp(); while (elapsed < 3000000 && (mSPS == null || mPPS == null)) { // Some encoders won't give us the SPS and PPS unless they receive something to encode first... int bufferIndex = mEncoder.DequeueInputBuffer(1000000 / FRAMERATE); if (bufferIndex >= 0) { check(inputBuffers[bufferIndex].Capacity() >= mData.Length, "The input buffer is not big enough."); inputBuffers[bufferIndex].Clear(); inputBuffers[bufferIndex].Put(mData, 0, mData.Length); mEncoder.QueueInputBuffer(bufferIndex, 0, mData.Length, timestamp(), 0); } else { if (VERBOSE) { Log.e(TAG, "No buffer available !"); } } // We are looking for the SPS and the PPS here. As always, Android is very inconsistent, I have observed that some // encoders will give those parameters through the MediaFormat object (that is the normal behaviour). // But some other will not, in that case we try to find a NAL unit of type 7 or 8 in the byte stream outputed by the encoder... int index = mEncoder.DequeueOutputBuffer(info, 1000000 / FRAMERATE); if (index == (int)MediaCodecInfoState.OutputFormatChanged) { // The PPS and PPS shoud be there MediaFormat format = mEncoder.OutputFormat; ByteBuffer spsb = format.GetByteBuffer("csd-0"); ByteBuffer ppsb = format.GetByteBuffer("csd-1"); mSPS = new byte[spsb.Capacity() - 4]; spsb.Position(4); spsb.Get(mSPS, 0, mSPS.Length); mPPS = new byte[ppsb.Capacity() - 4]; ppsb.Position(4); ppsb.Get(mPPS, 0, mPPS.Length); break; } else if (index == (int)MediaCodecInfoState.OutputBuffersChanged) { outputBuffers = mEncoder.GetOutputBuffers(); } else if (index >= 0) { len = info.Size; if (len < 128) { outputBuffers[index].Get(csd, 0, len); if (len > 0 && csd[0] == 0 && csd[1] == 0 && csd[2] == 0 && csd[3] == 1) { // Parses the SPS and PPS, they could be in two different packets and in a different order //depending on the phone so we don't make any assumption about that while (p < len) { while (!(csd[p + 0] == 0 && csd[p + 1] == 0 && csd[p + 2] == 0 && csd[p + 3] == 1) && p + 3 < len) { p++; } if (p + 3 >= len) { p = len; } if ((csd[q] & 0x1F) == 7) { mSPS = new byte[p - q]; JavaSystem.Arraycopy(csd, q, mSPS, 0, p - q); } else { mPPS = new byte[p - q]; JavaSystem.Arraycopy(csd, q, mPPS, 0, p - q); } p += 4; q = p; } } } mEncoder.ReleaseOutputBuffer(index, false); } elapsed = timestamp() - now; } check(mPPS != null && mSPS != null, "Could not determine the SPS & PPS."); mB64PPS = Base64.EncodeToString(mPPS, 0, mPPS.Length, Base64Flags.NoWrap); mB64SPS = Base64.EncodeToString(mSPS, 0, mSPS.Length, Base64Flags.NoWrap); return(elapsed); }
/** * Checks the video data. * * @return the number of bad frames */ private int checkVideoData(VideoChunks inputData, MediaCodec decoder, OutputSurface surface) { const int TIMEOUT_USEC = 1000; ByteBuffer[] decoderInputBuffers = decoder.GetInputBuffers(); ByteBuffer[] decoderOutputBuffers = decoder.GetOutputBuffers(); MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); int inputChunk = 0; int checkIndex = 0; int badFrames = 0; bool outputDone = false; bool inputDone = false; while (!outputDone) { if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "check loop"); } // Feed more data to the decoder. if (!inputDone) { int inputBufIndex = decoder.DequeueInputBuffer(TIMEOUT_USEC); if (inputBufIndex >= 0) { if (inputChunk == inputData.NumChunks) { // End of stream -- send empty frame with EOS flag set. decoder.QueueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodec.BufferFlagEndOfStream); inputDone = true; if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "sent input EOS"); } } else { // Copy a chunk of input to the decoder. The first chunk should have // the BUFFER_FLAG_CODEC_CONFIG flag set. ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex]; inputBuf.Clear(); inputData.getChunkData(inputChunk, inputBuf); int flags = inputData.getChunkFlags(inputChunk); long time = inputData.getChunkTime(inputChunk); decoder.QueueInputBuffer(inputBufIndex, 0, inputBuf.Position(), time, (MediaCodecBufferFlags)flags); if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "submitted frame " + inputChunk + " to dec, size=" + inputBuf.Position() + " flags=" + flags); } inputChunk++; } } else { if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "input buffer not available"); } } } if (!outputDone) { int decoderStatus = decoder.DequeueOutputBuffer(info, TIMEOUT_USEC); if (decoderStatus == (int)MediaCodec.InfoTryAgainLater) { // no output available yet if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "no output from decoder available"); } } else if (decoderStatus == (int)MediaCodec.InfoOutputBuffersChanged) { decoderOutputBuffers = decoder.GetOutputBuffers(); if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "decoder output buffers changed"); } } else if (decoderStatus == (int)MediaCodec.InfoOutputFormatChanged) { MediaFormat newFormat = decoder.OutputFormat; if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "decoder output format changed: " + newFormat); } } else if (decoderStatus < 0) { fail("unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus); } else // decoderStatus >= 0 { ByteBuffer decodedData = decoderOutputBuffers[decoderStatus]; if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "surface decoder given buffer " + decoderStatus + " (size=" + info.Size + ")"); } if ((info.Flags & MediaCodec.BufferFlagEndOfStream) != 0) { if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "output EOS"); } outputDone = true; } bool doRender = (info.Size != 0); // As soon as we call releaseOutputBuffer, the buffer will be forwarded // to SurfaceTexture to convert to a texture. The API doesn't guarantee // that the texture will be available before the call returns, so we // need to wait for the onFrameAvailable callback to fire. decoder.ReleaseOutputBuffer(decoderStatus, doRender); if (doRender) { if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "awaiting frame " + checkIndex); } assertEquals("Wrong time stamp", computePresentationTime(checkIndex), info.PresentationTimeUs); surface.AwaitNewImage(); surface.DrawImage(); if (!checkSurfaceFrame(checkIndex++)) { badFrames++; } } } } } return(badFrames); }
/** * Edits a stream of video data. */ private void editVideoData(VideoChunks inputData, MediaCodec decoder, OutputSurface outputSurface, InputSurface inputSurface, MediaCodec encoder, VideoChunks outputData) { const int TIMEOUT_USEC = 10000; ByteBuffer[] decoderInputBuffers = decoder.GetInputBuffers(); ByteBuffer[] encoderOutputBuffers = encoder.GetOutputBuffers(); MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); int inputChunk = 0; int outputCount = 0; bool outputDone = false; bool inputDone = false; bool decoderDone = false; while (!outputDone) { if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "edit loop"); } // Feed more data to the decoder. if (!inputDone) { int inputBufIndex = decoder.DequeueInputBuffer(TIMEOUT_USEC); if (inputBufIndex >= 0) { if (inputChunk == inputData.NumChunks) { // End of stream -- send empty frame with EOS flag set. decoder.QueueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodecBufferFlags.EndOfStream); inputDone = true; if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "sent input EOS (with zero-length frame)"); } } else { // Copy a chunk of input to the decoder. The first chunk should have // the BUFFER_FLAG_CODEC_CONFIG flag set. ByteBuffer inputBuf = decoderInputBuffers[inputBufIndex]; inputBuf.Clear(); inputData.getChunkData(inputChunk, inputBuf); int flags = inputData.getChunkFlags(inputChunk); long time = inputData.getChunkTime(inputChunk); decoder.QueueInputBuffer(inputBufIndex, 0, inputBuf.Position(), time, (MediaCodecBufferFlags)flags); // TODO: Not sure if it's MediaCodecBufferFlags, verify. if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "submitted frame " + inputChunk + " to dec, size=" + inputBuf.Position() + " flags=" + flags); } inputChunk++; } } else { if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "input buffer not available"); } } } // Assume output is available. Loop until both assumptions are false. bool decoderOutputAvailable = !decoderDone; bool encoderOutputAvailable = true; while (decoderOutputAvailable || encoderOutputAvailable) { // Start by draining any pending output from the encoder. It's important to // do this before we try to stuff any more data in. int encoderStatus = encoder.DequeueOutputBuffer(info, TIMEOUT_USEC); if (encoderStatus == (int)MediaCodecInfoState.TryAgainLater) { // no output available yet if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "no output from encoder available"); } encoderOutputAvailable = false; } else if (encoderStatus == (int)MediaCodecInfoState.OutputBuffersChanged) { encoderOutputBuffers = encoder.GetOutputBuffers(); if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "encoder output buffers changed"); } } else if (encoderStatus == (int)MediaCodecInfoState.OutputFormatChanged) { MediaFormat newFormat = encoder.OutputFormat; if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "encoder output format changed: " + newFormat); } } else if (encoderStatus < 0) { fail("unexpected result from encoder.dequeueOutputBuffer: " + encoderStatus); } else // encoderStatus >= 0 { ByteBuffer encodedData = encoderOutputBuffers[encoderStatus]; if (encodedData == null) { fail("encoderOutputBuffer " + encoderStatus + " was null"); } // Write the data to the output "file". if (info.Size != 0) { encodedData.Position(info.Offset); encodedData.Limit(info.Offset + info.Size); outputData.addChunk(encodedData, (int)info.Flags, info.PresentationTimeUs); outputCount++; if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "encoder output " + info.Size + " bytes"); } } outputDone = (info.Flags & MediaCodec.BufferFlagEndOfStream) != 0; encoder.ReleaseOutputBuffer(encoderStatus, false); } if (encoderStatus != (int)MediaCodec.InfoTryAgainLater) { // Continue attempts to drain output. continue; } // Encoder is drained, check to see if we've got a new frame of output from // the decoder. (The output is going to a Surface, rather than a ByteBuffer, // but we still get information through BufferInfo.) if (!decoderDone) { int decoderStatus = decoder.DequeueOutputBuffer(info, TIMEOUT_USEC); if (decoderStatus == (int)MediaCodec.InfoTryAgainLater) { // no output available yet if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "no output from decoder available"); } decoderOutputAvailable = false; } else if (decoderStatus == (int)MediaCodec.InfoOutputBuffersChanged) { //decoderOutputBuffers = decoder.getOutputBuffers(); if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "decoder output buffers changed (we don't care)"); } } else if (decoderStatus == (int)MediaCodec.InfoOutputFormatChanged) { // expected before first buffer of data MediaFormat newFormat = decoder.OutputFormat; if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "decoder output format changed: " + newFormat); } } else if (decoderStatus < 0) { fail("unexpected result from decoder.dequeueOutputBuffer: " + decoderStatus); } else // decoderStatus >= 0 { if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "surface decoder given buffer " + decoderStatus + " (size=" + info.Size + "("); } // The ByteBuffers are null references, but we still get a nonzero // size for the decoded data. bool doRender = (info.Size != 0); // As soon as we call releaseOutputBuffer, the buffer will be forwarded // to SurfaceTexture to convert to a texture. The API doesn't // guarantee that the texture will be available before the call // returns, so we need to wait for the onFrameAvailable callback to // fire. If we don't wait, we risk rendering from the previous frame. decoder.ReleaseOutputBuffer(decoderStatus, doRender); if (doRender) { // This waits for the image and renders it after it arrives. if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "awaiting frame"); } outputSurface.AwaitNewImage(); outputSurface.DrawImage(); // Send it to the encoder. inputSurface.SetPresentationTime(info.PresentationTimeUs * 1000); if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "swapBuffers"); } inputSurface.SwapBuffers(); } if ((info.Flags & MediaCodec.BufferFlagEndOfStream) != 0) { // forward decoder EOS to encoder if (AppSettings.Logging.SendToConsole) { Log.Debug(TAG, "signaling input EOS"); } if (WORK_AROUND_BUGS) { // Bail early, possibly dropping a frame. return; } else { encoder.SignalEndOfInputStream(); } } } } } } if (inputChunk != outputCount) { throw new RuntimeException("frame lost: " + inputChunk + " in, " + outputCount + " out"); } }
private bool ExtractSomeAudioData(out bool endOfFile) { endOfFile = extractionOutputDone; if (endOfFile) { return(false); } var hasExtractedData = false; int TimeoutUs = 20000; MediaCodec.BufferInfo info = new MediaCodec.BufferInfo(); if (!extractionInputDone) { int inputBufIndex = audioMediaDecoder.DequeueInputBuffer(TimeoutUs); if (inputBufIndex >= 0) { Java.Nio.ByteBuffer inputBuffer = audioMediaDecoder.GetInputBuffer(inputBufIndex); //Read the sample data into the ByteBuffer. This neither respects nor updates inputBuf's position, limit, etc. int chunkSize = audioMediaExtractor.ReadSampleData(inputBuffer, 0); if (chunkSize < 0) { //End of stream: send empty frame with EOS flag set audioMediaDecoder.QueueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodecBufferFlags.EndOfStream); extractionInputDone = true; //Logger.Verbose("sent input EOS"); } else { if (audioMediaExtractor.SampleTrackIndex != trackIndexAudio) { Logger.Warning(string.Format("got audio sample from track {0}, expected {1}", audioMediaExtractor.SampleTrackIndex, trackIndexAudio)); } var presentationTimeMicroSeconds = audioMediaExtractor.SampleTime; audioMediaDecoder.QueueInputBuffer(inputBufIndex, 0, chunkSize, presentationTimeMicroSeconds, 0); audioMediaExtractor.Advance(); } } else { //do nothing: the input buffer queue is full (we need to output them first) //continue; } } int decoderStatus = audioMediaDecoder.DequeueOutputBuffer(info, TimeoutUs); switch (decoderStatus) { case (int)MediaCodecInfoState.TryAgainLater: { Logger.Verbose("no output from decoder available"); break; } case (int)MediaCodecInfoState.OutputFormatChanged: { MediaFormat newFormat = audioMediaDecoder.OutputFormat; string newFormatStr = newFormat.ToString(); Logger.Verbose("audio decoder output format changed: " + newFormatStr); break; } case (int)MediaCodecInfoState.OutputBuffersChanged: { //deprecated: we just ignore it break; } default: { if (decoderStatus < 0) { throw new InvalidOperationException(string.Format("unexpected result from audio decoder.DequeueOutputBuffer: {0}", decoderStatus)); } if ((info.Flags & MediaCodecBufferFlags.EndOfStream) != 0) { Logger.Verbose("audio: output EOS"); extractionOutputDone = true; } if (info.Size > 0) { hasExtractedData = true; var buffer = audioMediaDecoder.GetOutputBuffer(decoderStatus); var presentationTime = TimeSpanExtensions.FromMicroSeconds(info.PresentationTimeUs); if (StorageBuffer.CountDataBytes + info.Size <= StorageBuffer.Data.Length) { buffer.Get(StorageBuffer.Data, StorageBuffer.CountDataBytes, info.Size); // Read the buffer all at once buffer.Clear(); // MUST DO!!! OTHERWISE THE NEXT TIME YOU GET THIS SAME BUFFER BAD THINGS WILL HAPPEN buffer.Position(0); if (StorageBuffer.CountDataBytes == 0) { StorageBuffer.PresentationTime = presentationTime; } StorageBuffer.CountDataBytes += info.Size; } else { Logger.Error("The storage buffer has reached full capacity. Current data will be dropped"); } } audioMediaDecoder.ReleaseOutputBuffer(decoderStatus, false); break; } } endOfFile = extractionOutputDone; return(hasExtractedData); }
private void ExtractMedia() { if (MediaDecoder == null) { throw new InvalidOperationException("The Media Codec Extractor has not been initialized"); } if (!isInitialized) { throw new InvalidOperationException("The Media Codec has not been initialized for a media"); } var bufferInfo = new MediaCodec.BufferInfo(); var waitDefaultTime = TimeSpan.FromMilliseconds(10); MediaDecoder.Start(); while (true) { var waitTime = waitDefaultTime; // time to wait at the end of the loop iteration //Process the commands if (ProcessCommandsAndUpdateCurrentState()) { waitTime = TimeSpan.Zero; } // terminate the thread on disposal if (currentState == SchedulerAsyncCommandEnum.Dispose) { return; } //================================================================================================= //Extract video inputs if (!inputExtractionDone) { int inputBufIndex = MediaDecoder.DequeueInputBuffer(0); if (inputBufIndex >= 0) { waitTime = TimeSpan.Zero; var inputBuffer = MediaDecoder.GetInputBuffer(inputBufIndex); // Read the sample data into the ByteBuffer. This neither respects nor updates inputBuf's position, limit, etc. int chunkSize = mediaExtractor.ReadSampleData(inputBuffer, 0); if (chunkSize > 0) { if (mediaExtractor.SampleTrackIndex != mediaTrackIndex) { throw new Exception($"Got media sample from track {mediaExtractor.SampleTrackIndex}, track expected {mediaTrackIndex}"); } MediaDecoder.QueueInputBuffer(inputBufIndex, 0, chunkSize, mediaExtractor.SampleTime, 0); mediaExtractor.Advance(); } else // End of stream -- send empty frame with EOS flag set. { MediaDecoder.QueueInputBuffer(inputBufIndex, 0, 0, 0L, MediaCodecBufferFlags.EndOfStream); inputExtractionDone = true; } } else { //do nothing: the input buffer queue is full (we need to output them first) } } //================================================================================================= // Process the output buffers if (ShouldProcessDequeueOutput(ref waitTime)) { int indexOutput = MediaDecoder.DequeueOutputBuffer(bufferInfo, 0); switch (indexOutput) { case (int)MediaCodecInfoState.TryAgainLater: // decoder not ready yet (haven't processed input yet) case (int)MediaCodecInfoState.OutputBuffersChanged: //deprecated: we just ignore it break; case (int)MediaCodecInfoState.OutputFormatChanged: Logger.Verbose("decoder output format changed: " + MediaDecoder.OutputFormat.ToString()); break; default: // the index of the output buffer if (indexOutput < 0) { Logger.Warning("unexpected index from decoder.dequeueOutputBuffer: " + indexOutput); isEOF = true; break; } if ((bufferInfo.Flags & MediaCodecBufferFlags.EndOfStream) != 0) { isEOF = true; MediaDecoder.ReleaseOutputBuffer(indexOutput, false); break; } MediaCurrentTime = TimeSpanExtensions.FromMicroSeconds(bufferInfo.PresentationTimeUs); ProcessOutputBuffer(bufferInfo, indexOutput); break; } } if (waitTime > TimeSpan.Zero) { // sleep required time to avoid active looping // Note: do not sleep more than 'waitDefaultTime' to continue processing play commands Utilities.Sleep(TimeSpanExtensions.Min(waitDefaultTime, waitTime)); } } }
public void Decode(MediaCodec _Decoder, MediaExtractor extractor) { Stopwatch s = new Stopwatch(); s.Start(); int TIMEOUT_USEC = 10000; ByteBuffer[] encoderInputBuffers = _Decoder.GetInputBuffers(); ByteBuffer[] outputBuffers = _Decoder.GetOutputBuffers(); var mBufferInfo = new MediaCodec.BufferInfo(); bool inputDone = false; var index = 0; try { while (true) { if (!inputDone) { int inputBufIndex = _Decoder.DequeueInputBuffer(TIMEOUT_USEC); if (inputBufIndex >= 0) { ByteBuffer buffer = encoderInputBuffers[inputBufIndex]; //long ptsUsec = computePresentationTime(frameIndex); int sampleSize = extractor.ReadSampleData(buffer, 0); if (sampleSize < 0) { // Send an empty frame with the end-of-stream flag set. If we set EOS on a frame with data, that frame data will be ignored, and the output will be short one frame. _Decoder.QueueInputBuffer(inputBufIndex, 0, 0, 0, MediaCodec.BufferFlagEndOfStream); inputDone = true; Log.Info(TAG, "sent input EOS (with zero-length frame)"); } else { Log.Info(TAG, "adding encoded video to decoder input "); _Decoder.QueueInputBuffer(inputBufIndex, 0, sampleSize, extractor.SampleTime, 0); extractor.Advance(); } } else { // either all in use, or we timed out during initial setup Log.Warn(TAG, "input buffer not available"); } } //ByteBuffer[] encoderOutputBuffers = _Decoder.GetOutputBuffers(); int encoderStatus = _Decoder.DequeueOutputBuffer(mBufferInfo, TIMEOUT_USEC); if (encoderStatus == (int)MediaCodecInfoState.TryAgainLater) { Log.Info(TAG, "no output available, spinning to await EOS"); } else if (encoderStatus == (int)MediaCodecInfoState.OutputBuffersChanged) { // not expected for an encoder Log.Warn(TAG, "not expected OutputBuffersChanged happened"); outputBuffers = _Decoder.GetOutputBuffers(); } else if (encoderStatus == (int)MediaCodecInfoState.OutputFormatChanged) { // should happen before receiving buffers, and should only happen once //if (_MuxerStarted) //{ // Log.Error(TAG, "format changed twice and should never happen"); // throw new RuntimeException("format changed twice"); //} //MediaFormat newFormat = _Decoder.OutputFormat; //Log.Info(TAG, "format changed and starting MUX"); //_TrackIndex = _Muxer.AddTrack(newFormat); //_Muxer.Start(); //_MuxerStarted = true; } else if (encoderStatus < 0) { Log.Warn(TAG, "unexpected but lets ignore"); // let's ignore it } else { ByteBuffer encodedData = outputBuffers[encoderStatus]; if (encodedData == null) { Log.Error(TAG, string.Format("encoderOutputBuffer {0} was null!!", encoderStatus)); throw new RuntimeException(string.Format("encoderOutputBuffer {0} was null!!", encoderStatus)); } if ((mBufferInfo.Flags & MediaCodecBufferFlags.CodecConfig) != 0) { // The codec config data was pulled out and fed to the muxer when we got // the INFO_OUTPUT_FORMAT_CHANGED status. Ignore it. mBufferInfo.Size = 0; } if (mBufferInfo.Size != 0) { //if (!_MuxerStarted) //{ // Log.Error(TAG, "muxer hasnt started!!"); // throw new RuntimeException("muxer hasnt started"); //} // adjust the ByteBuffer values to match BufferInfo (not needed?) old //encodedData.Position(mBufferInfo.Offset); //encodedData.Limit(mBufferInfo.Offset + this.mBufferInfo.Size); try { //byte[] dst = new byte[outputBuffers[encoderStatus].Capacity()]; //outputBuffers[encoderStatus].Get(dst); //ByteBuffer buffer = outputBuffers[encoderStatus]; //byte[] ba = new byte[encodedData.Remaining()]; //encodedData.Get(ba); //ByteBuffer buffer = outputBuffers[encoderStatus]; //buffer.Position(mBufferInfo.Offset); //buffer.Limit(mBufferInfo.Offset + mBufferInfo.Size); //byte[] ba = new byte[buffer.Remaining()]; //buffer.Get(ba); //if (index < 10) //{ YuvImage yuv = Utils.GetYUVImage(encodedData, _CameraColorFormat, _Width, _Height); //var imagedata = yuv.GetYuvData(); //Utils.swapNV21_NV12(ref imagedata, _Width, _Height); //Image might need to be corrected later //Bitmap b = Utils.GetBitmap(yuv, _Width, _Height); //Bitmap bmp = BitmapFactory.DecodeByteArray(ba, 0, ba.Length);// this return null //var createfilepath = new File(_downloadsfilesdir, DateTime.Now.Ticks + ".png").AbsolutePath; //using (FileStream bos = new FileStream(createfilepath, FileMode.CreateNew)) //{ // b.Compress(Bitmap.CompressFormat.Png, 100, bos); //} //b.Recycle(); //} index++; //writeFrameToSDCard(dst, i, dst.length); //i++; } catch (Exception e) { //Log("iDecodeActivity", "Error while creating bitmap with: "); } _Decoder.ReleaseOutputBuffer(encoderStatus, false); } if ((mBufferInfo.Flags & MediaCodecBufferFlags.EndOfStream) != 0) { Log.Info(TAG, "End of Stream Reached!!"); break; } } } s.Stop(); Log.Info("inner STOPWATCH!!!!:", string.Format("numberofframes = {0}, totaltime = {1}", index, s.ElapsedMilliseconds)); } catch (Exception e) { Log.Error(TAG, "Decode or Muxer failed", e, e.Message); throw; } }