Converting ByteBuffer obtained from Mediacodec to a bitmap to store it in SD card

Question

I'm using MediaCodec to read a video file and store the frames on SD card. However it is storing a green rectangle instead of the actual frame. Here is the code:

    int outIndex = decoder.dequeueOutputBuffer(info, 10000);
    switch (outIndex) {
    case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
        Log.d("DecodeActivity", "INFO_OUTPUT_BUFFERS_CHANGED");
        outputBuffers = decoder.getOutputBuffers();
        break;
    case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
        Log.d("DecodeActivity", "New format " + decoder.getOutputFormat());
        break;
    case MediaCodec.INFO_TRY_AGAIN_LATER:
        Log.d("DecodeActivity", "dequeueOutputBuffer timed out!");
        break;
    default:
        ByteBuffer buffer = outputBuffers[outIndex]; //The bytebuffer i want to convert to bitmap
        Log.v("DecodeActivity", "We can't use this buffer but render it due to the API limit, " + buffer);
        buffer.position(info.offset);
        buffer.limit(info.offset + info.size);
        byte[] ba = new byte[buffer.remaining()]; //converting bytebuffer to byte array
        buffer.get(ba);
        Log.d("ba", ba.length + "");
        // I have to convert the byte array to a bitmap. So I'm doing this : 
        YuvImage yuvimage = new YuvImage(ba, ImageFormat.NV21, 1280, 720, null);        
        ByteArrayOutputStream baos = new ByteArrayOutputStream();
        yuvimage.compressToJpeg(new Rect(0, 0, 1280, 720), 80, baos);
        byte[] jdata = baos.toByteArray();
        Bitmap bmp = BitmapFactory.decodeByteArray(jdata, 0, jdata.length);
        if (bmp != null) {
            //Store the bitmap as JPEG in SD card
            Log.d("Barcode",barcodeNumber+"");
            String baseDir = Environment.getExternalStorageDirectory().getAbsolutePath();
            File videoFile = new File(baseDir + "/MyPhotos2/image" + barcodeNumber + ".jpg");
            barcodeNumber ++;
            FileOutputStream out = null;
            Log.d("success","yes");
            try {
                out = new FileOutputStream(videoFile);
                bmp.compress(Bitmap.CompressFormat.JPEG, 100, out);
            } catch (Exception e) {
                e.printStackTrace();
            } finally {
                try {
                        if (out != null) {
                        out.close();
                    }
                } catch (IOException e) {
                    e.printStackTrace();
                }
                }
            }
            else if(bmp==null)
            {
                Log.d("null","null");
            }

    // We use a very simple clock to keep the video FPS, or the video
    // playback will be too fast
    while (info.presentationTimeUs / 1000 > System.currentTimeMillis() - startMs) {
            try {
                sleep(10);
            } catch (InterruptedException e) {
                e.printStackTrace();
                break;
            }
    }
    decoder.releaseOutputBuffer(outIndex, true);
    break;
    }

    // All decoded frames have been rendered, we can stop playing now
    if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
        Log.d("DecodeActivity", "OutputBuffer BUFFER_FLAG_END_OF_STREAM");
                break;
            }
    }
    decoder.stop();
    decoder.release();
    extractor.release();
}

I think the problem is with the format of the bitmap. I'm not sure if what I'm doing is right here. Please correct any mistakes in this or suggest a way in which I can extract the bitmap from the ByteBuffer. I decided to use mediacodec because MediaMetadataRetriever is too slow.

Answer 1

My goal was to handle the ByteBuffer myself so that later I can save the image while displaying it in Surface. Here's how I did it. I also note that in MediaCodec I do not transfer Surface that I can could processed ByteBuffer . Maybe you will find something that will help you.

public class MediaCodecSdCardControl extends BaseControl implements SurfaceHolder.Callback {

private static final String TAG = "MediaCodecSdCardControl";
private static final String SAMPLE = Environment.getExternalStorageDirectory() + "/record.mp4";
private PlayerThread mPlayer = null;
private SurfaceView surfaceView;
private ImageView imageView;
// Rect of the source bitmap to draw
private Rect srcRect = new Rect();
// Rect of the destination canvas to draw to
private Rect dstRect = new Rect();

MediaCodecSdCardControl(BaseActivity activity, ActivityMainBinding binding) {
    super(activity);
    this.surfaceView = binding.surfaceView;
    this.imageView = binding.imageView;
    surfaceView.getHolder().addCallback(this);


}

@Override
public void onPause() {
    super.onStop();
    if (mPlayer != null) {
        Log.e(TAG, "onPause");
        mPlayer.needStop = true;
        mPlayer.interrupt();
        mPlayer = null;
    }
}

@Override
public void onDestroy() {
    super.onDestroy();

}

@Override
public void surfaceCreated(SurfaceHolder holder) {
}

@Override
public void surfaceChanged(SurfaceHolder holder, int format, int width, int height) {
    if (mPlayer == null) {
        dstRect.right = width;
        dstRect.bottom = height;
        mPlayer = new PlayerThread(holder.getSurface());
        mPlayer.start();
    }
}

@Override
public void surfaceDestroyed(SurfaceHolder holder) {

}

private class PlayerThread extends Thread {
    private MediaExtractor extractor;
    private MediaCodec decoder;
    private Surface surface;
    private boolean needStop = false;

    public PlayerThread(Surface surface) {
        this.surface = surface;
    }

    @Override
    public void run() {
        extractor = new MediaExtractor();
        try {
            extractor.setDataSource(SAMPLE);
        } catch (IOException e) {
            Log.e(TAG, "ERROR STEP 1", e);
        }

        for (int i = 0; i < extractor.getTrackCount(); i++) {
            MediaFormat format = extractor.getTrackFormat(i);
            String mime = format.getString(MediaFormat.KEY_MIME);
            if (mime.startsWith("video/")) {
                extractor.selectTrack(i);
                try {
                    decoder = MediaCodec.createDecoderByType(mime);
                } catch (IOException e) {
                    Log.e(TAG, "ERROR STEP 2", e);
                }
                decoder.configure(format, /*surface*/ null, null, 0);
                break;
            }
        }

        if (decoder == null) {
            Log.e(TAG, "Can't find video info!");
            return;
        }

        decoder.start();

        ByteBuffer[] inputBuffers = decoder.getInputBuffers();
        ByteBuffer[] outputBuffers = decoder.getOutputBuffers();
        MediaCodec.BufferInfo info = new MediaCodec.BufferInfo();
        boolean isEOS = false;
        long startMs = System.currentTimeMillis();

        while (!Thread.interrupted() && !needStop) {
            if (!isEOS) {
                int inIndex = -1;
                try {
                    inIndex = decoder.dequeueInputBuffer(10000);
                } catch (IllegalStateException e) {
                    Log.e(TAG, "ERROR STEP 3", e);
                }

                if (inIndex >= 0) {
                    ByteBuffer buffer = inputBuffers[inIndex];
                    int sampleSize = extractor.readSampleData(buffer, 0);
                    if (sampleSize < 0) {
                        // We shouldn't stop the playback at this point, just pass the EOS
                        // flag to decoder, we will get it again from the
                        // dequeueOutputBuffer
                        Log.d(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
                        if (!needStop) {
                            decoder.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
                            isEOS = true;
                        }
                    } else {
                        try {
                            if (!needStop) {
                                decoder.queueInputBuffer(inIndex, 0, sampleSize, extractor.getSampleTime(), 0);
                                extractor.advance();
                            }
                        } catch (IllegalStateException e) {
                            Log.e(TAG, "ERROR STEP 3_1", e);
                        }

                    }

                }
            }
            int outIndex = MediaCodec.INFO_TRY_AGAIN_LATER;
            try {
                if (!needStop) {
                    outIndex = decoder.dequeueOutputBuffer(info, 10000);
                }
            } catch (IllegalStateException e) {
                Log.e(TAG, "ERROR STEP 4", e);
            }

            switch (outIndex) {
                case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
                    Log.d("DecodeActivity", "INFO_OUTPUT_BUFFERS_CHANGED");
                    outputBuffers = decoder.getOutputBuffers();
                    break;
                case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
                    Log.d(TAG, "New format " + decoder.getOutputFormat());
                    break;
                case MediaCodec.INFO_TRY_AGAIN_LATER:
                    Log.d(TAG, "dequeueOutputBuffer timed out!");
                    break;
                default:
                    ByteBuffer buffer = outputBuffers[outIndex];
                    Log.v(TAG, "We can't use this buffer but render it due to the API limit, " + buffer);

                    buffer.position(info.offset);
                    buffer.limit(info.offset + info.size);
                    byte[] ba = new byte[buffer.remaining()]; //converting bytebuffer to byte array
                    buffer.get(ba);

                    Log.e(TAG, "ba = " + ba.length + "");
                    YuvImage yuvimage = new YuvImage(ba, ImageFormat.NV21, 1280, 720, null);
                    ByteArrayOutputStream baos = new ByteArrayOutputStream();
                    yuvimage.compressToJpeg(new Rect(0, 0, 1280, 720), 100, baos);
                    byte[] jdata = baos.toByteArray();
                    final Bitmap bmp = BitmapFactory.decodeByteArray(jdata, 0, jdata.length);
                    if (bmp != null) {
                        Log.e(TAG, "bmp = GOOD");
                        imageView.post(new Runnable() {
                            @Override
                            public void run() {
                                imageView.setImageBitmap(bmp);
                            }
                        });
                        srcRect.left = 0;
                        srcRect.top = 0;
                        srcRect.bottom = 720;
                        srcRect.right = 1280;
                        Canvas canvas = surfaceView.getHolder().lockCanvas();
                        try {
                            if (canvas != null) {
                                canvas.drawBitmap(bmp, srcRect, dstRect, null);
                            }
                        } finally {
                            if (canvas != null) {
                                surfaceView.getHolder().unlockCanvasAndPost(canvas);
                            }
                        }
                    } else {
                        Log.e(TAG, "bmp = BAD");
                    }
                    while (info.presentationTimeUs / 1000 > System.currentTimeMillis() -
                            startMs && !needStop) {
                        try {
                            sleep(10);
                        } catch (InterruptedException e) {
                            //NOT NORMAL WORK if call thread interrupt
                            PlayerThread.this.interrupt();
                            e.printStackTrace();
                            break;
                        }
                    }
                    decoder.releaseOutputBuffer(outIndex, false);

                    break;
            }

            // All decoded frames have been rendered, we can stop playing now
            if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
                Log.d("DecodeActivity", "OutputBuffer BUFFER_FLAG_END_OF_STREAM");
                break;
            }
        }
        decoder.stop();
        decoder.release();
        extractor.release();
    }
}

}