Access violation in native code with Android MediaCodec hardware accelerator

I want to use Android MediaCodec to decode a video stream, and then use the output images to further process the images in my own code.

Platform: ASUS TF700T Android 4.1.1. Test Stream: H.264 full HD @ 24 frm / s

With the Tegra-3 SoC inside, I rely on hardware support for video decoding. Functionally, my application works as expected: I can really access the decoder images and process them properly. However, I am experiencing a very high decoder processor load.

In the following experiments, the process / thread load is measured by "top -m 32 -t" in the adb shell. In order to get reliable output from the “top”, all 4 processor cores are forcibly activated, starting several thread cycles forever with the lowest priority. This is confirmed by the repeated execution of "cat / sys / devices / system / cpu / cpu [0-3] / online". To make everything simple, there is only video decoding, there is no sound; and there is no time management, so the decoder works as fast as it can.

First experiment: run the application by calling the JNI processing function, but all further processing calls are commented out. Results:

• throughput: 25 frm / s
• 1% Download VideoDecoder Stream Application
• 24% load of the Binder_3 thread of the process / system / bin / mediaserver

, (25% )... . : .

, MediaCodec . . :

• 55 frm/s (!!)
• 2% VideoDecoder
• 1% - //bin/mediaserver

, . , ...

, MediaCodec , Surface?

. Surface ( , ). ! .

! http://developer.android.com/reference/android/media/MediaCodec.html. - google http://www.youtube.com/watch?v=RQws6vsoav8.

: Android MediaCodec ? ? ! .

, MediaExtractor MediaCodec , ok ( Surface). , API , -)

, - mSurface: null "mDecoder.configure(mediaFormat, mSurface, null, 0);"

:

mExtractor = new MediaExtractor();
mExtractor.setDataSource(mPath);

// Locate first video stream
for (int i = 0; i < mExtractor.getTrackCount(); i++) {
    mediaFormat = mExtractor.getTrackFormat(i);
    String mime = mediaFormat.getString(MediaFormat.KEY_MIME);
    Log.i(TAG, String.format("Stream %d/%d %s", i, mExtractor.getTrackCount(), mime));
    if (streamId == -1 && mime.startsWith("video/")) {
        streamId = i;
    }
}

if (streamId == -1) {
    Log.e(TAG, "Can't find video info in " + mPath);
    return;
}

mExtractor.selectTrack(streamId);
mediaFormat = mExtractor.getTrackFormat(streamId);

mDecoder = MediaCodec.createDecoderByType(mediaFormat.getString(MediaFormat.KEY_MIME));
mDecoder.configure(mediaFormat, mSurface, null, 0);

width = mediaFormat.getInteger(MediaFormat.KEY_WIDTH);
height = mediaFormat.getInteger(MediaFormat.KEY_HEIGHT);
Log.i(TAG, String.format("Image size: %dx%d format: %s", width, height, mediaFormat.toString()));
JniGlue.decoutStart(width, height);

( ):

ByteBuffer[] inputBuffers = mDecoder.getInputBuffers();
ByteBuffer[] outputBuffers = mDecoder.getOutputBuffers();

while (!isEOS && !Thread.interrupted()) {
    int inIndex = mDecoder.dequeueInputBuffer(10000);
    if (inIndex >= 0) {
        // Valid buffer returned
        int sampleSize = mExtractor.readSampleData(inputBuffers[inIndex], 0);
        if (sampleSize < 0) {
            Log.i(TAG, "InputBuffer BUFFER_FLAG_END_OF_STREAM");
            mDecoder.queueInputBuffer(inIndex, 0, 0, 0, MediaCodec.BUFFER_FLAG_END_OF_STREAM);
            isEOS = true;
        } else {
            mDecoder.queueInputBuffer(inIndex, 0, sampleSize, mExtractor.getSampleTime(), 0);
            mExtractor.advance();
        }
    }

    int outIndex = mDecoder.dequeueOutputBuffer(info, 10000);
    if (outIndex >= 0) {
        // Valid buffer returned
        ByteBuffer buffer = outputBuffers[outIndex];
        JniGlue.decoutFrame(buffer, info.offset, info.size);
        mDecoder.releaseOutputBuffer(outIndex, true);
    } else {
        // Some INFO_* value returned
        switch (outIndex) {
        case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED:
            Log.i(TAG, "RunDecoder: INFO_OUTPUT_BUFFERS_CHANGED");
            outputBuffers = mDecoder.getOutputBuffers();
            break;
        case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED:
            Log.i(TAG, "RunDecoder: New format " + mDecoder.getOutputFormat());
            break;
        case MediaCodec.INFO_TRY_AGAIN_LATER:
            // Timeout - simply ignore
            break;
        default:
            // Some other value, simply ignore
            break;
        }
    }

    if ((info.flags & MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
        Log.d(TAG, "RunDecoder: OutputBuffer BUFFER_FLAG_END_OF_STREAM");
        isEOS = true;
    }
}