android - Android MediaCodec recording produces a video which cannot be played on huawei

Question

I use MediaCodec.inputSurface and MediaMixuer for recording of my android view. Everything is good on the most of devices, but not on Huawei. For some reason it produces a video which cannot be played on this devices. But for some unknown reason 1 out of 10 times it generates a good video. Here's link to the broken video and to the normal video. It is also weird that both videos can be played on my mac laptop.

Our users reported the issue from multiple huawei models and I can confirm it from my phone: HUAWEI P8 lite 2017, android 7.0. Also it happens on new phones too with any android version.

Here's code of how I manage the recording:

/**
 * Stages:
 * 1. Draw canvas to bitmap
 * 2. Take bitmap pixels and convert them to YUV
 * 3. Write bitmap pixels as a frame to MediaCodec
 * 4. Take mediaCodec and write to mediaMuxer to receive file
 */

class VideoEncoder(
    val width: Int,
    val height: Int,
    val frameRate: Int,
    val file: File,
    val durationUs: Long,
    val handler: Handler,
    val videoRecordingFinished: () -> Unit,
    val onError: (MediaCodec.CodecException) -> Unit
) : KoinComponent {

    var mediaMuxer: MediaMuxer? = null
    var videoCodec: MediaCodec? = null
    var videoTrackIndex = 0

    var surface: Surface? = null
    val videoBufferInfo by lazy { MediaCodec.BufferInfo() }
    var writingVideoFinished = false

    private var currentFrame = 0

    var audioEncoder: AudioEncoder? = null

    var writingAudioFinished: Boolean
        get() = audioEncoder?.writingAudioFinished ?: true
        set(value) {
            audioEncoder?.writingAudioFinished = value
        }

    var videoFormatInited: Boolean = false

    val allFormatsInited: Boolean
        get() = videoFormatInited && (audioEncoder?.audioFormatInited != false)


    private val pendingVEncoderInfos = LinkedList<MediaCodec.BufferInfo>()
    private val pendingVEncoderIndices = LinkedList<Int>()

    val logger: KLogger by inject {
        parametersOf("video-encoder")
    }

    private fun createVideoFormat(mimeType: String, desiredColorFormat: Int): MediaFormat {
        val mediaFormat =
            MediaFormat.createVideoFormat(mimeType, width, height)
        mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, ENCODING_VIDEO_BITRATE)
        mediaFormat.setInteger(MediaFormat.KEY_FRAME_RATE, this.frameRate)
        mediaFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 1)
        mediaFormat.setInteger(MediaFormat.KEY_COLOR_FORMAT, desiredColorFormat)

        return mediaFormat
    }

    private fun findCorrectVideoFormat(): MediaFormat {

        val mimeType = POSSIBLE_MIME_TYPES[0]
        val desiredColorFormat = MediaCodecInfo.CodecCapabilities.COLOR_FormatSurface
        val mediaFormat = createVideoFormat(mimeType, desiredColorFormat)

        val encoderForFormat =
            MediaCodecList(MediaCodecList.REGULAR_CODECS).findEncoderForFormat(mediaFormat)

        if (encoderForFormat == null) {
            logger.info { "encoderForFormatIsNull!!! width = $width, height = $height" }

            videoCodec = MediaCodec.createEncoderByType(mimeType)
        } else {
            videoCodec = MediaCodec.createByCodecName(encoderForFormat)
        }
        val codecInfo = videoCodec!!.codecInfo

        if (codecInfo.isEncoder && codecInfo.supportedTypes.contains(mimeType) &&
            codecInfo.getCapabilitiesForType(mimeType).colorFormats
                .contains(desiredColorFormat)
        ) {

        } else {
            throw IllegalStateException("MediaCodec is wrong = ${codecInfo}")
        }

        val errorMessage = checkIsColorFormatSupported(mediaFormat, desiredColorFormat, mimeType)
        if (errorMessage != null)
            throw IllegalStateException(errorMessage)

        return mediaFormat
    }

    //return error message if false
    fun checkIsColorFormatSupported(
        mediaFormat: MediaFormat,
        desiredColorFormat: Int,
        mimeType: String
    ): String? {
        var colorFormats = videoCodec!!.codecInfo.getCapabilitiesForType(mimeType).colorFormats
        var colorFormatSize = colorFormats.size
        var counterColorFormat = 0
        val colorFormatCorrect: Boolean
        while (true) {
            if (counterColorFormat >= colorFormatSize) {
                colorFormatCorrect = false
                break
            }
            if (colorFormats[counterColorFormat] == desiredColorFormat) {
                colorFormatCorrect = true
                break
            }
            ++counterColorFormat
        }

        if (!colorFormatCorrect) {
            var message = "NO COLOR FORMAT COMPATIBLE\\n$mediaFormat"
            colorFormats = videoCodec!!.codecInfo.getCapabilitiesForType(mimeType).colorFormats
            colorFormatSize = colorFormats.size
            counterColorFormat = 0
            while (counterColorFormat < colorFormatSize) {
                val sb = StringBuilder()
                sb.append(message)
                sb.append("\\n")
                sb.append(colorFormats[counterColorFormat])
                message = sb.toString()
                logger.debug { message }
                ++counterColorFormat
            }
            return message
        }

        return null

    }

    private fun printVideoCodecInfo() {
        logger.debug {
            val json = JSONObject()
            json.put("codec_name", videoCodec!!.name)
            json.put("codec_info_name", videoCodec!!.codecInfo.name)
            json.put("codec_supported_types", videoCodec!!.codecInfo.supportedTypes)
            json.put("output_width", width)
            json.put("output_height", height)
            json.toString()
        }
    }

    @Throws(Exception::class)
    fun initialize(videoAsyncEncoder: Boolean) {

        val filePath = file.canonicalPath

        val mediaFormat = findCorrectVideoFormat()

        printVideoCodecInfo()

        if (videoAsyncEncoder) {
            videoCodec!!.setCallback(object : MediaCodec.Callback() {
                override fun onInputBufferAvailable(codec: MediaCodec, index: Int) {

                }

                override fun onOutputBufferAvailable(
                    codec: MediaCodec,
                    index: Int,
                    info: MediaCodec.BufferInfo
                ) {

                    pendingVEncoderIndices.add(index)
                    pendingVEncoderInfos.add(info)

                    if (allFormatsInited)
                        checkVideoOutputAvailable()
                }

                override fun onError(codec: MediaCodec, e: MediaCodec.CodecException) {
                    writingVideoFinished = true
                    e.printDebug()
                    onError.invoke(e)
                }

                override fun onOutputFormatChanged(codec: MediaCodec, format: MediaFormat) {
                    onVideoFormatChanged(format)
                }

            }, handler)
        }

        videoCodec!!.configure(
            mediaFormat, null, null,
            MediaCodec.CONFIGURE_FLAG_ENCODE
        )

        surface = videoCodec!!.createInputSurface()
        mediaMuxer = MediaMuxer(filePath, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4)
    }

    fun initAudio(
        path: String,
        startTimeUs: Long,
        volume: Int,
        audioRecordingFinished: () -> Unit
    ) {
        audioEncoder = AudioEncoder(
            mediaMuxer!!,
            handler,
            durationUs,
            ::checkFormatsInited,
            audioRecordingFinished
        )
        audioEncoder!!.initAudio(path, startTimeUs, volume)
        audioEncoder!!.startAudioCodec()
    }

    fun canWriteAudio() {
        audioEncoder?.canWriteAudio()
    }

    fun getCurrentAudioTime() = audioEncoder?.getCurrentAudioTime()

    private fun onVideoFormatChanged(format: MediaFormat) {
        videoTrackIndex =
            mediaMuxer!!.addTrack(format)
        videoFormatInited = true
        checkFormatsInited()
    }


    fun checkFormatsInited() {
        if (allFormatsInited) {
            mediaMuxer!!.start()
            checkVideoOutputAvailable()
        }
    }


    @Throws(IllegalStateException::class)
    fun writeToMuxerSyncMode(currentFrame: Int = -1): Boolean {
        var success = false

        while (videoCodec != null && mediaMuxer != null) {

            val outputBufferIndex = videoCodec!!.dequeueOutputBuffer(videoBufferInfo, 0L)

            logger.info {
                "writeToMuxer, outputBufferIndex = ${outputBufferIndex}, bufferFlag = ${videoBufferInfo.flags}," +
                        " presentationTime = ${((currentFrame * 1000000L) / frameRate)}," +
                        " bufferInfo.size ${videoBufferInfo.size}, bufferInfo.offset ${videoBufferInfo.offset}"
            }

            if (outputBufferIndex == MediaCodec.INFO_OUTPUT_FORMAT_CHANGED) {
                onVideoFormatChanged(videoCodec!!.outputFormat)
            } else {
                if (outputBufferIndex < 0) {
                    return success
                }
                success = true

                val bufferInfo = videoBufferInfo

                if (bufferInfo.offset >= 0 && bufferInfo.size > 0) {
                    val outputBuffer = videoCodec!!.getOutputBuffer(outputBufferIndex)!!

                    outputBuffer.position(this.videoBufferInfo.offset)
                    outputBuffer.limit(bufferInfo.offset + bufferInfo.size)

                    if (currentFrame != -1) {
                        if (videoBufferInfo.flags == MediaCodec.BUFFER_FLAG_CODEC_CONFIG)
                            success = false
                        else
                            bufferInfo.presentationTimeUs = (currentFrame * 1000000L) / frameRate
                    }

                    mediaMuxer!!.writeSampleData(
                        videoTrackIndex,
                        outputBuffer,
                        this.videoBufferInfo
                    )
                }
                videoCodec!!.releaseOutputBuffer(outputBufferIndex, false)

                if (bufferInfo.flags.and(MediaCodec.BUFFER_FLAG_END_OF_STREAM) != 0) {
                    return success
                }
            }
        }
        return success
    }

    private fun onVideoWritingFinished() {
        writingVideoFinished = true
        videoRecordingFinished.invoke()
    }

    private fun checkVideoOutputAvailable() {
        while (pendingVEncoderIndices.size > 0 &&
            pendingVEncoderInfos.size > 0 && videoCodec != null
        ) {

            val index = pendingVEncoderIndices.removeFirst()
            val info = pendingVEncoderInfos.removeFirst()

            onVideoOutputAvailable(videoCodec!!, index, info)
        }
    }

    private fun onVideoOutputAvailable(codec: MediaCodec, index: Int, info: MediaCodec.BufferInfo) {

        if (videoCodec == null)
            return

        if (info.flags and MediaCodec.BUFFER_FLAG_END_OF_STREAM != 0) {

            codec.releaseOutputBuffer(index, false)
            onVideoWritingFinished()

        } else {
            val outputBuffer = codec.getOutputBuffer(index)!!

            outputBuffer.position(info.offset)
            outputBuffer.limit(info.offset + info.size)

            info.presentationTimeUs = (currentFrame * 1000000L) / frameRate

            if (info.flags != MediaCodec.BUFFER_FLAG_CODEC_CONFIG) {
                currentFrame++
            }

            logger.info {
                "videoOutputAvailable time ${info.presentationTimeUs}, flags ${info.flags}," +
                        " size ${info.size}, offset ${info.offset}"
            }

            mediaMuxer!!.writeSampleData(
                videoTrackIndex,
                outputBuffer, info
            )

            codec.releaseOutputBuffer(index, false)
        }
    }

    fun startVideoCodec() {
        videoCodec?.start()
    }

    fun stop() {

        audioEncoder?.stop()

        pendingVEncoderInfos.clear()
        pendingVEncoderIndices.clear()
        surface?.release()
        surface = null

        if (videoCodec != null) {
            try {
                videoCodec?.stop()
            } catch (e: IllegalStateException) {
            } finally {
                videoCodec?.release()
                videoCodec = null
            }
        }
        if (mediaMuxer != null) {
            try {
                mediaMuxer?.release()
            } catch (e: IllegalStateException) {
                logger.error(e)
            } finally {
                mediaMuxer = null
            }
        }
    }

    fun sendEndOfStreamSurface() {
        videoCodec?.signalEndOfInputStream()
        if (!ThreadRecord.VIDEO_CODEC_ASYNC) {
            onVideoWritingFinished()
        }
    }

    companion object {
        const val ENCODING_VIDEO_BITRATE = 12000000
        val POSSIBLE_MIME_TYPES = arrayOf("video/avc", "video/hevc", "video/x-vnd.on2.vp8")
    }

}

标签： androidandroid-mediacodec