Texture convertation on Samsung Galaxy Ultra series (S 22, Tab 8) with SM8450

[Vault13san]

trafficstars

System information

• Devices: Samsung Galaxy S22 Ultra(only with SM8450, S5E9925 works fine), Samsung Galaxy Tab 8 Ultra
• Android version: 12
• OneUI version: 4.1.1
• Mediapipe version: 0.8.10.2

Use case: I need to handle some image from different sources (not only from camera), so i can't use CameraHelper. I create my own converter class for bitmap and it works fine for all devices, but not for Samsung Ultra series powered by Snapdragon cpu.

I do this steps:

1. Create bitmap from image
2. Create openGL texture from bitmap (by using ShaderUtil.createRgbaTexture method)
3. Create AppTextureFrame
4. Pass it to graph
5. Convert to CPU frame
6. Draw result

Expected behavior: Image from the graph output will be the same as source bitmap

Current behavior: Image from the graph output with almost correct borders, but wrong colors. It seems like output frame contains from some color rectangles (look screenshot example)

I made this code example for tests.

Simple graph

input_stream: "input_video"
output_stream: "output_video_cpu"

node {
  calculator: "GpuBufferToImageFrameCalculator"
  input_stream: "input_video"
  output_stream: "output_video_cpu"
}

BitmapConverter

class BitmapConverter @JvmOverloads constructor(
    parentContext: EGLContext?,
    numBuffers: Int = DEFAULT_NUM_BUFFERS
) : TextureFrameProducer, PipelineFrameAvailableListener {

    private val renderThread = RenderThread(parentContext, numBuffers)

    override fun setConsumer(next: TextureFrameConsumer) {
        renderThread.setConsumer(next)
    }

    fun close() {
        renderThread.quitSafely()
        renderThread.join()
    }

    override fun onFrame(bitmap: Bitmap?) {
        if (renderThread.isAlive) renderThread.onFrame(bitmap)
    }

    private class RenderThread(parentContext: EGLContext?, numBuffers: Int) :
        GlThread(parentContext), PipelineFrameAvailableListener {
        private val consumers: MutableList<TextureFrameConsumer?> = ArrayList()
        private val frameList: MutableList<AppTextureFrame?> = ArrayList()
        private var frameIndex = -1
        private var nextFrameTimestampOffset: Long = 0
        private var prevTimestamp: Long = 0
        private var cpuFrame: Bitmap? = null
        private var prevTimestampValid = false
        private var cpuFrameWidth = 0
        private var cpuFrameHeight = 0
        private var timestamp = 1L

        fun setConsumer(consumer: TextureFrameConsumer?) {
            synchronized(consumers) {
                consumers.clear()
                consumers.add(consumer)
            }
        }

        override fun releaseGl() {
            for (i in frameList.indices) teardown(i)
            super.releaseGl()
        }

        private fun teardown(index: Int) {
            if (frameList[index] != null) {
                frameList[index]?.waitUntilReleased()
                GLES20.glDeleteTextures(1, intArrayOf(frameList[index]!!.textureName), 0)
                frameList[index] = null
            }
        }

        override fun onFrame(bitmap: Bitmap?) {
            this.cpuFrame = bitmap
            handler.post { renderNext() }
        }

        private fun renderNext() {
            if (cpuFrame == null) {
                return
            }
            synchronized(consumers) {
                var frameUpdated = false
                for (consumer in consumers) {
                    val outputFrame = createNextGpuFrame(cpuFrame!!)
                    updateGpuFrame(outputFrame)
                    frameUpdated = true
                    if (consumer != null) {
                        outputFrame!!.setInUse()
                        consumer.onNewFrame(outputFrame)
                    }
                }
                if (!frameUpdated) {
                    val outputFrame = createNextGpuFrame(cpuFrame!!)
                    updateGpuFrame(outputFrame)
                }
            }
        }

        private fun createNextGpuFrame(bitmap: Bitmap): AppTextureFrame? {
            val textureId = ShaderUtil.createRgbaTexture(bitmap)
            frameIndex = (frameIndex + 1) % frameList.size
            cpuFrameHeight = bitmap.height
            cpuFrameWidth = bitmap.width
            teardown(frameIndex)
            frameList[frameIndex] = AppTextureFrame(textureId, cpuFrameWidth, cpuFrameHeight)
            val outputFrame = frameList[frameIndex]
            outputFrame?.waitUntilReleased()
            return outputFrame
        }

        private fun updateGpuFrame(gpuFrame: AppTextureFrame?) {
            timestamp += 1
            val texTimestamp = timestamp / NANOS_PER_MICRO
            if (prevTimestampValid && texTimestamp + nextFrameTimestampOffset <= prevTimestamp) {
                nextFrameTimestampOffset = prevTimestamp + 1 - texTimestamp
            }
            gpuFrame!!.timestamp = texTimestamp + nextFrameTimestampOffset
            prevTimestamp = gpuFrame.timestamp
            prevTimestampValid = true
        }


        init {
            frameList.addAll(Collections.nCopies(numBuffers, null))
        }

        companion object {
            private const val NANOS_PER_MICRO: Long = 1000
        }
    }

    init {
        renderThread.start()
        renderThread.waitUntilReady()
    }

    companion object {
        private const val DEFAULT_NUM_BUFFERS = 2
    }
}