javacv
javacv copied to clipboard
How can i convert avframe to ImageBitmap?(Compose for Desktop)
Here is an example code:
@Composable
fun App() {
val videoUrl by remember { mutableStateOf("<videoUrl>") }
var imageBitmap by remember { mutableStateOf<ImageBitmap?>(null) }
LaunchedEffect(Unit) {
val pFormatContext = avformat_alloc_context()
avformat_open_input(pFormatContext, videoUrl, null, null)
avformat_find_stream_info(pFormatContext, null as PointerPointer<*>?)
var pCodecParameters = AVCodecParameters(null)
for (i in 0 until pFormatContext.nb_streams()) {
val pLocalCodecParameters = pFormatContext.streams(i).codecpar()
if(pLocalCodecParameters.codec_type() == avutil.AVMEDIA_TYPE_VIDEO) {
pCodecParameters = pLocalCodecParameters
}
}
val pCodec = avcodec_find_decoder(pCodecParameters.codec_id())
val pCodecContext = avcodec_alloc_context3(pCodec)
avcodec_parameters_to_context(pCodecContext, pCodecParameters)
avcodec_open2(pCodecContext, pCodec, null as PointerPointer<*>?)
val pPacket = av_packet_alloc()
val pFrame = avutil.av_frame_alloc()
while(av_read_frame(pFormatContext, pPacket) >= 0) {
avcodec_send_packet(pCodecContext, pPacket)
avcodec_receive_frame(pCodecContext, pFrame)
val buff = pPacket.data().position(0).capacity(pPacket.size().toLong()).asByteBuffer()
val arr = ByteArray(buff.capacity())
buff.get(arr)
imageBitmap = Image.makeFromEncoded(arr).toComposeImageBitmap()
delay(500)
}
}
MaterialTheme {
imageBitmap?.let {
Image(
bitmap = it,
contentDescription = "Video"
)
}
}
}
But it doesn't work
For a working example of reading data out of AVFrame, please take a look at FFmpegFrameGrabber: https://github.com/bytedeco/javacv/blob/master/src/main/java/org/bytedeco/javacv/FFmpegFrameGrabber.java
I tried as you said, but the weight still does not work. Skia can't encode:
Loader.load(avutil::class.java)
Loader.load(swresample::class.java)
Loader.load(avcodec::class.java)
Loader.load(avformat::class.java)
Loader.load(swscale::class.java)
// Register all formats and codecs
avcodec.av_jni_set_java_vm(Loader.getJavaVM(), null)
avcodec.avcodec_register_all()
avformat.av_register_all()
avformat.avformat_network_init()
Loader.load(avdevice::class.java)
avdevice.avdevice_register_all()
val pFormatContext = avformat.avformat_alloc_context()
avformat.avformat_open_input(pFormatContext, videoUrl, null, null)
avformat.avformat_find_stream_info(pFormatContext, null as PointerPointer<*>?)
var pCodecParameters = AVCodecParameters(null)
var videoStream = -1
for (i in 0 until pFormatContext.nb_streams()) {
val pLocalCodecParameters = pFormatContext.streams(i).codecpar()
if(pLocalCodecParameters.codec_type() == avutil.AVMEDIA_TYPE_VIDEO) {
pCodecParameters = pLocalCodecParameters
videoStream = i
}
}
val pCodec = avcodec.avcodec_find_decoder(pCodecParameters.codec_id())
val pCodecContext = avcodec.avcodec_alloc_context3(pCodec)
avcodec.avcodec_parameters_to_context(pCodecContext, pCodecParameters)
avcodec.avcodec_open2(pCodecContext, pCodec, null as PointerPointer<*>?)
val pPacket = avcodec.av_packet_alloc()
val pFrame = avutil.av_frame_alloc()
val pFrameRgb = avutil.av_frame_alloc()
var imgConvertCtx = SwsContext(null)
imgConvertCtx = swscale.sws_getCachedContext(
imgConvertCtx,
pCodecContext.width(), pCodecContext.height(), pCodecContext.pix_fmt(),
pCodecContext.width(), pCodecContext.height(), pCodecContext.pix_fmt(),
swscale.SWS_BILINEAR, null, null, null as DoublePointer?
)
while(avformat.av_read_frame(pFormatContext, pPacket) >= 0) {
avcodec.avcodec_send_packet(pCodecContext, pPacket)
avcodec.avcodec_receive_frame(pCodecContext, pFrame)
if(pPacket.stream_index() == videoStream) {
val fmt = pCodecContext.pix_fmt()
val height = pCodecContext.height()
val width = pCodecContext.width()
// work around bug in swscale: https://trac.ffmpeg.org/ticket/1031
val align = 32
var stride = width
var i = 1
while (i <= align) {
stride = width + (i - 1) and (i - 1).inv()
avutil.av_image_fill_linesizes(pFrameRgb.linesize(), fmt, stride)
if (pFrameRgb.linesize(0) and align - 1 == 0) {
break
}
i += i
}
val size = avutil.av_image_get_buffer_size(fmt, stride, height, 1)
val image_ptr = arrayOf(BytePointer(avutil.av_malloc(size.toLong())).capacity(size.toLong()))
val image_buf = arrayOf<Buffer>(image_ptr[0].asByteBuffer())
pFrameRgb.format(fmt)
pFrameRgb.width(width)
pFrameRgb.height(height)
swscale.sws_scale(
imgConvertCtx, PointerPointer<AVFrame?>(pFrame), pFrame.linesize(), 0,
pCodecContext.height(), PointerPointer<AVFrame?>(pFrameRgb), pFrameRgb.linesize()
)
val buff = image_ptr[0].asByteBuffer()
if(buff != null) {
val arr = ByteArray(buff.capacity())
buff.get(arr)
imageBitmap = Image.makeFromEncoded(arr).toComposeImageBitmap()
println("Success, Buff: $image_buf")
delay(500)
}
}
}
First time I'm writing something like this. Could you help me?