Unable to overlay audio over video

[pawaom]

I want to work with following feature in our app

1. set Resolution , bitrate, FRAME_INTERVAL, FRAME_RATE, for video
2. set CHANNEL_COUNT,SAMPLE_RATE,KEY_BIT_RATE for audio
3. set duration for Audio and video
4. overlay audio over video

My video has audio and the audio file should overlay over it

I am trying this code

public class MainActivity extends AppCompatActivity {
   
    MediaMuxerMediaTarget mediaTarget;
    MediaExtractorMediaSource videoMediaSource, audioMediaSource;
    Uri sourceVideoUri = Uri.parse("content://media/external/video/media/1000000034");
    Uri sourceAudioUri = Uri.parse("content://media/external/audio/media/53");

    @Override
    protected void onCreate(@Nullable Bundle savedInstanceState) {
        super.onCreate(savedInstanceState);
        setContentView(R.layout.frag2);
        // common target
        String outputPath = Environment.getExternalStoragePublicDirectory(Environment.DIRECTORY_MOVIES) + "/vidos1.mp4";
        File file = new File(outputPath);
        if (file.exists()) {
            file.delete();
        }
        try {
            file.createNewFile();
        } catch (IOException e) {
            throw new RuntimeException(e);
        }
        try {
            mediaTarget = new MediaMuxerMediaTarget(outputPath, 2, 0, MediaMuxer.OutputFormat.MUXER_OUTPUT_MPEG_4);
        } catch (MediaTargetException e) {
            throw new RuntimeException(e);
        }
        try {
            videoMediaSource = new MediaExtractorMediaSource(MainActivity.this, sourceVideoUri);
            MediaRange mediaRange;
            mediaRange = new MediaRange(0, getMediaDuration(sourceVideoUri));
            audioMediaSource = new MediaExtractorMediaSource(MainActivity.this, sourceAudioUri, mediaRange);
        } catch (MediaSourceException e) {
            throw new RuntimeException(e);
        }
        // video track config
        MediaFormat targetVideoFormat = MediaFormat.createVideoFormat("video/avc", 1280, 720); // assuming 720p landscape video
        targetVideoFormat.setInteger(MediaFormat.KEY_BIT_RATE, 5_000_000);
        targetVideoFormat.setInteger(MediaFormat.KEY_I_FRAME_INTERVAL, 3);
        targetVideoFormat.setInteger(MediaFormat.KEY_FRAME_RATE, 30);
        targetVideoFormat.setInteger(KEY_ROTATION, 0);
        // video track transform
        TrackTransform videoTrackTransform = new TrackTransform.Builder(videoMediaSource, 0, mediaTarget) // assuming that video track index in source file is 0
                .setTargetTrack(0)
                .setDecoder(new MediaCodecDecoder())
                .setEncoder(new MediaCodecEncoder())
                .setRenderer(new GlVideoRenderer(null))
                .setTargetFormat(targetVideoFormat)
                .build();
        // audio track transform
        Encoder encoder = new MediaCodecEncoder();
        TrackTransform audioTrackTransform = new TrackTransform.Builder(audioMediaSource, 0, mediaTarget) // assuming that audio track index in source file is 0
                .setTargetTrack(1)
                .setDecoder(new MediaCodecDecoder())
                .setEncoder(encoder)
                .setTargetFormat(createAudioMediaFormat())
                .setRenderer(
                        new AudioRenderer(
                                encoder,
                                Collections.singletonList(new AudioOverlayFilter(MainActivity.this, sourceAudioUri))
                        ))
                .build();
        TransformationListener listener = new TransformationListener() {

            @Override
            public void onStarted(String id) {
// TODO: Implement this method
                Toast.makeText(MainActivity.this, "Started ", Toast.LENGTH_SHORT).show();
            }

            @Override
            public void onProgress(String id, float progress) {
// TODO: Implement this method
                Toast.makeText(MainActivity.this, "progress " + progress, Toast.LENGTH_SHORT).show();
            }

            @Override
            public void onCompleted(String id, List trackTransformationInfos) {
                //sm("ok");
                Toast.makeText(MainActivity.this, "Completed ", Toast.LENGTH_SHORT).show();
            }

            @Override
            public void onCancelled(String id, List trackTransformationInfos) {
// TODO: Implement this method
            }

            @Override
            public void onError(String id, Throwable cause, List trackTransformationInfos) {
                // sm("error=" + id);
                Log.e(TAG, "error= eeeeeeeeeeeeeeeeeee" + cause);
                Toast.makeText(MainActivity.this, "error=" + cause, Toast.LENGTH_SHORT).show();
            }
        };
        Button TransFormButton = (Button) findViewById(R.id.TransformButton);
        TransFormButton.setOnClickListener(view -> {
            MediaTransformer mediaTransformer = new MediaTransformer(getApplicationContext());
            List trackTransforms = new ArrayList<>();
            trackTransforms.add(videoTrackTransform);
            trackTransforms.add(audioTrackTransform);
            mediaTransformer.transform(
                    "663201",
                    trackTransforms,
                    listener,
                    MediaTransformer.GRANULARITY_DEFAULT);
        });
    }

    @NonNull
    private MediaFormat createAudioMediaFormat() {
        MediaFormat mediaFormat = new MediaFormat();
        for (int i = 0; i < audioMediaSource.getTrackCount(); i++) {
            MediaFormat trackFormat = audioMediaSource.getTrackFormat(i);
            mediaFormat.setString(MediaFormat.KEY_MIME, AUDIO_AAC);
            mediaFormat.setInteger(MediaFormat.KEY_CHANNEL_COUNT, trackFormat.getInteger(MediaFormat.KEY_CHANNEL_COUNT));
            mediaFormat.setInteger(MediaFormat.KEY_SAMPLE_RATE, trackFormat.getInteger(MediaFormat.KEY_SAMPLE_RATE));
            mediaFormat.setInteger(MediaFormat.KEY_BIT_RATE, trackFormat.getInteger(MediaFormat.KEY_BIT_RATE));
            mediaFormat.setLong(MediaFormat.KEY_DURATION, getMediaDuration(sourceVideoUri));
        }
        return mediaFormat;
    }

    private long getMediaDuration(@NonNull Uri uri) {
        MediaMetadataRetriever mediaMetadataRetriever = new MediaMetadataRetriever();
        mediaMetadataRetriever.setDataSource(MainActivity.this, uri);
        String durationStr = mediaMetadataRetriever.extractMetadata(MediaMetadataRetriever.METADATA_KEY_DURATION);
        return Long.parseLong(durationStr);
    }
}

However the overlay doesnot work even the original audio from video file is removed.

When we remove the

 .setRenderer(
                        new AudioRenderer(
                                encoder,
                                Collections.singletonList(new AudioOverlayFilter(MainActivity.this, sourceAudioUri))
                        ))

It replaces the audio of video with the audio from audio file.

How can we keep original audio and overlay new audio over it.

[izzytwosheds]

You are using audioMediaSource for track 1, you should be using videoMediaSource with correct sourceIndex of its audio track.