[QUESTION] how to add sub video track in kvs webrtc

[remainfox]

I want to use main stream (1080P) and sub stream (720P) to transmit video data in KVS WEBRTC.

I added the substream with the following code, but I couldn't view the video, neither the substream nor the main stream

RtcMediaStreamTrack videoTrack, audioTrack, subVideoTrack;
RtcRtpTransceiverInit subVideoRtpTransceiverInit;
subVideoTrack.kind = MEDIA_STREAM_TRACK_KIND_VIDEO;
subVideoTrack.codec = pCodecParam->video_codec_type;
subVideoRtpTransceiverInit.direction = RTC_RTP_TRANSCEIVER_DIRECTION_SENDONLY;
STRCPY(subVideoTrack.streamId, "myKvsSubVideoStream");
STRCPY(subVideoTrack.trackId, "mySubVideoTrack");
CHK_STATUS(addTransceiver(pSampleStreamingSession->pPeerConnection, &subVideoTrack, &subVideoRtpTransceiverInit,
                              &pSampleStreamingSession->pSubVideoRtcRtpTransceiver));
CHK_STATUS(transceiverOnBandwidthEstimation(pSampleStreamingSession->pSubVideoRtcRtpTransceiver, 
              (UINT64) pSampleStreamingSession,sampleBandwidthEstimationHandler));


frame_out.trackID = SUB_VIDEO_TRACK_ID;
pVideoRtcRtpTransceiver = pSampleStreamingSession->pSubVideoRtcRtpTransceiver;
retStatus = writeFrame(pVideoRtcRtpTransceiver, &frame_out);

[hassanctech]

Can you please first try to run the same as is with a single stream and see if that works for you, so running the provided samples without modifications. Then try to send only the main stream you wish to send and see if that works. This will help to pinpoint the issue.

[remainfox]

@hassanctech thanks, A single stream is fine, adding a track will not be normal viewing

[niyatim23]

Hi @remainfox , when you add the sub-stream, do you make any other changes to the sample application? Can you please share verbose logs with SDP from when you add an additional stream?

[github-actions[bot]]

It looks like this issue has not been active for 10 days. If the issue is not resolved, please add an update to the ticket, else it will be automatically resolved in a few days.

[disa6302]

Closing due to no response. Feel free to reopen if you have more details

[remainfox]

@niyatim23
This is an Answer SDP with a new trackid

v=0
o=- 598246763 2 IN IP4 127.0.0.1
s=-
t=0 0
a=group:BUNDLE 0 1 2 3
a=msid-semantic: WMS myKvsVideoStream
m=video 9 UDP/TLS/RTP/SAVPF 127 123
c=IN IP4 127.0.0.1
a=candidate:0 1 udp 2130706431 172.25.194.147 43665 typ host raddr 0.0.0.0 rport 0 generation 0 network-cost 999
a=msid:myKvsVideoStream myVideoTrack-WEBRTC_CODEC_H265RTX
a=ssrc-group:FID 1849923102 343418703
a=ssrc:1849923102 cname:EGeCy0xagkNlCge0
a=ssrc:1849923102 msid:myKvsVideoStream myVideoTrack-WEBRTC_CODEC_H265
a=ssrc:1849923102 mslabel:myKvsVideoStream
a=ssrc:1849923102 label:myVideoTrack-WEBRTC_CODEC_H265
a=ssrc:343418703 cname:EGeCy0xagkNlCge0
a=ssrc:343418703 msid:myKvsVideoStream myVideoTrack-WEBRTC_CODEC_H265RTX
a=ssrc:343418703 mslabel:myKvsVideoStreamRTX
a=ssrc:343418703 label:myVideoTrack-WEBRTC_CODEC_H265RTX
a=rtcp:9 IN IP4 0.0.0.0
a=ice-ufrag:+iOk
a=ice-pwd:dgpp4UJvql6XWbqZDpOryU7A
a=ice-options:trickle
a=fingerprint:sha-256 
a=setup:active
a=mid:0
a=sendrecv
a=rtcp-mux
a=rtcp-rsize
a=rtpmap:127 H265/90000
a=rtpmap:123 rtx/90000
a=fmtp:123 apt=127
a=rtcp-fb:127 nack
a=rtcp-fb:127 goog-remb
a=rtcp-fb:127 transport-cc
m=audio 9 UDP/TLS/RTP/SAVPF 8
c=IN IP4 127.0.0.1
a=candidate:0 1 udp 2130706431 172.25.194.147 43665 typ host raddr 0.0.0.0 rport 0 generation 0 network-cost 999
a=msid:myKvsVideoStream myAudioTrack-WEBRTC_CODEC_ALAW
a=ssrc:1912761015 cname:EGeCy0xagkNlCge0
a=ssrc:1912761015 msid:myKvsVideoStream myAudioTrack-WEBRTC_CODEC_ALAW
a=ssrc:1912761015 mslabel:myKvsVideoStream
a=ssrc:1912761015 label:myAudioTrack-WEBRTC_CODEC_ALAW
a=rtcp:9 IN IP4 0.0.0.0
a=ice-ufrag:+iOk
a=ice-pwd:dgpp4UJvql6XWbqZDpOryU7A
a=ice-options:trickle
a=fingerprint:sha-256 
a=setup:active
a=mid:1
a=sendonly
a=rtcp-mux
a=rtcp-rsize
a=rtpmap:8 PCMA/8000
a=rtcp-fb:8 nack
a=rtcp-fb:8 goog-remb
a=rtcp-fb:8 transport-cc
m=video 9 UDP/TLS/RTP/SAVPF 100 101
c=IN IP4 127.0.0.1
a=candidate:0 1 udp 2130706431 172.25.194.147 43665 typ host raddr 0.0.0.0 rport 0 generation 0 network-cost 999
a=msid:myKvsVideoStream myVideoTrack-WEBRTC_CODEC_H264RTX
a=ssrc-group:FID 1256275938 1215173870
a=ssrc:1256275938 cname:EGeCy0xagkNlCge0
a=ssrc:1256275938 msid:myKvsVideoStream myVideoTrack-WEBRTC_CODEC_H264
a=ssrc:1256275938 mslabel:myKvsVideoStream
a=ssrc:1256275938 label:myVideoTrack-WEBRTC_CODEC_H264
a=ssrc:1215173870 cname:EGeCy0xagkNlCge0
a=ssrc:1215173870 msid:myKvsVideoStream myVideoTrack-WEBRTC_CODEC_H264RTX
a=ssrc:1215173870 mslabel:myKvsVideoStreamRTX
a=ssrc:1215173870 label:myVideoTrack-WEBRTC_CODEC_H264RTX
a=rtcp:9 IN IP4 0.0.0.0
a=ice-ufrag:+iOk
a=ice-pwd:dgpp4UJvql6XWbqZDpOryU7A
a=ice-options:trickle
a=fingerprint:sha-256 
a=setup:active
a=mid:2
a=
a=rtcp-mux
a=rtcp-rsize
a=rtpmap:100 H264/90000
a=fmtp:100 level-asymmetry-allowed=1;packetization-mode=1;profile-level-id=42e01f
a=rtpmap:101 rtx/90000
a=fmtp:101 apt=100
a=rtcp-fb:100 nack
a=rtcp-fb:100 goog-remb
a=rtcp-fb:100 transport-cc
m=application 9 UDP/DTLS/SCTP webrtc-datachannel
c=IN IP4 127.0.0.1
a=candidate:0 1 udp 2130706431 172.25.194.147 43665 typ host raddr 0.0.0.0 rport 0 generation 0 network-cost 999
a=rtcp:9 IN IP4 0.0.0.0
a=ice-ufrag:+iOk
a=ice-pwd:dgpp4UJvql6XWbqZDpOryU7A
a=fingerprint:sha-256 
a=setup:active
a=mid:3
a=sctp-port:5000

The code of adding trackid is as follows

int32_t createRtpTransfer(PSampleConfiguration pSampleConfiguration, PSampleStreamingSession pSampleStreamingSession)
{
    PARAM_CHECK(pSampleConfiguration && pSampleStreamingSession, EN_RTC_ERR_CODE_PARAM_INVALID);
    
    INT32 i = 0;
    STATUS retStatus = STATUS_SUCCESS;
    CHAR   codecsName[128] = "";
    PCHAR  peerId = pSampleStreamingSession->peerId;
    webrtc_codec_param user_codec_param;
    webrtc_codec_param *rtc_codec_param = &pSampleStreamingSession->codecParam;
    
    RtcMediaStreamTrack videoTrack, audioTrack;
    RtcRtpTransceiverInit audioRtpTransceiverInit;
    RtcRtpTransceiverInit videoRtpTransceiverInit;
    WEBRTC_CODEC_TYPE video_codec_type;
    WEBRTC_CODEC_TYPE audio_codec_type;

    if(pSampleConfiguration->api_callbacks.avcodec_cb)
    {
        memset(&user_codec_param,0,sizeof(user_codec_param));
        pSampleConfiguration->api_callbacks.avcodec_cb(peerId,&user_codec_param);
    }

    CHK(webrtc_check_codec(pSampleStreamingSession,&user_codec_param) == TRUE,EN_RTC_ERR_CODE_CODEC_PARAM_INVALID);

    for (i = 0; i < rtc_codec_param->video_codec_num; i++)
    {
        video_codec_type = rtc_codec_param->video_codec_type[i];
        CHK_STATUS(addSupportedCodec(pSampleStreamingSession->pPeerConnection, video_codec_type));
        
        videoTrack.kind = MEDIA_STREAM_TRACK_KIND_VIDEO;
        videoTrack.codec = video_codec_type;
        videoRtpTransceiverInit.direction = RTC_RTP_TRANSCEIVER_DIRECTION_SENDONLY;
        
        STRCPY(videoTrack.streamId, "myKvsVideoStream");
        snprintf(videoTrack.trackId,sizeof(videoTrack.trackId),"myVideoTrack-%s",
        webrtc_get_codec_name_inner(video_codec_type));

        CHK_STATUS(addTransceiver(pSampleStreamingSession->pPeerConnection, &videoTrack, &videoRtpTransceiverInit,
                                &pSampleStreamingSession->pVideoRtcRtpTransceiver));
    }
    
    for (i = 0; i < rtc_codec_param->audio_codec_num; i++)
    {
        audio_codec_type = rtc_codec_param->audio_codec_type[i];
        CHK_STATUS(addSupportedCodec(pSampleStreamingSession->pPeerConnection, audio_codec_type));

        audioTrack.kind = MEDIA_STREAM_TRACK_KIND_AUDIO;
        audioTrack.codec = audio_codec_type;
        audioRtpTransceiverInit.direction = RTC_RTP_TRANSCEIVER_DIRECTION_SENDRECV;

        STRCPY(audioTrack.streamId, "myKvsVideoStream");
        snprintf(audioTrack.trackId,sizeof(audioTrack.trackId),"myAudioTrack-%s",
        webrtc_get_codec_name_inner(audio_codec_type));
        
        CHK_STATUS(addTransceiver(pSampleStreamingSession->pPeerConnection, &audioTrack, &audioRtpTransceiverInit,
                                &pSampleStreamingSession->pAudioRtcRtpTransceiver));
    }

CleanUp:
    LEAVES();
    return retStatus;
}