[BUG]: Video Stream does not show on the client in version 2.4.0-exp.4

[jwielebnowski]

Package version

2.4.0-exp.4

Environment

* OS: Windows 10
* Unity version: 2020.3.27f1

Steps To Reproduce

Code (snippets) below works on all the webRTC versions up to 2.4.0.-exp.3. In version 2.4.0-exp.4 the video stream seems to be going to the client (it shows in webRTC internals) but there is no video. I have looked at the changes between exp.3 and exp.4 and I cannot figure out what might be wrong (codec?). Maybe you could reproduce it using below code snippets and see what change/update is needed/

// Initialize WebRTC
public void Awake()
    {

        DontDestroyOnLoad(this.gameObject);
        Instance = this;
        WebRTC.Initialize(EncoderType.Hardware);
        EnhancedTouchSupport.Enable();

    }

   public void Update()
    {
        StartCoroutine(CaptureScreen());
    }

   // Create video stream track if not created. 
  // Get screen pixels
  // Call 'Update' delegate on the VideoStreamTrack
 //
    private IEnumerator CaptureScreen()
    {


        // Are we shutting down?
        if (Instance == null) yield return null;

        if (m_screen == null || (Screen.width != m_screen.width || Screen.height != m_screen.height))
        {
            CreateOrUpdateTrack();
        }
        

        yield return new WaitForEndOfFrame();
        m_screen.ReadPixels(new Rect(0, 0, Screen.width, Screen.height), 0, 0);
        m_screen.Apply();

        foreach (var update in m_listVideoStreamTrackUpdaters.Values)
        {
            try
            {
                update();
            }
            catch (Exception e)
            {
                Debug.LogErrorFormat("Failure during track update {0}", e);
            }
        }

    }

private void CreateOrUpdateTrack()
    {
        if (m_screen == null)
        {
            Screen.SetResolution(m_width, m_height, false, 60);
            m_screen = new Texture2D(Screen.width, Screen.height, TextureFormat.BGRA32, false);
        }
        else
        {
            m_screen.Resize(Screen.width, Screen.height);
        }

        var videoTrack = new VideoStreamTrack("screen", m_screen);
        // WebRTC.Update begins an infinite loop and does not allow us to perform our own capture before calling Update on the individual tracks.
        // Additionally, Track.Update is internal. This should be addressed.
        var trackUpdateMethod = videoTrack.GetType().GetMethod("Update", BindingFlags.NonPublic | BindingFlags.Instance);
        var update = (Action)trackUpdateMethod.CreateDelegate(typeof(Action), videoTrack);
        m_listVideoStreamTrackUpdaters.Add(videoTrack, update);
    }

////////// THE FOLLOWING PART IS TO SHOW WEBRTC NEGOTIATION, CONFIGURATION AND HOW AND WHEN THE VIDEO TRACK IS ADDED TO THE RTCPeerConnection.

    void BeginNegotiation(Signaling signaling, PlatformSessionDescription e)
    {
        RTCSessionDescription offer;
        offer.type = RTCSdpType.Offer;
        offer.sdp = e.sdp;
        var connectionId = e.playerId;
        if (m_mapConnectionIdAndPeer.TryGetValue(connectionId, out RTCPeerConnection target))
        {
            Debug.Log("Removing existing player: " + connectionId);
            target.Close();
            m_mapConnectionIdAndPeer.Remove(connectionId);
        }

        var pc = new RTCPeerConnection();
        if (!m_mapConnectionIdAndPeer.ContainsKey(e.playerId))
        {
            m_mapConnectionIdAndPeer.Add(e.playerId, pc);
        }

        pc.OnDataChannel = new DelegateOnDataChannel(channel => { OnDataChannel(pc, e.playerId, channel); });
        pc.OnIceCandidate = new DelegateOnIceCandidate(candidate =>
        {
            signaling.SendCandidate(e.playerId, candidate);
        });
        pc.OnIceConnectionChange = new DelegateOnIceConnectionChange(state =>
        {
            Debug.Log("PeerConnection ICE Connection State:" + state);
            if (state == RTCIceConnectionState.Disconnected)
            {
                pc.Close();
                m_mapConnectionIdAndPeer.Remove(e.playerId);
                if (e.playerId == 1)
                {
                    signaling.Close();
                }
            }
        });

        var conf = new RTCConfiguration
        {
            iceServers = m_clientConfiguration.peerConnectionOptions.iceServers,
            iceCandidatePoolSize = m_clientConfiguration.peerConnectionOptions.iceCandidatePoolSize == 0 ? 4 : m_clientConfiguration.peerConnectionOptions.iceCandidatePoolSize,
            iceTransportPolicy = m_clientConfiguration.peerConnectionOptions.iceTransportPolicy == "relay" ? RTCIceTransportPolicy.Relay : RTCIceTransportPolicy.All
        };

        Debug.Log("SetConfiguration: " + JsonUtility.ToJson(conf));

        var configError = pc.SetConfiguration(ref conf);
        Debug.Log("Config error: " + configError);
        if (configError != RTCErrorType.None)
        {
            throw new Exception("Unable to set WebRTC configuration");
        }

        // PLAT-1009
        // The WebRTC package supports receiving as well as sending video streams, but with different codecs (H264 for sending, VP8 for receiving).
        // The Hardware Encoder only supports H264 (which is what we want anyways). 
        // Forcing this into receive (from the browser pov)  forces the correct codecs.
        offer.sdp = offer.sdp.Replace("a=sendrecv", "a=recvonly");
        var setRemoteDescriptionOp = pc.SetRemoteDescription(ref offer);
        WaitForAsyncOperation(setRemoteDescriptionOp);

        foreach (var pair in m_listVideoStreamTrackUpdaters)
        {
            var sender = pc.AddTrack(pair.Key);
            m_rtpVideoSenders[pc] = sender;
        }

        if (m_audioStream != null)
        {
            foreach (var track in m_audioStream.GetTracks())
            {
                pc.AddTrack(track);
            }
        }

        RTCAnswerOptions options = default;
        var createAnswserOp = pc.CreateAnswer(ref options);
        WaitForAsyncOperation(createAnswserOp);


        var desc = createAnswserOp.Desc;
        signaling.SendAnswer(connectionId, desc);

        var setLocationDescriptionOp = pc.SetLocalDescription(ref desc);
        WaitForAsyncOperation(setLocationDescriptionOp);
    }

Current Behavior

Stream does not show on the browser client. WebRTC-internals show that the frames are going through fine but nothing on the client shows up.

Expected Behavior

No response

Anything else?

Please let me know what else I can provide to help.

Thank you,

Jacek

[karasusan]

@jwielebnowski You mean the code snippet in the document is wrong. is it right?

[jwielebnowski]

@karasusan The code snippet works with 2.4.0-exp.3, but does not with 2.4.0-exp.4. I looked at the changes between exp.3 and exp.4 and did not Find anything obvious. Everything looks fine, the game connects and sends the stream, webRTC-internals in the browser show frames coming but the Video stream does not show on the browser.

[karasusan]

@jwielebnowski I think the cause of the issue is the direction.

Could you share your project to reproduce the issue?

[jwielebnowski]

@karasusan

1. Create a project. I am using Unity 2020.3.27f1

2. Download platform-plugin-unity-trim.zip Extract it and import from the disk into the project. It will bring the plugin and webRTC v 2.4.0-exp.3 .

3. Add empty game object. Add to this object imported PureWebStreaming script.

4. Set PUREWEB_STREAMER_PORT=9210 environment variable.

5. In your command window run:

npx @pureweb/platform-streaming-agent --agentToken=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0eXBlIjoiYWdlbnQiLCJ1aWQiOiJ1cy13ZXN0LTI6ZTgwNjI3ZWEtMjQxZC00ZjU0LWE3NmYtNThiYjNiMDZiMjlmIiwicGlkIjoiN2YwZWVkYTMtNWQxOC00MzdhLThjMjEtNGI4NDViMjYzNWNhIiwibG5nIjoiLTExNC4wNDg4MCIsImxhdCI6IjUxLjEwOTQwIiwiaWF0IjoxNjQ0NjA3Mzg0LCJleHAiOjE2NDcxOTkzODQsImF1ZCI6IjdmOGFlMTFlLTZiNDAtNDI4ZC1hZmEwLTVlMDhhMDkxNGY1ZiIsImlzcyI6Imh0dHBzOi8vYXBpLWRldi5qdy5wdXJld2ViLmlvIiwic3ViIjoiNzQ1MGFhNTAtMTdhOS00M2FlLTlkOWItNTBhZTE4MmUxYTUwIiwianRpIjoiMDA1ZjFlOWEtN2NjMi00NzE5LTkzOTktZDdhMTE5ZGVhMzVkIn0.0iJCgHykSfIdtRc1bM1rzk_Ok6U9-SuW0WAAewiSxtU --debugLevel=debug

When you run the game it should connect to the streaming-agent.

5. In your browser go to: https://preview-dev.jw.pureweb.io/7f0eeda3-5d18-437a-8c21-4b845b2635ca?environmentId=7f8ae11e-6b40-428d-afa0-5e08a0914f5f&local=true

The game should stream fine into the browser client.

THE ISSUE HERE ***********************************************************************

1. Upgrade unity WebRTC in the project to 2.4.0-exp.4

2. Fix scripts based on the changes that were done between 2.4.0-exp.3 and exp.4 . <It would be awesome if you could send me the code update to the Audio stream. I could not figure out how to change it after the changes you have done in 2.4.0-exp.4.>

3. In your command window run again:

npx @pureweb/platform-streaming-agent --agentToken=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0eXBlIjoiYWdlbnQiLCJ1aWQiOiJ1cy13ZXN0LTI6ZTgwNjI3ZWEtMjQxZC00ZjU0LWE3NmYtNThiYjNiMDZiMjlmIiwicGlkIjoiN2YwZWVkYTMtNWQxOC00MzdhLThjMjEtNGI4NDViMjYzNWNhIiwibG5nIjoiLTExNC4wNDg4MCIsImxhdCI6IjUxLjEwOTQwIiwiaWF0IjoxNjQ0NjA3Mzg0LCJleHAiOjE2NDcxOTkzODQsImF1ZCI6IjdmOGFlMTFlLTZiNDAtNDI4ZC1hZmEwLTVlMDhhMDkxNGY1ZiIsImlzcyI6Imh0dHBzOi8vYXBpLWRldi5qdy5wdXJld2ViLmlvIiwic3ViIjoiNzQ1MGFhNTAtMTdhOS00M2FlLTlkOWItNTBhZTE4MmUxYTUwIiwianRpIjoiMDA1ZjFlOWEtN2NjMi00NzE5LTkzOTktZDdhMTE5ZGVhMzVkIn0.0iJCgHykSfIdtRc1bM1rzk_Ok6U9-SuW0WAAewiSxtU --debugLevel=debug

4. Run the game. It should connect to the streaming-agent

5. In your browser again go to: https://preview-dev.jw.pureweb.io/7f0eeda3-5d18-437a-8c21-4b845b2635ca?environmentId=7f8ae11e-6b40-428d-afa0-5e08a0914f5f&local=true

You will see that the stream connects fine and it shows in webrtc-internals that the frames are going across fine but there is no picture.

I wonder if there is something that changed in exp.4 . I looked at your changes between exp.3 and exp.4 and I could not figure out what might be the problem. The code (our plugin) works fine with all the previous versions down to 2.3.0.

Please let me know what else I could provide or if you have any problems recreating the issue. I really appreciate your help.

[karasusan]

@jwielebnowski This sample is so complicated for reproducing the issue. Could you make simpler?

[jwielebnowski]

Broken project with webRTC 2.4.0-exp.4: https://drive.google.com/file/d/1JPdLcD_S9UIRgieKJfPQAxshvDQVehCi/view?usp=sharing

1. Run the game

2. In your command window run:

npx @pureweb/platform-streaming-agent --agentToken=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0eXBlIjoiYWdlbnQiLCJ1aWQiOiJ1cy13ZXN0LTI6ZTgwNjI3ZWEtMjQxZC00ZjU0LWE3NmYtNThiYjNiMDZiMjlmIiwicGlkIjoiN2YwZWVkYTMtNWQxOC00MzdhLThjMjEtNGI4NDViMjYzNWNhIiwibG5nIjoiLTExNC4wNDg4MCIsImxhdCI6IjUxLjEwOTQwIiwiaWF0IjoxNjQ0NjA3Mzg0LCJleHAiOjE2NDcxOTkzODQsImF1ZCI6IjdmOGFlMTFlLTZiNDAtNDI4ZC1hZmEwLTVlMDhhMDkxNGY1ZiIsImlzcyI6Imh0dHBzOi8vYXBpLWRldi5qdy5wdXJld2ViLmlvIiwic3ViIjoiNzQ1MGFhNTAtMTdhOS00M2FlLTlkOWItNTBhZTE4MmUxYTUwIiwianRpIjoiMDA1ZjFlOWEtN2NjMi00NzE5LTkzOTktZDdhMTE5ZGVhMzVkIn0.0iJCgHykSfIdtRc1bM1rzk_Ok6U9-SuW0WAAewiSxtU --debugLevel=debug

3. In your browser go to: https://preview-dev.jw.pureweb.io/7f0eeda3-5d18-437a-8c21-4b845b2635ca?environmentId=7f8ae11e-6b40-428d-afa0-5e08a0914f5f&local=true

Working project with webRTC 2.4.0-exp.3:

https://drive.google.com/file/d/1V3Na_2uuL0F_HQ7u2LvuRzow5EjpoIZf/view?usp=sharing

1. Run the game

2. In your command window run:

npx @pureweb/platform-streaming-agent --agentToken=eyJhbGciOiJIUzI1NiIsInR5cCI6IkpXVCJ9.eyJ0eXBlIjoiYWdlbnQiLCJ1aWQiOiJ1cy13ZXN0LTI6ZTgwNjI3ZWEtMjQxZC00ZjU0LWE3NmYtNThiYjNiMDZiMjlmIiwicGlkIjoiN2YwZWVkYTMtNWQxOC00MzdhLThjMjEtNGI4NDViMjYzNWNhIiwibG5nIjoiLTExNC4wNDg4MCIsImxhdCI6IjUxLjEwOTQwIiwiaWF0IjoxNjQ0NjA3Mzg0LCJleHAiOjE2NDcxOTkzODQsImF1ZCI6IjdmOGFlMTFlLTZiNDAtNDI4ZC1hZmEwLTVlMDhhMDkxNGY1ZiIsImlzcyI6Imh0dHBzOi8vYXBpLWRldi5qdy5wdXJld2ViLmlvIiwic3ViIjoiNzQ1MGFhNTAtMTdhOS00M2FlLTlkOWItNTBhZTE4MmUxYTUwIiwianRpIjoiMDA1ZjFlOWEtN2NjMi00NzE5LTkzOTktZDdhMTE5ZGVhMzVkIn0.0iJCgHykSfIdtRc1bM1rzk_Ok6U9-SuW0WAAewiSxtU --debugLevel=debug

3. In your browser go to: https://preview-dev.jw.pureweb.io/7f0eeda3-5d18-437a-8c21-4b845b2635ca?environmentId=7f8ae11e-6b40-428d-afa0-5e08a0914f5f&local=true

[karasusan]

@jwielebnowski Is there the way to reproduce the issue without connecting pureweb? We are not a expert of pureweb.

[jwielebnowski]

@karasusan Of course you do not need to connect to our signaling service. The unity game can connect to any. I do not mind modifying the code for you. What I gave you was the simplest and fastest way to see the problem and check what might be wrong.

Please tell me what is your signaling service that you are using and your client in your development environment and how to set it up. If you can give me the protocol of the packages that you send with webRTC objects (offers/answers/iceCandidates) that would be great. There should be no problem setting it up.

[karasusan]

@jwielebnowski I understood your situation. We need more time to reproduce and fix your issue because need to understand the process in Pureweb.

[karasusan]

memo: WRS-318