Video and/or audio often not working in our web app

[sunweiyang]

Our web app uses this library for a 1-to-1 video chat, and we are very often observing video and/or audio not working in our web app. We have the following requirements:

• Agora Cloud Proxy in Force TCP mode
• Initially not publishing audio, and then publishing audio later (when a chat agenda modal is dismissed -- the modal's visibility status is handled in Redux)

Are we missing anything important in the Agora client lifecycle?

import React, { useState, useEffect, useCallback, useRef } from "react";
import AgoraRTC, {
  AgoraVideoPlayer,
  createClient,
  createMicrophoneAndCameraTracks,
  ClientConfig,
  IAgoraRTCRemoteUser,
  IMicrophoneAudioTrack,
  ICameraVideoTrack,
  AREAS,
} from "agora-rtc-react";

import { useDispatch, useSelector } from "react-redux";
import { ActionCreators } from "actions";

...

const config: ClientConfig = {
  mode: "rtc",
  codec: "vp8",
};
const useClient = createClient(config);
const useMicrophoneAndCameraTracks = createMicrophoneAndCameraTracks();

...

export default function Chat() {
  const dispatch = useDispatch();

  const chatAgendaModalVisible = useSelector(
    (state: any) => state.chatAgendaModalVisible
  );
  const [chatStarted, setChatStarted] = useState(false);
  const [chatEnded, setChatEnded] = useState(false);
  const [agoraRemoteUsers, setAgoraRemoteUsers] = useState<
    IAgoraRTCRemoteUser[]
  >([]);

  const agoraClient = useClient();
  const { ready, tracks } = useMicrophoneAndCameraTracks();
  const readyRef = useRef<boolean>();
  readyRef.current = ready;
  const tracksRef = useRef<[IMicrophoneAudioTrack, ICameraVideoTrack] | null>();
  tracksRef.current = tracks;

  const _handleInitAgoraRtcEngine = async () => {
    try {
      AgoraRTC.setArea({ areaCode: [AREAS.NORTH_AMERICA] });
      agoraClient.startProxyServer(5); // Force TCP Cloud Proxy mode
      agoraClient.on("connection-state-change", (state) => {
        console.log("event connection-state-change", state);
        if (state === "CONNECTED") {
          setHasSelfJoinedChannel(true);
        }
      });
      agoraClient.on("user-published", (user, mediaType) => {
        console.log("event user-published", user, mediaType);
        const subscribe = async () => {
          await agoraClient.subscribe(user, mediaType);
          if (mediaType === "video") {
            setAgoraRemoteUsers((prevUsers) => {
              return [...prevUsers, user];
            });
            setHasPartnerJoinedChannel(true);
          }
          if (mediaType === "audio") {
            user.audioTrack?.play();
          }
        };
        subscribe();
      });
      agoraClient.on("user-unpublished", (user, type) => {
        console.log("event user-unpublished", user, type);
        if (type === "video") {
          setAgoraRemoteUsers((prevUsers) => {
            return prevUsers.filter((u) => u.uid !== user.uid);
          });
        }
        if (type === "audio") {
          user.audioTrack?.stop();
        }
      });
      agoraClient.on("user-left", (user) => {
        console.log("event user-left", user);
        setAgoraRemoteUsers((prevUsers) => {
          return prevUsers.filter((u) => u.uid !== user.uid);
        });
      });
      agoraClient.on("is-using-cloud-proxy", () => {
        console.log("is-using-cloud-proxy");
      });
      await agoraClient.join(
        process.env.REACT_APP_AGORA_APP_ID || "",
        matchObject.agoraChannel,
        matchObject.agoraToken,
        userId
      );
    } catch (error) {
      console.log("_handleInitAgoraRtcEngine error", error);
    }
  };

  const _handlePublishTrack = async (type: "audio" | "video") => {
    if (readyRef.current && tracksRef.current) {
      const trackIndex = type === "audio" ? 0 : 1;
      await tracksRef.current[trackIndex].setMuted(false);
      await agoraClient.publish(tracksRef.current[trackIndex]);
    }
  };

  const _handleCloseChat = useCallback(async () => {
    if (!chatEnded) {
      setChatEnded(true);
      await agoraClient.leave();
      agoraClient.removeAllListeners();
      if (tracksRef.current) {
        for (const track of tracksRef.current) {
          track.close();
        }
      }
      agoraClient.stopProxyServer();
    }
  }, [chatEnded, agoraClient, _handleClearAllTimers]);
...
  useEffect(() => {
    const initChat = async () => {
      await _handleInitAgoraRtcEngine();
    };
    initChat();
    return () => {
      _handleCloseChat();
    };
  }, []);

  useEffect(() => {
    // As soon as tracks are available and ready, start publishing video
    if (readyRef.current && tracksRef.current) {
      _handlePublishTrack("video");
    }
  }, [readyRef.current, tracksRef.current]);

  useEffect(() => {
    // Disable initial agenda modal and turn audio on
    if (chatStarted && !chatAgendaModalVisible) {
      const publishAudio = async () => {
        if (readyRef.current && tracksRef.current) {
          await tracksRef.current[0].setMuted(false);
          await agoraClient.publish(tracksRef.current[0]);
        }
      };
      publishAudio();
    }
  }, [
    chatAgendaModalVisible,
    chatStarted,
    readyRef.current,
    tracksRef.current,
  ]);

  ...

  return (

  ...

        {agoraRemoteUsers.length > 0 && agoraRemoteUsers[0]?.videoTrack ? (
          <AgoraVideoPlayer
            videoTrack={agoraRemoteUsers[0].videoTrack}
            style={{ height: "100%", width: "100%" }}
          />
        ) : null}

  ...

        {readyRef.current && tracksRef.current ? (
          <AgoraVideoPlayer
            videoTrack={tracksRef.current[1]}
            style={{ height: "100%", width: "100%" }}
          />
        ) : null}

  ...

  )
}

[plutoless]

do you have the console logs when video/audio is not working?

[sunweiyang]

@plutoless Yes, please see logs below:

• user1 can see user2, but can't hear user2
• user2 can hear user1 but can't see user1

user1_firefox-export-2022-4-26_9-2-54.txt user2_chrome-export-1650988943171.log