Severe latency in outbound video stream on some devices

[kirgy]

Steps to reproduce

The following is our specific tech setup scenario;

1. enter a call on the mobile app
2. in a web instance enter the same twilio room
3. mobile app's video out frame-rate drops to less than 1fps (roughly 1 frame every 15 seconds). From the app's point of view:

• audio operates perfectly fine
• remote video operates fine
• app's video appears severely attend on both app's view and remote view
• entering or leaving, joining in any combination does not improve latency at all

Note: the confusing experience here is this appears to be device specific. Some devices have zero issues, others experience the issue. One a OnePlus 7 pro Android 12, this issue never occurs . On a Pixel Fold (folded), Android 14, this occurs 100% of the time.

We have similar reports from users on both Android and iOS and we have no yet correlated a pattern.

Expected behaviour

Video frame rate should perform well and remain consistent

Actual behaviour

Frame rate drops to below 1FPS

Environment

• Node.js version: v16.20.1
• React Native version: 0.72.4
• React Native platform + platform version: Android 14, Google Pixel Fold

react-native-twilio-video-webrtc

Version: 3.2.0

import {
  RoomErrorEventCb,
  RoomEventCb,
  TrackEventCbArgs,
  TwilioVideo,
  TwilioVideoLocalView,
  TwilioVideoParticipantView
} from "react-native-twilio-video-webrtc";

import KeepAwake from "react-native-keep-awake";

interface Props extends NavigationComponentProps {
  identity: string;
  item: Appointment[number];
  room: string;
  token: string;
}

type ParticipantsMap = Map<
  string,
  {
    identity: string;
    name?: string;
    isAudioEnabled?: boolean;
    isVideoEnabled?: boolean;
    videoTrackSid?: string;
  }
>;

function VideoCallScreen({ identity, item, token, room, componentId }: Props) {
  const { theme } = useStyledTheme();
  const { str } = useTranslations();
  const userData = useSelector(getUserData);

  const twilioVideoRef = useRef<TwilioVideo>(null);
  const [participants, setParticipants] = useState<ParticipantsMap>(new Map());
  const [alertType, setAlertType] = useState<AlertType>();
  const [isAudioEnabled, setIsAudioEnabled] = useState<boolean>(true);
  const [isVideoEnabled, setIsVideoEnabled] = useState<boolean>(true);

  const joined = participants.size > 0;

  useEffect(() => {
    twilioVideoRef.current.connect({
      accessToken: token,
      roomName: room,
    });
  }, [twilioVideoRef.current]);

  function sendName() {
    const metaNameMessage: NameMetaMessage = {
      identity,
      name: userData.me?.name ?? "",
      type: "ParticipantIdentity",
    };

    const message = createMetaMessage(metaNameMessage);
    twilioVideoRef.current?.sendString(message);
  }

  function onDataTrackMessageReceived({ message }: { message: string }) {
    const metaMessage = parseMetaMessage(message);

    if (metaMessage == null) {
      return;
    }

    switch (metaMessage.type) {
      case "ParticipantIdentity": {
        const participant = Array.from(participants.entries()).find(
          ([, data]) => data.identity === metaMessage?.identity
        );

        if (participant == null) {
          return;
        }

        const newParticipants = new Map(participants);

        setParticipants(
          newParticipants.set(participant[0], {
            ...participant[1],
            name: metaMessage.name,
          })
        );

        break;
      }
    }
  }

  // Handles the adding of a participant audio track as well as the
  // enabling/disabling.
  // * Adding  - called when a participant joins and we can check if
  //            they joined with mic on/off.
  // * Enable  - called when a participant unmutes
  // * Disable - called when a participant mutes
  function onParticipantUpdatedAudioTrack(args: TrackEventCbArgs) {
    const newParticipants = new Map(participants);
    const participant = participants.get(args.participant.sid);

    setParticipants(
      newParticipants.set(args.participant.sid, {
        ...participant,
        identity: args.participant.identity,
        isAudioEnabled: args.track.enabled,
      })
    );
  }

  function onParticipantAddedDataTrack() {
    sendName();
  }

  // Handles the adding of a participant video track as well as the
  // enabling/disabling.
  // * Adding  - called when a participant joins and we can check if
  //            they joined with camera on/off.
  // * Enable  - called when a participant turns camera on
  // * Disable - called when a participant turns camera off
  function onParticipantUpdatedVideoTrack(args: TrackEventCbArgs) {
    const newParticipants = new Map(participants);
    const participant = participants.get(args.participant.sid);

    setParticipants(
      newParticipants.set(args.participant.sid, {
        ...participant,
        identity: args.participant.identity,
        isVideoEnabled: args.track.enabled,
        videoTrackSid: args.track.trackSid,
      })
    );
  }

  // Handles the removing of a participant video track .
  // This will usually be called when a participant leaves the call.
  function onParticipantRemovedVideoTrack(args: TrackEventCbArgs) {
    const newParticipants = new Map(participants);
    newParticipants.delete(args.participant.sid);

    setParticipants(newParticipants);
  }

  function onLeavePress() {
    setAlertType("Leave");
  }

  function onCameraPress() {
    if (twilioVideoRef.current) {
      const newIsVideoEnabled = !isVideoEnabled;
      void twilioVideoRef.current.setLocalVideoEnabled(newIsVideoEnabled);
      setIsVideoEnabled(newIsVideoEnabled);
    }
  }

  function onFlipCameraPress() {
    if (twilioVideoRef.current) {
      void twilioVideoRef.current.flipCamera();
    }
  }

  function onMutePress() {
    if (twilioVideoRef.current) {
      const newIsAudioEnabled = !isAudioEnabled;
      void twilioVideoRef.current.setLocalAudioEnabled(newIsAudioEnabled);
      setIsAudioEnabled(newIsAudioEnabled);
    }
  }

  function onLeaveAlert() {
    if (twilioVideoRef.current) {
      twilioVideoRef.current.disconnect();
    }
    pop(componentId);
  }

  const onRoomDidFailToConnect: RoomErrorEventCb = (t) => {
    setAlertType("Error");
  };

  function WaitingScreen({
    details,
  }: {
    details: GetAppointmentsQuery["me"]["appointments"][number];
  }) {
    return (
      <StyledWaitingSafeArea>
        <VideoCallControlsContainer>
          <VideoCallControls
            isAudioEnabled={isAudioEnabled}
            isVideoEnabled={isVideoEnabled}
            onCameraPress={onCameraPress}
            onFlipCameraPress={onFlipCameraPress}
            onLeavePress={onLeavePress}
            onMutePress={onMutePress}
          />
        </VideoCallControlsContainer>
      </StyledWaitingSafeArea>
    );
  }

  function CallScreen({ tracks }: { tracks: ParticipantsMap }) {
    return (
      <>
        <ParticipantVideoContainer>
          {Array.from(tracks, ([participantSid, videoTrack]) => (
          <TwilioVideoParticipantView
            trackIdentifier={{
              participantSid: participantSid,
              videoTrackSid: videoTrack.videoTrackSid,
            }}
          />
          ))}
        </ParticipantVideoContainer>
        <VideoCallControlsContainer>
          <VideoCallControls
            isAudioEnabled={isAudioEnabled}
            isVideoEnabled={isVideoEnabled}
            onCameraPress={onCameraPress}
            onFlipCameraPress={onFlipCameraPress}
            onLeavePress={onLeavePress}
            onMutePress={onMutePress}
          />
        </VideoCallControlsContainer>
      </>
    );
  }

  return (
    <>
      {joined ? (
        <CallScreen tracks={participants} />
      ) : (
        <Loading/>
      )}
      {isVideoEnabled && <TwilioVideoLocalView enabled />}
      <TwilioVideo
        ref={twilioVideoRef}
        onDataTrackMessageReceived={onDataTrackMessageReceived}
        onParticipantAddedAudioTrack={onParticipantUpdatedAudioTrack}
        onParticipantAddedDataTrack={onParticipantAddedDataTrack}
        onParticipantAddedVideoTrack={onParticipantUpdatedVideoTrack}
        onParticipantDisabledAudioTrack={onParticipantUpdatedAudioTrack}
        onParticipantDisabledVideoTrack={onParticipantUpdatedVideoTrack}
        onParticipantEnabledAudioTrack={onParticipantUpdatedAudioTrack}
        onParticipantEnabledVideoTrack={onParticipantUpdatedVideoTrack}
        onParticipantRemovedVideoTrack={onParticipantRemovedVideoTrack}
        onRoomDidFailToConnect={onRoomDidFailToConnect}
      />
      {!!alertType && (
        <VideoCallAlerts
          onLeave={onLeaveAlert}
        />
      )}
      <KeepAwake />
    </>
  );
}

export default VideoCallScreen;