From 1ea94327699b4d139bc4cd96c4cd895a6f3b7573 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 21 Oct 2022 17:24:56 +0100 Subject: [PATCH 1/4] Show tiles for members we're trying to connect to This should help give more context on what's going wrong in splitbrain scenarios. If users leave calls uncleanly, their tile will remain in until their member event times out, which will be an hour from when they joined the call. See https://github.com/vector-im/element-call/issues/639. Part of https://github.com/vector-im/element-call/issues/616 --- src/room/GroupCallView.tsx | 3 +- src/room/InCallView.tsx | 60 +++++++++++++++++---------- src/room/useGroupCall.ts | 10 +---- src/video-grid/AudioContainer.tsx | 6 +-- src/video-grid/VideoGrid.stories.tsx | 6 ++- src/video-grid/VideoGrid.tsx | 8 ++-- src/video-grid/VideoTile.tsx | 6 ++- src/video-grid/VideoTileContainer.tsx | 12 +++--- src/video-grid/useCallFeed.ts | 5 +-- src/video-grid/useFullscreen.tsx | 12 +++--- src/video-grid/useMediaStream.ts | 18 ++++++-- 11 files changed, 85 insertions(+), 61 deletions(-) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 8bf6a132..ee44ef60 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -76,7 +76,6 @@ export function GroupCallView({ toggleScreensharing, requestingScreenshare, isScreensharing, - localScreenshareFeed, screenshareFeeds, participants, unencryptedEventsFromUsers, @@ -221,6 +220,7 @@ export function GroupCallView({ client={client} roomName={groupCall.room.name} avatarUrl={avatarUrl} + participants={participants} microphoneMuted={microphoneMuted} localVideoMuted={localVideoMuted} toggleLocalVideoMuted={toggleLocalVideoMuted} @@ -230,7 +230,6 @@ export function GroupCallView({ onLeave={onLeave} toggleScreensharing={toggleScreensharing} isScreensharing={isScreensharing} - localScreenshareFeed={localScreenshareFeed} screenshareFeeds={screenshareFeeds} roomIdOrAlias={roomIdOrAlias} unencryptedEventsFromUsers={unencryptedEventsFromUsers} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index da6a6dc9..80be6883 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -70,6 +70,7 @@ const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); interface Props { client: MatrixClient; groupCall: GroupCall; + participants: RoomMember[]; roomName: string; avatarUrl: string; microphoneMuted: boolean; @@ -82,14 +83,16 @@ interface Props { onLeave: () => void; isScreensharing: boolean; screenshareFeeds: CallFeed[]; - localScreenshareFeed: CallFeed; roomIdOrAlias: string; unencryptedEventsFromUsers: Set; hideHeader: boolean; } -export interface Participant { +// Represents something that should get a tile on the layout, +// ie. a user's video feed or a screen share feed. +export interface TileDescriptor { id: string; + member: RoomMember; focused: boolean; presenter: boolean; callFeed?: CallFeed; @@ -99,6 +102,7 @@ export interface Participant { export function InCallView({ client, groupCall, + participants, roomName, avatarUrl, microphoneMuted, @@ -111,7 +115,6 @@ export function InCallView({ toggleScreensharing, isScreensharing, screenshareFeeds, - localScreenshareFeed, roomIdOrAlias, unencryptedEventsFromUsers, hideHeader, @@ -185,39 +188,48 @@ export function InCallView({ }, [setLayout]); const items = useMemo(() => { - const participants: Participant[] = []; + const tileDescriptors: TileDescriptor[] = []; - for (const callFeed of userMediaFeeds) { - participants.push({ - id: callFeed.stream.id, - callFeed, - focused: - screenshareFeeds.length === 0 && callFeed.userId === activeSpeaker, - isLocal: callFeed.isLocal(), + // one tile for each participants, to start with (we want a tile for everyone we + // think should be in the call, even if we don't have a media feed for them yet) + for (const p of participants) { + const userMediaFeed = userMediaFeeds.find((f) => f.userId === p.userId); + + // NB. this assumes that the same user can't join more than once from multiple + // devices, but the participants are just RoomMembers, so this assumption is baked + // into GroupCall itself. + tileDescriptors.push({ + id: p.userId, + member: p, + callFeed: userMediaFeed, + focused: screenshareFeeds.length === 0 && p.userId === activeSpeaker, + isLocal: p.userId === client.getUserId(), presenter: false, }); } - for (const callFeed of screenshareFeeds) { - const userMediaItem = participants.find( - (item) => item.callFeed.userId === callFeed.userId + // add the screenshares too + for (const screenshareFeed of screenshareFeeds) { + const userMediaItem = tileDescriptors.find( + (item) => item.member.userId === screenshareFeed.userId ); if (userMediaItem) { userMediaItem.presenter = true; } - participants.push({ - id: callFeed.stream.id, - callFeed, + tileDescriptors.push({ + id: screenshareFeed.stream.id, + member: userMediaItem?.member, + callFeed: screenshareFeed, focused: true, - isLocal: callFeed.isLocal(), + isLocal: screenshareFeed.isLocal(), presenter: false, }); } - return participants; - }, [userMediaFeeds, activeSpeaker, screenshareFeeds]); + return tileDescriptors; + }, [client, participants, userMediaFeeds, activeSpeaker, screenshareFeeds]); // The maximised participant: either the participant that the user has // manually put in fullscreen, or the focused (active) participant if the @@ -281,7 +293,13 @@ export function InCallView({ return ( - {({ item, ...rest }: { item: Participant; [x: string]: unknown }) => ( + {({ + item, + ...rest + }: { + item: TileDescriptor; + [x: string]: unknown; + }) => ( = ({ }; interface AudioContainerProps { - items: Participant[]; + items: TileDescriptor[]; audioContext: AudioContext; audioDestination: AudioNode; } diff --git a/src/video-grid/VideoGrid.stories.tsx b/src/video-grid/VideoGrid.stories.tsx index 916ba685..2fee26a4 100644 --- a/src/video-grid/VideoGrid.stories.tsx +++ b/src/video-grid/VideoGrid.stories.tsx @@ -16,11 +16,12 @@ limitations under the License. import React, { useState } from "react"; import { useMemo } from "react"; +import { RoomMember } from "matrix-js-sdk"; import { VideoGrid, useVideoGridLayout } from "./VideoGrid"; import { VideoTile } from "./VideoTile"; import { Button } from "../button"; -import { Participant } from "../room/InCallView"; +import { TileDescriptor } from "../room/InCallView"; export default { title: "VideoGrid", @@ -33,10 +34,11 @@ export const ParticipantsTest = () => { const { layout, setLayout } = useVideoGridLayout(false); const [participantCount, setParticipantCount] = useState(1); - const items: Participant[] = useMemo( + const items: TileDescriptor[] = useMemo( () => new Array(participantCount).fill(undefined).map((_, i) => ({ id: (i + 1).toString(), + member: new RoomMember("!fake:room.id", `@user${i}:fake.dummy`), focused: false, presenter: false, })), diff --git a/src/video-grid/VideoGrid.tsx b/src/video-grid/VideoGrid.tsx index d9b2123b..b8108c88 100644 --- a/src/video-grid/VideoGrid.tsx +++ b/src/video-grid/VideoGrid.tsx @@ -23,7 +23,7 @@ import { ReactDOMAttributes } from "@use-gesture/react/dist/declarations/src/typ import styles from "./VideoGrid.module.css"; import { Layout } from "../room/GridLayoutMenu"; -import { Participant } from "../room/InCallView"; +import { TileDescriptor } from "../room/InCallView"; interface TilePosition { x: number; @@ -36,7 +36,7 @@ interface TilePosition { interface Tile { key: Key; order: number; - item: Participant; + item: TileDescriptor; remove: boolean; focused: boolean; presenter: boolean; @@ -693,12 +693,12 @@ interface ChildrenProperties extends ReactDOMAttributes { }; width: number; height: number; - item: Participant; + item: TileDescriptor; [index: string]: unknown; } interface VideoGridProps { - items: Participant[]; + items: TileDescriptor[]; layout: Layout; disableAnimations?: boolean; children: (props: ChildrenProperties) => React.ReactNode; diff --git a/src/video-grid/VideoTile.tsx b/src/video-grid/VideoTile.tsx index 901d2453..1a6b2301 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/video-grid/VideoTile.tsx @@ -26,6 +26,7 @@ import { AudioButton, FullscreenButton } from "../button/Button"; interface Props { name: string; + hasFeed: Boolean; speaking?: boolean; audioMuted?: boolean; videoMuted?: boolean; @@ -47,6 +48,7 @@ export const VideoTile = forwardRef( ( { name, + hasFeed, speaking, audioMuted, videoMuted, @@ -90,6 +92,8 @@ export const VideoTile = forwardRef( } } + const caption = hasFeed ? name : t("{{name}} (Connecting...)", { name }); + return ( (
{audioMuted && !videoMuted && } {videoMuted && } - {name} + {caption}
))}