From 4242d45ba2c7336d02813d0629cf2a0fae769ed8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 16 Aug 2023 18:41:27 +0100 Subject: [PATCH 01/21] WIP refactor for removing m.call events --- src/Header.tsx | 37 +------- src/IncompatibleVersionModal.tsx | 60 ------------ src/matrix-utils.ts | 1 - src/room/GroupCallLoader.tsx | 6 +- src/room/GroupCallView.tsx | 132 ++++++++++++++------------ src/room/InCallView.tsx | 64 +++++-------- src/room/RoomPage.tsx | 6 +- src/room/useGroupCall.ts | 18 ++-- src/room/useLoadGroupCall.ts | 70 ++------------ src/room/useSentryGroupCallHandler.ts | 46 --------- src/useMatrixRTCSessionJoinState.ts | 44 +++++++++ src/useMatrixRTCSessionMemberships.ts | 48 ++++++++++ 12 files changed, 209 insertions(+), 323 deletions(-) delete mode 100644 src/IncompatibleVersionModal.tsx delete mode 100644 src/room/useSentryGroupCallHandler.ts create mode 100644 src/useMatrixRTCSessionJoinState.ts create mode 100644 src/useMatrixRTCSessionMemberships.ts diff --git a/src/Header.tsx b/src/Header.tsx index 8d754b92..80a6ab15 100644 --- a/src/Header.tsx +++ b/src/Header.tsx @@ -15,17 +15,13 @@ limitations under the License. */ import classNames from "classnames"; -import { HTMLAttributes, ReactNode, useCallback } from "react"; +import { HTMLAttributes, ReactNode } from "react"; import { Link } from "react-router-dom"; -import { Room } from "matrix-js-sdk/src/models/room"; import { useTranslation } from "react-i18next"; import styles from "./Header.module.css"; -import { useModalTriggerState } from "./Modal"; -import { Button } from "./button"; import { ReactComponent as Logo } from "./icons/Logo.svg"; import { Subtitle } from "./typography/Typography"; -import { IncompatibleVersionModal } from "./IncompatibleVersionModal"; interface HeaderProps extends HTMLAttributes { children: ReactNode; @@ -125,34 +121,3 @@ export function RoomHeaderInfo({ roomName }: RoomHeaderInfo) { ); } - -interface VersionMismatchWarningProps { - users: Set; - room: Room; -} - -export function VersionMismatchWarning({ - users, - room, -}: VersionMismatchWarningProps) { - const { t } = useTranslation(); - const { modalState, modalProps } = useModalTriggerState(); - - const onDetailsClick = useCallback(() => { - modalState.open(); - }, [modalState]); - - if (users.size === 0) return null; - - return ( - - {t("Incompatible versions!")} - - {modalState.isOpen && ( - - )} - - ); -} diff --git a/src/IncompatibleVersionModal.tsx b/src/IncompatibleVersionModal.tsx deleted file mode 100644 index d11822de..00000000 --- a/src/IncompatibleVersionModal.tsx +++ /dev/null @@ -1,60 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { Room } from "matrix-js-sdk/src/models/room"; -import { FC, useMemo } from "react"; -import { Trans, useTranslation } from "react-i18next"; - -import { Modal, ModalContent } from "./Modal"; -import { Body } from "./typography/Typography"; - -interface Props { - userIds: Set; - room: Room; - onClose: () => void; -} - -export const IncompatibleVersionModal: FC = ({ - userIds, - room, - onClose, - ...rest -}) => { - const { t } = useTranslation(); - const userLis = useMemo( - () => [...userIds].map((u) =>
  • {room.getMember(u)?.name ?? u}
    • ), - [userIds, room] - ); - - return ( - - - - - Other users are trying to join this call from incompatible versions. - These users should ensure that they have refreshed their browsers: -
      {userLis}
    -     - -     -     - ); -}; diff --git a/src/matrix-utils.ts b/src/matrix-utils.ts index 5f672c57..73ab4151 100644 --- a/src/matrix-utils.ts +++ b/src/matrix-utils.ts @@ -172,7 +172,6 @@ export async function initClient( localTimeoutMs: 5000, useE2eForGroupCall: e2eEnabled, fallbackICEServerAllowed: fallbackICEServerAllowed, - useLivekitForGroupCalls: true, }); try { diff --git a/src/room/GroupCallLoader.tsx b/src/room/GroupCallLoader.tsx index ca52a163..78fa780c 100644 --- a/src/room/GroupCallLoader.tsx +++ b/src/room/GroupCallLoader.tsx @@ -16,8 +16,8 @@ limitations under the License. import { ReactNode } from "react"; import { MatrixClient } from "matrix-js-sdk/src/client"; -import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall"; import { useTranslation } from "react-i18next"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import { useLoadGroupCall } from "./useLoadGroupCall"; import { ErrorView, FullScreenView } from "../FullScreenView"; @@ -26,7 +26,7 @@ interface Props { client: MatrixClient; roomIdOrAlias: string; viaServers: string[]; - children: (groupCall: GroupCall) => ReactNode; + children: (rtcSession: MatrixRTCSession) => ReactNode; createPtt: boolean; } @@ -53,7 +53,7 @@ export function GroupCallLoader({ ); case "loaded": - return <>{children(groupCallState.groupCall)}; + return <>{children(groupCallState.rtcSession)}; case "failed": return ; } diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 8f533d72..9a91bfcf 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -16,33 +16,35 @@ limitations under the License. import { useCallback, useEffect, useMemo, useRef, useState } from "react"; import { useHistory } from "react-router-dom"; -import { GroupCall, GroupCallState } from "matrix-js-sdk/src/webrtc/groupCall"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { useTranslation } from "react-i18next"; import { Room } from "livekit-client"; import { logger } from "matrix-js-sdk/src/logger"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { Focus } from "matrix-js-sdk/src/matrixrtc/focus"; import type { IWidgetApiRequest } from "matrix-widget-api"; import { widget, ElementWidgetActions, JoinCallData } from "../widget"; -import { useGroupCall } from "./useGroupCall"; import { ErrorView, FullScreenView } from "../FullScreenView"; import { LobbyView } from "./LobbyView"; import { MatrixInfo } from "./VideoPreview"; import { CallEndedView } from "./CallEndedView"; -import { useSentryGroupCallHandler } from "./useSentryGroupCallHandler"; import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; import { useProfile } from "../profile/useProfile"; import { E2EEConfig } from "../livekit/useLiveKit"; import { findDeviceByName } from "../media-utils"; -import { OpenIDLoader } from "../livekit/OpenIDLoader"; +//import { OpenIDLoader } from "../livekit/OpenIDLoader"; import { ActiveCall } from "./InCallView"; -import { Config } from "../config/Config"; import { MuteStates, useMuteStates } from "./MuteStates"; import { useMediaDevices, MediaDevices } from "../livekit/MediaDevicesContext"; +import { LivekitFocus } from "../livekit/LivekitFocus"; +import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; +import { enterRTCSession, leaveRTCSession } from "../rtcSessionHelpers"; +import { useMatrixRTCSessionJoinState } from "../useMatrixRTCSessionJoinState"; declare global { interface Window { - groupCall?: GroupCall; + rtcSession?: MatrixRTCSession; } } @@ -52,7 +54,7 @@ interface Props { isEmbedded: boolean; preload: boolean; hideHeader: boolean; - groupCall: GroupCall; + rtcSession: MatrixRTCSession; } export function GroupCallView({ @@ -61,9 +63,9 @@ export function GroupCallView({ isEmbedded, preload, hideHeader, - groupCall, + rtcSession, }: Props) { - const { + /*const { state, error, enter, @@ -71,33 +73,36 @@ export function GroupCallView({ participants, unencryptedEventsFromUsers, otelGroupCallMembership, - } = useGroupCall(groupCall, client); + } = useGroupCall(groupCall, client);*/ + + const memberships = useMatrixRTCSessionMemberships(rtcSession); + const isJoined = useMatrixRTCSessionJoinState(rtcSession); const { t } = useTranslation(); useEffect(() => { - window.groupCall = groupCall; + window.rtcSession = rtcSession; return () => { - delete window.groupCall; + delete window.rtcSession; }; - }, [groupCall]); + }, [rtcSession]); const { displayName, avatarUrl } = useProfile(client); const matrixInfo = useMemo((): MatrixInfo => { return { displayName: displayName!, avatarUrl: avatarUrl!, - roomId: groupCall.room.roomId, - roomName: groupCall.room.name, - roomAlias: groupCall.room.getCanonicalAlias(), + roomId: rtcSession.room.roomId, + roomName: rtcSession.room.name, + roomAlias: rtcSession.room.getCanonicalAlias(), }; - }, [displayName, avatarUrl, groupCall]); + }, [displayName, avatarUrl, rtcSession]); const deviceContext = useMediaDevices(); const latestDevices = useRef(); latestDevices.current = deviceContext; - const muteStates = useMuteStates(participants.size); + const muteStates = useMuteStates(memberships.length); const latestMuteStates = useRef(); latestMuteStates.current = muteStates; @@ -154,10 +159,13 @@ export function GroupCallView({ } } - await enter(); + enterRTCSession(rtcSession); PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); - PosthogAnalytics.instance.eventCallStarted.track(groupCall.groupCallId); + // we only have room sessions right now, so call ID is the emprty string - we use the room ID + PosthogAnalytics.instance.eventCallStarted.track( + rtcSession.room.roomId + ); await Promise.all([ widget!.api.setAlwaysOnScreen(true), @@ -170,19 +178,18 @@ export function GroupCallView({ widget!.lazyActions.off(ElementWidgetActions.JoinCall, onJoin); }; } - }, [groupCall, preload, enter]); + }, [rtcSession, preload]); useEffect(() => { if (isEmbedded && !preload) { // In embedded mode, bypass the lobby and just enter the call straight away - enter(); + enterRTCSession(rtcSession); PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); - PosthogAnalytics.instance.eventCallStarted.track(groupCall.groupCallId); + // use the room ID as above + PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); } - }, [groupCall, isEmbedded, preload, enter]); - - useSentryGroupCallHandler(groupCall); + }, [rtcSession, isEmbedded, preload]); const [left, setLeft] = useState(false); const [leaveError, setLeaveError] = useState(undefined); @@ -193,21 +200,16 @@ export function GroupCallView({ setLeaveError(leaveError); setLeft(true); - let participantCount = 0; - for (const deviceMap of groupCall.participants.values()) { - participantCount += deviceMap.size; - } - // In embedded/widget mode the iFrame will be killed right after the call ended prohibiting the posthog event from getting sent, // therefore we want the event to be sent instantly without getting queued/batched. const sendInstantly = !!widget; PosthogAnalytics.instance.eventCallEnded.track( - groupCall.groupCallId, - participantCount, + rtcSession.room.roomId, + rtcSession.memberships.length, sendInstantly ); - leave(); + leaveRTCSession(rtcSession); if (widget) { // we need to wait until the callEnded event is tracked. Otherwise the iFrame gets killed before the callEnded event got tracked. await new Promise((resolve) => window.setTimeout(resolve, 10)); // 10ms @@ -224,13 +226,13 @@ export function GroupCallView({ history.push("/"); } }, - [groupCall, leave, isPasswordlessUser, isEmbedded, history] + [rtcSession, isPasswordlessUser, isEmbedded, history] ); useEffect(() => { - if (widget && state === GroupCallState.Entered) { + if (widget && isJoined) { const onHangup = async (ev: CustomEvent) => { - leave(); + leaveRTCSession(rtcSession); await widget!.api.transport.reply(ev.detail, {}); widget!.api.setAlwaysOnScreen(false); }; @@ -239,7 +241,7 @@ export function GroupCallView({ widget!.lazyActions.off(ElementWidgetActions.HangupCall, onHangup); }; } - }, [groupCall, state, leave]); + }, [isJoined, rtcSession]); const [e2eeConfig, setE2EEConfig] = useState( undefined @@ -248,36 +250,40 @@ export function GroupCallView({ const onReconnect = useCallback(() => { setLeft(false); setLeaveError(undefined); - groupCall.enter(); - }, [groupCall]); + rtcSession.joinRoomSession(); + }, [rtcSession]); - const livekitServiceURL = - groupCall.livekitServiceURL ?? Config.get().livekit?.livekit_service_url; - if (!livekitServiceURL) { - return ; + const focus: Focus | undefined = rtcSession + .getOldestMembership() + ?.getActiveFoci()?.[0]; + if ( + !focus || + focus.type !== "livekit" || + !(focus as LivekitFocus).livekit_alias || + !(focus as LivekitFocus).livekit_service_url + ) { + logger.error("Incompatible focus on call", focus); + return ; } - if (error) { - return ; - } else if (state === GroupCallState.Entered) { + if (isJoined) { return ( - - - + >*/ + + // ); } else if (left) { // The call ended view is shown for two reasons: prompting guests to create @@ -293,7 +299,7 @@ export function GroupCallView({ ) { return ( { setE2EEConfig(e2eeConfig); - enter(); + enterRTCSession(rtcSession); }} isEmbedded={isEmbedded} hideHeader={hideHeader} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index d8a9ef17..556ec4c4 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -27,7 +27,6 @@ import classNames from "classnames"; import { DisconnectReason, Room, RoomEvent, Track } from "livekit-client"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { RoomMember } from "matrix-js-sdk/src/models/room-member"; -import { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall"; import { Ref, useCallback, useEffect, useMemo, useRef } from "react"; import { useTranslation } from "react-i18next"; import useMeasure from "react-use-measure"; @@ -35,6 +34,8 @@ import { OverlayTriggerState } from "@react-stately/overlays"; import { JoinRule } from "matrix-js-sdk/src/@types/partials"; import { logger } from "matrix-js-sdk/src/logger"; import { RoomEventCallbacks } from "livekit-client/dist/src/room/Room"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { CallMembership } from "matrix-js-sdk/src/matrixrtc/CallMembership"; import type { IWidgetApiRequest } from "matrix-widget-api"; import { @@ -45,22 +46,13 @@ import { SettingsButton, InviteButton, } from "../button"; -import { - Header, - LeftNav, - RightNav, - RoomHeaderInfo, - VersionMismatchWarning, -} from "../Header"; +import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header"; import { useVideoGridLayout, TileDescriptor, VideoGrid, } from "../video-grid/VideoGrid"; -import { - useShowInspector, - useShowConnectionStats, -} from "../settings/useSetting"; +import { useShowConnectionStats } from "../settings/useSetting"; import { useModalTriggerState } from "../Modal"; import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; import { useUrlParams } from "../UrlParams"; @@ -68,10 +60,8 @@ import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts"; import { usePrefersReducedMotion } from "../usePrefersReducedMotion"; import { ElementWidgetActions, widget } from "../widget"; import { GridLayoutMenu } from "./GridLayoutMenu"; -import { GroupCallInspector } from "./GroupCallInspector"; import styles from "./InCallView.module.css"; import { useJoinRule } from "./useJoinRule"; -import { ParticipantInfo } from "./useGroupCall"; import { ItemData, TileContent, VideoTile } from "../video-grid/VideoTile"; import { NewVideoGrid } from "../video-grid/NewVideoGrid"; import { OTelGroupCallMembership } from "../otel/OTelGroupCallMembership"; @@ -120,24 +110,22 @@ export function ActiveCall(props: ActiveCallProps) { export interface InCallViewProps { client: MatrixClient; - groupCall: GroupCall; + rtcSession: MatrixRTCSession; livekitRoom: Room; muteStates: MuteStates; - participants: Map>; + memberships: CallMembership[]; onLeave: (error?: Error) => void; - unencryptedEventsFromUsers: Set; hideHeader: boolean; otelGroupCallMembership?: OTelGroupCallMembership; } export function InCallView({ client, - groupCall, + rtcSession, livekitRoom, muteStates, - participants, + memberships, onLeave, - unencryptedEventsFromUsers, hideHeader, otelGroupCallMembership, }: InCallViewProps) { @@ -161,7 +149,7 @@ export function InCallView({ screenSharingTracks.length > 0 ); - const [showInspector] = useShowInspector(); + //const [showInspector] = useShowInspector(); const [showConnectionStats] = useShowConnectionStats(); const { hideScreensharing } = useUrlParams(); @@ -179,7 +167,7 @@ export function InCallView({ [muteStates] ); - const joinRule = useJoinRule(groupCall.room); + const joinRule = useJoinRule(rtcSession.room); // This function incorrectly assumes that there is a camera and microphone, which is not always the case. // TODO: Make sure that this module is resilient when it comes to camera/microphone availability! @@ -250,7 +238,7 @@ export function InCallView({ const reducedControls = boundsValid && bounds.width <= 400; const noControls = reducedControls && bounds.height <= 400; - const items = useParticipantTiles(livekitRoom, participants); + const items = useParticipantTiles(livekitRoom, memberships); const { fullscreenItem, toggleFullscreen, exitFullscreen } = useFullscreen(items); @@ -324,7 +312,7 @@ export function InCallView({ const { modalState: rageshakeRequestModalState, modalProps: rageshakeRequestModalProps, - } = useRageshakeRequestModal(groupCall.room.roomId); + } = useRageshakeRequestModal(rtcSession.room.roomId); const { modalState: settingsModalState, @@ -419,11 +407,7 @@ export function InCallView({ {!hideHeader && maximisedParticipant === null && (
    - - + @@ -439,31 +423,31 @@ export function InCallView({ {renderContent()} {footer} - {otelGroupCallMembership && ( + {/*otelGroupCallMembership && ( - )} + )*/} {rageshakeRequestModalState.isOpen && !noControls && ( )} {settingsModalState.isOpen && ( )} {inviteModalState.isOpen && ( @@ -474,22 +458,26 @@ export function InCallView({ function useParticipantTiles( livekitRoom: Room, - participants: Map> + memberships: CallMembership[] ): TileDescriptor[] { const sfuParticipants = useParticipants({ room: livekitRoom, }); const items = useMemo(() => { - // The IDs of the participants who published membership event to the room (i.e. are present from Matrix perspective). const matrixParticipants: Map = new Map( + memberships.map((m) => [`${m.member.userId}:${m.deviceId}`, m.member]) + ); + + // The IDs of the participants who published membership event to the room (i.e. are present from Matrix perspective). + /*const matrixParticipants: Map = new Map( [...participants.entries()].flatMap(([user, devicesMap]) => { return [...devicesMap.keys()].map((deviceId) => [ `${user.userId}:${deviceId}`, user, ]); }) - ); + );*/ const hasPresenter = sfuParticipants.find((p) => p.isScreenShareEnabled) !== undefined; @@ -558,7 +546,7 @@ function useParticipantTiles( // If every item is a ghost, that probably means we're still connecting and // shouldn't bother showing anything yet return allGhosts ? [] : tiles; - }, [participants, sfuParticipants]); + }, [memberships, sfuParticipants]); return items; } diff --git a/src/room/RoomPage.tsx b/src/room/RoomPage.tsx index ddf92959..50156e70 100644 --- a/src/room/RoomPage.tsx +++ b/src/room/RoomPage.tsx @@ -15,8 +15,8 @@ limitations under the License. */ import { FC, useEffect, useState, useCallback } from "react"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; -import type { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall"; import { useClientLegacy } from "../ClientContext"; import { ErrorView, LoadingView } from "../FullScreenView"; import { RoomAuthView } from "./RoomAuthView"; @@ -73,10 +73,10 @@ export const RoomPage: FC = () => { ]); const groupCallView = useCallback( - (groupCall: GroupCall) => ( + (rtcSession: MatrixRTCSession) => ( >; hasLocalParticipant: boolean; - unencryptedEventsFromUsers: Set; otelGroupCallMembership?: OTelGroupCallMembership; } @@ -103,14 +103,14 @@ interface State { let groupCallOTelMembership: OTelGroupCallMembership | undefined; let groupCallOTelMembershipGroupCallId: string; -function getParticipants( - groupCall: GroupCall +/*function getParticipants( + rtcSession: MatrixRTCSession ): Map> { const participants = new Map>(); - for (const [member, participantsStateMap] of groupCall.participants) { + for (const membership of rtcSession.memberships) { const participantInfoMap = new Map(); - participants.set(member, participantInfoMap); + participants.set(membership.member, participantInfoMap); for (const [deviceId, participant] of participantsStateMap) { const feed = groupCall.userMediaFeeds.find( @@ -141,10 +141,10 @@ function getParticipants( } return participants; -} +}*/ export function useGroupCall( - groupCall: GroupCall, + rtcSession: MatrixRTCSession, client: MatrixClient ): UseGroupCallReturnType { const [ @@ -171,7 +171,7 @@ export function useGroupCall( isScreensharing: false, screenshareFeeds: [], requestingScreenshare: false, - participants: getParticipants(groupCall), + participants: getParticipants(rtcSession), hasLocalParticipant: false, }); diff --git a/src/room/useLoadGroupCall.ts b/src/room/useLoadGroupCall.ts index 4aa82f11..cae3d409 100644 --- a/src/room/useLoadGroupCall.ts +++ b/src/room/useLoadGroupCall.ts @@ -15,28 +15,19 @@ limitations under the License. */ import { useState, useEffect } from "react"; -import { EventType } from "matrix-js-sdk/src/@types/event"; -import { - GroupCallType, - GroupCallIntent, -} from "matrix-js-sdk/src/webrtc/groupCall"; -import { GroupCallEventHandlerEvent } from "matrix-js-sdk/src/webrtc/groupCallEventHandler"; import { logger } from "matrix-js-sdk/src/logger"; import { ClientEvent, MatrixClient } from "matrix-js-sdk/src/client"; import { SyncState } from "matrix-js-sdk/src/sync"; import { useTranslation } from "react-i18next"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import type { Room } from "matrix-js-sdk/src/models/room"; import type { GroupCall } from "matrix-js-sdk/src/webrtc/groupCall"; import { isLocalRoomId, createRoom, roomNameFromRoomId } from "../matrix-utils"; -import { translatedError } from "../TranslatedError"; -import { widget } from "../widget"; - -const STATS_COLLECT_INTERVAL_TIME_MS = 10000; export type GroupCallLoaded = { kind: "loaded"; - groupCall: GroupCall; + rtcSession: MatrixRTCSession; }; export type GroupCallLoadFailed = { @@ -115,61 +106,12 @@ export const useLoadGroupCall = ( } }; - const fetchOrCreateGroupCall = async (): Promise => { + const fetchOrCreateGroupCall = async (): Promise => { const room = await fetchOrCreateRoom(); logger.debug(`Fetched / joined room ${roomIdOrAlias}`); - let groupCall = client.getGroupCallForRoom(room.roomId); - logger.debug("Got group call", groupCall?.groupCallId); - if (groupCall) { - groupCall.setGroupCallStatsInterval(STATS_COLLECT_INTERVAL_TIME_MS); - return groupCall; - } - - if ( - !widget && - room.currentState.mayClientSendStateEvent( - EventType.GroupCallPrefix, - client - ) - ) { - // The call doesn't exist, but we can create it - console.log( - `No call found in ${roomIdOrAlias}: creating ${ - createPtt ? "PTT" : "video" - } call` - ); - groupCall = await client.createGroupCall( - room.roomId, - createPtt ? GroupCallType.Voice : GroupCallType.Video, - createPtt, - GroupCallIntent.Room - ); - groupCall.setGroupCallStatsInterval(STATS_COLLECT_INTERVAL_TIME_MS); - return groupCall; - } - - // We don't have permission to create the call, so all we can do is wait - // for one to come in - return new Promise((resolve, reject) => { - const onGroupCallIncoming = (groupCall: GroupCall) => { - if (groupCall?.room.roomId === room.roomId) { - clearTimeout(timeout); - groupCall.setGroupCallStatsInterval(STATS_COLLECT_INTERVAL_TIME_MS); - client.off( - GroupCallEventHandlerEvent.Incoming, - onGroupCallIncoming - ); - resolve(groupCall); - } - }; - client.on(GroupCallEventHandlerEvent.Incoming, onGroupCallIncoming); - - const timeout = setTimeout(() => { - client.off(GroupCallEventHandlerEvent.Incoming, onGroupCallIncoming); - reject(translatedError("Fetching group call timed out.", t)); - }, 30000); - }); + const rtcSession = client.matrixRTC.getRoomSession(room); + return rtcSession; }; const waitForClientSyncing = async () => { @@ -192,7 +134,7 @@ export const useLoadGroupCall = ( waitForClientSyncing() .then(fetchOrCreateGroupCall) - .then((groupCall) => setState({ kind: "loaded", groupCall })) + .then((rtcSession) => setState({ kind: "loaded", rtcSession })) .catch((error) => setState({ kind: "failed", error })); }, [client, roomIdOrAlias, viaServers, createPtt, t]); diff --git a/src/room/useSentryGroupCallHandler.ts b/src/room/useSentryGroupCallHandler.ts deleted file mode 100644 index 188a2934..00000000 --- a/src/room/useSentryGroupCallHandler.ts +++ /dev/null @@ -1,46 +0,0 @@ -/* -Copyright 2022 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { useEffect } from "react"; -import * as Sentry from "@sentry/react"; -import { GroupCall, GroupCallEvent } from "matrix-js-sdk/src/webrtc/groupCall"; -import { CallEvent, MatrixCall } from "matrix-js-sdk/src/webrtc/call"; - -export function useSentryGroupCallHandler(groupCall: GroupCall) { - useEffect(() => { - function onHangup(call: MatrixCall) { - if (call.hangupReason === "ice_failed") { - Sentry.captureException(new Error("Call hangup due to ICE failure.")); - } - } - - function onError(error: Error) { - Sentry.captureException(error); - } - - if (groupCall) { - groupCall.on(CallEvent.Hangup, onHangup); - groupCall.on(GroupCallEvent.Error, onError); - } - - return () => { - if (groupCall) { - groupCall.removeListener(CallEvent.Hangup, onHangup); - groupCall.removeListener(GroupCallEvent.Error, onError); - } - }; - }, [groupCall]); -} diff --git a/src/useMatrixRTCSessionJoinState.ts b/src/useMatrixRTCSessionJoinState.ts new file mode 100644 index 00000000..7afd6b52 --- /dev/null +++ b/src/useMatrixRTCSessionJoinState.ts @@ -0,0 +1,44 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { useCallback, useEffect, useState } from "react"; + +export function useMatrixRTCSessionJoinState( + rtcSession: MatrixRTCSession +): boolean { + const [isJoined, setJoined] = useState(rtcSession.isJoined()); + + const onJoinStateChanged = useCallback(() => { + setJoined(rtcSession.isJoined()); + }, [rtcSession]); + + useEffect(() => { + rtcSession.on(MatrixRTCSessionEvent.JoinStateChanged, onJoinStateChanged); + + return () => { + rtcSession.off( + MatrixRTCSessionEvent.JoinStateChanged, + onJoinStateChanged + ); + }; + }, [rtcSession, onJoinStateChanged]); + + return isJoined; +} diff --git a/src/useMatrixRTCSessionMemberships.ts b/src/useMatrixRTCSessionMemberships.ts new file mode 100644 index 00000000..149aa8f6 --- /dev/null +++ b/src/useMatrixRTCSessionMemberships.ts @@ -0,0 +1,48 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { CallMembership } from "matrix-js-sdk/src/matrixrtc/CallMembership"; +import { + MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { useCallback, useEffect, useState } from "react"; + +export function useMatrixRTCSessionMemberships( + rtcSession: MatrixRTCSession +): CallMembership[] { + const [memberships, setMemberships] = useState(rtcSession.memberships); + + const onMembershipsChanged = useCallback(() => { + setMemberships(rtcSession.memberships); + }, [rtcSession]); + + useEffect(() => { + rtcSession.on( + MatrixRTCSessionEvent.MembershipsChanged, + onMembershipsChanged + ); + + return () => { + rtcSession.off( + MatrixRTCSessionEvent.MembershipsChanged, + onMembershipsChanged + ); + }; + }, [rtcSession, onMembershipsChanged]); + + return memberships; +} From e39d00154d50ad7dc87d0fab1e583d362262b3ea Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 18 Aug 2023 09:03:21 +0100 Subject: [PATCH 02/21] More hacking on rtcsession --- src/livekit/LivekitFocus.ts | 23 +++++++ src/livekit/openIDSFU.ts | 87 ++++++++++++--------------- src/room/GroupCallView.tsx | 40 ++---------- src/room/useActiveFocus.ts | 68 +++++++++++++++++++++ src/rtcSessionHelpers.ts | 53 ++++++++++++++++ src/useMatrixRTCSessionJoinState.ts | 6 ++ src/useMatrixRTCSessionMemberships.ts | 4 ++ 7 files changed, 196 insertions(+), 85 deletions(-) create mode 100644 src/livekit/LivekitFocus.ts create mode 100644 src/room/useActiveFocus.ts create mode 100644 src/rtcSessionHelpers.ts diff --git a/src/livekit/LivekitFocus.ts b/src/livekit/LivekitFocus.ts new file mode 100644 index 00000000..f3a1e532 --- /dev/null +++ b/src/livekit/LivekitFocus.ts @@ -0,0 +1,23 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { Focus } from "matrix-js-sdk/src/matrixrtc/focus"; + +export interface LivekitFocus extends Focus { + type: "livekit"; + livekit_service_url: string; + livekit_alias: string; +} diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index 083e95b2..c122a844 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -14,10 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { GroupCall, IOpenIDToken, MatrixClient } from "matrix-js-sdk"; +import { IOpenIDToken, MatrixClient } from "matrix-js-sdk"; import { logger } from "matrix-js-sdk/src/logger"; +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { useEffect, useState } from "react"; -import { Config } from "../config/Config"; +import { LivekitFocus } from "./LivekitFocus"; +import { useActiveFocus } from "../room/useActiveFocus"; export interface SFUConfig { url: string; @@ -30,66 +33,52 @@ export type OpenIDClientParts = Pick< "getOpenIdToken" | "getDeviceId" >; +export function useOpenIDSFU( + client: OpenIDClientParts, + rtcSession: MatrixRTCSession +) { + const [sfuConfig, setSFUConfig] = useState(undefined); + + const activeFocus = useActiveFocus(rtcSession); + + useEffect(() => { + (async () => { + const sfuConfig = activeFocus + ? await getSFUConfigWithOpenID(client, activeFocus) + : undefined; + setSFUConfig(sfuConfig); + })(); + }, [client, activeFocus]); + + return sfuConfig; +} + export async function getSFUConfigWithOpenID( client: OpenIDClientParts, - groupCall: GroupCall, - roomName: string -): Promise { + activeFocus: LivekitFocus +): Promise { const openIdToken = await client.getOpenIdToken(); logger.debug("Got openID token", openIdToken); - // if the call has a livekit service URL, try it. - if (groupCall.livekitServiceURL) { - try { - logger.info( - `Trying to get JWT from call's configured URL of ${groupCall.livekitServiceURL}...` - ); - const sfuConfig = await getLiveKitJWT( - client, - groupCall.livekitServiceURL, - roomName, - openIdToken - ); - logger.info(`Got JWT from call state event URL.`); - - return sfuConfig; - } catch (e) { - logger.warn( - `Failed to get JWT from group call's configured URL of ${groupCall.livekitServiceURL}.`, - e - ); - } - } - - // otherwise, try our configured one and, if it works, update the call's service URL in the state event - // NB. This wuill update it for everyone so we may end up with multiple clients updating this when they - // join at similar times, but we don't have a huge number of options here. - const urlFromConf = Config.get().livekit!.livekit_service_url; - logger.info(`Trying livekit service URL from our config: ${urlFromConf}...`); try { + logger.info( + `Trying to get JWT from call's active focus URL of ${activeFocus.livekit_service_url}...` + ); const sfuConfig = await getLiveKitJWT( client, - urlFromConf, - roomName, + activeFocus.livekit_service_url, + activeFocus.livekit_alias, openIdToken ); - - logger.info( - `Got JWT, updating call livekit service URL with: ${urlFromConf}...` - ); - try { - await groupCall.updateLivekitServiceURL(urlFromConf); - logger.info(`Call livekit service URL updated.`); - } catch (e) { - logger.warn( - `Failed to update call livekit service URL: continuing anyway.` - ); - } + logger.info(`Got JWT from call's active focus URL.`); return sfuConfig; } catch (e) { - logger.error("Failed to get JWT from URL defined in Config.", e); - throw e; + logger.warn( + `Failed to get JWT from RTC session's active focus URL of ${activeFocus.livekit_service_url}.`, + e + ); + return undefined; } } diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index eaf334ce..57182b04 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -21,7 +21,6 @@ import { useTranslation } from "react-i18next"; import { Room } from "livekit-client"; import { logger } from "matrix-js-sdk/src/logger"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; -import { Focus } from "matrix-js-sdk/src/matrixrtc/focus"; import type { IWidgetApiRequest } from "matrix-widget-api"; import { widget, ElementWidgetActions, JoinCallData } from "../widget"; @@ -32,11 +31,9 @@ import { CallEndedView } from "./CallEndedView"; import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; import { useProfile } from "../profile/useProfile"; import { findDeviceByName } from "../media-utils"; -//import { OpenIDLoader } from "../livekit/OpenIDLoader"; import { ActiveCall } from "./InCallView"; import { MuteStates, useMuteStates } from "./MuteStates"; import { useMediaDevices, MediaDevices } from "../livekit/MediaDevicesContext"; -import { LivekitFocus } from "../livekit/LivekitFocus"; import { useMatrixRTCSessionMemberships } from "../useMatrixRTCSessionMemberships"; import { enterRTCSession, leaveRTCSession } from "../rtcSessionHelpers"; import { useMatrixRTCSessionJoinState } from "../useMatrixRTCSessionJoinState"; @@ -69,21 +66,11 @@ export function GroupCallView({ hideHeader, rtcSession, }: Props) { - /*const { - state, - error, - enter, - leave, - participants, - unencryptedEventsFromUsers, - otelGroupCallMembership, - } = useGroupCall(groupCall, client);*/ - const memberships = useMatrixRTCSessionMemberships(rtcSession); const isJoined = useMatrixRTCSessionJoinState(rtcSession); - const e2eeSharedKey = useManageRoomSharedKey(groupCall.room.roomId); - const isRoomE2EE = useIsRoomE2EE(groupCall.room.roomId); + const e2eeSharedKey = useManageRoomSharedKey(rtcSession.room.roomId); + const isRoomE2EE = useIsRoomE2EE(rtcSession.room.roomId); const { t } = useTranslation(); @@ -260,22 +247,9 @@ export function GroupCallView({ const onReconnect = useCallback(() => { setLeft(false); setLeaveError(undefined); - rtcSession.joinRoomSession(); + enterRTCSession(rtcSession); }, [rtcSession]); - const focus: Focus | undefined = rtcSession - .getOldestMembership() - ?.getActiveFoci()?.[0]; - if ( - !focus || - focus.type !== "livekit" || - !(focus as LivekitFocus).livekit_alias || - !(focus as LivekitFocus).livekit_service_url - ) { - logger.error("Incompatible focus on call", focus); - return ; - } - if (e2eeEnabled && isRoomE2EE && !e2eeSharedKey) { return ( */ - // ); } else if (left) { // The call ended view is shown for two reasons: prompting guests to create @@ -351,7 +319,7 @@ export function GroupCallView({ enter()} + onEnter={() => enterRTCSession(rtcSession)} isEmbedded={isEmbedded} hideHeader={hideHeader} /> diff --git a/src/room/useActiveFocus.ts b/src/room/useActiveFocus.ts new file mode 100644 index 00000000..a62ffafd --- /dev/null +++ b/src/room/useActiveFocus.ts @@ -0,0 +1,68 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + MatrixRTCSession, + MatrixRTCSessionEvent, +} from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; +import { useCallback, useEffect, useState } from "react"; +import { deepCompare } from "matrix-js-sdk/src/utils"; + +import { LivekitFocus } from "../livekit/LivekitFocus"; + +function getActiveFocus( + rtcSession: MatrixRTCSession +): LivekitFocus | undefined { + const oldestMembership = rtcSession.getOldestMembership(); + return oldestMembership?.getActiveFoci()[0] as LivekitFocus; +} + +/** + * Gets the currently active (livekit) focus for a MatrixRTC session + * This logic is specific to livekit foci where the whole call must use one + * and the same focus. + */ +export function useActiveFocus( + rtcSession: MatrixRTCSession +): LivekitFocus | undefined { + const [activeFocus, setActiveFocus] = useState(() => + getActiveFocus(rtcSession) + ); + + const onMembershipsChanged = useCallback(() => { + const newActiveFocus = getActiveFocus(rtcSession); + + if (!deepCompare(activeFocus, newActiveFocus)) { + setActiveFocus(newActiveFocus); + } + }, [activeFocus, rtcSession]); + + useEffect(() => { + rtcSession.on( + MatrixRTCSessionEvent.MembershipsChanged, + onMembershipsChanged + ); + + return () => { + rtcSession.off( + MatrixRTCSessionEvent.MembershipsChanged, + onMembershipsChanged + ); + }; + }); + + return activeFocus; +} diff --git a/src/rtcSessionHelpers.ts b/src/rtcSessionHelpers.ts new file mode 100644 index 00000000..3d62f980 --- /dev/null +++ b/src/rtcSessionHelpers.ts @@ -0,0 +1,53 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; + +import { PosthogAnalytics } from "./analytics/PosthogAnalytics"; +import { LivekitFocus } from "./livekit/LivekitFocus"; +import { Config } from "./config/Config"; + +function makeFocus(livekitAlias: string): LivekitFocus { + const urlFromConf = Config.get().livekit!.livekit_service_url; + if (!urlFromConf) { + throw new Error("No livekit_service_url is configured!"); + } + + return { + type: "livekit", + livekit_service_url: urlFromConf, + livekit_alias: livekitAlias, + }; +} + +export function enterRTCSession(rtcSession: MatrixRTCSession) { + PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); + PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId); + + // This must be called before we start trying to join the call, as we need to + // have started tracking by the time calls start getting created. + //groupCallOTelMembership?.onJoinCall(); + + // right now we asume everything is a room-scoped call + const livekitAlias = rtcSession.room.roomId; + + rtcSession.joinRoomSession([makeFocus(livekitAlias)]); +} + +export function leaveRTCSession(rtcSession: MatrixRTCSession) { + //groupCallOTelMembership?.onLeaveCall(); + rtcSession.leaveRoomSession(); +} diff --git a/src/useMatrixRTCSessionJoinState.ts b/src/useMatrixRTCSessionJoinState.ts index 7afd6b52..c7c54563 100644 --- a/src/useMatrixRTCSessionJoinState.ts +++ b/src/useMatrixRTCSessionJoinState.ts @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { logger } from "matrix-js-sdk/src/logger"; import { MatrixRTCSession, MatrixRTCSessionEvent, @@ -26,6 +27,11 @@ export function useMatrixRTCSessionJoinState( const [isJoined, setJoined] = useState(rtcSession.isJoined()); const onJoinStateChanged = useCallback(() => { + logger.info( + `Session in room ${rtcSession.room.roomId} changed to ${ + rtcSession.isJoined() ? "joined" : "left" + }` + ); setJoined(rtcSession.isJoined()); }, [rtcSession]); diff --git a/src/useMatrixRTCSessionMemberships.ts b/src/useMatrixRTCSessionMemberships.ts index 149aa8f6..a0d5a513 100644 --- a/src/useMatrixRTCSessionMemberships.ts +++ b/src/useMatrixRTCSessionMemberships.ts @@ -14,6 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ +import { logger } from "matrix-js-sdk/src/logger"; import { CallMembership } from "matrix-js-sdk/src/matrixrtc/CallMembership"; import { MatrixRTCSession, @@ -27,6 +28,9 @@ export function useMatrixRTCSessionMemberships( const [memberships, setMemberships] = useState(rtcSession.memberships); const onMembershipsChanged = useCallback(() => { + logger.info( + `Memberships changed for call in room ${rtcSession.room.roomId} (${rtcSession.memberships.length} members)` + ); setMemberships(rtcSession.memberships); }, [rtcSession]); From 274349f2e2e50fab851766747aceac6b3207767f Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 18 Aug 2023 10:20:14 +0100 Subject: [PATCH 03/21] Use the right sfu config hook --- src/room/InCallView.tsx | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index e8e6c71c..61146d79 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -72,13 +72,13 @@ import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { E2EEConfig, useLiveKit } from "../livekit/useLiveKit"; import { useFullscreen } from "./useFullscreen"; import { useLayoutStates } from "../video-grid/Layout"; -import { useSFUConfig } from "../livekit/OpenIDLoader"; import { E2EELock } from "../E2EELock"; import { useEventEmitterThree } from "../useEvents"; import { useWakeLock } from "../useWakeLock"; import { useMergedRefs } from "../useMergedRefs"; import { MuteStates } from "./MuteStates"; import { useIsRoomE2EE } from "../e2ee/sharedKeyManagement"; +import { useOpenIDSFU } from "../livekit/openIDSFU"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // There is currently a bug in Safari our our code with cloning and sending MediaStreams @@ -91,7 +91,7 @@ export interface ActiveCallProps extends Omit { } export function ActiveCall(props: ActiveCallProps) { - const sfuConfig = useSFUConfig(); + const sfuConfig = useOpenIDSFU(props.client, props.rtcSession); const livekitRoom = useLiveKit(props.muteStates, sfuConfig, props.e2eeConfig); if (!livekitRoom) { From 7a197a2700f79ae5e8bae57087436936b92edfc9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 18 Aug 2023 14:14:24 +0100 Subject: [PATCH 04/21] Use js-sdk branch --- package.json | 2 +- yarn.lock | 26 +++++++++++++++++--------- 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/package.json b/package.json index eff3786d..5bee62c3 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "1.12.3", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#b698217445318f453e0b1086364a33113eaa85d9", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#f5f63d67f74a936f971199e76935000d2d938c21", "matrix-widget-api": "^1.3.1", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", diff --git a/yarn.lock b/yarn.lock index 2e9bfbdc..6f2b59b9 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2222,10 +2222,10 @@ "@react-hook/latest" "^1.0.3" clsx "^1.2.1" -"@matrix-org/matrix-sdk-crypto-js@^0.1.1": - version "0.1.4" - resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.4.tgz#c13c7c8c3a1d8da08e6ad195d25e5e61cc402df7" - integrity sha512-OxG84iSeR89zYLFeb+DCaFtZT+DDiIu+kTkqY8OYfhE5vpGLFX2sDVBRrAdos1IUqEoboDloDBR9+yU7hNRyog== +"@matrix-org/matrix-sdk-crypto-wasm@^1.2.1": + version "1.2.1" + resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-wasm/-/matrix-sdk-crypto-wasm-1.2.1.tgz#5b546c8a0e53b614f10b77b3b649818aed9d0db1" + integrity sha512-DCb7Q83PCQK0uav5vB3KNV/hJPlxAhT/ddar+VHz2kC39hMLKGzWYVhprpLYVcavaE/6OX+Q/xFkAoV/3QtUHQ== "@matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz": version "3.2.14" @@ -10943,19 +10943,19 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#b698217445318f453e0b1086364a33113eaa85d9": - version "26.2.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/b698217445318f453e0b1086364a33113eaa85d9" +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#f5f63d67f74a936f971199e76935000d2d938c21": + version "27.2.0" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/f5f63d67f74a936f971199e76935000d2d938c21" dependencies: "@babel/runtime" "^7.12.5" - "@matrix-org/matrix-sdk-crypto-js" "^0.1.1" + "@matrix-org/matrix-sdk-crypto-wasm" "^1.2.1" another-json "^0.2.0" bs58 "^5.0.0" content-type "^1.0.4" jwt-decode "^3.1.2" loglevel "^1.7.1" matrix-events-sdk "0.0.1" - matrix-widget-api "^1.3.1" + matrix-widget-api "^1.5.0" oidc-client-ts "^2.2.4" p-retry "4" sdp-transform "^2.14.1" @@ -10970,6 +10970,14 @@ matrix-widget-api@^1.3.1: "@types/events" "^3.0.0" events "^3.2.0" +matrix-widget-api@^1.5.0: + version "1.5.0" + resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.5.0.tgz#4ae3e46a7f2854f944ddaf8a5af63d72fba76c45" + integrity sha512-hKGfqQKK5qVMwW0Sp8l2TiuW8UuHafTvUZNSWBPghedB/rSFbVLlr0mufuEV0iq/pQ7ChW96q/WEC6Llie4SnA== + dependencies: + "@types/events" "^3.0.0" + events "^3.2.0" + md5.js@^1.3.4: version "1.3.5" resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f" From ee77820e5be143cb14d8f45d929944f48f9b67f9 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Aug 2023 09:59:02 +0100 Subject: [PATCH 05/21] Use latest js-sdk branch --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 5bee62c3..77d10cfa 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "1.12.3", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#f5f63d67f74a936f971199e76935000d2d938c21", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#0d46aeb5a4d44685885adc502f46688a4bed65b6", "matrix-widget-api": "^1.3.1", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", diff --git a/yarn.lock b/yarn.lock index 6f2b59b9..4c2c0dac 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10943,9 +10943,9 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#f5f63d67f74a936f971199e76935000d2d938c21": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#0d46aeb5a4d44685885adc502f46688a4bed65b6": version "27.2.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/f5f63d67f74a936f971199e76935000d2d938c21" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/0d46aeb5a4d44685885adc502f46688a4bed65b6" dependencies: "@babel/runtime" "^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm" "^1.2.1" From 095753c6a03c4ca223bdf717544d3d126d80ecab Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Aug 2023 10:03:05 +0100 Subject: [PATCH 06/21] i18n --- public/locales/en-GB/app.json | 5 ----- 1 file changed, 5 deletions(-) diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 076c65b8..fc461337 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -32,7 +32,6 @@ "Create account": "Create account", "Debug log": "Debug log", "Debug log request": "Debug log request", - "Details": "Details", "Developer": "Developer", "Developer Settings": "Developer Settings", "Display name": "Display name", @@ -44,7 +43,6 @@ "Exit full screen": "Exit full screen", "Expose developer settings in the settings window.": "Expose developer settings in the settings window.", "Feedback": "Feedback", - "Fetching group call timed out.": "Fetching group call timed out.", "Freedom": "Freedom", "Full screen": "Full screen", "Go": "Go", @@ -53,8 +51,6 @@ "How did it go?": "How did it go?", "If you are experiencing issues or simply would like to provide some feedback, please send us a short description below.": "If you are experiencing issues or simply would like to provide some feedback, please send us a short description below.", "Include debug logs": "Include debug logs", - "Incompatible versions": "Incompatible versions", - "Incompatible versions!": "Incompatible versions!", "Inspector": "Inspector", "Invite": "Invite", "Invite people": "Invite people", @@ -73,7 +69,6 @@ "No": "No", "Not now, return to home screen": "Not now, return to home screen", "Not registered yet? <2>Create an account": "Not registered yet? <2>Create an account", - "Other users are trying to join this call from incompatible versions. These users should ensure that they have refreshed their browsers:<1>{userLis}": "Other users are trying to join this call from incompatible versions. These users should ensure that they have refreshed their browsers:<1>{userLis}", "Password": "Password", "Passwords must match": "Passwords must match", "Profile": "Profile", From 918736e758082720d0a7c7585d8f261414d160fe Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Aug 2023 10:06:08 +0100 Subject: [PATCH 07/21] Removed now unused useGroupCall & OpenIDLoader --- src/livekit/OpenIDLoader.tsx | 92 ----- src/room/useGroupCall.ts | 644 ----------------------------------- 2 files changed, 736 deletions(-) delete mode 100644 src/livekit/OpenIDLoader.tsx delete mode 100644 src/room/useGroupCall.ts diff --git a/src/livekit/OpenIDLoader.tsx b/src/livekit/OpenIDLoader.tsx deleted file mode 100644 index 911a1ae7..00000000 --- a/src/livekit/OpenIDLoader.tsx +++ /dev/null @@ -1,92 +0,0 @@ -/* -Copyright 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { - ReactNode, - createContext, - useContext, - useEffect, - useState, -} from "react"; -import { logger } from "matrix-js-sdk/src/logger"; -import { GroupCall } from "matrix-js-sdk"; - -import { - OpenIDClientParts, - SFUConfig, - getSFUConfigWithOpenID, -} from "./openIDSFU"; -import { ErrorView, LoadingView } from "../FullScreenView"; - -interface Props { - client: OpenIDClientParts; - groupCall: GroupCall; - roomName: string; - children: ReactNode; -} - -const SFUConfigContext = createContext(undefined); - -export const useSFUConfig = () => useContext(SFUConfigContext); - -export function OpenIDLoader({ client, groupCall, roomName, children }: Props) { - const [state, setState] = useState< - SFUConfigLoading | SFUConfigLoaded | SFUConfigFailed - >({ kind: "loading" }); - - useEffect(() => { - (async () => { - try { - const result = await getSFUConfigWithOpenID( - client, - groupCall, - roomName - ); - setState({ kind: "loaded", sfuConfig: result }); - } catch (e) { - logger.error("Failed to fetch SFU config: ", e); - setState({ kind: "failed", error: e as Error }); - } - })(); - }, [client, groupCall, roomName]); - - switch (state.kind) { - case "loading": - return ; - case "failed": - return ; - case "loaded": - return ( - - {children} - - ); - } -} - -type SFUConfigLoading = { - kind: "loading"; -}; - -type SFUConfigLoaded = { - kind: "loaded"; - sfuConfig: SFUConfig; -}; - -type SFUConfigFailed = { - kind: "failed"; - error: Error; -}; diff --git a/src/room/useGroupCall.ts b/src/room/useGroupCall.ts deleted file mode 100644 index 4f6d3ad7..00000000 --- a/src/room/useGroupCall.ts +++ /dev/null @@ -1,644 +0,0 @@ -/* -Copyright 2022-2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { useCallback, useEffect, useReducer, useState } from "react"; -import * as Sentry from "@sentry/react"; -import { - GroupCallEvent, - GroupCallState, - GroupCall, - GroupCallErrorCode, - GroupCallUnknownDeviceError, - GroupCallError, - GroupCallStatsReportEvent, - GroupCallStatsReport, -} from "matrix-js-sdk/src/webrtc/groupCall"; -import { CallFeed, CallFeedEvent } from "matrix-js-sdk/src/webrtc/callFeed"; -import { RoomMember } from "matrix-js-sdk/src/models/room-member"; -import { useTranslation } from "react-i18next"; -import { IWidgetApiRequest } from "matrix-widget-api"; -import { MatrixClient, RoomStateEvent } from "matrix-js-sdk"; -import { - ByteSentStatsReport, - ConnectionStatsReport, - SummaryStatsReport, - CallFeedReport, -} from "matrix-js-sdk/src/webrtc/stats/statsReport"; -import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; - -import { usePageUnload } from "./usePageUnload"; -import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; -import { TranslatedError, translatedError } from "../TranslatedError"; -import { ElementWidgetActions, ScreenshareStartData, widget } from "../widget"; -import { OTelGroupCallMembership } from "../otel/OTelGroupCallMembership"; -import { ElementCallOpenTelemetry } from "../otel/otel"; -import { checkForParallelCalls } from "./checkForParallelCalls"; - -enum ConnectionState { - EstablishingCall = "establishing call", // call hasn't been established yet - WaitMedia = "wait_media", // call is set up, waiting for ICE to connect - Connected = "connected", // media is flowing -} - -export interface ParticipantInfo { - connectionState: ConnectionState; - presenter: boolean; -} - -interface UseGroupCallReturnType { - state: GroupCallState; - localCallFeed?: CallFeed; - activeSpeaker?: CallFeed; - userMediaFeeds: CallFeed[]; - microphoneMuted: boolean; - localVideoMuted: boolean; - error?: TranslatedError; - initLocalCallFeed: () => void; - enter: () => Promise; - leave: () => void; - toggleLocalVideoMuted: () => void; - toggleMicrophoneMuted: () => void; - toggleScreensharing: () => void; - setMicrophoneMuted: (muted: boolean) => void; - requestingScreenshare: boolean; - isScreensharing: boolean; - screenshareFeeds: CallFeed[]; - participants: Map>; - hasLocalParticipant: boolean; - otelGroupCallMembership?: OTelGroupCallMembership; -} - -interface State { - state: GroupCallState; - localCallFeed?: CallFeed; - activeSpeaker?: CallFeed; - userMediaFeeds: CallFeed[]; - error?: TranslatedError; - microphoneMuted: boolean; - localVideoMuted: boolean; - screenshareFeeds: CallFeed[]; - isScreensharing: boolean; - requestingScreenshare: boolean; - participants: Map>; - hasLocalParticipant: boolean; -} - -// This is a bit of a hack, but we keep the opentelemetry tracker object at the file -// level so that it doesn't pop in & out of existence as react mounts & unmounts -// components. The right solution is probably for this to live in the js-sdk and have -// the same lifetime as groupcalls themselves. -let groupCallOTelMembership: OTelGroupCallMembership | undefined; -let groupCallOTelMembershipGroupCallId: string; - -/*function getParticipants( - rtcSession: MatrixRTCSession -): Map> { - const participants = new Map>(); - - for (const membership of rtcSession.memberships) { - const participantInfoMap = new Map(); - participants.set(membership.member, participantInfoMap); - - for (const [deviceId, participant] of participantsStateMap) { - const feed = groupCall.userMediaFeeds.find( - (f) => f.userId === member.userId && f.deviceId === deviceId - ); - - let connectionState: ConnectionState; - // If we allow calls without media, we have no feeds and cannot read the connection status from them. - // @TODO: The connection state should generally not be determined by the feed. - if ( - groupCall.allowCallWithoutVideoAndAudio && - !feed && - !participant.screensharing - ) { - connectionState = ConnectionState.Connected; - } else { - connectionState = feed - ? feed.connected - ? ConnectionState.Connected - : ConnectionState.WaitMedia - : ConnectionState.EstablishingCall; - } - participantInfoMap.set(deviceId, { - connectionState, - presenter: participant.screensharing, - }); - } - } - - return participants; -}*/ - -export function useGroupCall( - rtcSession: MatrixRTCSession, - client: MatrixClient -): UseGroupCallReturnType { - const [ - { - state, - localCallFeed, - activeSpeaker, - userMediaFeeds, - error, - microphoneMuted, - localVideoMuted, - isScreensharing, - screenshareFeeds, - participants, - hasLocalParticipant, - requestingScreenshare, - }, - setState, - ] = useState({ - state: GroupCallState.LocalCallFeedUninitialized, - userMediaFeeds: [], - microphoneMuted: false, - localVideoMuted: false, - isScreensharing: false, - screenshareFeeds: [], - requestingScreenshare: false, - participants: getParticipants(rtcSession), - hasLocalParticipant: false, - }); - - if (groupCallOTelMembershipGroupCallId !== groupCall.groupCallId) { - if (groupCallOTelMembership) groupCallOTelMembership.dispose(); - - // If the user disables analytics, this will stay around until they leave the call - // so analytics will be disabled once they leave. - if (ElementCallOpenTelemetry.instance) { - groupCallOTelMembership = new OTelGroupCallMembership(groupCall, client); - groupCallOTelMembershipGroupCallId = groupCall.groupCallId; - } else { - groupCallOTelMembership = undefined; - } - } - - const [unencryptedEventsFromUsers, addUnencryptedEventUser] = useReducer( - (state: Set, newVal: string) => { - return new Set(state).add(newVal); - }, - new Set() - ); - - const updateState = useCallback( - (state: Partial) => setState((prev) => ({ ...prev, ...state })), - [setState] - ); - - const doNothingMediaActionCallback = useCallback( - (details: MediaSessionActionDetails) => {}, - [] - ); - - const leaveCall = useCallback(() => { - groupCallOTelMembership?.onLeaveCall(); - groupCall.leave(); - }, [groupCall]); - - useEffect(() => { - // disable the media action keys, otherwise audio elements get paused when - // the user presses media keys or unplugs headphones, etc. - // Note there are actions for muting / unmuting a microphone & hanging up - // which we could wire up. - const mediaActions: MediaSessionAction[] = [ - "play", - "pause", - "stop", - "nexttrack", - "previoustrack", - ]; - - for (const mediaAction of mediaActions) { - navigator.mediaSession?.setActionHandler( - mediaAction, - doNothingMediaActionCallback - ); - } - - return () => { - for (const mediaAction of mediaActions) { - navigator.mediaSession?.setActionHandler(mediaAction, null); - } - }; - }, [doNothingMediaActionCallback]); - - useEffect(() => { - function onGroupCallStateChanged() { - updateState({ - state: groupCall.state, - localCallFeed: groupCall.localCallFeed, - activeSpeaker: groupCall.activeSpeaker, - userMediaFeeds: [...groupCall.userMediaFeeds], - microphoneMuted: groupCall.isMicrophoneMuted(), - localVideoMuted: groupCall.isLocalVideoMuted(), - isScreensharing: groupCall.isScreensharing(), - screenshareFeeds: [...groupCall.screenshareFeeds], - }); - } - - const prevUserMediaFeeds = new Set(); - - function onUserMediaFeedsChanged(userMediaFeeds: CallFeed[]): void { - for (const feed of prevUserMediaFeeds) { - feed.off(CallFeedEvent.ConnectedChanged, onConnectedChanged); - } - prevUserMediaFeeds.clear(); - - for (const feed of userMediaFeeds) { - feed.on(CallFeedEvent.ConnectedChanged, onConnectedChanged); - prevUserMediaFeeds.add(feed); - } - - updateState({ - userMediaFeeds: [...userMediaFeeds], - participants: getParticipants(groupCall), - }); - } - - const prevScreenshareFeeds = new Set(); - - function onScreenshareFeedsChanged(screenshareFeeds: CallFeed[]): void { - for (const feed of prevScreenshareFeeds) { - feed.off(CallFeedEvent.ConnectedChanged, onConnectedChanged); - } - prevScreenshareFeeds.clear(); - - for (const feed of screenshareFeeds) { - feed.on(CallFeedEvent.ConnectedChanged, onConnectedChanged); - prevScreenshareFeeds.add(feed); - } - - updateState({ - screenshareFeeds: [...screenshareFeeds], - }); - } - - function onConnectedChanged(connected: boolean): void { - updateState({ - participants: getParticipants(groupCall), - }); - } - - function onActiveSpeakerChanged(activeSpeaker: CallFeed | undefined): void { - updateState({ - activeSpeaker: activeSpeaker, - }); - } - - function onLocalMuteStateChanged( - microphoneMuted: boolean, - localVideoMuted: boolean - ): void { - updateState({ - microphoneMuted, - localVideoMuted, - }); - } - - function onLocalScreenshareStateChanged( - isScreensharing: boolean, - _localScreenshareFeed?: CallFeed, - localDesktopCapturerSourceId?: string - ): void { - updateState({ - isScreensharing, - }); - } - - function onCallsChanged(): void { - updateState({ participants: getParticipants(groupCall) }); - } - - function onParticipantsChanged(): void { - updateState({ - participants: getParticipants(groupCall), - hasLocalParticipant: groupCall.hasLocalParticipant(), - }); - } - - function onError(e: GroupCallError): void { - Sentry.captureException(e); - if (e.code === GroupCallErrorCode.UnknownDevice) { - const unknownDeviceError = e as GroupCallUnknownDeviceError; - addUnencryptedEventUser(unknownDeviceError.userId); - } - } - - function onConnectionStatsReport( - report: GroupCallStatsReport - ): void { - groupCallOTelMembership?.onConnectionStatsReport(report); - } - - function onByteSentStatsReport( - report: GroupCallStatsReport - ): void { - groupCallOTelMembership?.onByteSentStatsReport(report); - } - - function onSummaryStatsReport( - report: GroupCallStatsReport - ): void { - groupCallOTelMembership?.onSummaryStatsReport(report); - } - - function onCallFeedStatsReport( - report: GroupCallStatsReport - ): void { - groupCallOTelMembership?.onCallFeedStatsReport(report); - } - - groupCall.on(GroupCallEvent.GroupCallStateChanged, onGroupCallStateChanged); - groupCall.on(GroupCallEvent.UserMediaFeedsChanged, onUserMediaFeedsChanged); - groupCall.on( - GroupCallEvent.ScreenshareFeedsChanged, - onScreenshareFeedsChanged - ); - groupCall.on(GroupCallEvent.ActiveSpeakerChanged, onActiveSpeakerChanged); - groupCall.on(GroupCallEvent.LocalMuteStateChanged, onLocalMuteStateChanged); - groupCall.on( - GroupCallEvent.LocalScreenshareStateChanged, - onLocalScreenshareStateChanged - ); - groupCall.on(GroupCallEvent.CallsChanged, onCallsChanged); - groupCall.on(GroupCallEvent.ParticipantsChanged, onParticipantsChanged); - groupCall.on(GroupCallEvent.Error, onError); - groupCall.on( - GroupCallStatsReportEvent.ConnectionStats, - onConnectionStatsReport - ); - groupCall.on( - GroupCallStatsReportEvent.ByteSentStats, - onByteSentStatsReport - ); - groupCall.on(GroupCallStatsReportEvent.SummaryStats, onSummaryStatsReport); - groupCall.on( - GroupCallStatsReportEvent.CallFeedStats, - onCallFeedStatsReport - ); - - groupCall.room.currentState.on( - RoomStateEvent.Update, - checkForParallelCalls - ); - - updateState({ - error: undefined, - state: groupCall.state, - localCallFeed: groupCall.localCallFeed, - activeSpeaker: groupCall.activeSpeaker, - userMediaFeeds: [...groupCall.userMediaFeeds], - microphoneMuted: groupCall.isMicrophoneMuted(), - localVideoMuted: groupCall.isLocalVideoMuted(), - isScreensharing: groupCall.isScreensharing(), - screenshareFeeds: [...groupCall.screenshareFeeds], - participants: getParticipants(groupCall), - hasLocalParticipant: groupCall.hasLocalParticipant(), - }); - - return () => { - groupCall.removeListener( - GroupCallEvent.GroupCallStateChanged, - onGroupCallStateChanged - ); - groupCall.removeListener( - GroupCallEvent.UserMediaFeedsChanged, - onUserMediaFeedsChanged - ); - groupCall.removeListener( - GroupCallEvent.ScreenshareFeedsChanged, - onScreenshareFeedsChanged - ); - groupCall.removeListener( - GroupCallEvent.ActiveSpeakerChanged, - onActiveSpeakerChanged - ); - groupCall.removeListener( - GroupCallEvent.LocalMuteStateChanged, - onLocalMuteStateChanged - ); - groupCall.removeListener( - GroupCallEvent.LocalScreenshareStateChanged, - onLocalScreenshareStateChanged - ); - groupCall.removeListener(GroupCallEvent.CallsChanged, onCallsChanged); - groupCall.removeListener( - GroupCallEvent.ParticipantsChanged, - onParticipantsChanged - ); - groupCall.removeListener(GroupCallEvent.Error, onError); - groupCall.removeListener( - GroupCallStatsReportEvent.ConnectionStats, - onConnectionStatsReport - ); - groupCall.removeListener( - GroupCallStatsReportEvent.ByteSentStats, - onByteSentStatsReport - ); - groupCall.removeListener( - GroupCallStatsReportEvent.SummaryStats, - onSummaryStatsReport - ); - groupCall.removeListener( - GroupCallStatsReportEvent.CallFeedStats, - onCallFeedStatsReport - ); - groupCall.room.currentState.off( - RoomStateEvent.Update, - checkForParallelCalls - ); - leaveCall(); - }; - }, [groupCall, updateState, leaveCall]); - - usePageUnload(() => { - leaveCall(); - }); - - const initLocalCallFeed = useCallback( - () => groupCall.initLocalCallFeed(), - [groupCall] - ); - - const enter = useCallback(async () => { - if ( - groupCall.state !== GroupCallState.LocalCallFeedUninitialized && - groupCall.state !== GroupCallState.LocalCallFeedInitialized - ) { - return; - } - - PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date()); - PosthogAnalytics.instance.eventCallStarted.track(groupCall.groupCallId); - - // This must be called before we start trying to join the call, as we need to - // have started tracking by the time calls start getting created. - groupCallOTelMembership?.onJoinCall(); - - await groupCall.enter().catch((error) => { - console.error(error); - updateState({ error }); - }); - }, [groupCall, updateState]); - - const toggleLocalVideoMuted = useCallback(() => { - const toggleToMute = !groupCall.isLocalVideoMuted(); - groupCall.setLocalVideoMuted(toggleToMute); - groupCallOTelMembership?.onToggleLocalVideoMuted(toggleToMute); - // TODO: These explict posthog calls should be unnecessary now with the posthog otel exporter? - PosthogAnalytics.instance.eventMuteCamera.track( - toggleToMute, - groupCall.groupCallId - ); - }, [groupCall]); - - const setMicrophoneMuted = useCallback( - (setMuted: boolean) => { - groupCall.setMicrophoneMuted(setMuted); - groupCallOTelMembership?.onSetMicrophoneMuted(setMuted); - PosthogAnalytics.instance.eventMuteMicrophone.track( - setMuted, - groupCall.groupCallId - ); - }, - [groupCall] - ); - - const toggleMicrophoneMuted = useCallback(() => { - const toggleToMute = !groupCall.isMicrophoneMuted(); - groupCallOTelMembership?.onToggleMicrophoneMuted(toggleToMute); - setMicrophoneMuted(toggleToMute); - }, [groupCall, setMicrophoneMuted]); - - const toggleScreensharing = useCallback(async () => { - groupCallOTelMembership?.onToggleScreensharing(!groupCall.isScreensharing); - - if (!groupCall.isScreensharing()) { - // toggling on - updateState({ requestingScreenshare: true }); - - try { - await groupCall.setScreensharingEnabled(true, { - audio: true, - throwOnFail: true, - }); - updateState({ requestingScreenshare: false }); - } catch (e) { - // this will fail in Electron because getDisplayMedia just throws a permission - // error, so if we have a widget API, try requesting via that. - if (widget) { - const reply = await widget.api.transport.send( - ElementWidgetActions.ScreenshareRequest, - {} - ); - if (!reply.pending) { - updateState({ requestingScreenshare: false }); - } - } - } - } else { - // toggling off - groupCall.setScreensharingEnabled(false); - } - }, [groupCall, updateState]); - - const onScreenshareStart = useCallback( - async (ev: CustomEvent) => { - updateState({ requestingScreenshare: false }); - - const data = ev.detail.data as unknown as ScreenshareStartData; - - await groupCall.setScreensharingEnabled(true, { - desktopCapturerSourceId: data.desktopCapturerSourceId as string, - audio: !data.desktopCapturerSourceId, - }); - await widget?.api.transport.reply(ev.detail, {}); - }, - [groupCall, updateState] - ); - - const onScreenshareStop = useCallback( - async (ev: CustomEvent) => { - updateState({ requestingScreenshare: false }); - await groupCall.setScreensharingEnabled(false); - await widget?.api.transport.reply(ev.detail, {}); - }, - [groupCall, updateState] - ); - - useEffect(() => { - if (widget) { - widget.lazyActions.on( - ElementWidgetActions.ScreenshareStart, - onScreenshareStart - ); - widget.lazyActions.on( - ElementWidgetActions.ScreenshareStop, - onScreenshareStop - ); - - return () => { - widget?.lazyActions.off( - ElementWidgetActions.ScreenshareStart, - onScreenshareStart - ); - widget?.lazyActions.off( - ElementWidgetActions.ScreenshareStop, - onScreenshareStop - ); - }; - } - }, [onScreenshareStart, onScreenshareStop]); - - const { t } = useTranslation(); - - useEffect(() => { - if (window.RTCPeerConnection === undefined) { - const error = translatedError( - "WebRTC is not supported or is being blocked in this browser.", - t - ); - console.error(error); - updateState({ error }); - } - }, [t, updateState]); - - return { - state, - localCallFeed, - activeSpeaker, - userMediaFeeds, - microphoneMuted, - localVideoMuted, - error, - initLocalCallFeed, - enter, - leave: leaveCall, - toggleLocalVideoMuted, - toggleMicrophoneMuted, - toggleScreensharing, - setMicrophoneMuted, - requestingScreenshare, - isScreensharing, - screenshareFeeds, - participants, - hasLocalParticipant, - unencryptedEventsFromUsers, - otelGroupCallMembership: groupCallOTelMembership, - }; -} From c516207199dd6e4d2430b38b670ab1c9efb612d4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Aug 2023 10:12:09 +0100 Subject: [PATCH 08/21] Remove matrix-widget-api as a dep & remove livekit stuff from widget init We'll always have matrix-widget-api as a dep through js-sdk so also specifyin it ourselves just means we'll end up using a different version when the js-sdk upgrade their copy and get wierd errors. We could add a peerDependency if we really felt the need? --- package.json | 1 - src/widget.ts | 16 ---------------- yarn.lock | 8 -------- 3 files changed, 25 deletions(-) diff --git a/package.json b/package.json index 77d10cfa..f592e9ee 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,6 @@ "livekit-client": "1.12.3", "lodash": "^4.17.21", "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#0d46aeb5a4d44685885adc502f46688a4bed65b6", - "matrix-widget-api": "^1.3.1", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", "pako": "^2.0.4", diff --git a/src/widget.ts b/src/widget.ts index de03d78e..324dbe4a 100644 --- a/src/widget.ts +++ b/src/widget.ts @@ -157,15 +157,6 @@ export const widget: WidgetHelpers | null = (() => { timelineSupport: true, useE2eForGroupCall: e2eEnabled, fallbackICEServerAllowed: allowIceFallback, - // XXX: The client expects the list of foci in its constructor, but we don't - // know this until we fetch the config file. However, we can't wait to construct - // the client object or we'll miss the 'capabilities' request from the host app. - // As of writing this, I have made the embedded widget client send the 'contentLoaded' - // message so that we can use the widget API in less racy mode, but we need to change - // element-web to use waitForIFrameLoad=false. Once that change has rolled out, - // we can just start the client after we've fetched the config. - livekitServiceURL: undefined, - useLivekitForGroupCalls: true, } ); @@ -174,13 +165,6 @@ export const widget: WidgetHelpers | null = (() => { // wait for the config file to be ready (we load very early on so it might not // be otherwise) await Config.init(); - const livekit = Config.get().livekit; - const focus = livekit?.livekit_service_url; - // Now we've fetched the config, be evil and use the getter to inject the focus - // into the client (see above XXX). - if (focus) { - client.setLivekitServiceURL(livekit.livekit_service_url); - } await client.startClient(); resolve(client); })(); diff --git a/yarn.lock b/yarn.lock index 4c2c0dac..2e336c79 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10962,14 +10962,6 @@ matrix-events-sdk@0.0.1: unhomoglyph "^1.0.6" uuid "9" -matrix-widget-api@^1.3.1: - version "1.4.0" - resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.4.0.tgz#e426ec16a013897f3a4a9c2bff423f54ab0ba745" - integrity sha512-dw0dRylGQzDUoiaY/g5xx1tBbS7aoov31PRtFMAvG58/4uerYllV9Gfou7w+I1aglwB6hihTREzKltVjARWV6A== - dependencies: - "@types/events" "^3.0.0" - events "^3.2.0" - matrix-widget-api@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.5.0.tgz#4ae3e46a7f2854f944ddaf8a5af63d72fba76c45" From af5e679089033cfffcdc31120a0d6b1c8285a019 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Aug 2023 10:15:29 +0100 Subject: [PATCH 09/21] i18n --- public/locales/en-GB/app.json | 1 - 1 file changed, 1 deletion(-) diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index fc461337..30c770dd 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -114,7 +114,6 @@ "Waiting for other participants…": "Waiting for other participants…", "Walkie-talkie call": "Walkie-talkie call", "Walkie-talkie call name": "Walkie-talkie call name", - "WebRTC is not supported or is being blocked in this browser.": "WebRTC is not supported or is being blocked in this browser.", "Yes, join call": "Yes, join call", "You were disconnected from the call": "You were disconnected from the call", "Your feedback": "Your feedback", From 1955765cfdabee709d8c07d824a1a12d54965f8e Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Aug 2023 14:03:38 +0100 Subject: [PATCH 10/21] Bump js-sdk --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index f592e9ee..bf6220df 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "1.12.3", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#0d46aeb5a4d44685885adc502f46688a4bed65b6", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#c8ae665eb07079d9f0d568aa1316e792062b5439", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", "pako": "^2.0.4", diff --git a/yarn.lock b/yarn.lock index 2e336c79..8cce81d8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10943,9 +10943,9 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#0d46aeb5a4d44685885adc502f46688a4bed65b6": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#c8ae665eb07079d9f0d568aa1316e792062b5439": version "27.2.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/0d46aeb5a4d44685885adc502f46688a4bed65b6" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/c8ae665eb07079d9f0d568aa1316e792062b5439" dependencies: "@babel/runtime" "^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm" "^1.2.1" From dc80a7e35072aaf55842605f35db24f87659488a Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 23 Aug 2023 15:04:41 +0100 Subject: [PATCH 11/21] Log ghosts and remove unsused code --- src/room/InCallView.tsx | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 61146d79..7e6aafd7 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -467,16 +467,6 @@ function useParticipantTiles( memberships.map((m) => [`${m.member.userId}:${m.deviceId}`, m.member]) ); - // The IDs of the participants who published membership event to the room (i.e. are present from Matrix perspective). - /*const matrixParticipants: Map = new Map( - [...participants.entries()].flatMap(([user, devicesMap]) => { - return [...devicesMap.keys()].map((deviceId) => [ - `${user.userId}:${deviceId}`, - user, - ]); - }) - );*/ - const hasPresenter = sfuParticipants.find((p) => p.isScreenShareEnabled) !== undefined; let allGhosts = true; @@ -493,6 +483,11 @@ function useParticipantTiles( const id = sfuParticipant.identity; const member = matrixParticipants.get(id); + if (member === undefined) { + logger.warn( + `Ruh, roh! No matrix member found for SFU participant ${id}: creating g-g-g-ghost!` + ); + } allGhosts &&= member === undefined; const userMediaTile = { From e4e35eecfd5bde60d8d4be0dcd2a21eb1bcb2f91 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 25 Aug 2023 13:19:23 +0100 Subject: [PATCH 12/21] Use Matrix room membership to look up room member Rather than the matrixRTC memberships. We're essentially trusting LiveKit's view of weho is connected here, so we may as well include the real names of anyone we don't think is a matrixRTC participant, for whatever reason. --- src/room/InCallView.tsx | 37 +++++++++++++++++++++++++++---------- 1 file changed, 27 insertions(+), 10 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 7e6aafd7..e5f56668 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -27,6 +27,7 @@ import classNames from "classnames"; import { DisconnectReason, Room, RoomEvent, Track } from "livekit-client"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { RoomMember } from "matrix-js-sdk/src/models/room-member"; +import { Room as MatrixRoom } from "matrix-js-sdk/src/models/room"; import { Ref, useCallback, useEffect, useMemo, useRef } from "react"; import { useTranslation } from "react-i18next"; import useMeasure from "react-use-measure"; @@ -241,7 +242,7 @@ export function InCallView({ const reducedControls = boundsValid && bounds.width <= 400; const noControls = reducedControls && bounds.height <= 400; - const items = useParticipantTiles(livekitRoom, memberships); + const items = useParticipantTiles(livekitRoom, rtcSession.room); const { fullscreenItem, toggleFullscreen, exitFullscreen } = useFullscreen(items); @@ -454,19 +455,33 @@ export function InCallView({ ); } +function findMatrixMember( + room: MatrixRoom, + id: string +): RoomMember | undefined { + const parts = id.split(":"); + if (parts.length < 2) { + logger.warn( + "Livekit participants ID doesn't look like a userId:deviceId combination" + ); + return undefined; + } + + parts.pop(); + const userId = parts.join(":"); + + return room.getMember(userId) ?? undefined; +} + function useParticipantTiles( livekitRoom: Room, - memberships: CallMembership[] + matrixRoom: MatrixRoom ): TileDescriptor[] { const sfuParticipants = useParticipants({ room: livekitRoom, }); const items = useMemo(() => { - const matrixParticipants: Map = new Map( - memberships.map((m) => [`${m.member.userId}:${m.deviceId}`, m.member]) - ); - const hasPresenter = sfuParticipants.find((p) => p.isScreenShareEnabled) !== undefined; let allGhosts = true; @@ -482,10 +497,12 @@ function useParticipantTiles( : false; const id = sfuParticipant.identity; - const member = matrixParticipants.get(id); - if (member === undefined) { + const member = findMatrixMember(matrixRoom, id); + // We always start with a local participant wit the empty string as their ID before we're + // connected, this is fine and we'll be in "all ghosts" mode. + if (id !== "" && member === undefined) { logger.warn( - `Ruh, roh! No matrix member found for SFU participant ${id}: creating g-g-g-ghost!` + `Ruh, roh! No matrix member found for SFU participant '${id}': creating g-g-g-ghost!` ); } allGhosts &&= member === undefined; @@ -539,7 +556,7 @@ function useParticipantTiles( // If every item is a ghost, that probably means we're still connecting and // shouldn't bother showing anything yet return allGhosts ? [] : tiles; - }, [memberships, sfuParticipants]); + }, [matrixRoom, sfuParticipants]); return items; } From 992e6aa2a324fb07e5ce2d314f89ea3cb705181f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Mon, 28 Aug 2023 17:01:58 +0200 Subject: [PATCH 13/21] Update js-sdk MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index bf6220df..7a0ec680 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "1.12.3", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#c8ae665eb07079d9f0d568aa1316e792062b5439", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#1a0718fc66d7f5684a1763c5fee9b9f5d0dafbb8", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", "pako": "^2.0.4", From 4cd274b91eb6cf165926a5198d9178f85945de6a Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 29 Aug 2023 12:44:30 +0100 Subject: [PATCH 14/21] Manually disconnect & reconnect the livekit call if our focus changes Without breaking the 'disconnected' screen --- src/livekit/openIDSFU.ts | 7 +++ src/livekit/useECConnectionState.ts | 90 +++++++++++++++++++++++++++++ src/livekit/useLiveKit.ts | 20 +++++-- src/room/InCallView.tsx | 45 +++++++-------- 4 files changed, 135 insertions(+), 27 deletions(-) create mode 100644 src/livekit/useECConnectionState.ts diff --git a/src/livekit/openIDSFU.ts b/src/livekit/openIDSFU.ts index c122a844..d10f56fb 100644 --- a/src/livekit/openIDSFU.ts +++ b/src/livekit/openIDSFU.ts @@ -27,6 +27,13 @@ export interface SFUConfig { jwt: string; } +export function sfuConfigEquals(a?: SFUConfig, b?: SFUConfig): boolean { + if (a === undefined && b === undefined) return true; + if (a === undefined || b === undefined) return false; + + return a.jwt === b.jwt && a.url === b.url; +} + // The bits we need from MatrixClient export type OpenIDClientParts = Pick< MatrixClient, diff --git a/src/livekit/useECConnectionState.ts b/src/livekit/useECConnectionState.ts new file mode 100644 index 00000000..2e8bb430 --- /dev/null +++ b/src/livekit/useECConnectionState.ts @@ -0,0 +1,90 @@ +/* +Copyright 2023 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { ConnectionState, Room, RoomEvent } from "livekit-client"; +import { useCallback, useEffect, useRef, useState } from "react"; +import { logger } from "matrix-js-sdk/src/logger"; + +import { SFUConfig, sfuConfigEquals } from "./openIDSFU"; + +/* + * Additional values for states that a call can be in, beyond what livekit + * provides in ConnectionState. Also reconnects the call if the SFU Config + * changes. + */ +export enum ECAddonConnectionState { + // We are switching from one focus to another (or between livekit room aliases on the same focus) + ECSwitchingFocus = "ec_switching_focus", + // The call has just been initialised and is waiting for credentials to arrive before attempting + // to connect. This distinguishes from the 'Disconected' state which is now just for when livekit + // gives up on connectivity and we consider the call to have failed. + ECWaiting = "ec_waiting", +} + +export type ECConnectionState = ConnectionState | ECAddonConnectionState; + +export function useECConnectionState( + livekitRoom?: Room, + sfuConfig?: SFUConfig +): ECConnectionState { + const [connState, setConnState] = useState( + sfuConfig && livekitRoom + ? livekitRoom.state + : ECAddonConnectionState.ECWaiting + ); + + const [isSwitchingFocus, setSwitchingFocus] = useState(false); + + const onConnStateChanged = useCallback((state: ConnectionState) => { + if (state == ConnectionState.Connected) setSwitchingFocus(false); + setConnState(state); + }, []); + + useEffect(() => { + const oldRoom = livekitRoom; + + if (livekitRoom) { + livekitRoom.on(RoomEvent.ConnectionStateChanged, onConnStateChanged); + } + + return () => { + if (oldRoom) + oldRoom.off(RoomEvent.ConnectionStateChanged, onConnStateChanged); + }; + }, [livekitRoom, onConnStateChanged]); + + const currentSFUConfig = useRef(Object.assign({}, sfuConfig)); + + useEffect(() => { + if ( + sfuConfig && + currentSFUConfig.current && + !sfuConfigEquals(currentSFUConfig.current, sfuConfig) + ) { + logger.info("JWT changed!"); + + (async () => { + setSwitchingFocus(true); + await livekitRoom?.disconnect(); + await livekitRoom?.connect(sfuConfig.url, sfuConfig.jwt); + })(); + } + + currentSFUConfig.current = Object.assign({}, sfuConfig); + }, [sfuConfig, livekitRoom]); + + return isSwitchingFocus ? ECAddonConnectionState.ECSwitchingFocus : connState; +} diff --git a/src/livekit/useLiveKit.ts b/src/livekit/useLiveKit.ts index 14acfff3..1f5704b3 100644 --- a/src/livekit/useLiveKit.ts +++ b/src/livekit/useLiveKit.ts @@ -22,7 +22,7 @@ import { RoomOptions, setLogLevel, } from "livekit-client"; -import { useConnectionState, useLiveKitRoom } from "@livekit/components-react"; +import { useLiveKitRoom } from "@livekit/components-react"; import { useEffect, useMemo, useRef } from "react"; import E2EEWorker from "livekit-client/e2ee-worker?worker"; import { logger } from "matrix-js-sdk/src/logger"; @@ -35,6 +35,10 @@ import { MediaDevices, useMediaDevices, } from "./MediaDevicesContext"; +import { + ECConnectionState, + useECConnectionState, +} from "./useECConnectionState"; export type E2EEConfig = { sharedKey: string; @@ -42,11 +46,16 @@ export type E2EEConfig = { setLogLevel("debug"); +interface UseLivekitResult { + livekitRoom?: Room; + connState: ECConnectionState; +} + export function useLiveKit( muteStates: MuteStates, sfuConfig?: SFUConfig, e2eeConfig?: E2EEConfig -): Room | undefined { +): UseLivekitResult { const e2eeOptions = useMemo(() => { if (!e2eeConfig?.sharedKey) return undefined; @@ -101,7 +110,7 @@ export function useLiveKit( room: roomWithoutProps, }); - const connectionState = useConnectionState(roomWithoutProps); + const connectionState = useECConnectionState(room, sfuConfig); useEffect(() => { // Sync the requested mute states with LiveKit's mute states. We do it this @@ -149,5 +158,8 @@ export function useLiveKit( } }, [room, devices, connectionState]); - return room; + return { + connState: connectionState, + livekitRoom: room, + }; } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index e5f56668..76cb7c07 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -24,7 +24,7 @@ import { } from "@livekit/components-react"; import { usePreventScroll } from "@react-aria/overlays"; import classNames from "classnames"; -import { DisconnectReason, Room, RoomEvent, Track } from "livekit-client"; +import { Room, Track, ConnectionState } from "livekit-client"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { RoomMember } from "matrix-js-sdk/src/models/room-member"; import { Room as MatrixRoom } from "matrix-js-sdk/src/models/room"; @@ -34,7 +34,6 @@ import useMeasure from "react-use-measure"; import { OverlayTriggerState } from "@react-stately/overlays"; import { JoinRule } from "matrix-js-sdk/src/@types/partials"; import { logger } from "matrix-js-sdk/src/logger"; -import { RoomEventCallbacks } from "livekit-client/dist/src/room/Room"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import { CallMembership } from "matrix-js-sdk/src/matrixrtc/CallMembership"; @@ -74,12 +73,12 @@ import { E2EEConfig, useLiveKit } from "../livekit/useLiveKit"; import { useFullscreen } from "./useFullscreen"; import { useLayoutStates } from "../video-grid/Layout"; import { E2EELock } from "../E2EELock"; -import { useEventEmitterThree } from "../useEvents"; import { useWakeLock } from "../useWakeLock"; import { useMergedRefs } from "../useMergedRefs"; import { MuteStates } from "./MuteStates"; import { useIsRoomE2EE } from "../e2ee/sharedKeyManagement"; import { useOpenIDSFU } from "../livekit/openIDSFU"; +import { ECConnectionState } from "../livekit/useECConnectionState"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // There is currently a bug in Safari our our code with cloning and sending MediaStreams @@ -87,13 +86,18 @@ const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // For now we can disable screensharing in Safari. const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); -export interface ActiveCallProps extends Omit { +export interface ActiveCallProps + extends Omit { e2eeConfig?: E2EEConfig; } export function ActiveCall(props: ActiveCallProps) { const sfuConfig = useOpenIDSFU(props.client, props.rtcSession); - const livekitRoom = useLiveKit(props.muteStates, sfuConfig, props.e2eeConfig); + const { livekitRoom, connState } = useLiveKit( + props.muteStates, + sfuConfig, + props.e2eeConfig + ); if (!livekitRoom) { return null; @@ -105,7 +109,7 @@ export function ActiveCall(props: ActiveCallProps) { return ( - + ); } @@ -119,6 +123,7 @@ export interface InCallViewProps { onLeave: (error?: Error) => void; hideHeader: boolean; otelGroupCallMembership?: OTelGroupCallMembership; + connState: ECConnectionState; } export function InCallView({ @@ -130,11 +135,20 @@ export function InCallView({ onLeave, hideHeader, otelGroupCallMembership, + connState, }: InCallViewProps) { const { t } = useTranslation(); usePreventScroll(); useWakeLock(); + useEffect(() => { + if (connState === ConnectionState.Disconnected) { + // annoyingly we don't get the disconnection reason this way, + // only by listening for the emitted event + onLeave(new Error("Disconnected from call server")); + } + }, [connState, onLeave]); + const isRoomE2EE = useIsRoomE2EE(rtcSession.room.roomId); const containerRef1 = useRef(null); @@ -182,27 +196,10 @@ export function InCallView({ async (muted) => await localParticipant.setMicrophoneEnabled(!muted) ); - const onDisconnected = useCallback( - (reason?: DisconnectReason) => { - PosthogAnalytics.instance.eventCallDisconnected.track(reason); - logger.info("Disconnected from livekit call with reason ", reason); - onLeave( - new Error("Disconnected from LiveKit call with reason " + reason) - ); - }, - [onLeave] - ); - const onLeavePress = useCallback(() => { onLeave(); }, [onLeave]); - useEventEmitterThree( - livekitRoom, - RoomEvent.Disconnected, - onDisconnected - ); - useEffect(() => { widget?.api.transport.send( layout === "freedom" @@ -459,6 +456,8 @@ function findMatrixMember( room: MatrixRoom, id: string ): RoomMember | undefined { + if (!id) return undefined; + const parts = id.split(":"); if (parts.length < 2) { logger.warn( From 192c3295c9593472853d6fa74dfe7676702118c4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 29 Aug 2023 14:19:08 +0100 Subject: [PATCH 15/21] Update js-sdk --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 7a0ec680..99e6588c 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "1.12.3", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#1a0718fc66d7f5684a1763c5fee9b9f5d0dafbb8", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#c444e374407cee40643438e01270e1e6e2835e82", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", "pako": "^2.0.4", diff --git a/yarn.lock b/yarn.lock index 8cce81d8..ffca4a3c 100644 --- a/yarn.lock +++ b/yarn.lock @@ -10943,9 +10943,9 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#c8ae665eb07079d9f0d568aa1316e792062b5439": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#c444e374407cee40643438e01270e1e6e2835e82": version "27.2.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/c8ae665eb07079d9f0d568aa1316e792062b5439" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/c444e374407cee40643438e01270e1e6e2835e82" dependencies: "@babel/runtime" "^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm" "^1.2.1" From b256755a0d98ee1d0d931fc8060fdcb397347212 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 31 Aug 2023 09:44:23 +0100 Subject: [PATCH 16/21] Don't treat empty object as a valid sfu config This was causing an extra reconnect cycle when the call was first joined because it thought the previous SFU config was valid. This was probably causing some client to fail to connect at all. --- src/livekit/useECConnectionState.ts | 18 ++++++++++++++---- 1 file changed, 14 insertions(+), 4 deletions(-) diff --git a/src/livekit/useECConnectionState.ts b/src/livekit/useECConnectionState.ts index 2e8bb430..01adb64d 100644 --- a/src/livekit/useECConnectionState.ts +++ b/src/livekit/useECConnectionState.ts @@ -36,6 +36,12 @@ export enum ECAddonConnectionState { export type ECConnectionState = ConnectionState | ECAddonConnectionState; +// This is mostly necessary because an empty useRef is an empty object +// which is truthy, so we can't just use Boolean(currentSFUConfig.current) +function sfuConfigValid(sfuConfig?: SFUConfig): boolean { + return Boolean(sfuConfig?.url) && Boolean(sfuConfig?.jwt); +} + export function useECConnectionState( livekitRoom?: Room, sfuConfig?: SFUConfig @@ -68,18 +74,22 @@ export function useECConnectionState( const currentSFUConfig = useRef(Object.assign({}, sfuConfig)); + // Id we are transitioning from a valid config to another valid one, we need + // to explicitly switch focus useEffect(() => { if ( - sfuConfig && - currentSFUConfig.current && + sfuConfigValid(sfuConfig) && + sfuConfigValid(currentSFUConfig.current) && !sfuConfigEquals(currentSFUConfig.current, sfuConfig) ) { - logger.info("JWT changed!"); + logger.info( + `SFU config changed! URL was ${currentSFUConfig.current?.url} now ${sfuConfig?.url}` + ); (async () => { setSwitchingFocus(true); await livekitRoom?.disconnect(); - await livekitRoom?.connect(sfuConfig.url, sfuConfig.jwt); + await livekitRoom?.connect(sfuConfig!.url, sfuConfig!.jwt); })(); } From 0535a35a1cc968f45fa044355067e4317550c894 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 31 Aug 2023 14:36:54 +0100 Subject: [PATCH 17/21] Remove unused memberships prop --- src/room/GroupCallView.tsx | 1 - src/room/InCallView.tsx | 3 --- 2 files changed, 4 deletions(-) diff --git a/src/room/GroupCallView.tsx b/src/room/GroupCallView.tsx index 57182b04..1a449e9d 100644 --- a/src/room/GroupCallView.tsx +++ b/src/room/GroupCallView.tsx @@ -271,7 +271,6 @@ export function GroupCallView({ void; hideHeader: boolean; otelGroupCallMembership?: OTelGroupCallMembership; @@ -131,7 +129,6 @@ export function InCallView({ rtcSession, livekitRoom, muteStates, - memberships, onLeave, hideHeader, otelGroupCallMembership, From 981a29ebf78a95fa73e5caf8e36435f27238bde6 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 31 Aug 2023 14:38:31 +0100 Subject: [PATCH 18/21] yarn --- yarn.lock | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/yarn.lock b/yarn.lock index b2f396ee..fb5a53a8 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11057,6 +11057,14 @@ matrix-events-sdk@0.0.1: unhomoglyph "^1.0.6" uuid "9" +matrix-widget-api@^1.3.1: + version "1.6.0" + resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.6.0.tgz#f0075411edffc6de339580ade7e6e6e6edb01af4" + integrity sha512-VXIJyAZ/WnBmT4C7ePqevgMYGneKMCP/0JuCOqntSsaNlCRHJvwvTxmqUU+ufOpzIF5gYNyIrAjbgrEbK3iqJQ== + dependencies: + "@types/events" "^3.0.0" + events "^3.2.0" + matrix-widget-api@^1.5.0: version "1.5.0" resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.5.0.tgz#4ae3e46a7f2854f944ddaf8a5af63d72fba76c45" From fe4d42f4d0de4d1959ab7e58cfabfe539f1290b3 Mon Sep 17 00:00:00 2001 From: David Baker Date: Thu, 31 Aug 2023 14:43:53 +0100 Subject: [PATCH 19/21] Update js-sdk --- package.json | 2 +- yarn.lock | 18 +++++------------- 2 files changed, 6 insertions(+), 14 deletions(-) diff --git a/package.json b/package.json index 7539a17c..6abbcba2 100644 --- a/package.json +++ b/package.json @@ -58,7 +58,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "^1.12.3", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#c444e374407cee40643438e01270e1e6e2835e82", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#fc2671d8538a3339ca5925ef42e3ba6102f7e8f2", "matrix-widget-api": "^1.3.1", "mermaid": "^9.0.0", "normalize.css": "^8.0.1", diff --git a/yarn.lock b/yarn.lock index fb5a53a8..62da65eb 100644 --- a/yarn.lock +++ b/yarn.lock @@ -11038,9 +11038,9 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#c444e374407cee40643438e01270e1e6e2835e82": - version "27.2.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/c444e374407cee40643438e01270e1e6e2835e82" +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#fc2671d8538a3339ca5925ef42e3ba6102f7e8f2": + version "28.0.0" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fc2671d8538a3339ca5925ef42e3ba6102f7e8f2" dependencies: "@babel/runtime" "^7.12.5" "@matrix-org/matrix-sdk-crypto-wasm" "^1.2.1" @@ -11050,14 +11050,14 @@ matrix-events-sdk@0.0.1: jwt-decode "^3.1.2" loglevel "^1.7.1" matrix-events-sdk "0.0.1" - matrix-widget-api "^1.5.0" + matrix-widget-api "^1.6.0" oidc-client-ts "^2.2.4" p-retry "4" sdp-transform "^2.14.1" unhomoglyph "^1.0.6" uuid "9" -matrix-widget-api@^1.3.1: +matrix-widget-api@^1.3.1, matrix-widget-api@^1.6.0: version "1.6.0" resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.6.0.tgz#f0075411edffc6de339580ade7e6e6e6edb01af4" integrity sha512-VXIJyAZ/WnBmT4C7ePqevgMYGneKMCP/0JuCOqntSsaNlCRHJvwvTxmqUU+ufOpzIF5gYNyIrAjbgrEbK3iqJQ== @@ -11065,14 +11065,6 @@ matrix-widget-api@^1.3.1: "@types/events" "^3.0.0" events "^3.2.0" -matrix-widget-api@^1.5.0: - version "1.5.0" - resolved "https://registry.yarnpkg.com/matrix-widget-api/-/matrix-widget-api-1.5.0.tgz#4ae3e46a7f2854f944ddaf8a5af63d72fba76c45" - integrity sha512-hKGfqQKK5qVMwW0Sp8l2TiuW8UuHafTvUZNSWBPghedB/rSFbVLlr0mufuEV0iq/pQ7ChW96q/WEC6Llie4SnA== - dependencies: - "@types/events" "^3.0.0" - events "^3.2.0" - md5.js@^1.3.4: version "1.3.5" resolved "https://registry.yarnpkg.com/md5.js/-/md5.js-1.3.5.tgz#b5d07b8e3216e3e27cd728d72f70d1e6a342005f" From 8f570b7893179988566501aaa2ce8cd3e472a4b7 Mon Sep 17 00:00:00 2001 From: David Baker Date: Wed, 6 Sep 2023 09:12:04 +0100 Subject: [PATCH 20/21] Include the colon in the user ID in the numebr of parts we check for --- src/room/InCallView.tsx | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index c7e17036..68e4e80a 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -456,7 +456,8 @@ function findMatrixMember( if (!id) return undefined; const parts = id.split(":"); - if (parts.length < 2) { + // must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon + if (parts.length < 3) { logger.warn( "Livekit participants ID doesn't look like a userId:deviceId combination" ); From a4d20f85a0b504f0ae0d98cb60e9bf855a7f6ea4 Mon Sep 17 00:00:00 2001 From: David Baker Date: Tue, 12 Sep 2023 16:25:59 +0100 Subject: [PATCH 21/21] Use js-sdk from latest develop --- package.json | 2 +- yarn.lock | 14 +++++++------- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/package.json b/package.json index 33395b98..acc6068d 100644 --- a/package.json +++ b/package.json @@ -59,7 +59,7 @@ "i18next-http-backend": "^1.4.4", "livekit-client": "^1.12.3", "lodash": "^4.17.21", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#fc2671d8538a3339ca5925ef42e3ba6102f7e8f2", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#6836720e1e1c2cb01d49d6e5fcfc01afc14834ca", "matrix-widget-api": "^1.3.1", "mermaid": "^9.0.0", "normalize.css": "^8.0.1", diff --git a/yarn.lock b/yarn.lock index 73954f7d..4568b3bf 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2296,10 +2296,10 @@ clsx "^2.0.0" usehooks-ts "^2.9.1" -"@matrix-org/matrix-sdk-crypto-wasm@^1.2.1": - version "1.2.1" - resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-wasm/-/matrix-sdk-crypto-wasm-1.2.1.tgz#5b546c8a0e53b614f10b77b3b649818aed9d0db1" - integrity sha512-DCb7Q83PCQK0uav5vB3KNV/hJPlxAhT/ddar+VHz2kC39hMLKGzWYVhprpLYVcavaE/6OX+Q/xFkAoV/3QtUHQ== +"@matrix-org/matrix-sdk-crypto-wasm@^1.2.3-alpha.0": + version "1.2.3-alpha.0" + resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-wasm/-/matrix-sdk-crypto-wasm-1.2.3-alpha.0.tgz#f6f93e3ee44c5f1e0e255badd26f4a7d3fb1dab8" + integrity sha512-BFLqfq/WbYZ+83r4UWLhwtBYvTp5DKTHNeWUSDBVvudFtqBvkntNAAUz+xmhmO1XkyNm+sBaElxF8IS9S8zdww== "@matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz": version "3.2.14" @@ -11632,12 +11632,12 @@ matrix-events-sdk@0.0.1: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd" integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#fc2671d8538a3339ca5925ef42e3ba6102f7e8f2": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#6836720e1e1c2cb01d49d6e5fcfc01afc14834ca": version "28.0.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/fc2671d8538a3339ca5925ef42e3ba6102f7e8f2" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/6836720e1e1c2cb01d49d6e5fcfc01afc14834ca" dependencies: "@babel/runtime" "^7.12.5" - "@matrix-org/matrix-sdk-crypto-wasm" "^1.2.1" + "@matrix-org/matrix-sdk-crypto-wasm" "^1.2.3-alpha.0" another-json "^0.2.0" bs58 "^5.0.0" content-type "^1.0.4"