rzerres/element-call
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

LiveKit Device Usage Refactoring (#1120)

Browse Source 
Signed-off-by: Timo K <toger5@hotmail.de>
Co-authored-by: Timo K <toger5@hotmail.de>
This commit is contained in:
Daniel Abramov
2023-06-16 18:07:13 +02:00
committed by GitHub
parent 4342f4b027
commit 5b4787cef6
12 changed files with 278 additions and 503 deletions
Download Patch File Download Diff File Expand all files Collapse all files

32
src/room/GroupCallView.tsx
View File

@@ -26,12 +26,12 @@ import { useGroupCall } from "./useGroupCall";
import { ErrorView, FullScreenView } from "../FullScreenView";
import { LobbyView } from "./LobbyView";
import { MatrixInfo } from "./VideoPreview";
import { InCallView } from "./InCallView";
import { ActiveCall } from "./InCallView";
import { CallEndedView } from "./CallEndedView";
import { useSentryGroupCallHandler } from "./useSentryGroupCallHandler";
import { PosthogAnalytics } from "../analytics/PosthogAnalytics";
import { useProfile } from "../profile/useProfile";
import { useLiveKit } from "../livekit/useLiveKit";
import { UserChoices } from "../livekit/useLiveKit";
declare global {
interface Window {
@@ -86,8 +86,6 @@ export function GroupCallView({
roomIdOrAlias,
};
const lkState = useLiveKit();
useEffect(() => {
if (widget && preload) {
// In preload mode, wait for a join action before entering
@@ -174,11 +172,15 @@ export function GroupCallView({
}
}, [groupCall, state, leave]);
const [userChoices, setUserChoices] = useState<UserChoices | undefined>(
undefined
);
if (error) {
return <ErrorView error={error} />;
} else if (state === GroupCallState.Entered) {
} else if (state === GroupCallState.Entered && userChoices) {
return (
<InCallView
<ActiveCall
groupCall={groupCall}
client={client}
participants={participants}
@@ -186,12 +188,7 @@ export function GroupCallView({
unencryptedEventsFromUsers={unencryptedEventsFromUsers}
hideHeader={hideHeader}
matrixInfo={matrixInfo}
mediaDevices={lkState.mediaDevices}
livekitRoom={lkState.room}
userChoices={{
videoMuted: lkState?.localMedia.video?.muted ?? true,
audioMuted: lkState?.localMedia.audio?.muted ?? true,
}}
userChoices={userChoices}
otelGroupCallMembership={otelGroupCallMembership}
/>
);
@@ -227,18 +224,17 @@ export function GroupCallView({
<h1>{t("Loading…")}</h1>
</FullScreenView>
);
} else if (lkState) {
} else {
return (
<LobbyView
matrixInfo={matrixInfo}
mediaDevices={lkState.mediaDevices}
localMedia={lkState.localMedia}
onEnter={enter}
onEnter={(choices: UserChoices) => {
setUserChoices(choices);
enter();
}}
isEmbedded={isEmbedded}
hideHeader={hideHeader}
/>
);
} else {
return null;
}
}

79
src/room/InCallView.tsx
View File

@@ -18,7 +18,6 @@ import { ResizeObserver } from "@juggle/resize-observer";
import {
useLocalParticipant,
useParticipants,
useToken,
useTracks,
} from "@livekit/components-react";
import { usePreventScroll } from "@react-aria/overlays";
@@ -58,7 +57,6 @@ import { useShowInspector } from "../settings/useSetting";
import { useModalTriggerState } from "../Modal";
import { PosthogAnalytics } from "../analytics/PosthogAnalytics";
import { useUrlParams } from "../UrlParams";
import { MediaDevicesState } from "../settings/mediaDevices";
import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
import { usePrefersReducedMotion } from "../usePrefersReducedMotion";
import { ElementWidgetActions, widget } from "../widget";
@@ -77,7 +75,8 @@ import { InviteModal } from "./InviteModal";
import { useRageshakeRequestModal } from "../settings/submit-rageshake";
import { RageshakeRequestModal } from "./RageshakeRequestModal";
import { VideoTile } from "../video-grid/VideoTile";
import { useRoom } from "./useRoom";
import { UserChoices, useLiveKit } from "../livekit/useLiveKit";
import { useMediaDevices } from "../livekit/useMediaDevices";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
// There is currently a bug in Safari our our code with cloning and sending MediaStreams
@@ -85,46 +84,43 @@ const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
// For now we can disable screensharing in Safari.
const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
const onConnectedCallback = (): void => {
console.log("connected to LiveKit room");
};
const onDisconnectedCallback = (): void => {
console.log("disconnected from LiveKit room");
};
const onErrorCallback = (err: Error): void => {
console.error("error connecting to LiveKit room", err);
};
interface ActiveCallProps extends Omit<Props, "livekitRoom"> {
userChoices: UserChoices;
}
interface LocalUserChoices {
videoMuted: boolean;
audioMuted: boolean;
export function ActiveCall(props: ActiveCallProps) {
const livekitRoom = useLiveKit(props.userChoices, {
sfuUrl: Config.get().livekit!.server_url,
jwtUrl: `${Config.get().livekit!.jwt_service_url}/token`,
roomName: props.matrixInfo.roomName,
userName: props.matrixInfo.userName,
userIdentity: `${props.client.getUserId()}:${props.client.getDeviceId()}`,
});
return livekitRoom && <InCallView {...props} livekitRoom={livekitRoom} />;
}
interface Props {
client: MatrixClient;
groupCall: GroupCall;
livekitRoom: Room;
participants: Map<RoomMember, Map<string, ParticipantInfo>>;
onLeave: () => void;
unencryptedEventsFromUsers: Set<string>;
hideHeader: boolean;
matrixInfo: MatrixInfo;
mediaDevices: MediaDevicesState;
livekitRoom: Room;
userChoices: LocalUserChoices;
otelGroupCallMembership: OTelGroupCallMembership;
}
export function InCallView({
client,
groupCall,
livekitRoom,
participants,
onLeave,
unencryptedEventsFromUsers,
hideHeader,
matrixInfo,
mediaDevices,
livekitRoom,
userChoices,
otelGroupCallMembership,
}: Props) {
const { t } = useTranslation();
@@ -142,41 +138,8 @@ export function InCallView({
[containerRef1, containerRef2]
);
const userId = client.getUserId();
const deviceId = client.getDeviceId();
const options = useMemo(
() => ({
userInfo: {
name: matrixInfo.userName,
identity: `${userId}:${deviceId}`,
},
}),
[matrixInfo.userName, userId, deviceId]
);
const token = useToken(
`${Config.get().livekit.jwt_service_url}/token`,
matrixInfo.roomName,
options
);
// TODO: move the room creation into the useRoom hook and out of the useLiveKit hook.
// This would than allow to not have those 4 lines
livekitRoom.options.audioCaptureDefaults.deviceId =
mediaDevices.state.get("audioinput").selectedId;
livekitRoom.options.videoCaptureDefaults.deviceId =
mediaDevices.state.get("videoinput").selectedId;
// Uses a hook to connect to the LiveKit room (on unmount the room will be left) and publish local media tracks (default).
useRoom({
token,
serverUrl: Config.get().livekit.server_url,
room: livekitRoom,
audio: !userChoices.audioMuted,
video: !userChoices.videoMuted,
simulateParticipants: 10,
onConnected: onConnectedCallback,
onDisconnected: onDisconnectedCallback,
onError: onErrorCallback,
});
// Managed media devices state coupled with an active room.
const roomMediaDevices = useMediaDevices(livekitRoom);
const screenSharingTracks = useTracks(
[{ source: Track.Source.ScreenShare, withPlaceholder: false }],
@@ -211,6 +174,8 @@ export function InCallView({
const joinRule = useJoinRule(groupCall.room);
// This function incorrectly assumes that there is a camera and microphone, which is not always the case.
// TODO: Make sure that this module is resilient when it comes to camera/microphone availability!
useCallViewKeyboardShortcuts(
containerRef1,
toggleMicrophone,
@@ -436,7 +401,7 @@ export function InCallView({
<SettingsModal
client={client}
roomId={groupCall.room.roomId}
mediaDevices={mediaDevices}
mediaDevices={roomMediaDevices}
{...settingsModalProps}
/>
)}

18
src/room/LobbyView.tsx
View File

@@ -15,7 +15,6 @@ limitations under the License.
*/
import React from "react";
import { PressEvent } from "@react-types/shared";
import { Trans, useTranslation } from "react-i18next";
import styles from "./LobbyView.module.css";
@@ -25,15 +24,13 @@ import { getRoomUrl } from "../matrix-utils";
import { UserMenuContainer } from "../UserMenuContainer";
import { Body, Link } from "../typography/Typography";
import { useLocationNavigation } from "../useLocationNavigation";
import { LocalMediaInfo, MatrixInfo, VideoPreview } from "./VideoPreview";
import { MediaDevicesState } from "../settings/mediaDevices";
import { MatrixInfo, VideoPreview } from "./VideoPreview";
import { UserChoices } from "../livekit/useLiveKit";
interface Props {
matrixInfo: MatrixInfo;
mediaDevices: MediaDevicesState;
localMedia: LocalMediaInfo;
onEnter: (e: PressEvent) => void;
onEnter: (userChoices: UserChoices) => void;
isEmbedded: boolean;
hideHeader: boolean;
}
@@ -49,6 +46,10 @@ export function LobbyView(props: Props) {
}
}, [joinCallButtonRef]);
const [userChoices, setUserChoices] = React.useState<UserChoices | undefined>(
undefined
);
return (
<div className={styles.room}>
{!props.hideHeader && (
@@ -68,15 +69,14 @@ export function LobbyView(props: Props) {
<div className={styles.joinRoomContent}>
<VideoPreview
matrixInfo={props.matrixInfo}
mediaDevices={props.mediaDevices}
localMediaInfo={props.localMedia}
onUserChoicesChanged={setUserChoices}
/>
<Trans>
<Button
ref={joinCallButtonRef}
className={styles.copyButton}
size="lg"
onPress={props.onEnter}
onPress={() => props.onEnter(userChoices!)}
data-testid="lobby_joinCall"
>
Join call now

2
src/room/VideoPreview.module.css
View File

@@ -37,7 +37,7 @@ limitations under the License.
top: 0;
left: 0;
right: 0;
bottom: 66px;
bottom: 0;
display: flex;
justify-content: center;
align-items: center;

110
src/room/VideoPreview.tsx
View File

@@ -17,16 +17,18 @@ limitations under the License.
import React, { useCallback } from "react";
import useMeasure from "react-use-measure";
import { ResizeObserver } from "@juggle/resize-observer";
import { Track } from "livekit-client";
import { OverlayTriggerState } from "@react-stately/overlays";
import { usePreviewDevice } from "@livekit/components-react";
import { MicButton, SettingsButton, VideoButton } from "../button";
import { Avatar } from "../Avatar";
import styles from "./VideoPreview.module.css";
import { useModalTriggerState } from "../Modal";
import { SettingsModal } from "../settings/SettingsModal";
import { MediaDevicesState } from "../settings/mediaDevices";
import { useClient } from "../ClientContext";
import { useMediaDevices } from "../livekit/useMediaDevices";
import { DeviceChoices, UserChoices } from "../livekit/useLiveKit";
import { useDefaultDevices } from "../settings/useSetting";
export type MatrixInfo = {
userName: string;
@@ -35,28 +37,12 @@ export type MatrixInfo = {
roomIdOrAlias: string;
};
export type MediaInfo = {
track: Track; // TODO: Replace it by a more generic `CallFeed` type from JS SDK once we generalise the types.
muted: boolean;
toggle: () => void;
};
export type LocalMediaInfo = {
audio?: MediaInfo;
video?: MediaInfo;
};
interface Props {
matrixInfo: MatrixInfo;
mediaDevices: MediaDevicesState;
localMediaInfo: LocalMediaInfo;
onUserChoicesChanged: (choices: UserChoices) => void;
}
export function VideoPreview({
matrixInfo,
mediaDevices,
localMediaInfo,
}: Props) {
export function VideoPreview({ matrixInfo, onUserChoicesChanged }: Props) {
const { client } = useClient();
const [previewRef, previewBounds] = useMeasure({ polyfill: ResizeObserver });
@@ -75,21 +61,85 @@ export function VideoPreview({
settingsModalState.open();
}, [settingsModalState]);
// Fetch user media devices.
const mediaDevices = useMediaDevices();
// Create local media tracks.
const [videoEnabled, setVideoEnabled] = React.useState<boolean>(true);
const [audioEnabled, setAudioEnabled] = React.useState<boolean>(true);
const [videoId, audioId] = [
mediaDevices.videoIn.selectedId,
mediaDevices.audioIn.selectedId,
];
const [defaultDevices] = useDefaultDevices();
const video = usePreviewDevice(
videoEnabled,
videoId != "" ? videoId : defaultDevices.videoinput,
"videoinput"
);
const audio = usePreviewDevice(
audioEnabled,
audioId != "" ? audioId : defaultDevices.audiooutput,
"audioinput"
);
const activeVideoId = video?.selectedDevice?.deviceId;
const activeAudioId = audio?.selectedDevice?.deviceId;
React.useEffect(() => {
const createChoices = (
enabled: boolean,
deviceId?: string
): DeviceChoices | undefined => {
if (deviceId === undefined) {
return undefined;
}
return {
selectedId: deviceId,
enabled,
};
};
onUserChoicesChanged({
video: createChoices(videoEnabled, activeVideoId),
audio: createChoices(audioEnabled, activeAudioId),
});
}, [
onUserChoicesChanged,
activeVideoId,
videoEnabled,
activeAudioId,
audioEnabled,
]);
const [selectVideo, selectAudio] = [
mediaDevices.videoIn.setSelected,
mediaDevices.audioIn.setSelected,
];
React.useEffect(() => {
if (activeVideoId && activeVideoId !== "") {
selectVideo(activeVideoId);
}
if (activeAudioId && activeAudioId !== "") {
selectAudio(activeAudioId);
}
}, [selectVideo, selectAudio, activeVideoId, activeAudioId]);
const mediaElement = React.useRef(null);
React.useEffect(() => {
if (mediaElement.current) {
localMediaInfo.video?.track.attach(mediaElement.current);
video?.localTrack?.attach(mediaElement.current);
}
return () => {
localMediaInfo.video?.track.detach();
video?.localTrack?.detach();
};
}, [localMediaInfo.video?.track, mediaElement]);
}, [video?.localTrack, mediaElement]);
return (
<div className={styles.preview} ref={previewRef}>
<video ref={mediaElement} muted playsInline disablePictureInPicture />
<>
{(localMediaInfo.video?.muted ?? true) && (
{(video ? !videoEnabled : true) && (
<div className={styles.avatarContainer}>
<Avatar
size={(previewBounds.height - 66) / 2}
@@ -99,16 +149,16 @@ export function VideoPreview({
</div>
)}
<div className={styles.previewButtons}>
{localMediaInfo.audio && (
{audio.localTrack && (
<MicButton
muted={localMediaInfo.audio?.muted}
onPress={localMediaInfo.audio?.toggle}
muted={!audioEnabled}
onPress={() => setAudioEnabled(!audioEnabled)}
/>
)}
{localMediaInfo.video && (
{video.localTrack && (
<VideoButton
muted={localMediaInfo.video?.muted}
onPress={localMediaInfo.video?.toggle}
muted={!videoEnabled}
onPress={() => setVideoEnabled(!videoEnabled)}
/>
)}
<SettingsButton onPress={openSettings} />

135
src/room/useRoom.ts
View File

@@ -1,135 +0,0 @@
import * as React from "react";
import {
ConnectionState,
MediaDeviceFailure,
Room,
RoomEvent,
} from "livekit-client";
import { LiveKitRoomProps } from "@livekit/components-react/src/components/LiveKitRoom";
import { logger } from "matrix-js-sdk/src/logger";
const defaultRoomProps: Partial<LiveKitRoomProps> = {
connect: true,
audio: false,
video: false,
};
export function useRoom(props: LiveKitRoomProps) {
const {
token,
serverUrl,
options,
room: passedRoom,
connectOptions,
connect,
audio,
video,
screen,
onConnected,
onDisconnected,
onError,
onMediaDeviceFailure,
} = { ...defaultRoomProps, ...props };
if (options && passedRoom) {
logger.warn(
"when using a manually created room, the options object will be ignored. set the desired options directly when creating the room instead."
);
}
const [room, setRoom] = React.useState<Room | undefined>();
React.useEffect(() => {
setRoom(passedRoom ?? new Room(options));
}, [options, passedRoom]);
React.useEffect(() => {
if (!room) return;
const onSignalConnected = () => {
const localP = room.localParticipant;
try {
logger.debug("trying to publish local tracks");
localP.setMicrophoneEnabled(
!!audio,
typeof audio !== "boolean" ? audio : undefined
);
localP.setCameraEnabled(
!!video,
typeof video !== "boolean" ? video : undefined
);
localP.setScreenShareEnabled(
!!screen,
typeof screen !== "boolean" ? screen : undefined
);
} catch (e) {
logger.warn(e);
onError?.(e as Error);
}
};
const onMediaDeviceError = (e: Error) => {
const mediaDeviceFailure = MediaDeviceFailure.getFailure(e);
onMediaDeviceFailure?.(mediaDeviceFailure);
};
room.on(RoomEvent.SignalConnected, onSignalConnected);
room.on(RoomEvent.MediaDevicesError, onMediaDeviceError);
return () => {
room.off(RoomEvent.SignalConnected, onSignalConnected);
room.off(RoomEvent.MediaDevicesError, onMediaDeviceError);
};
}, [room, audio, video, screen, onError, onMediaDeviceFailure]);
React.useEffect(() => {
if (!room) return;
if (!token) {
logger.debug("no token yet");
return;
}
if (!serverUrl) {
logger.warn("no livekit url provided");
onError?.(Error("no livekit url provided"));
return;
}
if (connect) {
logger.debug("connecting");
room.connect(serverUrl, token, connectOptions).catch((e) => {
logger.warn(e);
onError?.(e as Error);
});
} else {
logger.debug("disconnecting because connect is false");
room.disconnect();
}
}, [connect, token, connectOptions, room, onError, serverUrl]);
React.useEffect(() => {
if (!room) return;
const connectionStateChangeListener = (state: ConnectionState) => {
switch (state) {
case ConnectionState.Disconnected:
if (onDisconnected) onDisconnected();
break;
case ConnectionState.Connected:
if (onConnected) onConnected();
break;
default:
break;
}
};
room.on(RoomEvent.ConnectionStateChanged, connectionStateChangeListener);
return () => {
room.off(RoomEvent.ConnectionStateChanged, connectionStateChangeListener);
};
}, [token, onConnected, onDisconnected, room]);
React.useEffect(() => {
if (!room) return;
return () => {
logger.info("disconnecting on onmount");
room.disconnect();
};
}, [room]);
return { room };
}