From e33fbd77d14daeec4b506da4a2d68f97409e6128 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 16 May 2024 12:32:18 -0400 Subject: [PATCH] Split local and remote user media into different classes --- src/room/InCallView.tsx | 4 +- src/state/CallViewModel.ts | 23 ++- src/state/MediaViewModel.ts | 122 ++++++++----- src/tile/GridTile.tsx | 332 ++++++++++++++++++++---------------- src/tile/SpotlightTile.tsx | 21 ++- 5 files changed, 292 insertions(+), 210 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 1ccdb2ec..f2e08e5c 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -384,7 +384,7 @@ export const InCallView: FC = subscribe( targetHeight={targetHeight} className={className} style={style} - showSpeakingIndicator={showSpeakingIndicators} + showSpeakingIndicators={showSpeakingIndicators} /> ); }, @@ -424,7 +424,7 @@ export const InCallView: FC = subscribe( targetHeight={gridBounds.height} targetWidth={gridBounds.width} key={maximisedParticipant.id} - showSpeakingIndicator={false} + showSpeakingIndicators={false} onOpenProfile={openProfile} /> ); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index e6d55b72..df3ca42a 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -61,9 +61,11 @@ import { } from "../livekit/useECConnectionState"; import { usePrevious } from "../usePrevious"; import { + LocalUserMediaViewModel, MediaViewModel, - UserMediaViewModel, + RemoteUserMediaViewModel, ScreenShareViewModel, + UserMediaViewModel, } from "./MediaViewModel"; import { finalizeValue } from "../observable-utils"; import { ObservableScope } from "./ObservableScope"; @@ -151,7 +153,10 @@ class UserMedia { participant: LocalParticipant | RemoteParticipant, callEncrypted: boolean, ) { - this.vm = new UserMediaViewModel(id, member, participant, callEncrypted); + this.vm = + participant instanceof LocalParticipant + ? new LocalUserMediaViewModel(id, member, participant, callEncrypted) + : new RemoteUserMediaViewModel(id, member, participant, callEncrypted); this.speaker = this.vm.speaking.pipeState( // Require 1 s of continuous speaking to become a speaker, and 60 s of @@ -520,7 +525,19 @@ export class CallViewModel extends ViewModel { const userMediaVm = tilesById.get(userMediaId)?.data ?? - new UserMediaViewModel(userMediaId, member, p, this.encrypted); + (p instanceof LocalParticipant + ? new LocalUserMediaViewModel( + userMediaId, + member, + p, + this.encrypted, + ) + : new RemoteUserMediaViewModel( + userMediaId, + member, + p, + this.encrypted, + )); tilesById.delete(userMediaId); const userMediaTile: TileDescriptor = { diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index f1e772da..7f7307f4 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -153,29 +153,14 @@ abstract class BaseMediaViewModel extends ViewModel { * Some participant's media. */ export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel; +export type UserMediaViewModel = + | LocalUserMediaViewModel + | RemoteUserMediaViewModel; /** * Some participant's user media. */ -export class UserMediaViewModel extends BaseMediaViewModel { - /** - * Whether the video should be mirrored. - */ - public readonly mirror = state( - this.video.pipe( - switchMap((v) => { - const track = v.publication?.track; - if (!(track instanceof LocalTrack)) return of(false); - // Watch for track restarts, because they indicate a camera switch - return fromEvent(track, TrackEvent.Restarted).pipe( - startWith(null), - // Mirror only front-facing cameras (those that face the user) - map(() => facingModeFromLocalTrack(track).facingMode === "user"), - ); - }), - ), - ); - +abstract class BaseUserMediaViewModel extends BaseMediaViewModel { /** * Whether the participant is speaking. */ @@ -186,19 +171,6 @@ export class UserMediaViewModel extends BaseMediaViewModel { ).pipe(map((p) => p.isSpeaking)), ); - private readonly _locallyMuted = new BehaviorSubject(false); - /** - * Whether we've disabled this participant's audio. - */ - public readonly locallyMuted = state(this._locallyMuted); - - private readonly _localVolume = new BehaviorSubject(1); - /** - * The volume to which we've set this participant's audio, as a scalar - * multiplier. - */ - public readonly localVolume = state(this._localVolume); - /** * Whether this participant is sending audio (i.e. is unmuted on their side). */ @@ -236,25 +208,83 @@ export class UserMediaViewModel extends BaseMediaViewModel { this.videoEnabled = state( media.pipe(map((m) => m.cameraTrack?.isMuted === false)), ); - - // Sync the local mute state and volume with LiveKit - if (!this.local) - combineLatest([this._locallyMuted, this._localVolume], (muted, volume) => - muted ? 0 : volume, - ) - .pipe(this.scope.bind()) - .subscribe((volume) => { - (this.participant as RemoteParticipant).setVolume(volume); - }); - } - - public toggleLocallyMuted(): void { - this._locallyMuted.next(!this._locallyMuted.value); } public toggleFitContain(): void { this._cropVideo.next(!this._cropVideo.value); } +} + +/** + * The local participant's user media. + */ +export class LocalUserMediaViewModel extends BaseUserMediaViewModel { + /** + * Whether the video should be mirrored. + */ + public readonly mirror = state( + this.video.pipe( + switchMap((v) => { + const track = v.publication?.track; + if (!(track instanceof LocalTrack)) return of(false); + // Watch for track restarts, because they indicate a camera switch + return fromEvent(track, TrackEvent.Restarted).pipe( + startWith(null), + // Mirror only front-facing cameras (those that face the user) + map(() => facingModeFromLocalTrack(track).facingMode === "user"), + ); + }), + ), + ); + + public constructor( + id: string, + member: RoomMember | undefined, + participant: LocalParticipant, + callEncrypted: boolean, + ) { + super(id, member, participant, callEncrypted); + } +} + +/** + * A remote participant's user media. + */ +export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { + private readonly _locallyMuted = new BehaviorSubject(false); + /** + * Whether we've disabled this participant's audio. + */ + public readonly locallyMuted = state(this._locallyMuted); + + private readonly _localVolume = new BehaviorSubject(1); + /** + * The volume to which we've set this participant's audio, as a scalar + * multiplier. + */ + public readonly localVolume = state(this._localVolume); + + public constructor( + id: string, + member: RoomMember | undefined, + participant: RemoteParticipant, + callEncrypted: boolean, + ) { + super(id, member, participant, callEncrypted); + + // Sync the local mute state and volume with LiveKit + combineLatest([this._locallyMuted, this._localVolume], (muted, volume) => + muted ? 0 : volume, + ) + .pipe(this.scope.bind()) + .subscribe((volume) => { + (this.participant as RemoteParticipant).setVolume(volume); + }); + } + + public toggleLocallyMuted(): void { + this._locallyMuted.next(!this._locallyMuted.value); + } public setLocalVolume(value: number): void { this._localVolume.next(value); diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index d88b189f..b0753aca 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -14,7 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { ComponentProps, forwardRef, useCallback, useState } from "react"; +import { + ComponentProps, + ReactNode, + forwardRef, + useCallback, + useState, +} from "react"; import { animated } from "@react-spring/web"; import classNames from "classnames"; import { useTranslation } from "react-i18next"; @@ -33,7 +39,7 @@ import { ToggleMenuItem, Menu, } from "@vector-im/compound-web"; -import { useStateObservable } from "@react-rxjs/core"; +import { useObservableEagerState } from "observable-hooks"; import styles from "./GridTile.module.css"; import { @@ -41,71 +47,92 @@ import { MediaViewModel, UserMediaViewModel, useNameData, + LocalUserMediaViewModel, + RemoteUserMediaViewModel, } from "../state/MediaViewModel"; -import { subscribe } from "../state/subscribe"; import { Slider } from "../Slider"; import { MediaView } from "./MediaView"; -interface UserMediaTileProps { - vm: UserMediaViewModel; +interface TileProps { className?: string; style?: ComponentProps["style"]; targetWidth: number; targetHeight: number; maximised: boolean; - onOpenProfile: () => void; - showSpeakingIndicator: boolean; + displayName: string; + nameTag: string; } -const UserMediaTile = subscribe( +interface MediaTileProps + extends TileProps, + Omit, "className"> { + vm: MediaViewModel; + videoEnabled: boolean; + videoFit: "contain" | "cover"; + nameTagLeadingIcon?: ReactNode; + primaryButton: ReactNode; + secondaryButton?: ReactNode; +} + +const MediaTile = forwardRef( + ({ vm, className, maximised, ...props }, ref) => { + const video = useObservableEagerState(vm.video); + const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); + + return ( + + ); + }, +); + +MediaTile.displayName = "MediaTile"; + +interface UserMediaTileProps extends TileProps { + vm: UserMediaViewModel; + showSpeakingIndicators: boolean; + menuStart?: ReactNode; + menuEnd?: ReactNode; +} + +const UserMediaTile = forwardRef( ( { vm, + showSpeakingIndicators, + menuStart, + menuEnd, className, - style, - targetWidth, - targetHeight, - maximised, - onOpenProfile, - showSpeakingIndicator, + nameTag, + ...props }, ref, ) => { const { t } = useTranslation(); - const { displayName, nameTag } = useNameData(vm); - const video = useStateObservable(vm.video); - const audioEnabled = useStateObservable(vm.audioEnabled); - const videoEnabled = useStateObservable(vm.videoEnabled); - const unencryptedWarning = useStateObservable(vm.unencryptedWarning); - const mirror = useStateObservable(vm.mirror); - const speaking = useStateObservable(vm.speaking); - const locallyMuted = useStateObservable(vm.locallyMuted); - const cropVideo = useStateObservable(vm.cropVideo); - const localVolume = useStateObservable(vm.localVolume); - const onChangeMute = useCallback(() => vm.toggleLocallyMuted(), [vm]); + const audioEnabled = useObservableEagerState(vm.audioEnabled); + const videoEnabled = useObservableEagerState(vm.videoEnabled); + const speaking = useObservableEagerState(vm.speaking); + const cropVideo = useObservableEagerState(vm.cropVideo); const onChangeFitContain = useCallback(() => vm.toggleFitContain(), [vm]); - const onSelectMute = useCallback((e: Event) => e.preventDefault(), []); const onSelectFitContain = useCallback( (e: Event) => e.preventDefault(), [], ); - const onChangeLocalVolume = useCallback( - (v: number) => vm.setLocalVolume(v), - [vm], - ); - const MicIcon = audioEnabled ? MicOnSolidIcon : MicOffSolidIcon; - const VolumeIcon = locallyMuted ? VolumeOffIcon : VolumeOnIcon; const [menuOpen, setMenuOpen] = useState(false); - const menu = vm.local ? ( + const menu = ( <> - + {menuStart} ( onChange={onChangeFitContain} onSelect={onSelectFitContain} /> - - ) : ( - <> - - - {/* TODO: Figure out how to make this slider keyboard accessible */} - - - + {menuEnd} ); const tile = ( - ( /> } nameTag={nameTag} - displayName={displayName} primaryButton={ ( {menu} } + {...props} /> ); @@ -202,35 +193,102 @@ const UserMediaTile = subscribe( UserMediaTile.displayName = "UserMediaTile"; -interface ScreenShareTileProps { +interface LocalUserMediaTileProps extends TileProps { + vm: LocalUserMediaViewModel; + onOpenProfile: () => void; + showSpeakingIndicators: boolean; +} + +const LocalUserMediaTile = forwardRef( + ({ vm, onOpenProfile, className, ...props }, ref) => { + const { t } = useTranslation(); + const mirror = useObservableEagerState(vm.mirror); + + return ( + + } + className={classNames(className, { [styles.mirror]: mirror })} + {...props} + /> + ); + }, +); + +LocalUserMediaTile.displayName = "LocalUserMediaTile"; + +interface RemoteUserMediaTileProps extends TileProps { + vm: RemoteUserMediaViewModel; + showSpeakingIndicators: boolean; +} + +const RemoteUserMediaTile = forwardRef< + HTMLDivElement, + RemoteUserMediaTileProps +>(({ vm, ...props }, ref) => { + const { t } = useTranslation(); + const locallyMuted = useObservableEagerState(vm.locallyMuted); + const localVolume = useObservableEagerState(vm.localVolume); + const onChangeMute = useCallback(() => vm.toggleLocallyMuted(), [vm]); + const onSelectMute = useCallback((e: Event) => e.preventDefault(), []); + const onChangeLocalVolume = useCallback( + (v: number) => vm.setLocalVolume(v), + [vm], + ); + + const VolumeIcon = locallyMuted ? VolumeOffIcon : VolumeOnIcon; + + return ( + + + {/* TODO: Figure out how to make this slider keyboard accessible */} + + + + + } + {...props} + /> + ); +}); + +RemoteUserMediaTile.displayName = "RemoteUserMediaTile"; + +interface ScreenShareTileProps extends TileProps { vm: ScreenShareViewModel; - className?: string; - style?: ComponentProps["style"]; - targetWidth: number; - targetHeight: number; - maximised: boolean; fullscreen: boolean; onToggleFullscreen: (itemId: string) => void; } -const ScreenShareTile = subscribe( - ( - { - vm, - className, - style, - targetWidth, - targetHeight, - maximised, - fullscreen, - onToggleFullscreen, - }, - ref, - ) => { +const ScreenShareTile = forwardRef( + ({ vm, fullscreen, onToggleFullscreen, ...props }, ref) => { const { t } = useTranslation(); - const { displayName, nameTag } = useNameData(vm); - const video = useStateObservable(vm.video); - const unencryptedWarning = useStateObservable(vm.unencryptedWarning); const onClickFullScreen = useCallback( () => onToggleFullscreen(vm.id), [onToggleFullscreen, vm], @@ -239,23 +297,10 @@ const ScreenShareTile = subscribe( const FullScreenIcon = fullscreen ? CollapseIcon : ExpandIcon; return ( - ( ) } + videoEnabled + {...props} /> ); }, @@ -277,7 +324,7 @@ const ScreenShareTile = subscribe( ScreenShareTile.displayName = "ScreenShareTile"; -interface Props { +interface GridTileProps { vm: MediaViewModel; maximised: boolean; fullscreen: boolean; @@ -287,51 +334,34 @@ interface Props { targetHeight: number; className?: string; style?: ComponentProps["style"]; - showSpeakingIndicator: boolean; + showSpeakingIndicators: boolean; } -export const GridTile = forwardRef( - ( - { - vm, - maximised, - fullscreen, - onToggleFullscreen, - onOpenProfile, - className, - style, - targetWidth, - targetHeight, - showSpeakingIndicator, - }, - ref, - ) => { - if (vm instanceof UserMediaViewModel) { +export const GridTile = forwardRef( + ({ vm, fullscreen, onToggleFullscreen, onOpenProfile, ...props }, ref) => { + const nameData = useNameData(vm); + + if (vm instanceof LocalUserMediaViewModel) { return ( - ); + } else if (vm instanceof RemoteUserMediaViewModel) { + return ; } else { return ( ); } diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 6abf0cdd..e4bb085d 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -37,8 +37,9 @@ import { MediaView } from "./MediaView"; import styles from "./SpotlightTile.module.css"; import { subscribe } from "../state/subscribe"; import { + LocalUserMediaViewModel, MediaViewModel, - UserMediaViewModel, + RemoteUserMediaViewModel, useNameData, } from "../state/MediaViewModel"; import { useInitial } from "../useInitial"; @@ -48,11 +49,11 @@ import { useReactiveState } from "../useReactiveState"; import { useLatest } from "../useLatest"; // Screen share video is always enabled -const screenShareVideoEnabled = state(of(true)); +const videoEnabledDefault = state(of(true)); // Never mirror screen share video -const screenShareMirror = state(of(false)); +const mirrorDefault = state(of(false)); // Never crop screen share video -const screenShareCropVideo = state(of(false)); +const cropVideoDefault = state(of(false)); interface SpotlightItemProps { vm: MediaViewModel; @@ -72,15 +73,19 @@ const SpotlightItem = subscribe( const { displayName, nameTag } = useNameData(vm); const video = useStateObservable(vm.video); const videoEnabled = useStateObservable( - vm instanceof UserMediaViewModel + vm instanceof LocalUserMediaViewModel || + vm instanceof RemoteUserMediaViewModel ? vm.videoEnabled - : screenShareVideoEnabled, + : videoEnabledDefault, ); const mirror = useStateObservable( - vm instanceof UserMediaViewModel ? vm.mirror : screenShareMirror, + vm instanceof LocalUserMediaViewModel ? vm.mirror : mirrorDefault, ); const cropVideo = useStateObservable( - vm instanceof UserMediaViewModel ? vm.cropVideo : screenShareCropVideo, + vm instanceof LocalUserMediaViewModel || + vm instanceof RemoteUserMediaViewModel + ? vm.cropVideo + : cropVideoDefault, ); const unencryptedWarning = useStateObservable(vm.unencryptedWarning);