From ffbbc74a9654f8703a502760abd7db9b859e3e68 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 17 May 2024 16:38:00 -0400 Subject: [PATCH] Implement the new spotlight layout --- src/grid/CallLayout.ts | 55 ++++++++++++++++++ src/grid/GridLayout.module.css | 6 +- src/grid/GridLayout.tsx | 25 +++----- src/grid/SpotlightLayout.module.css | 89 +++++++++++++++++++++++++++++ src/grid/SpotlightLayout.tsx | 89 +++++++++++++++++++++++++++++ src/room/InCallView.module.css | 4 +- src/room/InCallView.tsx | 76 ++++++++++++++---------- src/state/CallViewModel.ts | 47 +++++++++++---- src/tile/GridTile.tsx | 11 ++-- 9 files changed, 337 insertions(+), 65 deletions(-) create mode 100644 src/grid/CallLayout.ts create mode 100644 src/grid/SpotlightLayout.module.css create mode 100644 src/grid/SpotlightLayout.tsx diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts new file mode 100644 index 00000000..287f116d --- /dev/null +++ b/src/grid/CallLayout.ts @@ -0,0 +1,55 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { BehaviorSubject, Observable } from "rxjs"; + +import { MediaViewModel } from "../state/MediaViewModel"; +import { LayoutSystem } from "./Grid"; +import { Alignment } from "../room/InCallView"; + +export interface Bounds { + width: number; + height: number; +} + +export interface CallLayoutInputs { + /** + * The minimum bounds of the layout area. + */ + minBounds: Observable; + /** + * The alignment of the floating tile, if any. + */ + floatingAlignment: BehaviorSubject; +} + +export interface CallLayoutOutputs { + /** + * The visually fixed (non-scrolling) layer of the layout. + */ + fixed: LayoutSystem; + /** + * The layer of the layout that can overflow and be scrolled. + */ + scrolling: LayoutSystem; +} + +/** + * A layout system for media tiles. + */ +export type CallLayout = ( + inputs: CallLayoutInputs, +) => CallLayoutOutputs; diff --git a/src/grid/GridLayout.module.css b/src/grid/GridLayout.module.css index ef234b33..084b56bd 100644 --- a/src/grid/GridLayout.module.css +++ b/src/grid/GridLayout.module.css @@ -14,6 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ +.fixed, +.scrolling { + margin-inline: var(--inline-content-inset); +} + .scrolling { box-sizing: border-box; block-size: 100%; @@ -22,7 +27,6 @@ limitations under the License. justify-content: center; align-content: center; gap: var(--gap); - box-sizing: border-box; } .scrolling > .slot { diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 6b673e64..75f1e726 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -15,21 +15,15 @@ limitations under the License. */ import { CSSProperties, forwardRef, useMemo } from "react"; -import { BehaviorSubject, Observable, distinctUntilChanged } from "rxjs"; +import { distinctUntilChanged } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import { GridLayout as GridLayoutModel } from "../state/CallViewModel"; -import { MediaViewModel } from "../state/MediaViewModel"; -import { LayoutSystem, Slot } from "./Grid"; +import { Slot } from "./Grid"; import styles from "./GridLayout.module.css"; import { useReactiveState } from "../useReactiveState"; -import { Alignment } from "../room/InCallView"; import { useInitial } from "../useInitial"; - -export interface Bounds { - width: number; - height: number; -} +import { CallLayout } from "./CallLayout"; interface GridCSSProperties extends CSSProperties { "--gap": string; @@ -37,19 +31,14 @@ interface GridCSSProperties extends CSSProperties { "--height": string; } -interface GridLayoutSystems { - scrolling: LayoutSystem; - fixed: LayoutSystem; -} - const slotMinHeight = 130; const slotMaxAspectRatio = 17 / 9; const slotMinAspectRatio = 4 / 3; -export const gridLayoutSystems = ( - minBounds: Observable, - floatingAlignment: BehaviorSubject, -): GridLayoutSystems => ({ +export const makeGridLayout: CallLayout = ({ + minBounds, + floatingAlignment, +}) => ({ // The "fixed" (non-scrolling) part of the layout is where the spotlight tile // lives fixed: { diff --git a/src/grid/SpotlightLayout.module.css b/src/grid/SpotlightLayout.module.css new file mode 100644 index 00000000..bbce45cf --- /dev/null +++ b/src/grid/SpotlightLayout.module.css @@ -0,0 +1,89 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.fixed, +.scrolling { + margin-inline: var(--inline-content-inset); + display: grid; + --grid-slot-width: 180px; + --grid-gap: 20px; + grid-template-columns: 1fr calc( + var(--grid-columns) * var(--grid-slot-width) + (var(--grid-columns) - 1) * + var(--grid-gap) + ); + grid-template-rows: minmax(1fr, auto); + gap: 30px; +} + +.scrolling { + block-size: 100%; +} + +.spotlight { + container: spotlight / size; + display: grid; + place-items: center; +} + +/* CSS makes us put a condition here, even though all we want to do is +unconditionally select the container so we can use cq units */ +@container spotlight (width > 0) { + .spotlight > .slot { + inline-size: min(100cqi, 100cqb * (17 / 9)); + block-size: min(100cqb, 100cqi / (4 / 3)); + } +} + +.grid { + display: flex; + flex-wrap: wrap; + gap: var(--grid-gap); + justify-content: center; + align-content: center; +} + +.grid > .slot { + inline-size: var(--grid-slot-width); + block-size: 135px; +} + +@media (max-width: 600px) { + .fixed, + .scrolling { + margin-inline: 0; + display: block; + } + + .spotlight { + inline-size: 100%; + aspect-ratio: 16 / 9; + margin-block-end: var(--cpd-space-4x); + } + + .grid { + margin-inline: var(--inline-content-inset); + align-content: start; + } + + .grid > .slot { + --grid-columns: 2; + --grid-slot-width: calc( + (100% - (var(--grid-columns) - 1) * var(--grid-gap)) / var(--grid-columns) + ); + block-size: unset; + aspect-ratio: 4 / 3; + } +} diff --git a/src/grid/SpotlightLayout.tsx b/src/grid/SpotlightLayout.tsx new file mode 100644 index 00000000..38bc6e37 --- /dev/null +++ b/src/grid/SpotlightLayout.tsx @@ -0,0 +1,89 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { CSSProperties, forwardRef } from "react"; +import { useObservableEagerState } from "observable-hooks"; + +import { CallLayout } from "./CallLayout"; +import { SpotlightLayout as SpotlightLayoutModel } from "../state/CallViewModel"; +import { useReactiveState } from "../useReactiveState"; +import styles from "./SpotlightLayout.module.css"; +import { Slot } from "./Grid"; + +interface GridCSSProperties extends CSSProperties { + "--grid-columns": number; +} + +const getGridColumns = (gridLength: number): number => + gridLength > 20 ? 2 : 1; + +export const makeSpotlightLayout: CallLayout = ({ + minBounds, +}) => ({ + fixed: { + tiles: (model) => new Map([["spotlight", model.spotlight]]), + Layout: forwardRef(function SpotlightLayoutFixed({ model }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const gridColumns = getGridColumns(model.grid.length); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid.length, width, height], + ); + + return ( +
+
+ +
+
+
+ ); + }), + }, + + scrolling: { + tiles: (model) => new Map(model.grid.map((tile) => [tile.id, tile])), + Layout: forwardRef(function SpotlightLayoutScrolling({ model }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const gridColumns = getGridColumns(model.grid.length); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid, width, height], + ); + + return ( +
+
+
+ {model.grid.map((tile) => ( + + ))} +
+
+ ); + }), + }, +}); diff --git a/src/room/InCallView.module.css b/src/room/InCallView.module.css index f53ba025..76dc9bae 100644 --- a/src/room/InCallView.module.css +++ b/src/room/InCallView.module.css @@ -125,7 +125,7 @@ limitations under the License. .fixedGrid { position: absolute; - inline-size: calc(100% - 2 * var(--inline-content-inset)); + inline-size: 100%; align-self: center; /* Disable pointer events so the overlay doesn't block interaction with elements behind it */ @@ -139,6 +139,6 @@ limitations under the License. .scrollingGrid { position: relative; flex-grow: 1; - inline-size: calc(100% - 2 * var(--inline-content-inset)); + inline-size: 100%; align-self: center; } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index dba01e9d..b2aeb9d2 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -35,7 +35,7 @@ import { import useMeasure from "react-use-measure"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import classNames from "classnames"; -import { BehaviorSubject } from "rxjs"; +import { BehaviorSubject, map } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import { useTranslation } from "react-i18next"; @@ -73,17 +73,20 @@ import { ECConnectionState } from "../livekit/useECConnectionState"; import { useOpenIDSFU } from "../livekit/openIDSFU"; import { GridMode, + Layout, TileDescriptor, useCallViewModel, } from "../state/CallViewModel"; import { Grid, TileProps } from "../grid/Grid"; import { MediaViewModel } from "../state/MediaViewModel"; -import { gridLayoutSystems } from "../grid/GridLayout"; import { useObservable } from "../state/useObservable"; import { useInitial } from "../useInitial"; import { SpotlightTile } from "../tile/SpotlightTile"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; +import { makeGridLayout } from "../grid/GridLayout"; +import { makeSpotlightLayout } from "../grid/SpotlightLayout"; +import { CallLayout } from "../grid/CallLayout"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -302,6 +305,7 @@ export const InCallView: FC = ({ const [headerRef, headerBounds] = useMeasure(); const [footerRef, footerBounds] = useMeasure(); + const gridBounds = useMemo( () => ({ width: footerBounds.width, @@ -315,11 +319,29 @@ export const InCallView: FC = ({ ], ); const gridBoundsObservable = useObservable(gridBounds); + const floatingAlignment = useInitial( () => new BehaviorSubject(defaultAlignment), ); - const { fixed, scrolling } = useInitial(() => - gridLayoutSystems(gridBoundsObservable, floatingAlignment), + + const layoutSystem = useObservableEagerState( + useInitial(() => + vm.layout.pipe( + map((l) => { + let makeLayout: CallLayout; + if (l.type === "grid" && l.grid.length !== 2) + makeLayout = makeGridLayout as CallLayout; + else if (l.type === "spotlight") + makeLayout = makeSpotlightLayout as CallLayout; + else return null; // Not yet implemented + + return makeLayout({ + minBounds: gridBoundsObservable, + floatingAlignment, + }); + }), + ), + ), ); const setGridMode = useCallback( @@ -423,31 +445,9 @@ export const InCallView: FC = ({ ); } - // The only new layout we've implemented so far is grid layout for non-1:1 - // calls. All other layouts use the legacy grid system for now. - if ( - legacyLayout === "grid" && - layout.type === "grid" && - !(layout.grid.length === 2 && layout.spotlight === undefined) - ) { - return ( - <> - - - - ); - } else { + if (layoutSystem === null) { + // This new layout doesn't yet have an implemented layout system, so fall + // back to the legacy grid system return ( = ({ Tile={GridTileView} /> ); + } else { + return ( + <> + + + + ); } }; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index b0816dc2..cc5afc46 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -28,12 +28,13 @@ import { import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix"; import { useEffect, useRef } from "react"; import { - BehaviorSubject, EMPTY, Observable, + Subject, audit, combineLatest, concat, + concatMap, distinctUntilChanged, filter, map, @@ -48,6 +49,7 @@ import { switchMap, throttleTime, timer, + withLatestFrom, zip, } from "rxjs"; import { logger } from "matrix-js-sdk/src/logger"; @@ -371,6 +373,13 @@ export class CallViewModel extends ViewModel { private readonly screenShares: Observable = this.mediaItems.pipe( map((ms) => ms.filter((m): m is ScreenShare => m instanceof ScreenShare)), + shareReplay(1), + ); + + private readonly hasScreenShares: Observable = + this.screenShares.pipe( + map((ms) => ms.length > 0), + distinctUntilChanged(), ); private readonly spotlightSpeaker: Observable = @@ -385,11 +394,13 @@ export class CallViewModel extends ViewModel { scan<(readonly [UserMedia, boolean])[], UserMedia | null, null>( (prev, ms) => // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - ms.find(([m, s]) => m === prev && s)?.[0] ?? - // Otherwise, select anyone who is speaking - ms.find(([, s]) => s)?.[0] ?? + // If the previous speaker (not the local user) is still speaking, + // stick with them rather than switching eagerly to someone else + (prev === null || prev.vm.local + ? null + : ms.find(([m, s]) => m === prev && s)?.[0]) ?? + // Otherwise, select any remote user who is speaking + ms.find(([m, s]) => !m.vm.local && s)?.[0] ?? // Otherwise, stick with the person who was last speaking prev ?? // Otherwise, spotlight the local user @@ -398,7 +409,8 @@ export class CallViewModel extends ViewModel { null, ), distinctUntilChanged(), - throttleTime(800, undefined, { leading: true, trailing: true }), + shareReplay(1), + throttleTime(1600, undefined, { leading: true, trailing: true }), ); private readonly grid: Observable = this.userMedia.pipe( @@ -453,18 +465,31 @@ export class CallViewModel extends ViewModel { // orientation private readonly windowMode = of("normal"); - private readonly _gridMode = new BehaviorSubject("grid"); + private readonly gridModeUserSelection = new Subject(); /** * The layout mode of the media tile grid. */ - public readonly gridMode: Observable = this._gridMode; + public readonly gridMode: Observable = merge( + // Always honor a manual user selection + this.gridModeUserSelection, + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + this.hasScreenShares.pipe( + withLatestFrom(this.gridModeUserSelection.pipe(startWith(null))), + concatMap(([hasScreenShares, userSelection]) => + userSelection === "spotlight" + ? EMPTY + : of(hasScreenShares ? "spotlight" : "grid"), + ), + ), + ).pipe(distinctUntilChanged(), shareReplay(1)); public setGridMode(value: GridMode): void { - this._gridMode.next(value); + this.gridModeUserSelection.next(value); } public readonly layout: Observable = combineLatest( - [this._gridMode, this.windowMode], + [this.gridMode, this.windowMode], (gridMode, windowMode) => { switch (windowMode) { case "full screen": diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index ba615953..14f85831 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -71,6 +71,7 @@ interface MediaTileProps vm: MediaViewModel; videoEnabled: boolean; videoFit: "contain" | "cover"; + mirror: boolean; nameTagLeadingIcon?: ReactNode; primaryButton: ReactNode; secondaryButton?: ReactNode; @@ -87,7 +88,6 @@ const MediaTile = forwardRef( className={classNames(className, styles.tile)} data-maximised={maximised} video={video} - mirror={false} member={vm.member} unencryptedWarning={unencryptedWarning} {...props} @@ -100,6 +100,7 @@ MediaTile.displayName = "MediaTile"; interface UserMediaTileProps extends TileProps { vm: UserMediaViewModel; + mirror: boolean; showSpeakingIndicators: boolean; menuStart?: ReactNode; menuEnd?: ReactNode; @@ -202,7 +203,7 @@ interface LocalUserMediaTileProps extends TileProps { } const LocalUserMediaTile = forwardRef( - ({ vm, onOpenProfile, className, ...props }, ref) => { + ({ vm, onOpenProfile, ...props }, ref) => { const { t } = useTranslation(); const mirror = useObservableEagerState(vm.mirror); const alwaysShow = useObservableEagerState(vm.alwaysShow); @@ -220,6 +221,7 @@ const LocalUserMediaTile = forwardRef( ( onSelect={onOpenProfile} /> } - className={classNames(className, { [styles.mirror]: mirror })} {...props} /> ); @@ -270,6 +271,7 @@ const RemoteUserMediaTile = forwardRef< ( ( ) } - videoEnabled {...props} /> );