From e0b10d89b5c78ec5d11f1757c77ce722f610fdd9 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 7 Jun 2024 12:27:13 -0400 Subject: [PATCH 1/5] Add model for one-on-one layout --- src/room/InCallView.tsx | 5 +---- src/state/CallViewModel.ts | 37 +++++++++++++++++++++++++++++-------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 3a16e1c6..bab24991 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -330,10 +330,7 @@ export const InCallView: FC = ({ vm.layout.pipe( map((l) => { let makeLayout: CallLayout; - if ( - l.type === "grid" && - !(l.grid.length === 2 && l.spotlight === undefined) - ) + if (l.type === "grid") makeLayout = makeGridLayout as CallLayout; else if (l.type === "spotlight") makeLayout = makeSpotlightLayout as CallLayout; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 3d48da22..62c35a8e 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -102,6 +102,13 @@ export interface SpotlightLayout { grid: UserMediaViewModel[]; } +export interface OneOnOneLayout { + type: "one-on-one"; + spotlight?: ScreenShareViewModel[]; + local: LocalUserMediaViewModel; + remote: RemoteUserMediaViewModel; +} + export interface FullScreenLayout { type: "full screen"; spotlight: MediaViewModel[]; @@ -120,6 +127,7 @@ export interface PipLayout { export type Layout = | GridLayout | SpotlightLayout + | OneOnOneLayout | FullScreenLayout | PipLayout; @@ -501,14 +509,27 @@ export class CallViewModel extends ViewModel { case "grid": return combineLatest( [this.grid, this.spotlight, this.screenShares], - (grid, spotlight, screenShares): Layout => ({ - type: "grid", - spotlight: - screenShares.length > 0 || grid.length > 20 - ? spotlight - : undefined, - grid, - }), + (grid, spotlight, screenShares): Layout => + grid.length == 2 + ? { + type: "one-on-one", + spotlight: + screenShares.length > 0 ? spotlight : undefined, + local: grid.find( + (vm) => vm.local, + ) as LocalUserMediaViewModel, + remote: grid.find( + (vm) => !vm.local, + ) as RemoteUserMediaViewModel, + } + : { + type: "grid", + spotlight: + screenShares.length > 0 || grid.length > 20 + ? spotlight + : undefined, + grid, + }, ); case "spotlight": return combineLatest( From 7979493371a149ac2017c809b9703a395f61b19f Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 7 Jun 2024 16:59:56 -0400 Subject: [PATCH 2/5] Implement the new one-on-one layout --- src/grid/CallLayout.ts | 85 +++++++++++++++++- src/grid/GridLayout.module.css | 3 +- src/grid/GridLayout.tsx | 83 ++++------------- src/grid/OneOnOneLayout.module.css | 56 ++++++++++++ src/grid/OneOnOneLayout.tsx | 132 ++++++++++++++++++++++++++++ src/grid/SpotlightLayout.module.css | 5 +- src/grid/SpotlightLayout.tsx | 8 +- src/room/InCallView.tsx | 33 ++++--- 8 files changed, 312 insertions(+), 93 deletions(-) create mode 100644 src/grid/OneOnOneLayout.module.css create mode 100644 src/grid/OneOnOneLayout.tsx diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts index c4412677..8f8106de 100644 --- a/src/grid/CallLayout.ts +++ b/src/grid/CallLayout.ts @@ -19,22 +19,36 @@ import { ComponentType } from "react"; import { MediaViewModel } from "../state/MediaViewModel"; import { LayoutProps } from "./Grid"; -import { Alignment } from "../room/InCallView"; export interface Bounds { width: number; height: number; } +export interface Alignment { + inline: "start" | "end"; + block: "start" | "end"; +} + +export const defaultSpotlightAlignment: Alignment = { + inline: "end", + block: "end", +}; +export const defaultPipAlignment: Alignment = { inline: "end", block: "start" }; + export interface CallLayoutInputs { /** * The minimum bounds of the layout area. */ minBounds: Observable; /** - * The alignment of the floating tile, if any. + * The alignment of the floating spotlight tile, if present. */ - floatingAlignment: BehaviorSubject; + spotlightAlignment: BehaviorSubject; + /** + * The alignment of the small picture-in-picture tile, if present. + */ + pipAlignment: BehaviorSubject; } export interface GridTileModel { @@ -67,3 +81,68 @@ export interface CallLayoutOutputs { export type CallLayout = ( inputs: CallLayoutInputs, ) => CallLayoutOutputs; + +export interface GridArrangement { + tileWidth: number; + tileHeight: number; + gap: number; + columns: number; +} + +const tileMinHeight = 130; +const tileMaxAspectRatio = 17 / 9; +const tileMinAspectRatio = 4 / 3; + +/** + * Determine the ideal arrangement of tiles into a grid of a particular size. + */ +export function arrangeTiles( + width: number, + minHeight: number, + tileCount: number, +): GridArrangement { + // The goal here is to determine the grid size and padding that maximizes + // use of screen space for n tiles without making those tiles too small or + // too cropped (having an extreme aspect ratio) + const gap = width < 800 ? 16 : 20; + const tileMinWidth = width < 500 ? 150 : 180; + + let columns = Math.min( + // Don't create more columns than we have items for + tileCount, + // The ideal number of columns is given by a packing of equally-sized + // squares into a grid. + // width / column = height / row. + // columns * rows = number of squares. + // ∴ columns = sqrt(width / height * number of squares). + // Except we actually want 16:9-ish tiles rather than squares, so we + // divide the width-to-height ratio by the target aspect ratio. + Math.ceil(Math.sqrt((width / minHeight / tileMaxAspectRatio) * tileCount)), + ); + let rows = Math.ceil(tileCount / columns); + + let tileWidth = (width - (columns - 1) * gap) / columns; + let tileHeight = (minHeight - (rows - 1) * gap) / rows; + + // Impose a minimum width and height on the tiles + if (tileWidth < tileMinWidth) { + // In this case we want the tile width to determine the number of columns, + // not the other way around. If we take the above equation for the tile + // width (w = (W - (c - 1) * g) / c) and solve for c, we get + // c = (W + g) / (w + g). + columns = Math.floor((width + gap) / (tileMinWidth + gap)); + rows = Math.ceil(tileCount / columns); + tileWidth = (width - (columns - 1) * gap) / columns; + tileHeight = (minHeight - (rows - 1) * gap) / rows; + } + if (tileHeight < tileMinHeight) tileHeight = tileMinHeight; + // Impose a minimum and maximum aspect ratio on the tiles + const tileAspectRatio = tileWidth / tileHeight; + if (tileAspectRatio > tileMaxAspectRatio) + tileWidth = tileHeight * tileMaxAspectRatio; + else if (tileAspectRatio < tileMinAspectRatio) + tileHeight = tileWidth / tileMinAspectRatio; + // TODO: We might now be hitting the minimum height or width limit again + + return { tileWidth, tileHeight, gap, columns }; +} diff --git a/src/grid/GridLayout.module.css b/src/grid/GridLayout.module.css index 5e6aa9e1..33edc3be 100644 --- a/src/grid/GridLayout.module.css +++ b/src/grid/GridLayout.module.css @@ -17,11 +17,10 @@ limitations under the License. .fixed, .scrolling { margin-inline: var(--inline-content-inset); + block-size: 100%; } .scrolling { - box-sizing: border-box; - block-size: 100%; display: flex; flex-wrap: wrap; justify-content: center; diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 3861457e..4d499eed 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -22,7 +22,12 @@ import { GridLayout as GridLayoutModel } from "../state/CallViewModel"; import styles from "./GridLayout.module.css"; import { useReactiveState } from "../useReactiveState"; import { useInitial } from "../useInitial"; -import { CallLayout, GridTileModel, TileModel } from "./CallLayout"; +import { + CallLayout, + GridTileModel, + TileModel, + arrangeTiles, +} from "./CallLayout"; import { DragCallback } from "./Grid"; interface GridCSSProperties extends CSSProperties { @@ -31,13 +36,9 @@ interface GridCSSProperties extends CSSProperties { "--height": string; } -const slotMinHeight = 130; -const slotMaxAspectRatio = 17 / 9; -const slotMinAspectRatio = 4 / 3; - export const makeGridLayout: CallLayout = ({ minBounds, - floatingAlignment, + spotlightAlignment, }) => ({ // The "fixed" (non-scrolling) part of the layout is where the spotlight tile // lives @@ -45,7 +46,7 @@ export const makeGridLayout: CallLayout = ({ const { width, height } = useObservableEagerState(minBounds); const alignment = useObservableEagerState( useInitial(() => - floatingAlignment.pipe( + spotlightAlignment.pipe( distinctUntilChanged( (a1, a2) => a1.block === a2.block && a1.inline === a2.inline, ), @@ -68,7 +69,7 @@ export const makeGridLayout: CallLayout = ({ const onDragSpotlight: DragCallback = useCallback( ({ xRatio, yRatio }) => - floatingAlignment.next({ + spotlightAlignment.next({ block: yRatio < 0.5 ? "start" : "end", inline: xRatio < 0.5 ? "start" : "end", }), @@ -76,12 +77,7 @@ export const makeGridLayout: CallLayout = ({ ); return ( -
+
{tileModel && ( = ({ // The scrolling part of the layout is where all the grid tiles live scrolling: forwardRef(function GridLayout({ model, Slot }, ref) { const { width, height: minHeight } = useObservableEagerState(minBounds); - - // The goal here is to determine the grid size and padding that maximizes - // use of screen space for n tiles without making those tiles too small or - // too cropped (having an extreme aspect ratio) - const [gap, slotWidth, slotHeight] = useMemo(() => { - const gap = width < 800 ? 16 : 20; - const slotMinWidth = width < 500 ? 150 : 180; - - let columns = Math.min( - // Don't create more columns than we have items for - model.grid.length, - // The ideal number of columns is given by a packing of equally-sized - // squares into a grid. - // width / column = height / row. - // columns * rows = number of squares. - // ∴ columns = sqrt(width / height * number of squares). - // Except we actually want 16:9-ish slots rather than squares, so we - // divide the width-to-height ratio by the target aspect ratio. - Math.ceil( - Math.sqrt( - (width / minHeight / slotMaxAspectRatio) * model.grid.length, - ), - ), - ); - let rows = Math.ceil(model.grid.length / columns); - - let slotWidth = (width - (columns - 1) * gap) / columns; - let slotHeight = (minHeight - (rows - 1) * gap) / rows; - - // Impose a minimum width and height on the slots - if (slotWidth < slotMinWidth) { - // In this case we want the slot width to determine the number of columns, - // not the other way around. If we take the above equation for the slot - // width (w = (W - (c - 1) * g) / c) and solve for c, we get - // c = (W + g) / (w + g). - columns = Math.floor((width + gap) / (slotMinWidth + gap)); - rows = Math.ceil(model.grid.length / columns); - slotWidth = (width - (columns - 1) * gap) / columns; - slotHeight = (minHeight - (rows - 1) * gap) / rows; - } - if (slotHeight < slotMinHeight) slotHeight = slotMinHeight; - // Impose a minimum and maximum aspect ratio on the slots - const slotAspectRatio = slotWidth / slotHeight; - if (slotAspectRatio > slotMaxAspectRatio) - slotWidth = slotHeight * slotMaxAspectRatio; - else if (slotAspectRatio < slotMinAspectRatio) - slotHeight = slotWidth / slotMinAspectRatio; - // TODO: We might now be hitting the minimum height or width limit again - - return [gap, slotWidth, slotHeight]; - }, [width, minHeight, model.grid.length]); + const { gap, tileWidth, tileHeight } = useMemo( + () => arrangeTiles(width, minHeight, model.grid.length), + [width, minHeight, model.grid.length], + ); const [generation] = useReactiveState( (prev) => (prev === undefined ? 0 : prev + 1), @@ -170,8 +119,8 @@ export const makeGridLayout: CallLayout = ({ { width, "--gap": `${gap}px`, - "--width": `${Math.floor(slotWidth)}px`, - "--height": `${Math.floor(slotHeight)}px`, + "--width": `${Math.floor(tileWidth)}px`, + "--height": `${Math.floor(tileHeight)}px`, } as GridCSSProperties } > diff --git a/src/grid/OneOnOneLayout.module.css b/src/grid/OneOnOneLayout.module.css new file mode 100644 index 00000000..0d2ad4ff --- /dev/null +++ b/src/grid/OneOnOneLayout.module.css @@ -0,0 +1,56 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.layer { + margin-inline: var(--inline-content-inset); + block-size: 100%; + display: grid; + place-items: center; +} + +.container { + position: relative; +} + +.local { + position: absolute; + inline-size: 180px; + block-size: 135px; + inset: var(--cpd-space-4x); +} + +.spotlight { + position: absolute; + inline-size: 404px; + block-size: 233px; + inset: -12px; +} + +.slot[data-block-alignment="start"] { + inset-block-end: unset; +} + +.slot[data-block-alignment="end"] { + inset-block-start: unset; +} + +.slot[data-inline-alignment="start"] { + inset-inline-end: unset; +} + +.slot[data-inline-alignment="end"] { + inset-inline-start: unset; +} diff --git a/src/grid/OneOnOneLayout.tsx b/src/grid/OneOnOneLayout.tsx new file mode 100644 index 00000000..2eac1b7e --- /dev/null +++ b/src/grid/OneOnOneLayout.tsx @@ -0,0 +1,132 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { forwardRef, useCallback, useMemo } from "react"; +import { useObservableEagerState } from "observable-hooks"; +import classNames from "classnames"; + +import { OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewModel"; +import { + CallLayout, + GridTileModel, + SpotlightTileModel, + arrangeTiles, +} from "./CallLayout"; +import { useReactiveState } from "../useReactiveState"; +import styles from "./OneOnOneLayout.module.css"; +import { DragCallback } from "./Grid"; + +export const makeOneOnOneLayout: CallLayout = ({ + minBounds, + spotlightAlignment, + pipAlignment, +}) => ({ + fixed: forwardRef(function OneOnOneLayoutFixed({ model, Slot }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const spotlightAlignmentValue = useObservableEagerState(spotlightAlignment); + + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [width, height, model.spotlight === undefined, spotlightAlignmentValue], + ); + + const spotlightTileModel: SpotlightTileModel | undefined = useMemo( + () => + model.spotlight && { + type: "spotlight", + vms: model.spotlight, + maximised: false, + }, + [model.spotlight], + ); + + const onDragSpotlight: DragCallback = useCallback( + ({ xRatio, yRatio }) => + spotlightAlignment.next({ + block: yRatio < 0.5 ? "start" : "end", + inline: xRatio < 0.5 ? "start" : "end", + }), + [], + ); + + return ( +
+ {spotlightTileModel && ( + + )} +
+ ); + }), + + scrolling: forwardRef(function OneOnOneLayoutScrolling({ model, Slot }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const pipAlignmentValue = useObservableEagerState(pipAlignment); + const { tileWidth, tileHeight } = useMemo( + () => arrangeTiles(width, height, 1), + [width, height], + ); + + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [width, height, pipAlignmentValue], + ); + + const remoteTileModel: GridTileModel = useMemo( + () => ({ type: "grid", vm: model.remote }), + [model.remote], + ); + const localTileModel: GridTileModel = useMemo( + () => ({ type: "grid", vm: model.local }), + [model.local], + ); + + const onDragLocalTile: DragCallback = useCallback( + ({ xRatio, yRatio }) => + pipAlignment.next({ + block: yRatio < 0.5 ? "start" : "end", + inline: xRatio < 0.5 ? "start" : "end", + }), + [], + ); + + return ( +
+ + + +
+ ); + }), +}); diff --git a/src/grid/SpotlightLayout.module.css b/src/grid/SpotlightLayout.module.css index af43216c..d58a95a1 100644 --- a/src/grid/SpotlightLayout.module.css +++ b/src/grid/SpotlightLayout.module.css @@ -16,6 +16,7 @@ limitations under the License. .layer { margin-inline: var(--inline-content-inset); + block-size: 100%; display: grid; --grid-gap: 20px; gap: 30px; @@ -30,10 +31,6 @@ limitations under the License. grid-template-rows: minmax(1fr, auto); } -.scrolling { - block-size: 100%; -} - .spotlight { container: spotlight / size; display: grid; diff --git a/src/grid/SpotlightLayout.tsx b/src/grid/SpotlightLayout.tsx index 3e07a0b2..9ddbce10 100644 --- a/src/grid/SpotlightLayout.tsx +++ b/src/grid/SpotlightLayout.tsx @@ -69,10 +69,8 @@ export const makeSpotlightLayout: CallLayout = ({ ref={ref} data-generation={generation} data-orientation={layout.orientation} - className={classNames(styles.layer, styles.fixed)} - style={ - { "--grid-columns": layout.gridColumns, height } as GridCSSProperties - } + className={styles.layer} + style={{ "--grid-columns": layout.gridColumns } as GridCSSProperties} >
@@ -102,7 +100,7 @@ export const makeSpotlightLayout: CallLayout = ({ ref={ref} data-generation={generation} data-orientation={layout.orientation} - className={classNames(styles.layer, styles.scrolling)} + className={styles.layer} style={{ "--grid-columns": layout.gridColumns } as GridCSSProperties} >
; @@ -321,8 +321,11 @@ export const InCallView: FC = ({ ); const gridBoundsObservable = useObservable(gridBounds); - const floatingAlignment = useInitial( - () => new BehaviorSubject(defaultAlignment), + const spotlightAlignment = useInitial( + () => new BehaviorSubject(defaultSpotlightAlignment), + ); + const pipAlignment = useInitial( + () => new BehaviorSubject(defaultPipAlignment), ); const layoutSystem = useObservableEagerState( @@ -334,11 +337,14 @@ export const InCallView: FC = ({ makeLayout = makeGridLayout as CallLayout; else if (l.type === "spotlight") makeLayout = makeSpotlightLayout as CallLayout; + else if (l.type === "one-on-one") + makeLayout = makeOneOnOneLayout as CallLayout; else return null; // Not yet implemented return makeLayout({ minBounds: gridBoundsObservable, - floatingAlignment, + spotlightAlignment, + pipAlignment, }); }), ), @@ -491,7 +497,10 @@ export const InCallView: FC = ({ /> Date: Fri, 7 Jun 2024 17:29:48 -0400 Subject: [PATCH 3/5] Delete the legacy grid system --- public/locales/en-GB/app.json | 3 +- src/grid/CallLayout.ts | 4 +- src/grid/Grid.tsx | 8 +- src/grid/LegacyGrid.module.css | 22 - src/grid/LegacyGrid.tsx | 1405 -------------------------------- src/room/InCallView.tsx | 227 ++---- src/room/useFullscreen.ts | 26 +- src/state/CallViewModel.ts | 118 --- src/tile/GridTile.tsx | 112 +-- test/grid/LegacyGrid-test.ts | 69 -- 10 files changed, 102 insertions(+), 1892 deletions(-) delete mode 100644 src/grid/LegacyGrid.module.css delete mode 100644 src/grid/LegacyGrid.tsx delete mode 100644 test/grid/LegacyGrid-test.ts diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 922a4c79..b72e0bcb 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -162,6 +162,5 @@ "mute_for_me": "Mute for me", "sfu_participant_local": "You", "volume": "Volume" - }, - "waiting_for_participants": "Waiting for other participants…" + } } diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts index 8f8106de..e97b18a2 100644 --- a/src/grid/CallLayout.ts +++ b/src/grid/CallLayout.ts @@ -17,7 +17,7 @@ limitations under the License. import { BehaviorSubject, Observable } from "rxjs"; import { ComponentType } from "react"; -import { MediaViewModel } from "../state/MediaViewModel"; +import { MediaViewModel, UserMediaViewModel } from "../state/MediaViewModel"; import { LayoutProps } from "./Grid"; export interface Bounds { @@ -53,7 +53,7 @@ export interface CallLayoutInputs { export interface GridTileModel { type: "grid"; - vm: MediaViewModel; + vm: UserMediaViewModel; } export interface SpotlightTileModel { diff --git a/src/grid/Grid.tsx b/src/grid/Grid.tsx index 2e6a48ae..6c1bab9b 100644 --- a/src/grid/Grid.tsx +++ b/src/grid/Grid.tsx @@ -41,7 +41,6 @@ import styles from "./Grid.module.css"; import { useMergedRefs } from "../useMergedRefs"; import { TileWrapper } from "./TileWrapper"; import { usePrefersReducedMotion } from "../usePrefersReducedMotion"; -import { TileSpringUpdate } from "./LegacyGrid"; import { useInitial } from "../useInitial"; interface Rect { @@ -69,6 +68,13 @@ interface TileSpring { height: number; } +interface TileSpringUpdate extends Partial { + from?: Partial; + reset?: boolean; + immediate?: boolean | ((key: string) => boolean); + delay?: (key: string) => number; +} + interface DragState { tileId: string; tileX: number; diff --git a/src/grid/LegacyGrid.module.css b/src/grid/LegacyGrid.module.css deleted file mode 100644 index cad3e3c4..00000000 --- a/src/grid/LegacyGrid.module.css +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2022-2024 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -.grid { - position: relative; - overflow: hidden; - flex: 1; - touch-action: none; -} diff --git a/src/grid/LegacyGrid.tsx b/src/grid/LegacyGrid.tsx deleted file mode 100644 index f04cde78..00000000 --- a/src/grid/LegacyGrid.tsx +++ /dev/null @@ -1,1405 +0,0 @@ -/* -Copyright 2022-2024 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { - ComponentProps, - ComponentType, - MutableRefObject, - ReactNode, - Ref, - useCallback, - useEffect, - useRef, - useState, -} from "react"; -import { - EventTypes, - FullGestureState, - Handler, - useGesture, -} from "@use-gesture/react"; -import { - animated, - SpringRef, - SpringValues, - useSprings, -} from "@react-spring/web"; -import useMeasure from "react-use-measure"; -import { ResizeObserver as JuggleResizeObserver } from "@juggle/resize-observer"; -import { logger } from "matrix-js-sdk/src/logger"; - -import styles from "./LegacyGrid.module.css"; -import { Layout } from "../room/LayoutToggle"; -import { TileWrapper } from "./TileWrapper"; -import { TileDescriptor } from "../state/CallViewModel"; -import { TileProps } from "./Grid"; - -interface TilePosition { - x: number; - y: number; - width: number; - height: number; - zIndex: number; -} - -export interface Tile { - key: string; - order: number; - item: TileDescriptor; - remove: boolean; - focused: boolean; - isPresenter: boolean; - isSpeaker: boolean; - hasVideo: boolean; -} - -export interface TileSpring { - opacity: number; - scale: number; - shadow: number; - shadowSpread: number; - zIndex: number; - x: number; - y: number; - width: number; - height: number; -} - -export interface TileSpringUpdate extends Partial { - from?: Partial; - reset?: boolean; - immediate?: boolean | ((key: string) => boolean); - delay?: (key: string) => number; -} - -type LayoutDirection = "vertical" | "horizontal"; - -export function useLegacyGridLayout(hasScreenshareFeeds: boolean): { - layout: Layout; - setLayout: (layout: Layout) => void; -} { - const layoutRef = useRef("grid"); - const revertLayoutRef = useRef("grid"); - const prevHasScreenshareFeeds = useRef(hasScreenshareFeeds); - const [, forceUpdate] = useState({}); - - const setLayout = useCallback((layout: Layout) => { - // Store the user's set layout to revert to after a screenshare is finished - revertLayoutRef.current = layout; - layoutRef.current = layout; - forceUpdate({}); - }, []); - - // Note: We need the returned layout to update synchronously with a change in hasScreenshareFeeds - // so use refs and avoid useEffect. - if (prevHasScreenshareFeeds.current !== hasScreenshareFeeds) { - if (hasScreenshareFeeds) { - // Automatically switch to spotlight layout when there's a screenshare - layoutRef.current = "spotlight"; - } else { - // When the screenshares have ended, revert to the previous layout - layoutRef.current = revertLayoutRef.current; - } - } - - prevHasScreenshareFeeds.current = hasScreenshareFeeds; - - return { layout: layoutRef.current, setLayout }; -} - -const GAP = 20; - -function useIsMounted(): MutableRefObject { - const isMountedRef = useRef(false); - - useEffect(() => { - isMountedRef.current = true; - - return (): void => { - isMountedRef.current = false; - }; - }, []); - - return isMountedRef; -} - -function isInside([x, y]: number[], targetTile: TilePosition): boolean { - const left = targetTile.x; - const top = targetTile.y; - const bottom = targetTile.y + targetTile.height; - const right = targetTile.x + targetTile.width; - - if (x < left || x > right || y < top || y > bottom) { - return false; - } - - return true; -} - -const getPipGap = (gridAspectRatio: number, gridWidth: number): number => - gridAspectRatio < 1 || gridWidth < 700 ? 12 : 24; - -function getTilePositions( - tileCount: number, - focusedTileCount: number, - gridWidth: number, - gridHeight: number, - pipXRatio: number, - pipYRatio: number, - layout: Layout, -): TilePosition[] { - if (layout === "grid") { - if (tileCount === 2 && focusedTileCount === 0) { - return getOneOnOneLayoutTilePositions( - gridWidth, - gridHeight, - pipXRatio, - pipYRatio, - ); - } - - return getFreedomLayoutTilePositions( - tileCount, - focusedTileCount, - gridWidth, - gridHeight, - ); - } else { - return getSpotlightLayoutTilePositions(tileCount, gridWidth, gridHeight); - } -} - -function getOneOnOneLayoutTilePositions( - gridWidth: number, - gridHeight: number, - pipXRatio: number, - pipYRatio: number, -): TilePosition[] { - const [remotePosition] = getFreedomLayoutTilePositions( - 1, - 0, - gridWidth, - gridHeight, - ); - - const gridAspectRatio = gridWidth / gridHeight; - - const smallPip = gridAspectRatio < 1 || gridWidth < 700; - const maxPipWidth = smallPip ? 114 : 230; - const maxPipHeight = smallPip ? 163 : 155; - // Cap the PiP size at 1/3 the remote tile size, preserving aspect ratio - const pipScaleFactor = Math.min( - 1, - remotePosition.width / 3 / maxPipWidth, - remotePosition.height / 3 / maxPipHeight, - ); - const pipWidth = maxPipWidth * pipScaleFactor; - const pipHeight = maxPipHeight * pipScaleFactor; - const pipGap = getPipGap(gridAspectRatio, gridWidth); - - const pipMinX = remotePosition.x + pipGap; - const pipMinY = remotePosition.y + pipGap; - const pipMaxX = remotePosition.x + remotePosition.width - pipWidth - pipGap; - const pipMaxY = remotePosition.y + remotePosition.height - pipHeight - pipGap; - - return [ - { - // Apply the PiP position as a proportion of the available space - x: pipMinX + pipXRatio * (pipMaxX - pipMinX), - y: pipMinY + pipYRatio * (pipMaxY - pipMinY), - width: pipWidth, - height: pipHeight, - zIndex: 1, - }, - remotePosition, - ]; -} - -function getSpotlightLayoutTilePositions( - tileCount: number, - gridWidth: number, - gridHeight: number, -): TilePosition[] { - const tilePositions: TilePosition[] = []; - - const gridAspectRatio = gridWidth / gridHeight; - - if (gridAspectRatio < 1) { - // Vertical layout (mobile) - const spotlightTileHeight = - tileCount > 1 ? (gridHeight - GAP * 3) * (4 / 5) : gridHeight - GAP * 2; - const spectatorTileSize = - tileCount > 1 ? gridHeight - GAP * 3 - spotlightTileHeight : 0; - - for (let i = 0; i < tileCount; i++) { - if (i === 0) { - // Spotlight tile - tilePositions.push({ - x: GAP, - y: GAP, - width: gridWidth - GAP * 2, - height: spotlightTileHeight, - zIndex: 0, - }); - } else { - // Spectator tile - tilePositions.push({ - x: (GAP + spectatorTileSize) * (i - 1) + GAP, - y: spotlightTileHeight + GAP * 2, - width: spectatorTileSize, - height: spectatorTileSize, - zIndex: 0, - }); - } - } - } else { - // Horizontal layout (desktop) - const spotlightTileWidth = - tileCount > 1 ? ((gridWidth - GAP * 3) * 4) / 5 : gridWidth - GAP * 2; - const spectatorTileWidth = - tileCount > 1 ? gridWidth - GAP * 3 - spotlightTileWidth : 0; - const spectatorTileHeight = spectatorTileWidth * (9 / 16); - - for (let i = 0; i < tileCount; i++) { - if (i === 0) { - tilePositions.push({ - x: GAP, - y: GAP, - width: spotlightTileWidth, - height: gridHeight - GAP * 2, - zIndex: 0, - }); - } else { - tilePositions.push({ - x: GAP * 2 + spotlightTileWidth, - y: (GAP + spectatorTileHeight) * (i - 1) + GAP, - width: spectatorTileWidth, - height: spectatorTileHeight, - zIndex: 0, - }); - } - } - } - - return tilePositions; -} - -function getFreedomLayoutTilePositions( - tileCount: number, - focusedTileCount: number, - gridWidth: number, - gridHeight: number, -): TilePosition[] { - if (tileCount === 0) { - return []; - } - - if (tileCount > 12) { - logger.warn("Over 12 tiles is not currently supported"); - } - - const { layoutDirection, itemGridRatio } = getGridLayout( - tileCount, - focusedTileCount, - gridWidth, - gridHeight, - ); - - let itemGridWidth; - let itemGridHeight; - - if (layoutDirection === "vertical") { - itemGridWidth = gridWidth; - itemGridHeight = Math.round(gridHeight * itemGridRatio); - } else { - itemGridWidth = Math.round(gridWidth * itemGridRatio); - itemGridHeight = gridHeight; - } - - const itemTileCount = tileCount - focusedTileCount; - - const { - columnCount: itemColumnCount, - rowCount: itemRowCount, - tileAspectRatio: itemTileAspectRatio, - } = getSubGridLayout(itemTileCount, itemGridWidth, itemGridHeight); - - const itemGridPositions = getSubGridPositions( - itemTileCount, - itemColumnCount, - itemRowCount, - itemTileAspectRatio, - itemGridWidth, - itemGridHeight, - ); - const itemGridBounds = getSubGridBoundingBox(itemGridPositions); - - let focusedGridWidth: number; - let focusedGridHeight: number; - - if (focusedTileCount === 0) { - focusedGridWidth = 0; - focusedGridHeight = 0; - } else if (layoutDirection === "vertical") { - focusedGridWidth = gridWidth; - focusedGridHeight = - gridHeight - (itemGridBounds.height + (itemTileCount ? GAP * 2 : 0)); - } else { - focusedGridWidth = - gridWidth - (itemGridBounds.width + (itemTileCount ? GAP * 2 : 0)); - focusedGridHeight = gridHeight; - } - - const { - columnCount: focusedColumnCount, - rowCount: focusedRowCount, - tileAspectRatio: focusedTileAspectRatio, - } = getSubGridLayout(focusedTileCount, focusedGridWidth, focusedGridHeight); - - const focusedGridPositions = getSubGridPositions( - focusedTileCount, - focusedColumnCount, - focusedRowCount, - focusedTileAspectRatio, - focusedGridWidth, - focusedGridHeight, - ); - - const tilePositions = [...focusedGridPositions, ...itemGridPositions]; - - centerTiles(focusedGridPositions, focusedGridWidth, focusedGridHeight, 0, 0); - - if (layoutDirection === "vertical") { - centerTiles( - itemGridPositions, - gridWidth, - gridHeight - focusedGridHeight, - 0, - focusedGridHeight, - ); - } else { - centerTiles( - itemGridPositions, - gridWidth - focusedGridWidth, - gridHeight, - focusedGridWidth, - 0, - ); - } - - return tilePositions; -} - -function getSubGridBoundingBox(positions: TilePosition[]): { - left: number; - right: number; - top: number; - bottom: number; - width: number; - height: number; -} { - let left = 0; - let right = 0; - let top = 0; - let bottom = 0; - - for (let i = 0; i < positions.length; i++) { - const { x, y, width, height } = positions[i]; - - if (i === 0) { - left = x; - right = x + width; - top = y; - bottom = y + height; - } else { - if (x < left) { - left = x; - } - - if (y < top) { - top = y; - } - - if (x + width > right) { - right = x + width; - } - - if (y + height > bottom) { - bottom = y + height; - } - } - } - - return { - left, - right, - top, - bottom, - width: right - left, - height: bottom - top, - }; -} - -function isMobileBreakpoint(gridWidth: number, gridHeight: number): boolean { - const gridAspectRatio = gridWidth / gridHeight; - return gridAspectRatio < 1; -} - -function getGridLayout( - tileCount: number, - focusedTileCount: number, - gridWidth: number, - gridHeight: number, -): { itemGridRatio: number; layoutDirection: LayoutDirection } { - let layoutDirection: LayoutDirection = "horizontal"; - let itemGridRatio = 1; - - if (focusedTileCount === 0) { - return { itemGridRatio, layoutDirection }; - } - - if (isMobileBreakpoint(gridWidth, gridHeight)) { - layoutDirection = "vertical"; - itemGridRatio = 1 / 3; - } else { - layoutDirection = "horizontal"; - itemGridRatio = 1 / 3; - } - - return { itemGridRatio, layoutDirection }; -} - -function centerTiles( - positions: TilePosition[], - gridWidth: number, - gridHeight: number, - offsetLeft: number, - offsetTop: number, -): TilePosition[] { - const bounds = getSubGridBoundingBox(positions); - - const leftOffset = Math.round((gridWidth - bounds.width) / 2) + offsetLeft; - const topOffset = Math.round((gridHeight - bounds.height) / 2) + offsetTop; - - applyTileOffsets(positions, leftOffset, topOffset); - - return positions; -} - -function applyTileOffsets( - positions: TilePosition[], - leftOffset: number, - topOffset: number, -): TilePosition[] { - for (const position of positions) { - position.x += leftOffset; - position.y += topOffset; - } - - return positions; -} - -function getSubGridLayout( - tileCount: number, - gridWidth: number, - gridHeight: number, -): { columnCount: number; rowCount: number; tileAspectRatio: number } { - const gridAspectRatio = gridWidth / gridHeight; - - let columnCount: number; - let rowCount: number; - let tileAspectRatio: number = 16 / 9; - - if (gridAspectRatio < 3 / 4) { - // Phone - if (tileCount === 1) { - columnCount = 1; - rowCount = 1; - tileAspectRatio = 0; - } else if (tileCount <= 4) { - columnCount = 1; - rowCount = tileCount; - } else if (tileCount <= 12) { - columnCount = 2; - rowCount = Math.ceil(tileCount / columnCount); - tileAspectRatio = 0; - } else { - // Unsupported - columnCount = 3; - rowCount = Math.ceil(tileCount / columnCount); - tileAspectRatio = 1; - } - } else if (gridAspectRatio < 1) { - // Tablet - if (tileCount === 1) { - columnCount = 1; - rowCount = 1; - tileAspectRatio = 0; - } else if (tileCount <= 4) { - columnCount = 1; - rowCount = tileCount; - } else if (tileCount <= 12) { - columnCount = 2; - rowCount = Math.ceil(tileCount / columnCount); - } else { - // Unsupported - columnCount = 3; - rowCount = Math.ceil(tileCount / columnCount); - tileAspectRatio = 1; - } - } else if (gridAspectRatio < 17 / 9) { - // Computer - if (tileCount === 1) { - columnCount = 1; - rowCount = 1; - } else if (tileCount === 2) { - columnCount = 2; - rowCount = 1; - } else if (tileCount <= 4) { - columnCount = 2; - rowCount = 2; - } else if (tileCount <= 6) { - columnCount = 3; - rowCount = 2; - } else if (tileCount <= 8) { - columnCount = 4; - rowCount = 2; - tileAspectRatio = 1; - } else if (tileCount <= 12) { - columnCount = 4; - rowCount = 3; - tileAspectRatio = 1; - } else { - // Unsupported - columnCount = 4; - rowCount = 4; - } - } else if (gridAspectRatio <= 32 / 9) { - // Ultrawide - if (tileCount === 1) { - columnCount = 1; - rowCount = 1; - } else if (tileCount === 2) { - columnCount = 2; - rowCount = 1; - } else if (tileCount <= 4) { - columnCount = 2; - rowCount = 2; - } else if (tileCount <= 6) { - columnCount = 3; - rowCount = 2; - } else if (tileCount <= 8) { - columnCount = 4; - rowCount = 2; - } else if (tileCount <= 12) { - columnCount = 4; - rowCount = 3; - } else { - // Unsupported - columnCount = 4; - rowCount = 4; - } - } else { - // Super Ultrawide - if (tileCount <= 6) { - columnCount = tileCount; - rowCount = 1; - } else { - columnCount = Math.ceil(tileCount / 2); - rowCount = 2; - } - } - - return { columnCount, rowCount, tileAspectRatio }; -} - -function getSubGridPositions( - tileCount: number, - columnCount: number, - rowCount: number, - tileAspectRatio: number, - gridWidth: number, - gridHeight: number, -): TilePosition[] { - if (tileCount === 0) { - return []; - } - - const newTilePositions: TilePosition[] = []; - - const boxWidth = Math.round( - (gridWidth - GAP * (columnCount + 1)) / columnCount, - ); - const boxHeight = Math.round((gridHeight - GAP * (rowCount + 1)) / rowCount); - - let tileWidth: number; - let tileHeight: number; - - if (tileAspectRatio) { - const boxAspectRatio = boxWidth / boxHeight; - - if (boxAspectRatio > tileAspectRatio) { - tileWidth = boxHeight * tileAspectRatio; - tileHeight = boxHeight; - } else { - tileWidth = boxWidth; - tileHeight = boxWidth / tileAspectRatio; - } - } else { - tileWidth = boxWidth; - tileHeight = boxHeight; - } - - for (let i = 0; i < tileCount; i++) { - const verticalIndex = Math.floor(i / columnCount); - const top = verticalIndex * GAP + verticalIndex * tileHeight; - - let rowItemCount: number; - - if (verticalIndex + 1 === rowCount && tileCount % columnCount !== 0) { - rowItemCount = tileCount % columnCount; - } else { - rowItemCount = columnCount; - } - - const horizontalIndex = i % columnCount; - - let centeringPadding = 0; - - if (rowItemCount < columnCount) { - const subgridWidth = tileWidth * columnCount + (GAP * columnCount - 1); - centeringPadding = Math.round( - (subgridWidth - (tileWidth * rowItemCount + (GAP * rowItemCount - 1))) / - 2, - ); - } - - const left = - centeringPadding + GAP * horizontalIndex + tileWidth * horizontalIndex; - - newTilePositions.push({ - width: tileWidth, - height: tileHeight, - x: left, - y: top, - zIndex: 0, - }); - } - - return newTilePositions; -} - -// Calculates the number of possible tiles that can be displayed -function displayedTileCount( - layout: Layout, - tileCount: number, - gridWidth: number, - gridHeight: number, -): number { - let displayedTile = -1; - if (layout === "grid") { - return displayedTile; - } - if (tileCount < 2) { - return displayedTile; - } - - const gridAspectRatio = gridWidth / gridHeight; - - if (gridAspectRatio < 1) { - // Vertical layout (mobile) - const spotlightTileHeight = (gridHeight - GAP * 3) * (4 / 5); - const spectatorTileSize = gridHeight - GAP * 3 - spotlightTileHeight; - displayedTile = Math.round(gridWidth / spectatorTileSize); - } else { - const spotlightTileWidth = ((gridWidth - GAP * 3) * 4) / 5; - const spectatorTileWidth = gridWidth - GAP * 3 - spotlightTileWidth; - const spectatorTileHeight = spectatorTileWidth * (9 / 16); - displayedTile = Math.round(gridHeight / spectatorTileHeight); - } - - return displayedTile; -} - -// Sets the 'order' property on tiles based on the layout param and -// other properties of the tiles, eg. 'focused' and 'presenter' -export function reorderTiles( - tiles: Tile[], - layout: Layout, - displayedTile = -1, -): void { - // We use a special layout for 1:1 to always put the local tile first. - // We only do this if there are two tiles (obviously) and exactly one - // of them is local: during startup we can have tiles from other users - // but not our own, due to the order they're added, so without this we - // can assign multiple remote tiles order '1' and this persists through - // subsequent reorders because we preserve the order of the tiles. - if ( - layout === "grid" && - tiles.length === 2 && - tiles.filter((t) => t.item.local).length === 1 && - !tiles.some((t) => t.focused) - ) { - // 1:1 layout - tiles.forEach((tile) => (tile.order = tile.item.local ? 0 : 1)); - } else { - const focusedTiles: Tile[] = []; - const presenterTiles: Tile[] = []; - const onlyVideoTiles: Tile[] = []; - const otherTiles: Tile[] = []; - - const orderedTiles: Tile[] = new Array(tiles.length); - tiles.forEach((tile) => (orderedTiles[tile.order] = tile)); - - let firstLocalTile: Tile | undefined; - orderedTiles.forEach((tile) => { - if (tile.focused) { - focusedTiles.push(tile); - } else if (tile.isPresenter) { - presenterTiles.push(tile); - } else if (tile.hasVideo) { - if (tile.order === 0 && tile.item.local) { - firstLocalTile = tile; - } else { - onlyVideoTiles.push(tile); - } - } else { - if (tile.order === 0 && tile.item.local) { - firstLocalTile = tile; - } else { - otherTiles.push(tile); - } - } - }); - - if (firstLocalTile) { - if (firstLocalTile.hasVideo) { - onlyVideoTiles.push(firstLocalTile); - } else { - otherTiles.push(firstLocalTile); - } - } - - const reorderedTiles = [ - ...focusedTiles, - ...presenterTiles, - ...onlyVideoTiles, - ...otherTiles, - ]; - let nextSpeakerTileIndex = focusedTiles.length + presenterTiles.length; - - reorderedTiles.forEach((tile, i) => { - // If a speaker's natural ordering would place it outside the default - // visible area, promote them to the section dedicated to speakers - if (tile.isSpeaker && displayedTile <= i && nextSpeakerTileIndex < i) { - // Remove the tile from its current section - reorderedTiles.splice(i, 1); - // Insert it into the speaker section - reorderedTiles.splice(nextSpeakerTileIndex, 0, tile); - nextSpeakerTileIndex++; - } - }); - - reorderedTiles.forEach((tile, i) => (tile.order = i)); - } -} - -interface DragTileData { - offsetX: number; - offsetY: number; - key: string; - x: number; - y: number; -} - -export interface ChildrenProperties { - ref: Ref; - style: ComponentProps["style"]; - /** - * The width this tile will have once its animations have settled. - */ - targetWidth: number; - /** - * The height this tile will have once its animations have settled. - */ - targetHeight: number; - data: T; -} - -export interface LegacyGridProps { - items: TileDescriptor[]; - layout: Layout; - disableAnimations: boolean; - Tile: ComponentType>; -} - -export function LegacyGrid({ - items, - layout, - disableAnimations, - Tile, -}: LegacyGridProps): ReactNode { - // Place the PiP in the bottom right corner by default - const [pipXRatio, setPipXRatio] = useState(1); - const [pipYRatio, setPipYRatio] = useState(1); - - const [{ tiles, tilePositions }, setTileState] = useState<{ - tiles: Tile[]; - tilePositions: TilePosition[]; - }>({ - tiles: [], - tilePositions: [], - }); - const [scrollPosition, setScrollPosition] = useState(0); - const draggingTileRef = useRef(null); - const lastTappedRef = useRef<{ [index: string]: number }>({}); - const lastLayoutRef = useRef(layout); - const isMounted = useIsMounted(); - - // The 'polyfill' argument to useMeasure is not a polyfill at all but is the impl that is always used - // if passed, whether the browser has native support or not, so pass in either the browser native - // version or the ponyfill (yes, pony) because Juggle's resizeobserver ponyfill is being weirdly - // buggy for me on my dev env my never updating the size until the window resizes. - const [gridRef, gridBounds] = useMeasure({ - polyfill: window.ResizeObserver ?? JuggleResizeObserver, - }); - - useEffect(() => { - setTileState(({ tiles, ...rest }) => { - const newTiles: Tile[] = []; - const removedTileKeys: Set = new Set(); - - for (const tile of tiles) { - let item = items.find((item) => item.id === tile.key); - - let remove = false; - - if (!item) { - remove = true; - item = tile.item; - removedTileKeys.add(tile.key); - } - - let focused: boolean; - let isSpeaker: boolean; - let isPresenter: boolean; - let hasVideo: boolean; - if (layout === "spotlight") { - focused = item.focused; - isPresenter = item.isPresenter; - isSpeaker = item.isSpeaker; - hasVideo = item.hasVideo; - } else { - focused = layout === lastLayoutRef.current ? tile.focused : false; - isPresenter = false; - isSpeaker = false; - hasVideo = false; - } - - newTiles.push({ - key: item.id, - order: tile.order, - item, - remove, - focused, - isSpeaker: isSpeaker, - isPresenter: isPresenter, - hasVideo: hasVideo, - }); - } - - for (const item of items) { - const existingTileIndex = newTiles.findIndex( - ({ key }) => item.id === key, - ); - - const existingTile = newTiles[existingTileIndex]; - - if (existingTile && !existingTile.remove) { - continue; - } - - const newTile: Tile = { - key: item.id, - order: existingTile?.order ?? newTiles.length, - item, - remove: false, - focused: layout === "spotlight" && item.focused, - isPresenter: item.isPresenter, - isSpeaker: item.isSpeaker, - hasVideo: item.hasVideo, - }; - - if (existingTile) { - // Replace an existing tile - newTiles.splice(existingTileIndex, 1, newTile); - } else { - // Added tiles - newTiles.push(newTile); - } - } - - const presenter = newTiles.find((t) => t.isPresenter); - let displayedTile = -1; - // Only on screen share we will not move active displayed speaker - if (presenter !== undefined) { - displayedTile = displayedTileCount( - layout, - newTiles.length, - gridBounds.width, - gridBounds.height, - ); - } - - reorderTiles(newTiles, layout, displayedTile); - - if (removedTileKeys.size > 0) { - setTimeout(() => { - if (!isMounted.current) { - return; - } - - setTileState(({ tiles, ...rest }) => { - const newTiles: Tile[] = tiles - .filter((tile) => !removedTileKeys.has(tile.key)) - .map((tile) => ({ ...tile })); // clone before reordering - reorderTiles(newTiles, layout); - - const focusedTileCount = newTiles.reduce( - (count, tile) => count + (tile.focused ? 1 : 0), - 0, - ); - - return { - ...rest, - tiles: newTiles, - tilePositions: getTilePositions( - newTiles.length, - focusedTileCount, - gridBounds.width, - gridBounds.height, - pipXRatio, - pipYRatio, - layout, - ), - }; - }); - }, 250); - } - - const focusedTileCount = newTiles.reduce( - (count, tile) => count + (tile.focused ? 1 : 0), - 0, - ); - - lastLayoutRef.current = layout; - - return { - ...rest, - tiles: newTiles, - tilePositions: getTilePositions( - newTiles.length, - focusedTileCount, - gridBounds.width, - gridBounds.height, - pipXRatio, - pipYRatio, - layout, - ), - }; - }); - }, [items, gridBounds, layout, isMounted, pipXRatio, pipYRatio]); - - const tilePositionsValid = useRef(false); - - const animate = useCallback( - (tiles: Tile[]) => { - // Whether the tile positions were valid at the time of the previous - // animation - const tilePositionsWereValid = tilePositionsValid.current; - const oneOnOneLayout = - tiles.length === 2 && !tiles.some((t) => t.focused); - - return (tileIndex: number): TileSpringUpdate => { - const tile = tiles[tileIndex]; - const tilePosition = tilePositions[tile.order]; - const draggingTile = draggingTileRef.current; - const dragging = draggingTile && tile.key === draggingTile.key; - const remove = tile.remove; - tilePositionsValid.current = tilePosition.height > 0; - - if (dragging) { - return { - width: tilePosition.width, - height: tilePosition.height, - x: draggingTile.offsetX + draggingTile.x, - y: draggingTile.offsetY + draggingTile.y, - scale: 1.1, - opacity: 1, - zIndex: 2, - shadow: 15, - shadowSpread: 0, - immediate: (key: string): boolean => - disableAnimations || - key === "zIndex" || - key === "x" || - key === "y" || - key === "shadow" || - key === "shadowSpread", - from: { - shadow: 0, - scale: 0, - opacity: 0, - zIndex: 0, - }, - reset: false, - }; - } else { - const isMobile = isMobileBreakpoint( - gridBounds.width, - gridBounds.height, - ); - - const x = - tilePosition.x + - (layout === "spotlight" && tile.order !== 0 && isMobile - ? scrollPosition - : 0); - const y = - tilePosition.y + - (layout === "spotlight" && tile.order !== 0 && !isMobile - ? scrollPosition - : 0); - const from: { - shadow: number; - scale: number; - opacity: number; - zIndex?: number; - x?: number; - y?: number; - width?: number; - height?: number; - } = { shadow: 0, scale: 0, opacity: 0 }; - let reset = false; - - if (!tilePositionsWereValid) { - // This indicates that the component just mounted. We discard the - // previous keyframe by resetting the tile's position, so that it - // animates in from the right place on screen, rather than wherever - // the zero-height grid placed it. - from.x = x; - from.y = y; - from.width = tilePosition.width; - from.height = tilePosition.height; - reset = true; - } - - return { - x, - y, - width: tilePosition.width, - height: tilePosition.height, - scale: remove ? 0 : 1, - opacity: remove ? 0 : 1, - zIndex: tilePosition.zIndex, - shadow: oneOnOneLayout && tile.item.local ? 1 : 0, - shadowSpread: oneOnOneLayout && tile.item.local ? 1 : 0, - from, - reset, - immediate: (key: string): boolean => - disableAnimations || - key === "zIndex" || - key === "shadow" || - key === "shadowSpread", - // If we just stopped dragging a tile, give it time for the - // animation to settle before pushing its z-index back down - delay: (key: string): number => (key === "zIndex" ? 500 : 0), - }; - } - }; - }, - [tilePositions, disableAnimations, scrollPosition, layout, gridBounds], - ); - - const [springs, api] = useSprings(tiles.length, animate(tiles), [ - tilePositions, - tiles, - scrollPosition, - // react-spring's types are bugged and can't infer the spring type - ]) as unknown as [SpringValues[], SpringRef]; - - const onTap = useCallback( - (tileKey: string) => { - const lastTapped = lastTappedRef.current[tileKey]; - - if (!lastTapped || Date.now() - lastTapped > 500) { - lastTappedRef.current[tileKey] = Date.now(); - return; - } - - lastTappedRef.current[tileKey] = 0; - - const tile = tiles.find((tile) => tile.key === tileKey); - if (!tile || layout !== "grid") return; - const item = tile.item; - - setTileState(({ tiles, ...state }) => { - let focusedTileCount = 0; - const newTiles = tiles.map((tile) => { - const newTile = { ...tile }; // clone before reordering - - if (tile.item === item) { - newTile.focused = !tile.focused; - } - if (newTile.focused) { - focusedTileCount++; - } - - return newTile; - }); - - reorderTiles(newTiles, layout); - - return { - ...state, - tiles: newTiles, - tilePositions: getTilePositions( - newTiles.length, - focusedTileCount, - gridBounds.width, - gridBounds.height, - pipXRatio, - pipYRatio, - layout, - ), - }; - }); - }, - [tiles, layout, gridBounds.width, gridBounds.height, pipXRatio, pipYRatio], - ); - - // Callback for useDrag. We could call useDrag here, but the default - // pattern of spreading {...bind()} across the children to bind the gesture - // ends up breaking memoization and ruining this component's performance. - // Instead, we pass this callback to each tile via a ref, to let them bind the - // gesture using the much more sensible ref-based method. - const onTileDrag = ( - tileId: string, - { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - active, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - xy, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - movement, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - tap, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - last, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - event, - }: Parameters>[0], - ): void => { - event.preventDefault(); - - if (tap) { - onTap(tileId); - return; - } - - if (layout !== "grid") return; - - const dragTileIndex = tiles.findIndex((tile) => tile.key === tileId); - const dragTile = tiles[dragTileIndex]; - const dragTilePosition = tilePositions[dragTile.order]; - - const cursorPosition = [xy[0] - gridBounds.left, xy[1] - gridBounds.top]; - - let newTiles = tiles; - - if (tiles.length === 2 && !tiles.some((t) => t.focused)) { - // We're in 1:1 mode, so only the local tile should be draggable - if (!dragTile.item.local) return; - - // Position should only update on the very last event, to avoid - // compounding the offset on every drag event - if (last) { - const remotePosition = tilePositions[1]; - - const pipGap = getPipGap( - gridBounds.width / gridBounds.height, - gridBounds.width, - ); - const pipMinX = remotePosition.x + pipGap; - const pipMinY = remotePosition.y + pipGap; - const pipMaxX = - remotePosition.x + - remotePosition.width - - dragTilePosition.width - - pipGap; - const pipMaxY = - remotePosition.y + - remotePosition.height - - dragTilePosition.height - - pipGap; - - const newPipXRatio = - (dragTilePosition.x + movement[0] - pipMinX) / (pipMaxX - pipMinX); - const newPipYRatio = - (dragTilePosition.y + movement[1] - pipMinY) / (pipMaxY - pipMinY); - - setPipXRatio(Math.max(0, Math.min(1, newPipXRatio))); - setPipYRatio(Math.max(0, Math.min(1, newPipYRatio))); - } - } else { - const hoverTile = tiles.find( - (tile) => - tile.key !== tileId && - isInside(cursorPosition, tilePositions[tile.order]), - ); - - if (hoverTile) { - // Shift the tiles into their new order - newTiles = newTiles.map((tile) => { - let order = tile.order; - if (order < dragTile.order) { - if (order >= hoverTile.order) order++; - } else if (order > dragTile.order) { - if (order <= hoverTile.order) order--; - } else { - order = hoverTile.order; - } - - let focused; - if (tile === hoverTile) { - focused = dragTile.focused; - } else if (tile === dragTile) { - focused = hoverTile.focused; - } else { - focused = tile.focused; - } - - return { ...tile, order, focused }; - }); - - reorderTiles(newTiles, layout); - - setTileState((state) => ({ ...state, tiles: newTiles })); - } - } - - if (active) { - if (!draggingTileRef.current) { - draggingTileRef.current = { - key: dragTile.key, - offsetX: dragTilePosition.x, - offsetY: dragTilePosition.y, - x: movement[0], - y: movement[1], - }; - } else { - draggingTileRef.current.x = movement[0]; - draggingTileRef.current.y = movement[1]; - } - } else { - draggingTileRef.current = null; - } - - api.start(animate(newTiles)); - }; - - const onTileDragRef = useRef(onTileDrag); - onTileDragRef.current = onTileDrag; - - const onGridGesture = useCallback( - ( - e: - | Omit, "event"> - | Omit, "event">, - isWheel: boolean, - ) => { - if (layout !== "spotlight") { - return; - } - - const isMobile = isMobileBreakpoint(gridBounds.width, gridBounds.height); - - let movement = e.delta[isMobile ? 0 : 1]; - - if (isWheel) { - movement = -movement; - } - - let min = 0; - - if (tilePositions.length > 1) { - const lastTile = tilePositions[tilePositions.length - 1]; - min = isMobile - ? gridBounds.width - lastTile.x - lastTile.width - GAP - : gridBounds.height - lastTile.y - lastTile.height - GAP; - } - - setScrollPosition((scrollPosition) => - Math.min(Math.max(movement + scrollPosition, min), 0), - ); - }, - [layout, gridBounds, tilePositions], - ); - - const bindGrid = useGesture( - { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - onWheel: (e) => onGridGesture(e, true), - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - onDrag: (e) => onGridGesture(e, false), - }, - {}, - ); - - return ( -
- {springs.map((spring, i) => { - const tile = tiles[i]; - const tilePosition = tilePositions[tile.order]; - - return ( - - ); - })} -
- ); -} - -LegacyGrid.defaultProps = { - layout: "grid", -}; diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 90791de6..a1b3f4cc 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -18,10 +18,9 @@ import { RoomAudioRenderer, RoomContext, useLocalParticipant, - useTracks, } from "@livekit/components-react"; import { usePreventScroll } from "@react-aria/overlays"; -import { ConnectionState, Room, Track } from "livekit-client"; +import { ConnectionState, Room } from "livekit-client"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { FC, @@ -38,7 +37,6 @@ import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import classNames from "classnames"; import { BehaviorSubject, map } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; -import { useTranslation } from "react-i18next"; import LogoMark from "../icons/LogoMark.svg?react"; import LogoType from "../icons/LogoType.svg?react"; @@ -51,10 +49,8 @@ import { SettingsButton, } from "../button"; import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header"; -import { LegacyGrid, useLegacyGridLayout } from "../grid/LegacyGrid"; import { useUrlParams } from "../UrlParams"; import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts"; -import { usePrefersReducedMotion } from "../usePrefersReducedMotion"; import { ElementWidgetActions, widget } from "../widget"; import styles from "./InCallView.module.css"; import { GridTile } from "../tile/GridTile"; @@ -72,14 +68,8 @@ import { InviteButton } from "../button/InviteButton"; import { LayoutToggle } from "./LayoutToggle"; import { ECConnectionState } from "../livekit/useECConnectionState"; import { useOpenIDSFU } from "../livekit/openIDSFU"; -import { - GridMode, - Layout, - TileDescriptor, - useCallViewModel, -} from "../state/CallViewModel"; +import { GridMode, Layout, useCallViewModel } from "../state/CallViewModel"; import { Grid, TileProps } from "../grid/Grid"; -import { MediaViewModel } from "../state/MediaViewModel"; import { useObservable } from "../state/useObservable"; import { useInitial } from "../useInitial"; import { SpotlightTile } from "../tile/SpotlightTile"; @@ -89,7 +79,6 @@ import { makeGridLayout } from "../grid/GridLayout"; import { makeSpotlightLayout } from "../grid/SpotlightLayout"; import { CallLayout, - GridTileModel, TileModel, defaultPipAlignment, defaultSpotlightAlignment, @@ -98,10 +87,6 @@ import { makeOneOnOneLayout } from "../grid/OneOnOneLayout"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); -const dummySpotlightItem = { - id: "spotlight", -} as TileDescriptor; - export interface ActiveCallProps extends Omit { e2eeSystem: EncryptionSystem; @@ -155,11 +140,9 @@ export const InCallView: FC = ({ participantCount, onLeave, hideHeader, - otelGroupCallMembership, connState, onShareClick, }) => { - const { t } = useTranslation(); usePreventScroll(); useWakeLock(); @@ -177,15 +160,6 @@ export const InCallView: FC = ({ // Merge the refs so they can attach to the same element const containerRef = useMergedRefs(containerRef1, containerRef2); - const screenSharingTracks = useTracks( - [{ source: Track.Source.ScreenShare, withPlaceholder: false }], - { - room: livekitRoom, - }, - ); - const { layout: legacyLayout, setLayout: setLegacyLayout } = - useLegacyGridLayout(screenSharingTracks.length > 0); - const { hideScreensharing, showControls } = useUrlParams(); const { isScreenShareEnabled, localParticipant } = useLocalParticipant({ @@ -210,42 +184,6 @@ export const InCallView: FC = ({ (muted) => muteStates.audio.setEnabled?.(!muted), ); - useEffect(() => { - widget?.api.transport.send( - legacyLayout === "grid" - ? ElementWidgetActions.TileLayout - : ElementWidgetActions.SpotlightLayout, - {}, - ); - }, [legacyLayout]); - - useEffect(() => { - if (widget) { - const onTileLayout = (ev: CustomEvent): void => { - setLegacyLayout("grid"); - widget!.api.transport.reply(ev.detail, {}); - }; - const onSpotlightLayout = (ev: CustomEvent): void => { - setLegacyLayout("spotlight"); - widget!.api.transport.reply(ev.detail, {}); - }; - - widget.lazyActions.on(ElementWidgetActions.TileLayout, onTileLayout); - widget.lazyActions.on( - ElementWidgetActions.SpotlightLayout, - onSpotlightLayout, - ); - - return (): void => { - widget!.lazyActions.off(ElementWidgetActions.TileLayout, onTileLayout); - widget!.lazyActions.off( - ElementWidgetActions.SpotlightLayout, - onSpotlightLayout, - ); - }; - } - }, [setLegacyLayout]); - const mobile = boundsValid && bounds.width <= 660; const reducedControls = boundsValid && bounds.width <= 340; const noControls = reducedControls && bounds.height <= 400; @@ -256,15 +194,12 @@ export const InCallView: FC = ({ matrixInfo.e2eeSystem.kind !== E2eeType.NONE, connState, ); - const items = useObservableEagerState(vm.tiles); const layout = useObservableEagerState(vm.layout); + const gridMode = useObservableEagerState(vm.gridMode); const hasSpotlight = layout.spotlight !== undefined; - // Hack: We insert a dummy "spotlight" tile into the tiles we pass to - // useFullscreen so that we can control the fullscreen state of the - // spotlight tile in the new layouts with this same hook. const fullscreenItems = useMemo( - () => (hasSpotlight ? [...items, dummySpotlightItem] : items), - [items, hasSpotlight], + () => (hasSpotlight ? ["spotlight"] : []), + [hasSpotlight], ); const { fullscreenItem, toggleFullscreen, exitFullscreen } = useFullscreen(fullscreenItems); @@ -274,18 +209,9 @@ export const InCallView: FC = ({ ); // The maximised participant: either the participant that the user has - // manually put in fullscreen, or the focused (active) participant if the - // window is too small to show everyone - const maximisedParticipant = useMemo( - () => - fullscreenItem ?? - (noControls - ? items.find((item) => item.isSpeaker) ?? items.at(0) ?? null - : null), - [fullscreenItem, noControls, items], - ); - - const prefersReducedMotion = usePrefersReducedMotion(); + // manually put in fullscreen, or (TODO) the spotlight if the window is too + // small to show everyone + const maximisedParticipant = fullscreenItem; const [settingsModalOpen, setSettingsModalOpen] = useState(false); const [settingsTab, setSettingsTab] = useState(defaultSettingsTab); @@ -339,7 +265,7 @@ export const InCallView: FC = ({ makeLayout = makeSpotlightLayout as CallLayout; else if (l.type === "one-on-one") makeLayout = makeOneOnOneLayout as CallLayout; - else return null; // Not yet implemented + else throw new Error(`Unimplemented layout: ${l.type}`); return makeLayout({ minBounds: gridBoundsObservable, @@ -352,13 +278,46 @@ export const InCallView: FC = ({ ); const setGridMode = useCallback( - (mode: GridMode) => { - setLegacyLayout(mode); - vm.setGridMode(mode); - }, - [setLegacyLayout, vm], + (mode: GridMode) => vm.setGridMode(mode), + [vm], ); + useEffect(() => { + widget?.api.transport.send( + gridMode === "grid" + ? ElementWidgetActions.TileLayout + : ElementWidgetActions.SpotlightLayout, + {}, + ); + }, [gridMode]); + + useEffect(() => { + if (widget) { + const onTileLayout = (ev: CustomEvent): void => { + setGridMode("grid"); + widget!.api.transport.reply(ev.detail, {}); + }; + const onSpotlightLayout = (ev: CustomEvent): void => { + setGridMode("spotlight"); + widget!.api.transport.reply(ev.detail, {}); + }; + + widget.lazyActions.on(ElementWidgetActions.TileLayout, onTileLayout); + widget.lazyActions.on( + ElementWidgetActions.SpotlightLayout, + onSpotlightLayout, + ); + + return (): void => { + widget!.lazyActions.off(ElementWidgetActions.TileLayout, onTileLayout); + widget!.lazyActions.off( + ElementWidgetActions.SpotlightLayout, + onSpotlightLayout, + ); + }; + } + }, [setGridMode]); + const showSpotlightIndicators = useObservable(layout.type === "spotlight"); const showSpeakingIndicators = useObservable( layout.type === "spotlight" || @@ -419,33 +378,10 @@ export const InCallView: FC = ({ ], ); - const LegacyTile = useMemo( - () => - forwardRef< - HTMLDivElement, - PropsWithoutRef> - >(function LegacyTile({ model: legacyModel, ...props }, ref) { - const model: GridTileModel = useMemo( - () => ({ type: "grid", vm: legacyModel }), - [legacyModel], - ); - return ; - }), - [Tile], - ); - const renderContent = (): JSX.Element => { - if (items.length === 0) { - return ( -
-

{t("waiting_for_participants")}

-
- ); - } - if (maximisedParticipant !== null) { const fullscreen = maximisedParticipant === fullscreenItem; - if (maximisedParticipant.id === "spotlight") { + if (maximisedParticipant === "spotlight") { return ( = ({ /> ); } - return ( - - ); } - if (layoutSystem === null) { - // This new layout doesn't yet have an implemented layout system, so fall - // back to the legacy grid system - return ( - + - ); - } else { - return ( - <> - - - - ); - } + + + ); }; const rageshakeRequestModalProps = useRageshakeRequestModal( @@ -596,7 +505,7 @@ export const InCallView: FC = ({ {!mobile && !hideHeader && showControls && ( )} diff --git a/src/room/useFullscreen.ts b/src/room/useFullscreen.ts index db769fa5..650cc6ea 100644 --- a/src/room/useFullscreen.ts +++ b/src/room/useFullscreen.ts @@ -20,7 +20,6 @@ import { useCallback, useLayoutEffect, useRef } from "react"; import { useReactiveState } from "../useReactiveState"; import { useEventTarget } from "../useEvents"; -import { TileDescriptor } from "../state/CallViewModel"; const isFullscreen = (): boolean => Boolean(document.fullscreenElement) || @@ -55,31 +54,30 @@ function useFullscreenChange(onFullscreenChange: () => void): void { * Provides callbacks for controlling the full-screen view, which can hold one * item at a time. */ -export function useFullscreen(items: TileDescriptor[]): { - fullscreenItem: TileDescriptor | null; +// TODO: Simplify this. Nowadays we only allow the spotlight to be fullscreen, +// so we don't need to bother with multiple items. +export function useFullscreen(items: string[]): { + fullscreenItem: string | null; toggleFullscreen: (itemId: string) => void; exitFullscreen: () => void; } { - const [fullscreenItem, setFullscreenItem] = - useReactiveState | null>( - (prevItem) => - prevItem == null - ? null - : items.find((i) => i.id === prevItem.id) ?? null, - [items], - ); + const [fullscreenItem, setFullscreenItem] = useReactiveState( + (prevItem) => + prevItem == null ? null : items.find((i) => i === prevItem) ?? null, + [items], + ); - const latestItems = useRef[]>(items); + const latestItems = useRef(items); latestItems.current = items; - const latestFullscreenItem = useRef | null>(fullscreenItem); + const latestFullscreenItem = useRef(fullscreenItem); latestFullscreenItem.current = fullscreenItem; const toggleFullscreen = useCallback( (itemId: string) => { setFullscreenItem( latestFullscreenItem.current === null - ? latestItems.current.find((i) => i.id === itemId) ?? null + ? latestItems.current.find((i) => i === itemId) ?? null : null, ); }, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 62c35a8e..975d069b 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -74,22 +74,6 @@ import { ObservableScope } from "./ObservableScope"; // list again const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; -// Represents something that should get a tile on the layout, -// ie. a user's video feed or a screen share feed. -// TODO: This exposes too much information to the view layer, let's keep this -// information internal to the view model and switch to using Tile instead -export interface TileDescriptor { - id: string; - focused: boolean; - isPresenter: boolean; - isSpeaker: boolean; - hasVideo: boolean; - local: boolean; - largeBaseSize: boolean; - placeNear?: string; - data: T; -} - export interface GridLayout { type: "grid"; spotlight?: MediaViewModel[]; @@ -548,108 +532,6 @@ export class CallViewModel extends ViewModel { shareReplay(1), ); - /** - * The media tiles to be displayed in the call view. - */ - // TODO: Get rid of this field, replacing it with the 'layout' field above - // which keeps more details of the layout order internal to the view model - public readonly tiles: Observable[]> = - combineLatest([ - this.remoteParticipants, - observeParticipantMedia(this.livekitRoom.localParticipant), - ]).pipe( - scan((ts, [remoteParticipants, { participant: localParticipant }]) => { - const ps = [localParticipant, ...remoteParticipants]; - const tilesById = new Map(ts.map((t) => [t.id, t])); - const now = Date.now(); - let allGhosts = true; - - const newTiles = ps.flatMap((p) => { - const userMediaId = p.identity; - const member = findMatrixMember(this.matrixRoom, userMediaId); - allGhosts &&= member === undefined; - const spokeRecently = - p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000; - - // We always start with a local participant with the empty string as - // their ID before we're connected, this is fine and we'll be in - // "all ghosts" mode. - if (userMediaId !== "" && member === undefined) { - logger.warn( - `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`, - ); - } - - const userMediaVm = - tilesById.get(userMediaId)?.data ?? - (p instanceof LocalParticipant - ? new LocalUserMediaViewModel( - userMediaId, - member, - p, - this.encrypted, - ) - : new RemoteUserMediaViewModel( - userMediaId, - member, - p, - this.encrypted, - )); - tilesById.delete(userMediaId); - - const userMediaTile: TileDescriptor = { - id: userMediaId, - focused: false, - isPresenter: p.isScreenShareEnabled, - isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal, - hasVideo: p.isCameraEnabled, - local: p.isLocal, - largeBaseSize: false, - data: userMediaVm, - }; - - if (p.isScreenShareEnabled) { - const screenShareId = `${userMediaId}:screen-share`; - const screenShareVm = - tilesById.get(screenShareId)?.data ?? - new ScreenShareViewModel( - screenShareId, - member, - p, - this.encrypted, - ); - tilesById.delete(screenShareId); - - const screenShareTile: TileDescriptor = { - id: screenShareId, - focused: true, - isPresenter: false, - isSpeaker: false, - hasVideo: true, - local: p.isLocal, - largeBaseSize: true, - placeNear: userMediaId, - data: screenShareVm, - }; - return [userMediaTile, screenShareTile]; - } else { - return [userMediaTile]; - } - }); - - // Any tiles left in the map are unused and should be destroyed - for (const t of tilesById.values()) t.data.destroy(); - - // If every item is a ghost, that probably means we're still connecting - // and shouldn't bother showing anything yet - return allGhosts ? [] : newTiles; - }, [] as TileDescriptor[]), - finalizeValue((ts) => { - for (const t of ts) t.data.destroy(); - }), - shareReplay(1), - ); - public constructor( // A call is permanently tied to a single Matrix room and LiveKit room private readonly matrixRoom: MatrixRoom, diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 14f85831..4dd83567 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -33,7 +33,6 @@ import VolumeOffIcon from "@vector-im/compound-design-tokens/icons/volume-off.sv import VisibilityOnIcon from "@vector-im/compound-design-tokens/icons/visibility-on.svg?react"; import UserProfileIcon from "@vector-im/compound-design-tokens/icons/user-profile.svg?react"; import ExpandIcon from "@vector-im/compound-design-tokens/icons/expand.svg?react"; -import CollapseIcon from "@vector-im/compound-design-tokens/icons/collapse.svg?react"; import { ContextMenu, MenuItem, @@ -44,8 +43,6 @@ import { useObservableEagerState } from "observable-hooks"; import styles from "./GridTile.module.css"; import { - ScreenShareViewModel, - MediaViewModel, UserMediaViewModel, useNameData, LocalUserMediaViewModel, @@ -63,45 +60,12 @@ interface TileProps { maximised: boolean; displayName: string; nameTag: string; + showSpeakingIndicators: boolean; } -interface MediaTileProps - extends TileProps, - Omit, "className"> { - vm: MediaViewModel; - videoEnabled: boolean; - videoFit: "contain" | "cover"; - mirror: boolean; - nameTagLeadingIcon?: ReactNode; - primaryButton: ReactNode; - secondaryButton?: ReactNode; -} - -const MediaTile = forwardRef( - ({ vm, className, maximised, ...props }, ref) => { - const video = useObservableEagerState(vm.video); - const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); - - return ( - - ); - }, -); - -MediaTile.displayName = "MediaTile"; - interface UserMediaTileProps extends TileProps { vm: UserMediaViewModel; mirror: boolean; - showSpeakingIndicators: boolean; menuStart?: ReactNode; menuEnd?: ReactNode; } @@ -115,11 +79,14 @@ const UserMediaTile = forwardRef( menuEnd, className, nameTag, + maximised, ...props }, ref, ) => { const { t } = useTranslation(); + const video = useObservableEagerState(vm.video); + const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); const audioEnabled = useObservableEagerState(vm.audioEnabled); const videoEnabled = useObservableEagerState(vm.videoEnabled); const speaking = useObservableEagerState(vm.speaking); @@ -148,12 +115,14 @@ const UserMediaTile = forwardRef( ); const tile = ( - ( {menu} } + data-maximised={maximised} {...props} /> ); @@ -199,7 +169,6 @@ UserMediaTile.displayName = "UserMediaTile"; interface LocalUserMediaTileProps extends TileProps { vm: LocalUserMediaViewModel; onOpenProfile: () => void; - showSpeakingIndicators: boolean; } const LocalUserMediaTile = forwardRef( @@ -248,7 +217,6 @@ LocalUserMediaTile.displayName = "LocalUserMediaTile"; interface RemoteUserMediaTileProps extends TileProps { vm: RemoteUserMediaViewModel; - showSpeakingIndicators: boolean; } const RemoteUserMediaTile = forwardRef< @@ -303,53 +271,8 @@ const RemoteUserMediaTile = forwardRef< RemoteUserMediaTile.displayName = "RemoteUserMediaTile"; -interface ScreenShareTileProps extends TileProps { - vm: ScreenShareViewModel; - fullscreen: boolean; - onToggleFullscreen: (itemId: string) => void; -} - -const ScreenShareTile = forwardRef( - ({ vm, fullscreen, onToggleFullscreen, ...props }, ref) => { - const { t } = useTranslation(); - const onClickFullScreen = useCallback( - () => onToggleFullscreen(vm.id), - [onToggleFullscreen, vm], - ); - - const FullScreenIcon = fullscreen ? CollapseIcon : ExpandIcon; - - return ( - - - - ) - } - {...props} - /> - ); - }, -); - -ScreenShareTile.displayName = "ScreenShareTile"; - interface GridTileProps { - vm: MediaViewModel; + vm: UserMediaViewModel; maximised: boolean; fullscreen: boolean; onToggleFullscreen: (itemId: string) => void; @@ -375,19 +298,8 @@ export const GridTile = forwardRef( {...nameData} /> ); - } else if (vm instanceof RemoteUserMediaViewModel) { - return ; } else { - return ( - - ); + return ; } }, ); diff --git a/test/grid/LegacyGrid-test.ts b/test/grid/LegacyGrid-test.ts deleted file mode 100644 index e57adf9d..00000000 --- a/test/grid/LegacyGrid-test.ts +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2023-2024 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { TileDescriptor } from "../../src/state/CallViewModel"; -import { Tile, reorderTiles } from "../../src/grid/LegacyGrid"; - -const alice: Tile = { - key: "alice", - order: 0, - item: { local: false } as unknown as TileDescriptor, - remove: false, - focused: false, - isPresenter: false, - isSpeaker: false, - hasVideo: true, -}; -const bob: Tile = { - key: "bob", - order: 1, - item: { local: false } as unknown as TileDescriptor, - remove: false, - focused: false, - isPresenter: false, - isSpeaker: false, - hasVideo: false, -}; - -test("reorderTiles does not promote a non-speaker", () => { - const tiles = [{ ...alice }, { ...bob }]; - reorderTiles(tiles, "spotlight", 1); - expect(tiles).toEqual([ - expect.objectContaining({ key: "alice", order: 0 }), - expect.objectContaining({ key: "bob", order: 1 }), - ]); -}); - -test("reorderTiles promotes a speaker into the visible area", () => { - const tiles = [{ ...alice }, { ...bob, isSpeaker: true }]; - reorderTiles(tiles, "spotlight", 1); - expect(tiles).toEqual([ - expect.objectContaining({ key: "alice", order: 1 }), - expect.objectContaining({ key: "bob", order: 0 }), - ]); -}); - -test("reorderTiles keeps a promoted speaker in the visible area", () => { - const tiles = [ - { ...alice, order: 1 }, - { ...bob, isSpeaker: true, order: 0 }, - ]; - reorderTiles(tiles, "spotlight", 1); - expect(tiles).toEqual([ - expect.objectContaining({ key: "alice", order: 1 }), - expect.objectContaining({ key: "bob", order: 0 }), - ]); -}); From a16f2352779e6b9016b89c70b9331e510a18ea33 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 12 Jun 2024 15:26:00 -0400 Subject: [PATCH 4/5] Fix crash in spotlight mode while connecting Because we were hiding even the local participant during initial connection, there would be no participants, and therefore nothing to put in the spotlight. The designs don't really tell us what the connecting state should look like, so I've taken the liberty of restoring it to its former glory of showing the local participant immediately. --- src/grid/Grid.tsx | 13 +++++++------ src/state/CallViewModel.ts | 21 ++++++--------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/grid/Grid.tsx b/src/grid/Grid.tsx index 6c1bab9b..ea33a32d 100644 --- a/src/grid/Grid.tsx +++ b/src/grid/Grid.tsx @@ -268,12 +268,13 @@ export function Grid< ) as HTMLCollectionOf; for (const slot of slots) { const id = slot.getAttribute("data-id")!; - result.push({ - ...tiles.get(id)!, - ...offset(slot, gridRoot), - width: slot.offsetWidth, - height: slot.offsetHeight, - }); + if (slot.offsetWidth > 0 && slot.offsetHeight > 0) + result.push({ + ...tiles.get(id)!, + ...offset(slot, gridRoot), + width: slot.offsetWidth, + height: slot.offsetHeight, + }); } } diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 975d069b..6ed21a59 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -207,7 +207,8 @@ function findMatrixMember( room: MatrixRoom, id: string, ): RoomMember | undefined { - if (!id) return undefined; + if (id === "local") + return room.getMember(room.client.getUserId()!) ?? undefined; const parts = id.split(":"); // must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon @@ -307,23 +308,16 @@ export class CallViewModel extends ViewModel { ]).pipe( scan( (prevItems, [remoteParticipants, { participant: localParticipant }]) => { - let allGhosts = true; - const newItems = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { for (const p of [localParticipant, ...remoteParticipants]) { - const member = findMatrixMember(this.matrixRoom, p.identity); - allGhosts &&= member === undefined; - // We always start with a local participant with the empty string as - // their ID before we're connected, this is fine and we'll be in - // "all ghosts" mode. - if (p.identity !== "" && member === undefined) { + const userMediaId = p === localParticipant ? "local" : p.identity; + const member = findMatrixMember(this.matrixRoom, userMediaId); + if (member === undefined) logger.warn( `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`, ); - } - const userMediaId = p.identity; yield [ userMediaId, prevItems.get(userMediaId) ?? @@ -343,10 +337,7 @@ export class CallViewModel extends ViewModel { ); for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy(); - - // If every item is a ghost, that probably means we're still connecting - // and shouldn't bother showing anything yet - return allGhosts ? new Map() : newItems; + return newItems; }, new Map(), ), From 7526826b0c01c6f6e70db4176ece928b92083a85 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Jul 2024 11:01:21 -0400 Subject: [PATCH 5/5] Improve aspect ratios on mobile --- src/grid/CallLayout.ts | 11 +++++++++-- src/grid/OneOnOneLayout.module.css | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts index e97b18a2..e0054a1c 100644 --- a/src/grid/CallLayout.ts +++ b/src/grid/CallLayout.ts @@ -92,6 +92,7 @@ export interface GridArrangement { const tileMinHeight = 130; const tileMaxAspectRatio = 17 / 9; const tileMinAspectRatio = 4 / 3; +const tileMobileMinAspectRatio = 2 / 3; /** * Determine the ideal arrangement of tiles into a grid of a particular size. @@ -136,12 +137,18 @@ export function arrangeTiles( tileHeight = (minHeight - (rows - 1) * gap) / rows; } if (tileHeight < tileMinHeight) tileHeight = tileMinHeight; + // Impose a minimum and maximum aspect ratio on the tiles const tileAspectRatio = tileWidth / tileHeight; + // We enforce a different min aspect ratio in 1:1s on mobile + const minAspectRatio = + tileCount === 1 && width < 600 + ? tileMobileMinAspectRatio + : tileMinAspectRatio; if (tileAspectRatio > tileMaxAspectRatio) tileWidth = tileHeight * tileMaxAspectRatio; - else if (tileAspectRatio < tileMinAspectRatio) - tileHeight = tileWidth / tileMinAspectRatio; + else if (tileAspectRatio < minAspectRatio) + tileHeight = tileWidth / minAspectRatio; // TODO: We might now be hitting the minimum height or width limit again return { tileWidth, tileHeight, gap, columns }; diff --git a/src/grid/OneOnOneLayout.module.css b/src/grid/OneOnOneLayout.module.css index 0d2ad4ff..54c39d25 100644 --- a/src/grid/OneOnOneLayout.module.css +++ b/src/grid/OneOnOneLayout.module.css @@ -27,11 +27,18 @@ limitations under the License. .local { position: absolute; - inline-size: 180px; - block-size: 135px; + inline-size: 135px; + block-size: 160px; inset: var(--cpd-space-4x); } +@media (min-width: 600px) { + .local { + inline-size: 170px; + block-size: 110px; + } +} + .spotlight { position: absolute; inline-size: 404px;