From 20602c122b8b0a81a9654f65737d751da221e564 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 2 May 2024 18:44:36 -0400 Subject: [PATCH 01/31] Implement the new unified grid layout Here I've implemented an MVP for the new unified grid layout, which scales smoothly up to arbitrarily many participants. It doesn't yet have a special 1:1 layout, so in spotlight mode and 1:1s, we will still fall back to the legacy grid systems. Things that happened along the way: - The part of VideoTile that is common to both spotlight and grid tiles, I refactored into MediaView - VideoTile renamed to GridTile - Added SpotlightTile for the new, glassy spotlight designs - NewVideoGrid renamed to Grid, and refactored to be even more generic - I extracted the media name logic into a custom React hook - Deleted the BigGrid experiment --- public/locales/en-GB/app.json | 2 + src/Header.module.css | 1 + src/Header.tsx | 24 +- .../NewVideoGrid.module.css => grid/Grid.css} | 0 .../Grid.module.css} | 15 +- src/grid/Grid.tsx | 465 +++++++ src/grid/GridLayout.module.css | 54 + src/grid/GridLayout.tsx | 189 +++ .../LegacyGrid.module.css} | 2 +- .../VideoGrid.tsx => grid/LegacyGrid.tsx} | 31 +- src/grid/TileWrapper.module.css | 23 + src/{video-grid => grid}/TileWrapper.tsx | 77 +- src/room/InCallView.module.css | 37 +- src/room/InCallView.tsx | 307 +++-- src/room/VideoPreview.tsx | 23 +- src/state/CallViewModel.ts | 4 +- src/state/MediaViewModel.ts | 48 +- src/state/useObservable.ts | 15 +- src/tile/GridTile.module.css | 81 ++ .../VideoTile.tsx => tile/GridTile.tsx} | 199 +-- .../MediaView.module.css} | 98 +- src/tile/MediaView.tsx | 130 ++ src/tile/SpotlightTile.module.css | 153 +++ src/tile/SpotlightTile.tsx | 263 ++++ src/useInitial.ts | 26 + src/useLatest.ts | 31 + src/useReactiveState.ts | 2 +- src/video-grid/BigGrid.tsx | 1070 ----------------- src/video-grid/Layout.tsx | 195 --- src/video-grid/NewVideoGrid.tsx | 389 ------ .../LegacyGrid-test.ts} | 2 +- test/video-grid/BigGrid-test.ts | 493 -------- 32 files changed, 1863 insertions(+), 2586 deletions(-) rename src/{video-grid/NewVideoGrid.module.css => grid/Grid.css} (100%) rename src/{video-grid/BigGrid.module.css => grid/Grid.module.css} (71%) create mode 100644 src/grid/Grid.tsx create mode 100644 src/grid/GridLayout.module.css create mode 100644 src/grid/GridLayout.tsx rename src/{video-grid/VideoGrid.module.css => grid/LegacyGrid.module.css} (97%) rename src/{video-grid/VideoGrid.tsx => grid/LegacyGrid.tsx} (98%) create mode 100644 src/grid/TileWrapper.module.css rename src/{video-grid => grid}/TileWrapper.tsx (59%) create mode 100644 src/tile/GridTile.module.css rename src/{video-grid/VideoTile.tsx => tile/GridTile.tsx} (66%) rename src/{video-grid/VideoTile.module.css => tile/MediaView.module.css} (66%) create mode 100644 src/tile/MediaView.tsx create mode 100644 src/tile/SpotlightTile.module.css create mode 100644 src/tile/SpotlightTile.tsx create mode 100644 src/useInitial.ts create mode 100644 src/useLatest.ts delete mode 100644 src/video-grid/BigGrid.tsx delete mode 100644 src/video-grid/Layout.tsx delete mode 100644 src/video-grid/NewVideoGrid.tsx rename test/{video-grid/VideoGrid-test.ts => grid/LegacyGrid-test.ts} (96%) delete mode 100644 test/video-grid/BigGrid-test.ts diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 7fd742cb..7a804a5d 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -41,6 +41,7 @@ "analytics": "Analytics", "audio": "Audio", "avatar": "Avatar", + "back": "Back", "camera": "Camera", "copied": "Copied!", "display_name": "Display name", @@ -49,6 +50,7 @@ "home": "Home", "loading": "Loading…", "microphone": "Microphone", + "next": "Next", "options": "Options", "password": "Password", "profile": "Profile", diff --git a/src/Header.module.css b/src/Header.module.css index 53f51d3a..6aa609f7 100644 --- a/src/Header.module.css +++ b/src/Header.module.css @@ -22,6 +22,7 @@ limitations under the License. user-select: none; flex-shrink: 0; padding-inline: var(--inline-content-inset); + padding-block-end: var(--cpd-space-4x); } .nav { diff --git a/src/Header.tsx b/src/Header.tsx index 1bf8a4a7..ffb4731e 100644 --- a/src/Header.tsx +++ b/src/Header.tsx @@ -15,7 +15,7 @@ limitations under the License. */ import classNames from "classnames"; -import { FC, HTMLAttributes, ReactNode } from "react"; +import { FC, HTMLAttributes, ReactNode, forwardRef } from "react"; import { Link } from "react-router-dom"; import { useTranslation } from "react-i18next"; import { Heading, Text } from "@vector-im/compound-web"; @@ -32,13 +32,21 @@ interface HeaderProps extends HTMLAttributes { className?: string; } -export const Header: FC = ({ children, className, ...rest }) => { - return ( -
- {children} -
- ); -}; +export const Header = forwardRef( + ({ children, className, ...rest }, ref) => { + return ( +
+ {children} +
+ ); + }, +); + +Header.displayName = "Header"; interface LeftNavProps extends HTMLAttributes { children: ReactNode; diff --git a/src/video-grid/NewVideoGrid.module.css b/src/grid/Grid.css similarity index 100% rename from src/video-grid/NewVideoGrid.module.css rename to src/grid/Grid.css diff --git a/src/video-grid/BigGrid.module.css b/src/grid/Grid.module.css similarity index 71% rename from src/video-grid/BigGrid.module.css rename to src/grid/Grid.module.css index 2201295d..33e593be 100644 --- a/src/video-grid/BigGrid.module.css +++ b/src/grid/Grid.module.css @@ -1,5 +1,5 @@ /* -Copyright 2023 New Vector Ltd +Copyright 2023-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,15 +14,10 @@ See the License for the specific language governing permissions and limitations under the License. */ -.bigGrid { - display: grid; - grid-auto-rows: 130px; - gap: var(--cpd-space-2x); +.grid { + contain: layout style; } -@media (min-width: 800px) { - .bigGrid { - grid-auto-rows: 135px; - gap: var(--cpd-space-5x); - } +.slot { + contain: strict; } diff --git a/src/grid/Grid.tsx b/src/grid/Grid.tsx new file mode 100644 index 00000000..9aa7f95f --- /dev/null +++ b/src/grid/Grid.tsx @@ -0,0 +1,465 @@ +/* +Copyright 2023-2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + SpringRef, + TransitionFn, + animated, + useTransition, +} from "@react-spring/web"; +import { EventTypes, Handler, useScroll } from "@use-gesture/react"; +import { + CSSProperties, + ComponentProps, + ComponentType, + FC, + LegacyRef, + ReactNode, + useCallback, + useEffect, + useMemo, + useRef, + useState, +} from "react"; +import useMeasure from "react-use-measure"; +import classNames from "classnames"; + +import styles from "./Grid.module.css"; +import { useMergedRefs } from "../useMergedRefs"; +import { TileWrapper } from "./TileWrapper"; +import { usePrefersReducedMotion } from "../usePrefersReducedMotion"; +import { TileSpringUpdate } from "./LegacyGrid"; + +interface Rect { + x: number; + y: number; + width: number; + height: number; +} + +interface Tile extends Rect { + id: string; + model: Model; +} + +interface TileSpring { + opacity: number; + scale: number; + zIndex: number; + x: number; + y: number; + width: number; + height: number; +} + +interface DragState { + tileId: string; + tileX: number; + tileY: number; + cursorX: number; + cursorY: number; +} + +interface SlotProps extends ComponentProps<"div"> { + tile: string; + style?: CSSProperties; + className?: string; +} + +/** + * An invisible "slot" for a tile to go in. + */ +export const Slot: FC = ({ tile, style, className, ...props }) => ( +
+); + +export interface LayoutProps { + ref: LegacyRef; + model: Model; +} + +export interface TileProps { + ref: LegacyRef; + className?: string; + style?: ComponentProps["style"]; + /** + * The width this tile will have once its animations have settled. + */ + targetWidth: number; + /** + * The height this tile will have once its animations have settled. + */ + targetHeight: number; + model: Model; +} + +interface Drag { + /** + * The X coordinate of the dragged tile in grid space. + */ + x: number; + /** + * The Y coordinate of the dragged tile in grid space. + */ + y: number; + /** + * The X coordinate of the dragged tile, as a scalar of the grid width. + */ + xRatio: number; + /** + * The Y coordinate of the dragged tile, as a scalar of the grid height. + */ + yRatio: number; +} + +type DragCallback = (drag: Drag) => void; + +export interface LayoutSystem { + /** + * Defines the ID and model of each tile present in the layout. + */ + tiles: (model: LayoutModel) => Map; + /** + * A component which creates an invisible layout grid of "slots" for tiles to + * go in. The root element must have a data-generation attribute which + * increments whenever the layout may have changed. + */ + Layout: ComponentType>; + /** + * Gets a drag callback for the tile with the given ID. If this is not + * provided or it returns null, the tile is not draggable. + */ + onDrag?: (model: LayoutModel, tile: string) => DragCallback | null; +} + +interface Props< + LayoutModel, + TileModel, + LayoutRef extends HTMLElement, + TileRef extends HTMLElement, +> { + /** + * Data with which to populate the layout. + */ + model: LayoutModel; + /** + * The system by which to arrange the layout and respond to interactions. + */ + system: LayoutSystem; + /** + * The component used to render each tile in the layout. + */ + Tile: ComponentType>; + className?: string; + style?: CSSProperties; +} + +/** + * A grid of animated tiles. + */ +export function Grid< + LayoutModel, + TileModel, + LayoutRef extends HTMLElement, + TileRef extends HTMLElement, +>({ + model, + system: { tiles: getTileModels, Layout, onDrag }, + Tile, + className, + style, +}: Props): ReactNode { + // Overview: This component places tiles by rendering an invisible layout grid + // of "slots" for tiles to go in. Once rendered, it uses the DOM API to get + // the dimensions of each slot, feeding these numbers back into react-spring + // to let the actual tiles move freely atop the layout. + + // To tell us when the layout has changed, the layout system increments its + // data-generation attribute, which we watch with a MutationObserver. + + const [gridRef1, gridBounds] = useMeasure(); + const [gridRoot, gridRef2] = useState(null); + const gridRef = useMergedRefs(gridRef1, gridRef2); + + const [layoutRoot, setLayoutRoot] = useState(null); + const [generation, setGeneration] = useState(null); + const prefersReducedMotion = usePrefersReducedMotion(); + + const layoutRef = useCallback( + (e: HTMLElement | null) => { + setLayoutRoot(e); + if (e !== null) + setGeneration(parseInt(e.getAttribute("data-generation")!)); + }, + [setLayoutRoot, setGeneration], + ); + + useEffect(() => { + if (layoutRoot !== null) { + const observer = new MutationObserver((mutations) => { + if (mutations.some((m) => m.type === "attributes")) { + setGeneration(parseInt(layoutRoot.getAttribute("data-generation")!)); + } + }); + + observer.observe(layoutRoot, { attributes: true }); + return (): void => observer.disconnect(); + } + }, [layoutRoot, setGeneration]); + + const slotRects = useMemo(() => { + const rects = new Map(); + + if (layoutRoot !== null) { + const slots = layoutRoot.getElementsByClassName( + styles.slot, + ) as HTMLCollectionOf; + for (const slot of slots) + rects.set(slot.getAttribute("data-tile")!, { + x: slot.offsetLeft, + y: slot.offsetTop, + width: slot.offsetWidth, + height: slot.offsetHeight, + }); + } + + return rects; + // The rects may change due to the grid being resized or rerendered, but + // eslint can't statically verify this + // eslint-disable-next-line react-hooks/exhaustive-deps + }, [layoutRoot, generation]); + + const tileModels = useMemo( + () => getTileModels(model), + [getTileModels, model], + ); + + // Combine the tile models and slots together to create placed tiles + const tiles = useMemo[]>(() => { + const items: Tile[] = []; + for (const [id, model] of tileModels) { + const rect = slotRects.get(id); + if (rect !== undefined) items.push({ id, model, ...rect }); + } + return items; + }, [slotRects, tileModels]); + + const dragCallbacks = useMemo( + () => + new Map( + (function* (): Iterable<[string, DragCallback | null]> { + if (onDrag !== undefined) + for (const id of tileModels.keys()) yield [id, onDrag(model, id)]; + })(), + ), + [onDrag, tileModels, model], + ); + + // Drag state is stored in a ref rather than component state, because we use + // react-spring's imperative API during gestures to improve responsiveness + const dragState = useRef(null); + + const [tileTransitions, springRef] = useTransition( + tiles, + () => ({ + key: ({ id }: Tile): string => id, + from: ({ x, y, width, height }: Tile): TileSpringUpdate => ({ + opacity: 0, + scale: 0, + zIndex: 1, + x, + y, + width, + height, + immediate: prefersReducedMotion, + }), + enter: { opacity: 1, scale: 1, immediate: prefersReducedMotion }, + update: ({ + id, + x, + y, + width, + height, + }: Tile): TileSpringUpdate | null => + id === dragState.current?.tileId + ? null + : { + x, + y, + width, + height, + immediate: prefersReducedMotion, + }, + leave: { opacity: 0, scale: 0, immediate: prefersReducedMotion }, + config: { mass: 0.7, tension: 252, friction: 25 }, + }), + // react-spring's types are bugged and can't infer the spring type + ) as unknown as [ + TransitionFn, TileSpring>, + SpringRef, + ]; + + // Because we're using react-spring in imperative mode, we're responsible for + // firing animations manually whenever the tiles array updates + useEffect(() => { + springRef.start(); + }, [tiles, springRef]); + + const animateDraggedTile = ( + endOfGesture: boolean, + callback: DragCallback, + ): void => { + const { tileId, tileX, tileY } = dragState.current!; + const tile = tiles.find((t) => t.id === tileId)!; + + springRef.current + .find((c) => (c.item as Tile).id === tileId) + ?.start( + endOfGesture + ? { + scale: 1, + zIndex: 1, + x: tile.x, + y: tile.y, + width: tile.width, + height: tile.height, + immediate: + prefersReducedMotion || ((key): boolean => key === "zIndex"), + // Allow the tile's position to settle before pushing its + // z-index back down + delay: (key): number => (key === "zIndex" ? 500 : 0), + } + : { + scale: 1.1, + zIndex: 2, + x: tileX, + y: tileY, + immediate: + prefersReducedMotion || + ((key): boolean => + key === "zIndex" || key === "x" || key === "y"), + }, + ); + + if (endOfGesture) + callback({ + x: tileX, + y: tileY, + xRatio: tileX / (gridBounds.width - tile.width), + yRatio: tileY / (gridBounds.height - tile.height), + }); + }; + + // Callback for useDrag. We could call useDrag here, but the default + // pattern of spreading {...bind()} across the children to bind the gesture + // ends up breaking memoization and ruining this component's performance. + // Instead, we pass this callback to each tile via a ref, to let them bind the + // gesture using the much more sensible ref-based method. + const onTileDrag = ( + tileId: string, + + { + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + tap, + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + initial: [initialX, initialY], + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + delta: [dx, dy], + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + last, + }: Parameters>[0], + ): void => { + if (!tap) { + const tileController = springRef.current.find( + (c) => (c.item as Tile).id === tileId, + )!; + const callback = dragCallbacks.get(tileController.item.id); + + if (callback != null) { + if (dragState.current === null) { + const tileSpring = tileController.get(); + dragState.current = { + tileId, + tileX: tileSpring.x, + tileY: tileSpring.y, + cursorX: initialX - gridBounds.x, + cursorY: initialY - gridBounds.y + scrollOffset.current, + }; + } + + dragState.current.tileX += dx; + dragState.current.tileY += dy; + dragState.current.cursorX += dx; + dragState.current.cursorY += dy; + + animateDraggedTile(last, callback); + + if (last) dragState.current = null; + } + } + }; + + const onTileDragRef = useRef(onTileDrag); + onTileDragRef.current = onTileDrag; + + const scrollOffset = useRef(0); + + useScroll( + // eslint-disable-next-line @typescript-eslint/ban-ts-comment + // @ts-ignore + ({ xy: [, y], delta: [, dy] }) => { + scrollOffset.current = y; + + if (dragState.current !== null) { + dragState.current.tileY += dy; + dragState.current.cursorY += dy; + animateDraggedTile(false, onDrag!(model, dragState.current.tileId)!); + } + }, + { target: gridRoot ?? undefined }, + ); + + return ( +
+ + {tileTransitions((spring, { id, model, width, height }) => ( + + ))} +
+ ); +} diff --git a/src/grid/GridLayout.module.css b/src/grid/GridLayout.module.css new file mode 100644 index 00000000..ef234b33 --- /dev/null +++ b/src/grid/GridLayout.module.css @@ -0,0 +1,54 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.scrolling { + box-sizing: border-box; + block-size: 100%; + display: flex; + flex-wrap: wrap; + justify-content: center; + align-content: center; + gap: var(--gap); + box-sizing: border-box; +} + +.scrolling > .slot { + width: var(--width); + height: var(--height); +} + +.fixed > .slot { + position: absolute; + inline-size: 404px; + block-size: 233px; + inset: -12px; +} + +.fixed > .slot[data-block-alignment="start"] { + inset-block-end: unset; +} + +.fixed > .slot[data-block-alignment="end"] { + inset-block-start: unset; +} + +.fixed > .slot[data-inline-alignment="start"] { + inset-inline-end: unset; +} + +.fixed > .slot[data-inline-alignment="end"] { + inset-inline-start: unset; +} diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx new file mode 100644 index 00000000..8df7753b --- /dev/null +++ b/src/grid/GridLayout.tsx @@ -0,0 +1,189 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { CSSProperties, useMemo } from "react"; +import { StateObservable, state, useStateObservable } from "@react-rxjs/core"; +import { BehaviorSubject, distinctUntilChanged } from "rxjs"; + +import { GridLayout as GridLayoutModel } from "../state/CallViewModel"; +import { MediaViewModel } from "../state/MediaViewModel"; +import { LayoutSystem, Slot } from "./Grid"; +import styles from "./GridLayout.module.css"; +import { useReactiveState } from "../useReactiveState"; +import { subscribe } from "../state/subscribe"; +import { Alignment } from "../room/InCallView"; +import { useInitial } from "../useInitial"; + +export interface Bounds { + width: number; + height: number; +} + +interface GridCSSProperties extends CSSProperties { + "--gap": string; + "--width": string; + "--height": string; +} + +interface GridLayoutSystems { + scrolling: LayoutSystem; + fixed: LayoutSystem; +} + +const slotMinHeight = 130; +const slotMaxAspectRatio = 17 / 9; +const slotMinAspectRatio = 4 / 3; + +export const gridLayoutSystems = ( + minBounds: StateObservable, + floatingAlignment: BehaviorSubject, +): GridLayoutSystems => ({ + // The "fixed" (non-scrolling) part of the layout is where the spotlight tile + // lives + fixed: { + tiles: (model) => + new Map( + model.spotlight === undefined ? [] : [["spotlight", model.spotlight]], + ), + Layout: subscribe(function GridLayoutFixed({ model }, ref) { + const { width, height } = useStateObservable(minBounds); + const alignment = useStateObservable( + useInitial>(() => + state( + floatingAlignment.pipe( + distinctUntilChanged( + (a1, a2) => a1.block === a2.block && a1.inline === a2.inline, + ), + ), + ), + ), + ); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.spotlight === undefined, width, height, alignment], + ); + + return ( +
+ {model.spotlight && ( + + )} +
+ ); + }), + onDrag: + () => + ({ xRatio, yRatio }) => + floatingAlignment.next({ + block: yRatio < 0.5 ? "start" : "end", + inline: xRatio < 0.5 ? "start" : "end", + }), + }, + + // The scrolling part of the layout is where all the grid tiles live + scrolling: { + tiles: (model) => new Map(model.grid.map((tile) => [tile.id, tile])), + Layout: subscribe(function GridLayout({ model }, ref) { + const { width, height: minHeight } = useStateObservable(minBounds); + + // The goal here is to determine the grid size and padding that maximizes + // use of screen space for n tiles without making those tiles too small or + // too cropped (having an extreme aspect ratio) + const [gap, slotWidth, slotHeight] = useMemo(() => { + const gap = width < 800 ? 16 : 20; + const slotMinWidth = width < 500 ? 150 : 180; + + let columns = Math.min( + // Don't create more columns than we have items for + model.grid.length, + // The ideal number of columns is given by a packing of equally-sized + // squares into a grid. + // width / column = height / row. + // columns * rows = number of squares. + // ∴ columns = sqrt(width / height * number of squares). + // Except we actually want 16:9-ish slots rather than squares, so we + // divide the width-to-height ratio by the target aspect ratio. + Math.ceil( + Math.sqrt( + (width / minHeight / slotMaxAspectRatio) * model.grid.length, + ), + ), + ); + let rows = Math.ceil(model.grid.length / columns); + + let slotWidth = (width - (columns - 1) * gap) / columns; + let slotHeight = (minHeight - (rows - 1) * gap) / rows; + + // Impose a minimum width and height on the slots + if (slotWidth < slotMinWidth) { + // In this case we want the slot width to determine the number of columns, + // not the other way around. If we take the above equation for the slot + // width (w = (W - (c - 1) * g) / c) and solve for c, we get + // c = (W + g) / (w + g). + columns = Math.floor((width + gap) / (slotMinWidth + gap)); + rows = Math.ceil(model.grid.length / columns); + slotWidth = (width - (columns - 1) * gap) / columns; + slotHeight = (minHeight - (rows - 1) * gap) / rows; + } + if (slotHeight < slotMinHeight) slotHeight = slotMinHeight; + // Impose a minimum and maximum aspect ratio on the slots + const slotAspectRatio = slotWidth / slotHeight; + if (slotAspectRatio > slotMaxAspectRatio) + slotWidth = slotHeight * slotMaxAspectRatio; + else if (slotAspectRatio < slotMinAspectRatio) + slotHeight = slotWidth / slotMinAspectRatio; + // TODO: We might now be hitting the minimum height or width limit again + + return [gap, slotWidth, slotHeight]; + }, [width, minHeight, model.grid.length]); + + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid, width, minHeight], + ); + + return ( +
+ {model.grid.map((tile) => ( + + ))} +
+ ); + }), + }, +}); diff --git a/src/video-grid/VideoGrid.module.css b/src/grid/LegacyGrid.module.css similarity index 97% rename from src/video-grid/VideoGrid.module.css rename to src/grid/LegacyGrid.module.css index df6e4fa7..6e59e66e 100644 --- a/src/video-grid/VideoGrid.module.css +++ b/src/grid/LegacyGrid.module.css @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -.videoGrid { +.grid { position: relative; overflow: hidden; flex: 1; diff --git a/src/video-grid/VideoGrid.tsx b/src/grid/LegacyGrid.tsx similarity index 98% rename from src/video-grid/VideoGrid.tsx rename to src/grid/LegacyGrid.tsx index fa56bd88..9e1fb876 100644 --- a/src/video-grid/VideoGrid.tsx +++ b/src/grid/LegacyGrid.tsx @@ -16,6 +16,7 @@ limitations under the License. import { ComponentProps, + ComponentType, MutableRefObject, ReactNode, Ref, @@ -40,11 +41,11 @@ import useMeasure from "react-use-measure"; import { ResizeObserver as JuggleResizeObserver } from "@juggle/resize-observer"; import { logger } from "matrix-js-sdk/src/logger"; -import styles from "./VideoGrid.module.css"; +import styles from "./LegacyGrid.module.css"; import { Layout } from "../room/LayoutToggle"; import { TileWrapper } from "./TileWrapper"; -import { LayoutStatesMap } from "./Layout"; import { TileDescriptor } from "../state/CallViewModel"; +import { TileProps } from "./Grid"; interface TilePosition { x: number; @@ -86,7 +87,7 @@ export interface TileSpringUpdate extends Partial { type LayoutDirection = "vertical" | "horizontal"; -export function useVideoGridLayout(hasScreenshareFeeds: boolean): { +export function useLegacyGridLayout(hasScreenshareFeeds: boolean): { layout: Layout; setLayout: (layout: Layout) => void; } { @@ -838,20 +839,19 @@ export interface ChildrenProperties { data: T; } -export interface VideoGridProps { +export interface LegacyGridProps { items: TileDescriptor[]; layout: Layout; disableAnimations: boolean; - layoutStates: LayoutStatesMap; - children: (props: ChildrenProperties) => ReactNode; + Tile: ComponentType>; } -export function VideoGrid({ +export function LegacyGrid({ items, layout, disableAnimations, - children, -}: VideoGridProps): ReactNode { + Tile, +}: LegacyGridProps): ReactNode { // Place the PiP in the bottom right corner by default const [pipXRatio, setPipXRatio] = useState(1); const [pipYRatio, setPipYRatio] = useState(1); @@ -1378,7 +1378,7 @@ export function VideoGrid({ ); return ( -
+
{springs.map((spring, i) => { const tile = tiles[i]; const tilePosition = tilePositions[tile.order]; @@ -1387,20 +1387,19 @@ export function VideoGrid({ - {children as (props: ChildrenProperties) => ReactNode} - + /> ); })}
); } -VideoGrid.defaultProps = { +LegacyGrid.defaultProps = { layout: "grid", }; diff --git a/src/grid/TileWrapper.module.css b/src/grid/TileWrapper.module.css new file mode 100644 index 00000000..ed3acda3 --- /dev/null +++ b/src/grid/TileWrapper.module.css @@ -0,0 +1,23 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.tile.draggable { + cursor: grab; +} + +.tile.draggable:active { + cursor: grabbing; +} diff --git a/src/video-grid/TileWrapper.tsx b/src/grid/TileWrapper.tsx similarity index 59% rename from src/video-grid/TileWrapper.tsx rename to src/grid/TileWrapper.tsx index 5c771e6c..ded2be28 100644 --- a/src/video-grid/TileWrapper.tsx +++ b/src/grid/TileWrapper.tsx @@ -14,83 +14,76 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { memo, ReactNode, RefObject, useRef } from "react"; +import { ComponentType, memo, RefObject, useRef } from "react"; import { EventTypes, Handler, useDrag } from "@use-gesture/react"; -import { SpringValue, to } from "@react-spring/web"; +import { SpringValue } from "@react-spring/web"; +import classNames from "classnames"; -import { ChildrenProperties } from "./VideoGrid"; +import { TileProps } from "./Grid"; +import styles from "./TileWrapper.module.css"; -interface Props { +interface Props { id: string; - onDragRef: RefObject< + onDrag: RefObject< ( tileId: string, state: Parameters>[0], ) => void - >; + > | null; targetWidth: number; targetHeight: number; - data: T; + model: M; + Tile: ComponentType>; opacity: SpringValue; scale: SpringValue; - shadow: SpringValue; - shadowSpread: SpringValue; zIndex: SpringValue; x: SpringValue; y: SpringValue; width: SpringValue; height: SpringValue; - children: (props: ChildrenProperties) => ReactNode; } const TileWrapper_ = memo( - ({ + ({ id, - onDragRef, + onDrag, targetWidth, targetHeight, - data, + model, + Tile, opacity, scale, - shadow, - shadowSpread, zIndex, x, y, width, height, - children, - }: Props) => { - const ref = useRef(null); + }: Props) => { + const ref = useRef(null); - useDrag((state) => onDragRef?.current!(id, state), { + useDrag((state) => onDrag?.current!(id, state), { target: ref, filterTaps: true, preventScroll: true, }); return ( - <> - {children({ - ref, - style: { - opacity, - scale, - zIndex, - x, - y, - width, - height, - boxShadow: to( - [shadow, shadowSpread], - (s, ss) => `rgba(0, 0, 0, 0.5) 0px ${s}px ${2 * s}px ${ss}px`, - ), - }, - targetWidth, - targetHeight, - data, - })} - + ); }, ); @@ -104,4 +97,6 @@ TileWrapper_.displayName = "TileWrapper"; // We pretend this component is a simple function rather than a // NamedExoticComponent, because that's the only way we can fit in a type // parameter -export const TileWrapper = TileWrapper_ as (props: Props) => JSX.Element; +export const TileWrapper = TileWrapper_ as ( + props: Props, +) => JSX.Element; diff --git a/src/room/InCallView.module.css b/src/room/InCallView.module.css index 4bbb6f33..f53ba025 100644 --- a/src/room/InCallView.module.css +++ b/src/room/InCallView.module.css @@ -1,5 +1,5 @@ /* -Copyright 2021 New Vector Ltd +Copyright 2021-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -19,6 +19,7 @@ limitations under the License. flex-direction: column; height: 100%; width: 100%; + overflow-y: auto; } .controlsOverlay { @@ -46,16 +47,28 @@ limitations under the License. margin-bottom: 0; } +.header { + position: sticky; + inset-block-start: 0; + z-index: 1; + background: linear-gradient( + 0deg, + rgba(0, 0, 0, 0) 0%, + var(--cpd-color-bg-canvas-default) 100% + ); +} + .footer { position: sticky; inset-block-end: 0; + z-index: 1; display: grid; grid-template-columns: 1fr auto 1fr; grid-template-areas: "logo buttons layout"; align-items: center; gap: var(--cpd-space-3x); padding-block: var(--cpd-space-4x); - padding-inline: var(--inline-content-inset); + margin-inline: var(--inline-content-inset); background: linear-gradient( 180deg, rgba(0, 0, 0, 0) 0%, @@ -109,3 +122,23 @@ limitations under the License. .footerHidden { display: none; } + +.fixedGrid { + position: absolute; + inline-size: calc(100% - 2 * var(--inline-content-inset)); + align-self: center; + /* Disable pointer events so the overlay doesn't block interaction with + elements behind it */ + pointer-events: none; +} + +.fixedGrid > :not(:first-child) { + pointer-events: initial; +} + +.scrollingGrid { + position: relative; + flex-grow: 1; + inline-size: calc(100% - 2 * var(--inline-content-inset)); + align-self: center; +} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 94edace7..4407de38 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -14,7 +14,6 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { ResizeObserver } from "@juggle/resize-observer"; import { RoomAudioRenderer, RoomContext, @@ -26,19 +25,19 @@ import { ConnectionState, Room, Track } from "livekit-client"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { FC, - ReactNode, - Ref, + forwardRef, useCallback, useEffect, useMemo, useRef, useState, } from "react"; -import { useTranslation } from "react-i18next"; import useMeasure from "react-use-measure"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import classNames from "classnames"; -import { useStateObservable } from "@react-rxjs/core"; +import { state, useStateObservable } from "@react-rxjs/core"; +import { BehaviorSubject } from "rxjs"; +import { useTranslation } from "react-i18next"; import LogoMark from "../icons/LogoMark.svg?react"; import LogoType from "../icons/LogoType.svg?react"; @@ -51,21 +50,19 @@ import { SettingsButton, } from "../button"; import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header"; -import { useVideoGridLayout, VideoGrid } from "../video-grid/VideoGrid"; +import { LegacyGrid, useLegacyGridLayout } from "../grid/LegacyGrid"; import { useUrlParams } from "../UrlParams"; import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts"; import { usePrefersReducedMotion } from "../usePrefersReducedMotion"; import { ElementWidgetActions, widget } from "../widget"; import styles from "./InCallView.module.css"; -import { VideoTile } from "../video-grid/VideoTile"; -import { NewVideoGrid } from "../video-grid/NewVideoGrid"; +import { GridTile } from "../tile/GridTile"; import { OTelGroupCallMembership } from "../otel/OTelGroupCallMembership"; import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal"; import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { useLiveKit } from "../livekit/useLiveKit"; import { useFullscreen } from "./useFullscreen"; -import { useLayoutStates } from "../video-grid/Layout"; import { useWakeLock } from "../useWakeLock"; import { useMergedRefs } from "../useMergedRefs"; import { MuteStates } from "./MuteStates"; @@ -74,13 +71,33 @@ import { InviteButton } from "../button/InviteButton"; import { LayoutToggle } from "./LayoutToggle"; import { ECConnectionState } from "../livekit/useECConnectionState"; import { useOpenIDSFU } from "../livekit/openIDSFU"; -import { useCallViewModel } from "../state/CallViewModel"; +import { + GridMode, + TileDescriptor, + useCallViewModel, +} from "../state/CallViewModel"; import { subscribe } from "../state/subscribe"; +import { Grid, TileProps } from "../grid/Grid"; +import { MediaViewModel } from "../state/MediaViewModel"; +import { gridLayoutSystems } from "../grid/GridLayout"; +import { useObservable } from "../state/useObservable"; +import { useInitial } from "../useInitial"; +import { SpotlightTile } from "../tile/SpotlightTile"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); -const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent); + +export interface Alignment { + inline: "start" | "end"; + block: "start" | "end"; +} + +const defaultAlignment: Alignment = { inline: "end", block: "end" }; + +const dummySpotlightItem = { + id: "spotlight", +} as TileDescriptor; export interface ActiveCallProps extends Omit { @@ -153,7 +170,7 @@ export const InCallView: FC = subscribe( }, [connState, onLeave]); const containerRef1 = useRef(null); - const [containerRef2, bounds] = useMeasure({ polyfill: ResizeObserver }); + const [containerRef2, bounds] = useMeasure(); const boundsValid = bounds.height > 0; // Merge the refs so they can attach to the same element const containerRef = useMergedRefs(containerRef1, containerRef2); @@ -164,9 +181,8 @@ export const InCallView: FC = subscribe( room: livekitRoom, }, ); - const { layout, setLayout } = useVideoGridLayout( - screenSharingTracks.length > 0, - ); + const { layout: legacyLayout, setLayout: setLegacyLayout } = + useLegacyGridLayout(screenSharingTracks.length > 0); const { hideScreensharing, showControls } = useUrlParams(); @@ -194,23 +210,23 @@ export const InCallView: FC = subscribe( useEffect(() => { widget?.api.transport.send( - layout === "grid" + legacyLayout === "grid" ? ElementWidgetActions.TileLayout : ElementWidgetActions.SpotlightLayout, {}, ); - }, [layout]); + }, [legacyLayout]); useEffect(() => { if (widget) { const onTileLayout = (ev: CustomEvent): void => { - setLayout("grid"); + setLegacyLayout("grid"); widget!.api.transport.reply(ev.detail, {}); }; const onSpotlightLayout = ( ev: CustomEvent, ): void => { - setLayout("spotlight"); + setLegacyLayout("spotlight"); widget!.api.transport.reply(ev.detail, {}); }; @@ -231,7 +247,7 @@ export const InCallView: FC = subscribe( ); }; } - }, [setLayout]); + }, [setLegacyLayout]); const mobile = boundsValid && bounds.width <= 660; const reducedControls = boundsValid && bounds.width <= 340; @@ -244,8 +260,21 @@ export const InCallView: FC = subscribe( connState, ); const items = useStateObservable(vm.tiles); + const layout = useStateObservable(vm.layout); + const hasSpotlight = layout.spotlight !== undefined; + // Hack: We insert a dummy "spotlight" tile into the tiles we pass to + // useFullscreen so that we can control the fullscreen state of the + // spotlight tile in the new layouts with this same hook. + const fullscreenItems = useMemo( + () => [...items, ...(hasSpotlight ? [dummySpotlightItem] : [])], + [items, hasSpotlight], + ); const { fullscreenItem, toggleFullscreen, exitFullscreen } = - useFullscreen(items); + useFullscreen(fullscreenItems); + const toggleSpotlightFullscreen = useCallback( + () => toggleFullscreen("spotlight"), + [toggleFullscreen], + ); // The maximised participant: either the participant that the user has // manually put in fullscreen, or the focused (active) participant if the @@ -259,66 +288,8 @@ export const InCallView: FC = subscribe( [fullscreenItem, noControls, items], ); - const Grid = - items.length > 12 && layout === "grid" ? NewVideoGrid : VideoGrid; - const prefersReducedMotion = usePrefersReducedMotion(); - // This state is lifted out of NewVideoGrid so that layout states can be - // restored after a layout switch or upon exiting fullscreen - const layoutStates = useLayoutStates(); - - const renderContent = (): JSX.Element => { - if (items.length === 0) { - return ( -
-

{t("waiting_for_participants")}

-
- ); - } - if (maximisedParticipant) { - return ( - - ); - } - - return ( - - {({ data: vm, ...props }): ReactNode => ( - 2} - onOpenProfile={openProfile} - {...props} - ref={props.ref as Ref} - /> - )} - - ); - }; - - const rageshakeRequestModalProps = useRageshakeRequestModal( - rtcSession.room.roomId, - ); - const [settingsModalOpen, setSettingsModalOpen] = useState(false); const [settingsTab, setSettingsTab] = useState(defaultSettingsTab); @@ -336,6 +307,169 @@ export const InCallView: FC = subscribe( setSettingsModalOpen(true); }, [setSettingsTab, setSettingsModalOpen]); + const [headerRef, headerBounds] = useMeasure(); + const [footerRef, footerBounds] = useMeasure(); + const gridBounds = useMemo( + () => ({ + width: footerBounds.width, + height: bounds.height - headerBounds.height - footerBounds.height, + }), + [ + footerBounds.width, + bounds.height, + headerBounds.height, + footerBounds.height, + ], + ); + const gridBoundsObservable = useObservable(gridBounds); + const floatingAlignment = useInitial( + () => new BehaviorSubject(defaultAlignment), + ); + const { fixed, scrolling } = useInitial(() => + gridLayoutSystems(state(gridBoundsObservable), floatingAlignment), + ); + + const setGridMode = useCallback( + (mode: GridMode) => { + setLegacyLayout(mode); + vm.setGridMode(mode); + }, + [setLegacyLayout, vm], + ); + + const showSpeakingIndicators = + layout.type === "spotlight" || + (layout.type === "grid" && layout.grid.length > 2); + + const SpotlightTileView = useMemo( + () => + forwardRef>( + function SpotlightTileView( + { className, style, targetWidth, targetHeight, model }, + ref, + ) { + return ( + + ); + }, + ), + [toggleSpotlightFullscreen], + ); + const GridTileView = useMemo( + () => + forwardRef>( + function GridTileView( + { className, style, targetWidth, targetHeight, model }, + ref, + ) { + return ( + + ); + }, + ), + [toggleFullscreen, openProfile, showSpeakingIndicators], + ); + + const renderContent = (): JSX.Element => { + if (items.length === 0) { + return ( +
+

{t("waiting_for_participants")}

+
+ ); + } + + if (maximisedParticipant !== null) { + const fullscreen = maximisedParticipant === fullscreenItem; + if (maximisedParticipant.id === "spotlight") { + return ( + + ); + } + return ( + + ); + } + + // The only new layout we've implemented so far is grid layout for non-1:1 + // calls. All other layouts use the legacy grid system for now. + if ( + legacyLayout === "grid" && + layout.type === "grid" && + !(layout.grid.length === 2 && layout.spotlight === undefined) + ) { + return ( + <> + + + + ); + } else { + return ( + + ); + } + }; + + const rageshakeRequestModalProps = useRageshakeRequestModal( + rtcSession.room.roomId, + ); + const toggleScreensharing = useCallback(async () => { exitFullscreen(); await localParticipant.setScreenShareEnabled(!isScreenShareEnabled, { @@ -395,6 +529,7 @@ export const InCallView: FC = subscribe( ); footer = (
= subscribe( {!mobile && !hideHeader && showControls && ( )}
@@ -428,7 +563,7 @@ export const InCallView: FC = subscribe( return (
{!hideHeader && maximisedParticipant === null && ( -
+
= subscribe(
)} -
- - {renderContent()} - {footer} -
+ + {renderContent()} + {footer} {!noControls && ( )} diff --git a/src/room/VideoPreview.tsx b/src/room/VideoPreview.tsx index 2f6dbbbc..dd98421e 100644 --- a/src/room/VideoPreview.tsx +++ b/src/room/VideoPreview.tsx @@ -18,11 +18,7 @@ import { useEffect, useMemo, useRef, FC, ReactNode, useCallback } from "react"; import useMeasure from "react-use-measure"; import { ResizeObserver } from "@juggle/resize-observer"; import { usePreviewTracks } from "@livekit/components-react"; -import { - CreateLocalTracksOptions, - LocalVideoTrack, - Track, -} from "livekit-client"; +import { LocalVideoTrack, Track } from "livekit-client"; import classNames from "classnames"; import { logger } from "matrix-js-sdk/src/logger"; import { Glass } from "@vector-im/compound-web"; @@ -32,6 +28,7 @@ import styles from "./VideoPreview.module.css"; import { useMediaDevices } from "../livekit/MediaDevicesContext"; import { MuteStates } from "./MuteStates"; import { useMediaQuery } from "../useMediaQuery"; +import { useInitial } from "../useInitial"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; export type MatrixInfo = { @@ -63,10 +60,10 @@ export const VideoPreview: FC = ({ // Capture the audio options as they were when we first mounted, because // we're not doing anything with the audio anyway so we don't need to // re-open the devices when they change (see below). - const initialAudioOptions = useRef(); - initialAudioOptions.current ??= muteStates.audio.enabled && { - deviceId: devices.audioInput.selectedId, - }; + const initialAudioOptions = useInitial( + () => + muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId }, + ); const localTrackOptions = useMemo( () => ({ @@ -76,12 +73,16 @@ export const VideoPreview: FC = ({ // reference the initial values here. // We also pass in a clone because livekit mutates the object passed in, // which would cause the devices to be re-opened on the next render. - audio: Object.assign({}, initialAudioOptions.current), + audio: Object.assign({}, initialAudioOptions), video: muteStates.video.enabled && { deviceId: devices.videoInput.selectedId, }, }), - [devices.videoInput.selectedId, muteStates.video.enabled], + [ + initialAudioOptions, + devices.videoInput.selectedId, + muteStates.video.enabled, + ], ); const onError = useCallback( diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 4ad2f024..e6d55b72 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -154,11 +154,11 @@ class UserMedia { this.vm = new UserMediaViewModel(id, member, participant, callEncrypted); this.speaker = this.vm.speaking.pipeState( - // Require 1 s of continuous speaking to become a speaker, and 10 s of + // Require 1 s of continuous speaking to become a speaker, and 60 s of // continuous silence to stop being considered a speaker audit((s) => merge( - timer(s ? 1000 : 10000), + timer(s ? 1000 : 60000), // If the speaking flag resets to its original value during this time, // end the silencing window to stick with that original value this.vm.speaking.pipe(filter((s1) => s1 !== s)), diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index db11017e..f1e772da 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -32,7 +32,7 @@ import { TrackEvent, facingModeFromLocalTrack, } from "livekit-client"; -import { RoomMember } from "matrix-js-sdk/src/matrix"; +import { RoomMember, RoomMemberEvent } from "matrix-js-sdk/src/matrix"; import { BehaviorSubject, combineLatest, @@ -44,8 +44,49 @@ import { startWith, switchMap, } from "rxjs"; +import { useTranslation } from "react-i18next"; +import { useEffect } from "react"; import { ViewModel } from "./ViewModel"; +import { useReactiveState } from "../useReactiveState"; + +export interface NameData { + /** + * The display name of the participant. + */ + displayName: string; + /** + * The text to be shown on the participant's name tag. + */ + nameTag: string; +} + +// TODO: Move this naming logic into the view model +export function useNameData(vm: MediaViewModel): NameData { + const { t } = useTranslation(); + + const [displayName, setDisplayName] = useReactiveState( + () => vm.member?.rawDisplayName ?? "[👻]", + [vm.member], + ); + useEffect(() => { + if (vm.member) { + const updateName = (): void => { + setDisplayName(vm.member!.rawDisplayName); + }; + + vm.member!.on(RoomMemberEvent.Name, updateName); + return (): void => { + vm.member!.removeListener(RoomMemberEvent.Name, updateName); + }; + } + }, [vm.member, setDisplayName]); + const nameTag = vm.local + ? t("video_tile.sfu_participant_local") + : displayName; + + return { displayName, nameTag }; +} function observeTrackReference( participant: Participant, @@ -78,8 +119,9 @@ abstract class BaseMediaViewModel extends ViewModel { public readonly unencryptedWarning: StateObservable; public constructor( - // TODO: This is only needed for full screen toggling and can be removed as - // soon as that code is moved into the view models + /** + * An opaque identifier for this media. + */ public readonly id: string, /** * The Matrix room member to which this media belongs. diff --git a/src/state/useObservable.ts b/src/state/useObservable.ts index 92210e34..037c3bd5 100644 --- a/src/state/useObservable.ts +++ b/src/state/useObservable.ts @@ -14,9 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { useRef } from "react"; +import { Ref, useCallback, useRef } from "react"; import { BehaviorSubject, Observable } from "rxjs"; +import { useInitial } from "../useInitial"; + /** * React hook that creates an Observable from a changing value. The Observable * replays its current value upon subscription and emits whenever the value @@ -28,3 +30,14 @@ export function useObservable(value: T): Observable { if (value !== subject.current.value) subject.current.next(value); return subject.current; } + +/** + * React hook that creates a ref and an Observable that emits any values + * stored in the ref. The Observable replays the value currently stored in the + * ref upon subscription. + */ +export function useObservableRef(initialValue: T): [Observable, Ref] { + const subject = useInitial(() => new BehaviorSubject(initialValue)); + const ref = useCallback((value: T) => subject.next(value), [subject]); + return [subject, ref]; +} diff --git a/src/tile/GridTile.module.css b/src/tile/GridTile.module.css new file mode 100644 index 00000000..923c7633 --- /dev/null +++ b/src/tile/GridTile.module.css @@ -0,0 +1,81 @@ +/* +Copyright 2022-2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + +http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.tile { + position: absolute; + top: 0; + --media-view-border-radius: var(--cpd-space-4x); + transition: outline-color ease 0.15s; + outline: var(--cpd-border-width-2) solid rgb(0 0 0 / 0); +} + +/* Use a pseudo-element to create the expressive speaking border, since CSS +borders don't support gradients */ +.tile[data-maximised="false"]::before { + content: ""; + position: absolute; + z-index: -1; /* Put it below the outline */ + opacity: 0; /* Hidden unless speaking */ + transition: opacity ease 0.15s; + inset: calc(-1 * var(--cpd-border-width-4)); + border-radius: var(--cpd-space-5x); + background: linear-gradient( + 119deg, + rgba(13, 92, 189, 0.7) 0%, + rgba(13, 189, 168, 0.7) 100% + ), + linear-gradient( + 180deg, + rgba(13, 92, 189, 0.9) 0%, + rgba(13, 189, 168, 0.9) 100% + ); + background-blend-mode: overlay, normal; +} + +.tile[data-maximised="false"].speaking { + /* !important because speaking border should take priority over hover */ + outline: var(--cpd-border-width-1) solid var(--cpd-color-bg-canvas-default) !important; +} + +.tile[data-maximised="false"].speaking::before { + opacity: 1; +} + +@media (hover: hover) { + .tile[data-maximised="false"]:hover { + outline: var(--cpd-border-width-2) solid + var(--cpd-color-border-interactive-hovered); + } +} + +.tile[data-maximised="true"] { + position: relative; + flex-grow: 1; + --media-view-border-radius: 0; + --media-view-fg-inset: 10px; +} + +.muteIcon[data-muted="true"] { + color: var(--cpd-color-icon-secondary); +} + +.muteIcon[data-muted="false"] { + color: var(--cpd-color-icon-primary); +} + +.volumeSlider { + width: 100%; +} diff --git a/src/video-grid/VideoTile.tsx b/src/tile/GridTile.tsx similarity index 66% rename from src/video-grid/VideoTile.tsx rename to src/tile/GridTile.tsx index d4a7442e..d88b189f 100644 --- a/src/video-grid/VideoTile.tsx +++ b/src/tile/GridTile.tsx @@ -1,5 +1,5 @@ /* -Copyright 2022-2023 New Vector Ltd +Copyright 2022-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,29 +14,12 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { - ComponentProps, - ForwardedRef, - ReactNode, - forwardRef, - useCallback, - useEffect, - useState, -} from "react"; +import { ComponentProps, forwardRef, useCallback, useState } from "react"; import { animated } from "@react-spring/web"; import classNames from "classnames"; import { useTranslation } from "react-i18next"; -import { - TrackReferenceOrPlaceholder, - VideoTrack, -} from "@livekit/components-react"; -import { - RoomMember, - RoomMemberEvent, -} from "matrix-js-sdk/src/models/room-member"; import MicOnSolidIcon from "@vector-im/compound-design-tokens/icons/mic-on-solid.svg?react"; import MicOffSolidIcon from "@vector-im/compound-design-tokens/icons/mic-off-solid.svg?react"; -import ErrorIcon from "@vector-im/compound-design-tokens/icons/error.svg?react"; import MicOffIcon from "@vector-im/compound-design-tokens/icons/mic-off.svg?react"; import OverflowHorizontalIcon from "@vector-im/compound-design-tokens/icons/overflow-horizontal.svg?react"; import VolumeOnIcon from "@vector-im/compound-design-tokens/icons/volume-on.svg?react"; @@ -45,8 +28,6 @@ import UserProfileIcon from "@vector-im/compound-design-tokens/icons/user-profil import ExpandIcon from "@vector-im/compound-design-tokens/icons/expand.svg?react"; import CollapseIcon from "@vector-im/compound-design-tokens/icons/collapse.svg?react"; import { - Text, - Tooltip, ContextMenu, MenuItem, ToggleMenuItem, @@ -54,120 +35,16 @@ import { } from "@vector-im/compound-web"; import { useStateObservable } from "@react-rxjs/core"; -import { Avatar } from "../Avatar"; -import styles from "./VideoTile.module.css"; -import { useReactiveState } from "../useReactiveState"; +import styles from "./GridTile.module.css"; import { ScreenShareViewModel, MediaViewModel, UserMediaViewModel, + useNameData, } from "../state/MediaViewModel"; import { subscribe } from "../state/subscribe"; -import { useMergedRefs } from "../useMergedRefs"; import { Slider } from "../Slider"; - -interface TileProps { - tileRef?: ForwardedRef; - className?: string; - style?: ComponentProps["style"]; - targetWidth: number; - targetHeight: number; - video: TrackReferenceOrPlaceholder; - member: RoomMember | undefined; - videoEnabled: boolean; - maximised: boolean; - unencryptedWarning: boolean; - nameTagLeadingIcon?: ReactNode; - nameTag: string; - displayName: string; - primaryButton: ReactNode; - secondaryButton?: ReactNode; - [k: string]: unknown; -} - -const Tile = forwardRef( - ( - { - tileRef = null, - className, - style, - targetWidth, - targetHeight, - video, - member, - videoEnabled, - maximised, - unencryptedWarning, - nameTagLeadingIcon, - nameTag, - displayName, - primaryButton, - secondaryButton, - ...props - }, - ref, - ) => { - const { t } = useTranslation(); - const mergedRef = useMergedRefs(tileRef, ref); - - return ( - -
- - {video.publication !== undefined && ( - - )} -
-
-
- {nameTagLeadingIcon} - - {nameTag} - - {unencryptedWarning && ( - - - - )} -
- {primaryButton} - {secondaryButton} -
- - ); - }, -); - -Tile.displayName = "Tile"; +import { MediaView } from "./MediaView"; interface UserMediaTileProps { vm: UserMediaViewModel; @@ -175,8 +52,6 @@ interface UserMediaTileProps { style?: ComponentProps["style"]; targetWidth: number; targetHeight: number; - nameTag: string; - displayName: string; maximised: boolean; onOpenProfile: () => void; showSpeakingIndicator: boolean; @@ -190,8 +65,6 @@ const UserMediaTile = subscribe( style, targetWidth, targetHeight, - nameTag, - displayName, maximised, onOpenProfile, showSpeakingIndicator, @@ -199,6 +72,7 @@ const UserMediaTile = subscribe( ref, ) => { const { t } = useTranslation(); + const { displayName, nameTag } = useNameData(vm); const video = useStateObservable(vm.video); const audioEnabled = useStateObservable(vm.audioEnabled); const videoEnabled = useStateObservable(vm.videoEnabled); @@ -273,20 +147,20 @@ const UserMediaTile = subscribe( ); const tile = ( - ["style"]; targetWidth: number; targetHeight: number; - nameTag: string; - displayName: string; maximised: boolean; fullscreen: boolean; onToggleFullscreen: (itemId: string) => void; @@ -349,8 +221,6 @@ const ScreenShareTile = subscribe( style, targetWidth, targetHeight, - nameTag, - displayName, maximised, fullscreen, onToggleFullscreen, @@ -358,6 +228,7 @@ const ScreenShareTile = subscribe( ref, ) => { const { t } = useTranslation(); + const { displayName, nameTag } = useNameData(vm); const video = useStateObservable(vm.video); const unencryptedWarning = useStateObservable(vm.unencryptedWarning); const onClickFullScreen = useCallback( @@ -368,16 +239,20 @@ const ScreenShareTile = subscribe( const FullScreenIcon = fullscreen ? CollapseIcon : ExpandIcon; return ( - ( +export const GridTile = forwardRef( ( { vm, @@ -431,30 +306,6 @@ export const VideoTile = forwardRef( }, ref, ) => { - const { t } = useTranslation(); - - // Handle display name changes. - // TODO: Move this into the view model - const [displayName, setDisplayName] = useReactiveState( - () => vm.member?.rawDisplayName ?? "[👻]", - [vm.member], - ); - useEffect(() => { - if (vm.member) { - const updateName = (): void => { - setDisplayName(vm.member!.rawDisplayName); - }; - - vm.member!.on(RoomMemberEvent.Name, updateName); - return (): void => { - vm.member!.removeListener(RoomMemberEvent.Name, updateName); - }; - } - }, [vm.member, setDisplayName]); - const nameTag = vm.local - ? t("video_tile.sfu_participant_local") - : displayName; - if (vm instanceof UserMediaViewModel) { return ( ( vm={vm} targetWidth={targetWidth} targetHeight={targetHeight} - nameTag={nameTag} - displayName={displayName} maximised={maximised} onOpenProfile={onOpenProfile} showSpeakingIndicator={showSpeakingIndicator} @@ -480,8 +329,6 @@ export const VideoTile = forwardRef( vm={vm} targetWidth={targetWidth} targetHeight={targetHeight} - nameTag={nameTag} - displayName={displayName} maximised={maximised} fullscreen={fullscreen} onToggleFullscreen={onToggleFullscreen} @@ -491,4 +338,4 @@ export const VideoTile = forwardRef( }, ); -VideoTile.displayName = "VideoTile"; +GridTile.displayName = "GridTile"; diff --git a/src/video-grid/VideoTile.module.css b/src/tile/MediaView.module.css similarity index 66% rename from src/video-grid/VideoTile.module.css rename to src/tile/MediaView.module.css index b4da6e5e..65cf9fc7 100644 --- a/src/video-grid/VideoTile.module.css +++ b/src/tile/MediaView.module.css @@ -1,5 +1,5 @@ /* -Copyright 2022-2023 New Vector Ltd +Copyright 2022-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -14,63 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -.videoTile { - position: absolute; - top: 0; - container-name: videoTile; +.media { + container-name: mediaView; container-type: size; - border-radius: var(--cpd-space-4x); - transition: outline-color ease 0.15s; - outline: var(--cpd-border-width-2) solid rgb(0 0 0 / 0); + border-radius: var(--media-view-border-radius); } -/* Use a pseudo-element to create the expressive speaking border, since CSS -borders don't support gradients */ -.videoTile::before { - content: ""; - position: absolute; - z-index: -1; /* Put it below the outline */ - opacity: 0; /* Hidden unless speaking */ - transition: opacity ease 0.15s; - inset: calc(-1 * var(--cpd-border-width-4)); - border-radius: var(--cpd-space-5x); - background: linear-gradient( - 119deg, - rgba(13, 92, 189, 0.7) 0%, - rgba(13, 189, 168, 0.7) 100% - ), - linear-gradient( - 180deg, - rgba(13, 92, 189, 0.9) 0%, - rgba(13, 189, 168, 0.9) 100% - ); - background-blend-mode: overlay, normal; -} - -.videoTile.speaking { - /* !important because speaking border should take priority over hover */ - outline: var(--cpd-border-width-1) solid var(--cpd-color-bg-canvas-default) !important; -} - -.videoTile.speaking::before { - opacity: 1; -} - -@media (hover: hover) { - .videoTile:hover { - outline: var(--cpd-border-width-2) solid - var(--cpd-color-border-interactive-hovered); - } -} - -.videoTile.maximised { - position: relative; - border-radius: 0; - inline-size: 100%; - block-size: 100%; -} - -.videoTile video { +.media video { inline-size: 100%; block-size: 100%; object-fit: contain; @@ -81,19 +31,19 @@ borders don't support gradients */ transform: translate(0); } -.videoTile.mirror video { +.media.mirror video { transform: scaleX(-1); } -.videoTile.screenshare video { - object-fit: contain; -} - -.videoTile.cropVideo video { +.media[data-video-fit="cover"] video { object-fit: cover; } -.videoTile.videoMuted video { +.media[data-video-fit="contain"] video { + object-fit: contain; +} + +.media.videoMuted video { display: none; } @@ -114,13 +64,13 @@ borders don't support gradients */ pointer-events: none; } -.videoTile.videoMuted .avatar { +.media.videoMuted .avatar { display: initial; } /* CSS makes us put a condition here, even though all we want to do is unconditionally select the container so we can use cqmin units */ -@container videoTile (width > 0) { +@container mediaView (width > 0) { .avatar { /* Half of the smallest dimension of the tile */ inline-size: 50cqmin; @@ -137,7 +87,10 @@ unconditionally select the container so we can use cqmin units */ .fg { position: absolute; - inset: var(--cpd-space-1x); + inset: var( + --media-view-fg-inset, + calc(var(--media-view-border-radius) - var(--cpd-space-3x)) + ); display: grid; grid-template-columns: 1fr auto; grid-template-rows: 1fr auto; @@ -167,14 +120,6 @@ unconditionally select the container so we can use cqmin units */ flex-shrink: 0; } -.muteIcon[data-muted="true"] { - color: var(--cpd-color-icon-secondary); -} - -.muteIcon[data-muted="false"] { - color: var(--cpd-color-icon-primary); -} - .nameTag > .name { text-overflow: ellipsis; overflow: hidden; @@ -200,8 +145,7 @@ unconditionally select the container so we can use cqmin units */ transition: opacity ease 0.15s; } -.fg > button:focus-visible, -.fg > :focus-visible ~ button, +.fg:has(:focus-visible) > button, .fg > button[data-enabled="true"], .fg > button[data-state="open"] { opacity: 1; @@ -237,7 +181,3 @@ unconditionally select the container so we can use cqmin units */ .fg > button:nth-of-type(2) { grid-area: button2; } - -.volumeSlider { - width: 100%; -} diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx new file mode 100644 index 00000000..69c3591e --- /dev/null +++ b/src/tile/MediaView.tsx @@ -0,0 +1,130 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { TrackReferenceOrPlaceholder } from "@livekit/components-core"; +import { animated } from "@react-spring/web"; +import { RoomMember } from "matrix-js-sdk/src/matrix"; +import { ComponentProps, ReactNode, forwardRef } from "react"; +import { useTranslation } from "react-i18next"; +import classNames from "classnames"; +import { VideoTrack } from "@livekit/components-react"; +import { Text, Tooltip } from "@vector-im/compound-web"; +import ErrorIcon from "@vector-im/compound-design-tokens/icons/error.svg?react"; + +import styles from "./MediaView.module.css"; +import { Avatar } from "../Avatar"; + +interface Props extends ComponentProps { + className?: string; + style?: ComponentProps["style"]; + targetWidth: number; + targetHeight: number; + video: TrackReferenceOrPlaceholder; + videoFit: "cover" | "contain"; + mirror: boolean; + member: RoomMember | undefined; + videoEnabled: boolean; + unencryptedWarning: boolean; + nameTagLeadingIcon?: ReactNode; + nameTag: string; + displayName: string; + primaryButton?: ReactNode; + secondaryButton?: ReactNode; +} + +export const MediaView = forwardRef( + ( + { + className, + style, + targetWidth, + targetHeight, + video, + videoFit, + mirror, + member, + videoEnabled, + unencryptedWarning, + nameTagLeadingIcon, + nameTag, + displayName, + primaryButton, + secondaryButton, + ...props + }, + ref, + ) => { + const { t } = useTranslation(); + + return ( + +
+ + {video.publication !== undefined && ( + + )} +
+
+
+ {nameTagLeadingIcon} + + {nameTag} + + {unencryptedWarning && ( + + + + )} +
+ {primaryButton} + {secondaryButton} +
+ + ); + }, +); + +MediaView.displayName = "MediaView"; diff --git a/src/tile/SpotlightTile.module.css b/src/tile/SpotlightTile.module.css new file mode 100644 index 00000000..9d772c1d --- /dev/null +++ b/src/tile/SpotlightTile.module.css @@ -0,0 +1,153 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.tile { + position: absolute; + top: 0; + --border-width: var(--cpd-space-3x); +} + +.tile.maximised { + position: relative; + flex-grow: 1; + --border-width: 0px; +} + +.border { + box-sizing: border-box; + block-size: 100%; + inline-size: 100%; +} + +.tile.maximised .border { + display: contents; +} + +.contents { + display: flex; + border-radius: var(--cpd-space-6x); + contain: strict; + overflow: auto; + scrollbar-width: none; + scroll-snap-type: inline mandatory; + scroll-snap-stop: always; + /* It would be nice to use smooth scrolling here, but Firefox has a bug where + it will not re-snap if the snapping point changes while it's smoothly + animating to another snapping point. + scroll-behavior: smooth; */ +} + +.tile.maximised .contents { + border-radius: 0; +} + +.item { + height: 100%; + flex-basis: 100%; + flex-shrink: 0; + --media-view-fg-inset: 10px; +} + +.item.snap { + scroll-snap-align: start; +} + +.advance { + appearance: none; + cursor: pointer; + opacity: 0; + padding: calc(var(--cpd-space-3x) - var(--cpd-border-width-1)); + border: var(--cpd-border-width-1) solid + var(--cpd-color-border-interactive-secondary); + border-radius: var(--cpd-radius-pill-effect); + background: var(--cpd-color-alpha-gray-1400); + box-shadow: var(--small-drop-shadow); + transition-duration: 0.1s; + transition-property: opacity, background-color, border-color; + position: absolute; + z-index: 1; + /* Center the button vertically on the tile */ + top: 50%; + transform: translateY(-50%); +} + +.advance > svg { + display: block; + color: var(--cpd-color-icon-on-solid-primary); +} + +@media (hover) { + .advance:hover { + border-color: var(--cpd-color-bg-action-primary-hovered); + background: var(--cpd-color-bg-action-primary-hovered); + } +} + +.advance:active { + border-color: var(--cpd-color-bg-action-primary-pressed); + background: var(--cpd-color-bg-action-primary-pressed); +} + +.back { + inset-inline-start: var(--cpd-space-1x); +} + +.next { + inset-inline-end: var(--cpd-space-1x); +} + +.fullScreen { + appearance: none; + cursor: pointer; + opacity: 0; + padding: var(--cpd-space-2x); + border: none; + border-radius: var(--cpd-radius-pill-effect); + background: var(--cpd-color-alpha-gray-1400); + box-shadow: var(--small-drop-shadow); + transition-duration: 0.1s; + transition-property: opacity, background-color; + position: absolute; + z-index: 1; + --inset: calc(var(--border-width) + 6px); + inset-block-end: var(--inset); + inset-inline-end: var(--inset); +} + +.fullScreen > svg { + display: block; + color: var(--cpd-color-icon-on-solid-primary); +} + +@media (hover) { + .fullScreen:hover { + background: var(--cpd-color-bg-action-primary-hovered); + } +} + +.fullScreen:active { + background: var(--cpd-color-bg-action-primary-pressed); +} + +@media (hover) { + .tile:hover > button { + opacity: 1; + } +} + +.tile:has(:focus-visible) > button { + opacity: 1; +} diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx new file mode 100644 index 00000000..6abf0cdd --- /dev/null +++ b/src/tile/SpotlightTile.tsx @@ -0,0 +1,263 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { + ComponentProps, + forwardRef, + useCallback, + useEffect, + useRef, + useState, +} from "react"; +import { Glass } from "@vector-im/compound-web"; +import ExpandIcon from "@vector-im/compound-design-tokens/icons/expand.svg?react"; +import CollapseIcon from "@vector-im/compound-design-tokens/icons/collapse.svg?react"; +import ChevronLeftIcon from "@vector-im/compound-design-tokens/icons/chevron-left.svg?react"; +import ChevronRightIcon from "@vector-im/compound-design-tokens/icons/chevron-right.svg?react"; +import { animated } from "@react-spring/web"; +import { state, useStateObservable } from "@react-rxjs/core"; +import { Observable, map, of } from "rxjs"; +import { useTranslation } from "react-i18next"; +import classNames from "classnames"; + +import { MediaView } from "./MediaView"; +import styles from "./SpotlightTile.module.css"; +import { subscribe } from "../state/subscribe"; +import { + MediaViewModel, + UserMediaViewModel, + useNameData, +} from "../state/MediaViewModel"; +import { useInitial } from "../useInitial"; +import { useMergedRefs } from "../useMergedRefs"; +import { useObservableRef } from "../state/useObservable"; +import { useReactiveState } from "../useReactiveState"; +import { useLatest } from "../useLatest"; + +// Screen share video is always enabled +const screenShareVideoEnabled = state(of(true)); +// Never mirror screen share video +const screenShareMirror = state(of(false)); +// Never crop screen share video +const screenShareCropVideo = state(of(false)); + +interface SpotlightItemProps { + vm: MediaViewModel; + targetWidth: number; + targetHeight: number; + intersectionObserver: Observable; + /** + * Whether this item should act as a scroll snapping point. + */ + snap: boolean; +} + +const SpotlightItem = subscribe( + ({ vm, targetWidth, targetHeight, intersectionObserver, snap }, theirRef) => { + const ourRef = useRef(null); + const ref = useMergedRefs(ourRef, theirRef); + const { displayName, nameTag } = useNameData(vm); + const video = useStateObservable(vm.video); + const videoEnabled = useStateObservable( + vm instanceof UserMediaViewModel + ? vm.videoEnabled + : screenShareVideoEnabled, + ); + const mirror = useStateObservable( + vm instanceof UserMediaViewModel ? vm.mirror : screenShareMirror, + ); + const cropVideo = useStateObservable( + vm instanceof UserMediaViewModel ? vm.cropVideo : screenShareCropVideo, + ); + const unencryptedWarning = useStateObservable(vm.unencryptedWarning); + + // Hook this item up to the intersection observer + useEffect(() => { + const element = ourRef.current!; + let prevIo: IntersectionObserver | null = null; + const subscription = intersectionObserver.subscribe((io) => { + prevIo?.unobserve(element); + io.observe(element); + prevIo = io; + }); + return (): void => { + subscription.unsubscribe(); + prevIo?.unobserve(element); + }; + }, [intersectionObserver]); + + return ( + + ); + }, +); + +interface Props { + vms: MediaViewModel[]; + maximised: boolean; + fullscreen: boolean; + onToggleFullscreen: () => void; + targetWidth: number; + targetHeight: number; + className?: string; + style?: ComponentProps["style"]; +} + +export const SpotlightTile = forwardRef( + ( + { + vms, + maximised, + fullscreen, + onToggleFullscreen, + targetWidth, + targetHeight, + className, + style, + }, + theirRef, + ) => { + const { t } = useTranslation(); + const [root, ourRef] = useObservableRef(null); + const ref = useMergedRefs(ourRef, theirRef); + const [visibleId, setVisibleId] = useState(vms[0].id); + const latestVms = useLatest(vms); + const latestVisibleId = useLatest(visibleId); + const canGoBack = visibleId !== vms[0].id; + const canGoToNext = visibleId !== vms[vms.length - 1].id; + + // To keep track of which item is visible, we need an intersection observer + // hooked up to the root element and the items. Because the items will run + // their effects before their parent does, we need to do this dance with an + // Observable to actually give them the intersection observer. + const intersectionObserver = useInitial>( + () => + root.pipe( + map( + (r) => + new IntersectionObserver( + (entries) => { + const visible = entries.find((e) => e.isIntersecting); + if (visible !== undefined) + setVisibleId(visible.target.getAttribute("data-id")!); + }, + { root: r, threshold: 0.5 }, + ), + ), + ), + ); + + const [scrollToId, setScrollToId] = useReactiveState( + (prev) => + prev == null || prev === visibleId || vms.every((vm) => vm.id !== prev) + ? null + : prev, + [visibleId], + ); + + const onBackClick = useCallback(() => { + const vms = latestVms.current; + const visibleIndex = vms.findIndex( + (vm) => vm.id === latestVisibleId.current, + ); + if (visibleIndex > 0) setScrollToId(vms[visibleIndex - 1].id); + }, [latestVisibleId, latestVms, setScrollToId]); + + const onNextClick = useCallback(() => { + const vms = latestVms.current; + const visibleIndex = vms.findIndex( + (vm) => vm.id === latestVisibleId.current, + ); + if (visibleIndex !== -1 && visibleIndex !== vms.length - 1) + setScrollToId(vms[visibleIndex + 1].id); + }, [latestVisibleId, latestVms, setScrollToId]); + + const FullScreenIcon = fullscreen ? CollapseIcon : ExpandIcon; + + // We need a wrapper element because Glass doesn't provide an animated.div + return ( + + {canGoBack && ( + + )} + + {/* Similarly we need a wrapper element here because Glass expects a + single child */} +
+ {vms.map((vm) => ( + + ))} +
+ + + {canGoToNext && ( + + )} + + ); + }, +); + +SpotlightTile.displayName = "SpotlightTile"; diff --git a/src/useInitial.ts b/src/useInitial.ts new file mode 100644 index 00000000..3b794dd3 --- /dev/null +++ b/src/useInitial.ts @@ -0,0 +1,26 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { useRef } from "react"; + +/** + * React hook that returns the value given on the initial render. + */ +export function useInitial(getValue: () => T): T { + const ref = useRef<{ value: T }>(); + ref.current ??= { value: getValue() }; + return ref.current.value; +} diff --git a/src/useLatest.ts b/src/useLatest.ts new file mode 100644 index 00000000..a0e1ecc7 --- /dev/null +++ b/src/useLatest.ts @@ -0,0 +1,31 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { RefObject, useRef } from "react"; + +export interface LatestRef extends RefObject { + current: T; +} + +/** + * React hook that returns a ref containing the value given on the latest + * render. + */ +export function useLatest(value: T): LatestRef { + const ref = useRef(value); + ref.current = value; + return ref; +} diff --git a/src/useReactiveState.ts b/src/useReactiveState.ts index f5daa1fe..af18e84b 100644 --- a/src/useReactiveState.ts +++ b/src/useReactiveState.ts @@ -44,7 +44,7 @@ export const useReactiveState = ( if ( prevDeps.current === undefined || deps.length !== prevDeps.current.length || - deps.some((d, i) => d !== prevDeps.current![i]) + deps.some((d, i) => !Object.is(d, prevDeps.current![i])) ) { state.current = updateFn(state.current); } diff --git a/src/video-grid/BigGrid.tsx b/src/video-grid/BigGrid.tsx deleted file mode 100644 index bde7eda6..00000000 --- a/src/video-grid/BigGrid.tsx +++ /dev/null @@ -1,1070 +0,0 @@ -/* -Copyright 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import TinyQueue from "tinyqueue"; -import { RectReadOnly } from "react-use-measure"; -import { FC, memo, ReactNode } from "react"; -import { zip } from "lodash"; - -import { Slot } from "./NewVideoGrid"; -import { Layout } from "./Layout"; -import { count, findLastIndex } from "../array-utils"; -import styles from "./BigGrid.module.css"; -import { TileDescriptor } from "../state/CallViewModel"; - -/** - * A 1×1 cell in a grid which belongs to a tile. - */ -interface Cell { - /** - * The item displayed on the tile. - */ - readonly item: TileDescriptor; - /** - * Whether this cell is the origin (top left corner) of the tile. - */ - readonly origin: boolean; - /** - * The width, in columns, of the tile. - */ - readonly columns: number; - /** - * The height, in rows, of the tile. - */ - readonly rows: number; -} - -export interface Grid { - columns: number; - /** - * The cells of the grid, in left-to-right top-to-bottom order. - */ - cells: Cell[]; -} - -export interface SparseGrid { - columns: number; - /** - * The cells of the grid, in left-to-right top-to-bottom order. - * undefined = a gap in the grid. - */ - cells: (Cell | undefined)[]; -} - -/** - * Gets the paths that tiles should travel along in the grid to reach a - * particular destination. - * @param g The grid. - * @param dest The destination index. - * @param avoid A predicate defining the cells that paths should avoid going - * through. - * @returns An array in which each cell holds the index of the next cell to move - * to to reach the destination, or null if it is the destination or otherwise - * immovable. - */ -export function getPaths( - g: SparseGrid, - dest: number, - avoid: (cell: number) => boolean = (): boolean => false, -): (number | null)[] { - const destRow = row(dest, g); - const destColumn = column(dest, g); - - // This is Dijkstra's algorithm - - const distances = new Array(dest + 1).fill(Infinity); - distances[dest] = 0; - const edges = new Array(dest).fill(null); - edges[dest] = null; - const heap = new TinyQueue([dest], (i) => distances[i]); - - const visit = (curr: number, via: number, distanceVia: number): void => { - if (distanceVia < distances[curr]) { - distances[curr] = distanceVia; - edges[curr] = via; - heap.push(curr); - } - }; - - while (heap.length > 0) { - const via = heap.pop()!; - - if (!avoid(via)) { - const viaRow = row(via, g); - const viaColumn = column(via, g); - const viaCell = g.cells[via]; - const viaLargeTile = viaCell !== undefined && !is1By1(viaCell); - // Since it looks nicer to have paths go around large tiles, we impose an - // increased cost for moving through them - const distanceVia = distances[via] + (viaLargeTile ? 8 : 1); - - // Visit each neighbor - if (viaRow > 0) visit(via - g.columns, via, distanceVia); - if (viaColumn > 0) visit(via - 1, via, distanceVia); - if (viaColumn < (viaRow === destRow ? destColumn : g.columns - 1)) - visit(via + 1, via, distanceVia); - if ( - viaRow < destRow - 1 || - (viaRow === destRow - 1 && viaColumn <= destColumn) - ) - visit(via + g.columns, via, distanceVia); - } - } - - // The heap is empty, so we've generated all paths - return edges; -} - -const is1By1 = (c: Cell): boolean => c.columns === 1 && c.rows === 1; - -const findLast1By1Index = (g: SparseGrid): number | null => - findLastIndex(g.cells, (c) => c !== undefined && is1By1(c)); - -export function row(index: number, g: SparseGrid): number { - return Math.floor(index / g.columns); -} - -export function column(index: number, g: SparseGrid): number { - return ((index % g.columns) + g.columns) % g.columns; -} - -function inArea( - index: number, - start: number, - end: number, - g: SparseGrid, -): boolean { - const indexColumn = column(index, g); - const indexRow = row(index, g); - return ( - indexRow >= row(start, g) && - indexRow <= row(end, g) && - indexColumn >= column(start, g) && - indexColumn <= column(end, g) - ); -} - -function* cellsInArea( - start: number, - end: number, - g: SparseGrid, -): Generator { - const startColumn = column(start, g); - const endColumn = column(end, g); - for ( - let i = start; - i <= end; - i = - column(i, g) === endColumn - ? i + g.columns + startColumn - endColumn - : i + 1 - ) - yield i; -} - -export function forEachCellInArea( - start: number, - end: number, - g: G, - fn: (c: G["cells"][0], i: number) => void, -): void { - for (const i of cellsInArea(start, end, g)) fn(g.cells[i], i); -} - -function allCellsInArea( - start: number, - end: number, - g: G, - fn: (c: G["cells"][0], i: number) => boolean, -): boolean { - for (const i of cellsInArea(start, end, g)) { - if (!fn(g.cells[i], i)) return false; - } - - return true; -} - -/** - * Counts the number of cells in the area that satsify the given predicate. - */ -function countCellsInArea( - start: number, - end: number, - g: G, - predicate: (c: G["cells"][0], i: number) => boolean, -): number { - let count = 0; - for (const i of cellsInArea(start, end, g)) { - if (predicate(g.cells[i], i)) count++; - } - return count; -} - -const areaEnd = ( - start: number, - columns: number, - rows: number, - g: SparseGrid, -): number => start + columns - 1 + g.columns * (rows - 1); - -const cloneGrid = (g: G): G => ({ - ...g, - cells: [...g.cells], -}); - -/** - * Gets the index of the next gap in the grid that should be backfilled by 1×1 - * tiles. - */ -function getNextGap( - g: SparseGrid, - ignoreGap: (cell: number) => boolean, -): number | null { - const last1By1Index = findLast1By1Index(g); - if (last1By1Index === null) return null; - - for (let i = 0; i < last1By1Index; i++) { - // To make the backfilling process look natural when there are multiple - // gaps, we actually scan each row from right to left - const j = i; /* - (row(i, g) === row(last1By1Index, g) - ? last1By1Index - : (row(i, g) + 1) * g.columns) - - 1 - - column(i, g);*/ - - if (!ignoreGap(j) && g.cells[j] === undefined) return j; - } - - return null; -} - -/** - * Moves the tile at index "from" over to index "to", displacing other tiles - * along the way. - * Precondition: the destination area must consist of only 1×1 tiles. - */ -function moveTileUnchecked(g: SparseGrid, from: number, to: number): void { - const tile = g.cells[from]!; - const fromEnd = areaEnd(from, tile.columns, tile.rows, g); - const toEnd = areaEnd(to, tile.columns, tile.rows, g); - - const displacedTiles: Cell[] = []; - forEachCellInArea(to, toEnd, g, (c, i) => { - if (c !== undefined && !inArea(i, from, fromEnd, g)) - displacedTiles.push(c!); - }); - - const movingCells: Cell[] = []; - forEachCellInArea(from, fromEnd, g, (c, i) => { - movingCells.push(c!); - g.cells[i] = undefined; - }); - - forEachCellInArea( - to, - toEnd, - g, - (_c, i) => (g.cells[i] = movingCells.shift()), - ); - forEachCellInArea( - from, - fromEnd, - g, - (_c, i) => (g.cells[i] ??= displacedTiles.shift()), - ); -} - -/** - * Moves the tile at index "from" over to index "to", if there is space. - */ -export function moveTile( - g: G, - from: number, - to: number, -): G { - const tile = g.cells[from]!; - - if ( - to !== from && // Skip the operation if nothing would move - to >= 0 && - to < g.cells.length && - column(to, g) <= g.columns - tile.columns - ) { - const fromEnd = areaEnd(from, tile.columns, tile.rows, g); - const toEnd = areaEnd(to, tile.columns, tile.rows, g); - - // The contents of a given cell are 'displaceable' if it's empty, holds a - // 1×1 tile, or is part of the original tile we're trying to reposition - const displaceable = (c: Cell | undefined, i: number): boolean => - c === undefined || is1By1(c) || inArea(i, from, fromEnd, g); - - if (allCellsInArea(to, toEnd, g, displaceable)) { - // The target space is free; move - const gClone = cloneGrid(g); - moveTileUnchecked(gClone, from, to); - return gClone; - } - } - - // The target space isn't free; don't move - return g; -} - -/** - * Attempts to push a tile upwards by a number of rows, displacing 1×1 tiles. - * @returns The number of rows the tile was successfully pushed (may be less - * than requested if there are obstacles blocking movement). - */ -function pushTileUp( - g: SparseGrid, - from: number, - rows: number, - avoid: (cell: number) => boolean = (): boolean => false, -): number { - const tile = g.cells[from]!; - - for (let tryRows = rows; tryRows > 0; tryRows--) { - const to = from - tryRows * g.columns; - const toEnd = areaEnd(to, tile.columns, tile.rows, g); - - const cellsAboveAreDisplacable = - from - g.columns >= 0 && - allCellsInArea( - to, - Math.min(from - g.columns + tile.columns - 1, toEnd), - g, - (c, i) => (c === undefined || is1By1(c)) && !avoid(i), - ); - - if (cellsAboveAreDisplacable) { - moveTileUnchecked(g, from, to); - return tryRows; - } - } - - return 0; -} - -function trimTrailingGaps(g: SparseGrid): void { - // Shrink the array to remove trailing gaps - const newLength = (findLastIndex(g.cells, (c) => c !== undefined) ?? -1) + 1; - if (newLength !== g.cells.length) g.cells = g.cells.slice(0, newLength); -} - -/** - * Determines whether the given area is sufficiently clear of obstacles for - * vacateArea to work. - */ -function canVacateArea(g: SparseGrid, start: number, end: number): boolean { - const newCellCount = countCellsInArea(start, end, g, (c) => c !== undefined); - const newFullRows = Math.floor(newCellCount / g.columns); - return allCellsInArea( - start, - end - newFullRows * g.columns, - g, - (c) => c === undefined || is1By1(c), - ); -} - -/** - * Clears away all the tiles in a given area by pushing them elsewhere. - * Precondition: the area must first be checked with canVacateArea, and the only - * gaps in the given grid must lie either within the area being cleared, or - * after the last 1×1 tile. - */ -function vacateArea(g: SparseGrid, start: number, end: number): SparseGrid { - const newCellCount = countCellsInArea( - start, - end, - g, - (c, i) => c !== undefined || i >= g.cells.length, - ); - const newFullRows = Math.floor(newCellCount / g.columns); - const endRow = row(end, g); - - // To avoid subverting users' expectations, this operation should be the exact - // inverse of fillGaps. We do this by reverse-engineering a grid G with the - // area cleared out and structured such that fillGaps(G) = g. - - // A grid that will have the same structure as the final result, but be filled - // with fake data - const outputStructure: SparseGrid = { - columns: g.columns, - cells: new Array(g.cells.length + newCellCount), - }; - - // The first step in populating outputStructure is to copy over all the large - // tiles, pushing those tiles downwards that fillGaps would push upwards - g.cells.forEach((cell, fromStart) => { - if (cell?.origin && !is1By1(cell)) { - const fromEnd = areaEnd(fromStart, cell.columns, cell.rows, g); - const offset = - row(fromStart, g) + newFullRows > endRow ? newFullRows * g.columns : 0; - forEachCellInArea(fromStart, fromEnd, g, (c, i) => { - outputStructure.cells[i + offset] = c; - }); - } - }); - - // Then, we need to fill it in with the same number of 1×1 tiles as appear in - // the input - const oneByOneTileCount = count(g.cells, (c) => c !== undefined && is1By1(c)); - let oneByOneTilesDistributed = 0; - - for (let i = 0; i < outputStructure.cells.length; i++) { - if (outputStructure.cells[i] === undefined) { - if (inArea(i, start, end, g)) { - // Leave the requested area clear - outputStructure.cells[i] = undefined; - } else if (oneByOneTilesDistributed < oneByOneTileCount) { - outputStructure.cells[i] = { - // Fake data because we only care about the grid's structure - item: {} as unknown as TileDescriptor, - origin: true, - columns: 1, - rows: 1, - }; - oneByOneTilesDistributed++; - } - } - } - - // Lastly, handle the edge case where there were gaps in the input after the - // last 1×1 tile by resizing the cells array to delete these gaps - trimTrailingGaps(outputStructure); - - // outputStructure is now fully populated, and so running fillGaps on it - // should produce a grid with the same structure as the input - const inputStructure = fillGaps( - outputStructure, - false, - (i) => inArea(i, start, end, g) && g.cells[i] === undefined, - ); - - // We exploit the fact that g and inputStructure have the same structure to - // create a mapping between cells in the structure grids and cells in g - const structureMapping = new Map(zip(inputStructure.cells, g.cells)); - - // And finally, we can use that mapping to swap the fake data in - // outputStructure with the real thing - return { - columns: g.columns, - cells: outputStructure.cells.map((placeholder) => - structureMapping.get(placeholder), - ), - }; -} - -/** - * Backfill any gaps in the grid. - */ -export function fillGaps( - g: SparseGrid, - packLargeTiles?: true, - ignoreGap?: () => false, -): Grid; -export function fillGaps( - g: SparseGrid, - packLargeTiles?: boolean, - ignoreGap?: (cell: number) => boolean, -): SparseGrid; -export function fillGaps( - g: SparseGrid, - packLargeTiles = true, - ignoreGap: (cell: number) => boolean = (): boolean => false, -): SparseGrid { - const lastGap = findLastIndex( - g.cells, - (c, i) => c === undefined && !ignoreGap(i), - ); - if (lastGap === null) return g; // There are no gaps to fill - const lastGapRow = row(lastGap, g); - - const result = cloneGrid(g); - - // This will be the size of the grid after we're done here (assuming we're - // allowed to pack the large tiles into the rest of the grid as necessary) - let idealLength = count( - result.cells, - (c, i) => c !== undefined || ignoreGap(i), - ); - const fullRowsRemoved = Math.floor( - (g.cells.length - idealLength) / g.columns, - ); - - // Step 1: Push all large tiles below the last gap upwards, so that they move - // roughly the same distance that we're expecting 1×1 tiles to move - if (fullRowsRemoved > 0) { - for ( - let i = (lastGapRow + 1) * result.columns; - i < result.cells.length; - i++ - ) { - const cell = result.cells[i]; - if (cell?.origin && !is1By1(cell)) - pushTileUp(result, i, fullRowsRemoved, ignoreGap); - } - } - - // Step 2: Deal with any large tiles that are still hanging off the bottom - if (packLargeTiles) { - for (let i = result.cells.length - 1; i >= idealLength; i--) { - const cell = result.cells[i]; - if (cell !== undefined && !is1By1(cell)) { - // First, try to just push it upwards a bit more - const originIndex = - i - (cell.columns - 1) - result.columns * (cell.rows - 1); - const pushed = pushTileUp(result, originIndex, 1, ignoreGap) === 1; - - // If that failed, collapse the tile to 1×1 so it can be dealt with in - // step 3 - if (!pushed) { - const collapsedTile: Cell = { - item: cell.item, - origin: true, - columns: 1, - rows: 1, - }; - forEachCellInArea(originIndex, i, result, (_c, j) => { - result.cells[j] = undefined; - }); - result.cells[i] = collapsedTile; - // Collapsing the tile makes the final grid size smaller - idealLength -= cell.columns * cell.rows - 1; - } - } - } - } - - // Step 3: Fill all remaining gaps with 1×1 tiles - let gap = getNextGap(result, ignoreGap); - - if (gap !== null) { - const pathsToEnd = getPaths(result, findLast1By1Index(result)!, ignoreGap); - - do { - let filled = false; - let to = gap; - let from = pathsToEnd[gap]; - - // First, attempt to fill the gap by moving 1×1 tiles backwards from the - // end of the grid along a set path - while (from !== null) { - const toCell = result.cells[to] as Cell | undefined; - const fromCell = result.cells[from] as Cell | undefined; - - // Skip over slots that are already full - if (toCell !== undefined) { - to = pathsToEnd[to]!; - // Skip over large tiles. Also, we might run into gaps along the path - // created during the filling of previous gaps. Skip over those too; - // they'll be picked up on the next iteration of the outer loop. - } else if (fromCell === undefined || !is1By1(fromCell)) { - from = pathsToEnd[from]; - } else { - result.cells[to] = result.cells[from]; - result.cells[from] = undefined; - filled = true; - to = pathsToEnd[to]!; - from = pathsToEnd[from]; - } - } - - // In case the path approach failed, fall back to taking the very last 1×1 - // tile, and just dropping it into place - if (!filled) { - const last1By1Index = findLast1By1Index(result)!; - result.cells[gap] = result.cells[last1By1Index]; - result.cells[last1By1Index] = undefined; - } - - gap = getNextGap(result, ignoreGap); - } while (gap !== null); - } - - trimTrailingGaps(result); - return result; -} - -// TODO: replace all usages of this function with vacateArea, as this results in -// somewhat unpredictable movement -function createRows(g: SparseGrid, count: number, atRow: number): SparseGrid { - const result = { - columns: g.columns, - cells: new Array(g.cells.length + g.columns * count), - }; - const offsetAfterNewRows = g.columns * count; - - // Copy tiles from the original grid to the new one, with the new rows - // inserted at the target location - g.cells.forEach((c, from) => { - if (c?.origin) { - const offset = row(from, g) >= atRow ? offsetAfterNewRows : 0; - forEachCellInArea( - from, - areaEnd(from, c.columns, c.rows, g), - g, - (c, i) => { - result.cells[i + offset] = c; - }, - ); - } - }); - - return result; -} - -/** - * Adds a set of new items into the grid. - */ -export function addItems( - items: TileDescriptor[], - g: SparseGrid, -): SparseGrid { - let result: SparseGrid = cloneGrid(g); - - for (const item of items) { - const cell = { - item, - origin: true, - columns: 1, - rows: 1, - }; - - let placeAt: number; - - if (item.placeNear === undefined) { - // This item has no special placement requests, so let's put it - // uneventfully at the end of the grid - placeAt = result.cells.length; - } else { - // This item wants to be placed near another; let's put it on a row - // directly below the related tile - const placeNear = result.cells.findIndex( - (c) => c?.item.id === item.placeNear, - ); - if (placeNear === -1) { - // Can't find the related tile, so let's give up and place it at the end - placeAt = result.cells.length; - } else { - const placeNearCell = result.cells[placeNear]!; - const placeNearEnd = areaEnd( - placeNear, - placeNearCell.columns, - placeNearCell.rows, - result, - ); - - result = createRows(result, 1, row(placeNearEnd, result) + 1); - placeAt = - placeNear + - Math.floor(placeNearCell.columns / 2) + - result.columns * placeNearCell.rows; - } - } - - result.cells[placeAt] = cell; - - if (item.largeBaseSize) { - // Cycle the tile size once to set up the tile with its larger base size - // This also fills any gaps in the grid, hence no extra call to fillGaps - result = cycleTileSize(result, item); - } - } - - return result; -} - -const largeTileDimensions = (g: SparseGrid): [number, number] => [ - Math.min(3, Math.max(2, g.columns - 1)), - 2, -]; - -const extraLargeTileDimensions = (g: SparseGrid): [number, number] => - g.columns > 3 ? [4, 3] : [g.columns, 2]; - -export function cycleTileSize( - g: G, - tile: TileDescriptor, -): G { - const from = g.cells.findIndex((c) => c?.item === tile); - if (from === -1) return g; // Tile removed, no change - const fromCell = g.cells[from]!; - const fromWidth = fromCell.columns; - const fromHeight = fromCell.rows; - - const [baseDimensions, enlargedDimensions] = fromCell.item.largeBaseSize - ? [largeTileDimensions(g), extraLargeTileDimensions(g)] - : [[1, 1], largeTileDimensions(g)]; - // The target dimensions, which toggle between the base and enlarged sizes - const [toWidth, toHeight] = - fromWidth === baseDimensions[0] && fromHeight === baseDimensions[1] - ? enlargedDimensions - : baseDimensions; - - return setTileSize(g, from, toWidth, toHeight); -} - -/** - * Finds the cell nearest to 'nearestTo' that satsifies the given predicate. - * @param shouldScan A predicate constraining the bounds of the search. - */ -function findNearestCell( - g: G, - nearestTo: number, - shouldScan: (index: number) => boolean, - predicate: (cell: G["cells"][0], index: number) => boolean, -): number | null { - const scanLocations = new Set([nearestTo]); - - for (const scanLocation of scanLocations) { - if (shouldScan(scanLocation)) { - if (predicate(g.cells[scanLocation], scanLocation)) return scanLocation; - - // Scan outwards in all directions - const scanColumn = column(scanLocation, g); - const scanRow = row(scanLocation, g); - if (scanColumn > 0) scanLocations.add(scanLocation - 1); - if (scanColumn < g.columns - 1) scanLocations.add(scanLocation + 1); - if (scanRow > 0) scanLocations.add(scanLocation - g.columns); - scanLocations.add(scanLocation + g.columns); - } - } - - return null; -} - -/** - * Changes the size of a tile, rearranging the grid to make space. - * @param tileId The ID of the tile to modify. - * @param g The grid. - * @returns The updated grid. - */ -export function setTileSize( - g: G, - from: number, - toWidth: number, - toHeight: number, -): G { - const fromCell = g.cells[from]!; - const fromWidth = fromCell.columns; - const fromHeight = fromCell.rows; - const fromEnd = areaEnd(from, fromWidth, fromHeight, g); - const newGridSize = - g.cells.length + toWidth * toHeight - fromWidth * fromHeight; - - const toColumn = Math.max( - 0, - Math.min( - g.columns - toWidth, - column(from, g) + Math.trunc((fromWidth - toWidth) / 2), - ), - ); - const toRow = Math.max( - 0, - row(from, g) + Math.trunc((fromHeight - toHeight) / 2), - ); - const targetDest = toColumn + toRow * g.columns; - - const gridWithoutTile = cloneGrid(g); - forEachCellInArea(from, fromEnd, gridWithoutTile, (_c, i) => { - gridWithoutTile.cells[i] = undefined; - }); - - const placeTile = ( - to: number, - toEnd: number, - grid: Grid | SparseGrid, - ): void => { - forEachCellInArea(to, toEnd, grid, (_c, i) => { - grid.cells[i] = { - item: fromCell.item, - origin: i === to, - columns: toWidth, - rows: toHeight, - }; - }); - }; - - if (toWidth <= fromWidth && toHeight <= fromHeight) { - // The tile is shrinking, which can always happen in-place - const to = targetDest; - const toEnd = areaEnd(to, toWidth, toHeight, g); - - const result: SparseGrid = gridWithoutTile; - placeTile(to, toEnd, result); - return fillGaps(result, true, (i: number) => inArea(i, to, toEnd, g)) as G; - } else if (toWidth >= fromWidth && toHeight >= fromHeight) { - // The tile is growing, which might be able to happen in-place - const to = findNearestCell( - gridWithoutTile, - targetDest, - (i) => { - const end = areaEnd(i, toWidth, toHeight, g); - return ( - column(i, g) + toWidth - 1 < g.columns && - inArea(from, i, end, g) && - inArea(fromEnd, i, end, g) - ); - }, - (_c, i) => { - const end = areaEnd(i, toWidth, toHeight, g); - return end < newGridSize && canVacateArea(gridWithoutTile, i, end); - }, - ); - - if (to !== null) { - const toEnd = areaEnd(to, toWidth, toHeight, g); - const result = vacateArea(gridWithoutTile, to, toEnd); - - placeTile(to, toEnd, result); - return result as G; - } - } - - // Catch-all path for when the tile is neither strictly shrinking nor - // growing, or when there's not enough space for it to grow in-place - - const packedGridWithoutTile = fillGaps(gridWithoutTile, false); - - const to = findNearestCell( - packedGridWithoutTile, - targetDest, - (i) => i < newGridSize && column(i, g) + toWidth - 1 < g.columns, - (_c, i) => { - const end = areaEnd(i, toWidth, toHeight, g); - return end < newGridSize && canVacateArea(packedGridWithoutTile, i, end); - }, - ); - - if (to === null) return g; // There's no space anywhere; give up - - const toEnd = areaEnd(to, toWidth, toHeight, g); - const result = vacateArea(packedGridWithoutTile, to, toEnd); - - placeTile(to, toEnd, result); - return result as G; -} - -/** - * Resizes the grid to a new column width. - */ -export function resize(g: Grid, columns: number): Grid { - const result: SparseGrid = { columns, cells: [] }; - const [largeColumns, largeRows] = largeTileDimensions(result); - - // Copy each tile from the old grid to the resized one in the same order - - // The next index in the result grid to copy a tile to - let next = 0; - - for (const cell of g.cells) { - if (cell.origin) { - // TODO make aware of extra large tiles - const [nextColumns, nextRows] = is1By1(cell) - ? [1, 1] - : [largeColumns, largeRows]; - - // If there isn't enough space left on this row, jump to the next row - if (columns - column(next, result) < nextColumns) - next = columns * (Math.floor(next / columns) + 1); - const nextEnd = areaEnd(next, nextColumns, nextRows, result); - - // Expand the cells array as necessary - if (result.cells.length <= nextEnd) - result.cells.push(...new Array(nextEnd + 1 - result.cells.length)); - - // Copy the tile into place - forEachCellInArea(next, nextEnd, result, (_c, i) => { - result.cells[i] = { - item: cell.item, - origin: i === next, - columns: nextColumns, - rows: nextRows, - }; - }); - - next = nextEnd + 1; - } - } - - return fillGaps(result); -} - -/** - * Promotes speakers to the first page of the grid. - */ -export function promoteSpeakers(g: SparseGrid): void { - // This is all a bit of a hack right now, because we don't know if the designs - // will stick with this approach in the long run - // We assume that 4 rows are probably about 1 page - const firstPageEnd = g.columns * 4; - - for (let from = firstPageEnd; from < g.cells.length; from++) { - const fromCell = g.cells[from]; - // Don't bother trying to promote enlarged tiles - if (fromCell?.item.isSpeaker && is1By1(fromCell)) { - // Promote this tile by making 10 attempts to place it on the first page - for (let j = 0; j < 10; j++) { - const to = Math.floor(Math.random() * firstPageEnd); - const toCell = g.cells[to]; - if (toCell === undefined || is1By1(toCell)) { - moveTileUnchecked(g, from, to); - break; - } - } - } - } -} - -/** - * The algorithm for updating a grid with a new set of tiles. - */ -function updateTiles(g: Grid, tiles: TileDescriptor[]): Grid { - // Step 1: Update tiles that still exist, and remove tiles that have left - // the grid - const itemsById = new Map(tiles.map((i) => [i.id, i])); - const grid1: SparseGrid = { - ...g, - cells: g.cells.map((c) => { - if (c === undefined) return undefined; - const item = itemsById.get(c.item.id); - return item === undefined ? undefined : { ...c, item }; - }), - }; - - // Step 2: Add new tiles - const existingItemIds = new Set( - grid1.cells.filter((c) => c !== undefined).map((c) => c!.item.id), - ); - const newItems = tiles.filter((i) => !existingItemIds.has(i.id)); - const grid2 = addItems(newItems, grid1); - - // Step 3: Promote speakers to the top - promoteSpeakers(grid2); - - return fillGaps(grid2); -} - -function updateBounds(g: Grid, bounds: RectReadOnly): Grid { - const columns = Math.max(2, Math.floor(bounds.width * 0.0055)); - return columns === g.columns ? g : resize(g, columns); -} - -const Slots: FC<{ s: Grid }> = memo(({ s: g }) => { - const areas = new Array<(number | null)[]>( - Math.ceil(g.cells.length / g.columns), - ); - for (let i = 0; i < areas.length; i++) - areas[i] = new Array(g.columns).fill(null); - - let slotCount = 0; - for (let i = 0; i < g.cells.length; i++) { - const cell = g.cells[i]; - if (cell.origin) { - const slotEnd = i + cell.columns - 1 + g.columns * (cell.rows - 1); - forEachCellInArea( - i, - slotEnd, - g, - (_c, j) => (areas[row(j, g)][column(j, g)] = slotCount), - ); - slotCount++; - } - } - - const style = { - gridTemplateAreas: areas - .map( - (row) => - `'${row - .map((slotId) => (slotId === null ? "." : `s${slotId}`)) - .join(" ")}'`, - ) - .join(" "), - gridTemplateColumns: `repeat(${g.columns}, 1fr)`, - }; - - const slots = new Array(slotCount); - for (let i = 0; i < slotCount; i++) - slots[i] = ; - - return ( -
- {slots} -
- ); -}); - -Slots.displayName = "Slots"; - -/** - * Given a tile and numbers in the range [0, 1) describing a position within the - * tile, this returns the index of the specific cell in which that position - * lies. - */ -function positionOnTileToCell( - g: SparseGrid, - tileOriginIndex: number, - xPositionOnTile: number, - yPositionOnTile: number, -): number { - const tileOrigin = g.cells[tileOriginIndex]!; - const columnOnTile = Math.floor(xPositionOnTile * tileOrigin.columns); - const rowOnTile = Math.floor(yPositionOnTile * tileOrigin.rows); - return tileOriginIndex + columnOnTile + g.columns * rowOnTile; -} - -function dragTile( - g: Grid, - from: TileDescriptor, - to: TileDescriptor, - xPositionOnFrom: number, - yPositionOnFrom: number, - xPositionOnTo: number, - yPositionOnTo: number, -): Grid { - const fromOrigin = g.cells.findIndex((c) => c.item === from); - const toOrigin = g.cells.findIndex((c) => c.item === to); - const fromCell = positionOnTileToCell( - g, - fromOrigin, - xPositionOnFrom, - yPositionOnFrom, - ); - const toCell = positionOnTileToCell( - g, - toOrigin, - xPositionOnTo, - yPositionOnTo, - ); - - return moveTile(g, fromOrigin, fromOrigin + toCell - fromCell); -} - -export const BigGrid: Layout = { - emptyState: { columns: 4, cells: [] }, - updateTiles, - updateBounds, - getTiles: (g: Grid) => - g.cells.filter((c) => c.origin).map((c) => c!.item as T), - canDragTile: () => true, - dragTile, - toggleFocus: cycleTileSize, - Slots, - rememberState: false, -}; diff --git a/src/video-grid/Layout.tsx b/src/video-grid/Layout.tsx deleted file mode 100644 index b540cbe1..00000000 --- a/src/video-grid/Layout.tsx +++ /dev/null @@ -1,195 +0,0 @@ -/* -Copyright 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { ComponentType, ReactNode, useCallback, useMemo, useRef } from "react"; - -import type { RectReadOnly } from "react-use-measure"; -import { useReactiveState } from "../useReactiveState"; -import { TileDescriptor } from "../state/CallViewModel"; - -/** - * A video grid layout system with concrete states of type State. - */ -// Ideally State would be parameterized by the tile data type, but then that -// makes Layout a higher-kinded type, which isn't achievable in TypeScript -// (unless you invoke some dark type-level computation magic… 😏) -// So we're stuck with these types being a little too strong. -export interface Layout { - /** - * The layout state for zero tiles. - */ - readonly emptyState: State; - /** - * Updates/adds/removes tiles in a way that looks natural in the context of - * the given initial state. - */ - readonly updateTiles: (s: State, tiles: TileDescriptor[]) => State; - /** - * Adapts the layout to a new container size. - */ - readonly updateBounds: (s: State, bounds: RectReadOnly) => State; - /** - * Gets tiles in the order created by the layout. - */ - readonly getTiles: (s: State) => TileDescriptor[]; - /** - * Determines whether a tile is draggable. - */ - readonly canDragTile: (s: State, tile: TileDescriptor) => boolean; - /** - * Drags the tile 'from' to the location of the tile 'to' (if possible). - * The position parameters are numbers in the range [0, 1) describing the - * specific positions on 'from' and 'to' that the drag gesture is targeting. - */ - readonly dragTile: ( - s: State, - from: TileDescriptor, - to: TileDescriptor, - xPositionOnFrom: number, - yPositionOnFrom: number, - xPositionOnTo: number, - yPositionOnTo: number, - ) => State; - /** - * Toggles the focus of the given tile (if this layout has the concept of - * focus). - */ - readonly toggleFocus?: (s: State, tile: TileDescriptor) => State; - /** - * A React component generating the slot elements for a given layout state. - */ - readonly Slots: ComponentType<{ s: State }>; - /** - * Whether the state of this layout should be remembered even while a - * different layout is active. - */ - readonly rememberState: boolean; -} - -/** - * A version of Map with stronger types that allow us to save layout states in a - * type-safe way. - */ -export interface LayoutStatesMap { - get(layout: Layout): State | undefined; - set(layout: Layout, state: State): LayoutStatesMap; - delete(layout: Layout): boolean; -} - -/** - * Hook creating a Map to store layout states in. - */ -export const useLayoutStates = (): LayoutStatesMap => { - const layoutStates = useRef>(); - if (layoutStates.current === undefined) layoutStates.current = new Map(); - return layoutStates.current as LayoutStatesMap; -}; - -interface UseLayout { - state: State; - orderedItems: TileDescriptor[]; - generation: number; - canDragTile: (tile: TileDescriptor) => boolean; - dragTile: ( - from: TileDescriptor, - to: TileDescriptor, - xPositionOnFrom: number, - yPositionOnFrom: number, - xPositionOnTo: number, - yPositionOnTo: number, - ) => void; - toggleFocus: ((tile: TileDescriptor) => void) | undefined; - slots: ReactNode; -} - -/** - * Hook which uses the provided layout system to arrange a set of items into a - * concrete layout state, and provides callbacks for user interaction. - */ -export function useLayout( - layout: Layout, - items: TileDescriptor[], - bounds: RectReadOnly, - layoutStates: LayoutStatesMap, -): UseLayout { - const prevLayout = useRef>(); - const prevState = layoutStates.get(layout); - - const [state, setState] = useReactiveState(() => { - // If the bounds aren't known yet, don't add anything to the layout - if (bounds.width === 0) { - return layout.emptyState; - } else { - if ( - prevLayout.current !== undefined && - layout !== prevLayout.current && - !prevLayout.current.rememberState - ) - layoutStates.delete(prevLayout.current); - - const baseState = layoutStates.get(layout) ?? layout.emptyState; - return layout.updateTiles(layout.updateBounds(baseState, bounds), items); - } - }, [layout, items, bounds]); - - const generation = useRef(0); - if (state !== prevState) generation.current++; - - prevLayout.current = layout as Layout; - // No point in remembering an empty state, plus it would end up clobbering the - // real saved state while restoring a layout - if (state !== layout.emptyState) layoutStates.set(layout, state); - - return { - state, - orderedItems: useMemo(() => layout.getTiles(state), [layout, state]), - generation: generation.current, - canDragTile: useCallback( - (tile: TileDescriptor) => layout.canDragTile(state, tile), - [layout, state], - ), - dragTile: useCallback( - ( - from: TileDescriptor, - to: TileDescriptor, - xPositionOnFrom: number, - yPositionOnFrom: number, - xPositionOnTo: number, - yPositionOnTo: number, - ) => - setState((s) => - layout.dragTile( - s, - from, - to, - xPositionOnFrom, - yPositionOnFrom, - xPositionOnTo, - yPositionOnTo, - ), - ), - [layout, setState], - ), - toggleFocus: useMemo( - () => - layout.toggleFocus && - ((tile: TileDescriptor): void => - setState((s) => layout.toggleFocus!(s, tile))), - [layout, setState], - ), - slots: , - }; -} diff --git a/src/video-grid/NewVideoGrid.tsx b/src/video-grid/NewVideoGrid.tsx deleted file mode 100644 index 61813125..00000000 --- a/src/video-grid/NewVideoGrid.tsx +++ /dev/null @@ -1,389 +0,0 @@ -/* -Copyright 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { SpringRef, TransitionFn, useTransition } from "@react-spring/web"; -import { EventTypes, Handler, useScroll } from "@use-gesture/react"; -import { - CSSProperties, - FC, - ReactNode, - useEffect, - useMemo, - useRef, - useState, -} from "react"; -import useMeasure from "react-use-measure"; -import { zip } from "lodash"; - -import styles from "./NewVideoGrid.module.css"; -import { - VideoGridProps as Props, - TileSpring, - ChildrenProperties, - TileSpringUpdate, -} from "./VideoGrid"; -import { useReactiveState } from "../useReactiveState"; -import { useMergedRefs } from "../useMergedRefs"; -import { TileWrapper } from "./TileWrapper"; -import { BigGrid } from "./BigGrid"; -import { useLayout } from "./Layout"; -import { TileDescriptor } from "../state/CallViewModel"; - -interface Rect { - x: number; - y: number; - width: number; - height: number; -} - -interface Tile extends Rect { - item: TileDescriptor; -} - -interface DragState { - tileId: string; - tileX: number; - tileY: number; - cursorX: number; - cursorY: number; -} - -interface TapData { - tileId: string; - ts: number; -} - -interface SlotProps { - style?: CSSProperties; -} - -export const Slot: FC = ({ style }) => ( -
-); - -/** - * An interactive, animated grid of video tiles. - */ -export function NewVideoGrid({ - items, - disableAnimations, - layoutStates, - children, -}: Props): ReactNode { - // Overview: This component lays out tiles by rendering an invisible template - // grid of "slots" for tiles to go in. Once rendered, it uses the DOM API to - // get the dimensions of each slot, feeding these numbers back into - // react-spring to let the actual tiles move freely atop the template. - - // To know when the rendered grid becomes consistent with the layout we've - // requested, we give it a data-generation attribute which holds the ID of the - // most recently rendered generation of the grid, and watch it with a - // MutationObserver. - - const [slotsRoot, setSlotsRoot] = useState(null); - const [renderedGeneration, setRenderedGeneration] = useState(0); - - useEffect(() => { - if (slotsRoot !== null) { - setRenderedGeneration( - parseInt(slotsRoot.getAttribute("data-generation")!), - ); - - const observer = new MutationObserver((mutations) => { - if (mutations.some((m) => m.type === "attributes")) { - setRenderedGeneration( - parseInt(slotsRoot.getAttribute("data-generation")!), - ); - } - }); - - observer.observe(slotsRoot, { attributes: true }); - return (): void => observer.disconnect(); - } - }, [slotsRoot, setRenderedGeneration]); - - const [gridRef1, gridBounds] = useMeasure(); - const gridRef2 = useRef(null); - const gridRef = useMergedRefs(gridRef1, gridRef2); - - const slotRects = useMemo(() => { - if (slotsRoot === null) return []; - - const slots = slotsRoot.getElementsByClassName(styles.slot); - const rects = new Array(slots.length); - for (let i = 0; i < slots.length; i++) { - const slot = slots[i] as HTMLElement; - rects[i] = { - x: slot.offsetLeft, - y: slot.offsetTop, - width: slot.offsetWidth, - height: slot.offsetHeight, - }; - } - - return rects; - // The rects may change due to the grid being resized or rerendered, but - // eslint can't statically verify this - // eslint-disable-next-line react-hooks/exhaustive-deps - }, [slotsRoot, renderedGeneration, gridBounds]); - - // TODO: Implement more layouts and select the right one here - const layout = BigGrid; - const { - state: grid, - orderedItems, - generation, - canDragTile, - dragTile, - toggleFocus, - slots, - } = useLayout(layout, items, gridBounds, layoutStates); - - const [tiles] = useReactiveState[]>( - (prevTiles) => { - // If React hasn't yet rendered the current generation of the grid, skip - // the update, because grid and slotRects will be out of sync - if (renderedGeneration !== generation) return prevTiles ?? []; - - const tileRects = new Map( - zip(orderedItems, slotRects) as [TileDescriptor, Rect][], - ); - // In order to not break drag gestures, it's critical that we render tiles - // in a stable order (that of 'items') - return items.map((item) => ({ ...tileRects.get(item)!, item })); - }, - [slotRects, grid, renderedGeneration], - ); - - // Drag state is stored in a ref rather than component state, because we use - // react-spring's imperative API during gestures to improve responsiveness - const dragState = useRef(null); - - const [tileTransitions, springRef] = useTransition( - tiles, - () => ({ - key: ({ item }: Tile): string => item.id, - from: ({ x, y, width, height }: Tile): TileSpringUpdate => ({ - opacity: 0, - scale: 0, - shadow: 0, - shadowSpread: 0, - zIndex: 1, - x, - y, - width, - height, - immediate: disableAnimations, - }), - enter: { opacity: 1, scale: 1, immediate: disableAnimations }, - update: ({ - item, - x, - y, - width, - height, - }: Tile): TileSpringUpdate | null => - item.id === dragState.current?.tileId - ? null - : { - x, - y, - width, - height, - immediate: disableAnimations, - }, - leave: { opacity: 0, scale: 0, immediate: disableAnimations }, - config: { mass: 0.7, tension: 252, friction: 25 }, - }), - // react-spring's types are bugged and can't infer the spring type - ) as unknown as [TransitionFn, TileSpring>, SpringRef]; - - // Because we're using react-spring in imperative mode, we're responsible for - // firing animations manually whenever the tiles array updates - useEffect(() => { - springRef.start(); - }, [tiles, springRef]); - - const animateDraggedTile = (endOfGesture: boolean): void => { - const { tileId, tileX, tileY, cursorX, cursorY } = dragState.current!; - const tile = tiles.find((t) => t.item.id === tileId)!; - - springRef.current - .find((c) => (c.item as Tile).item.id === tileId) - ?.start( - endOfGesture - ? { - scale: 1, - zIndex: 1, - shadow: 0, - x: tile.x, - y: tile.y, - width: tile.width, - height: tile.height, - immediate: - disableAnimations || ((key): boolean => key === "zIndex"), - // Allow the tile's position to settle before pushing its - // z-index back down - delay: (key): number => (key === "zIndex" ? 500 : 0), - } - : { - scale: 1.1, - zIndex: 2, - shadow: 15, - x: tileX, - y: tileY, - immediate: - disableAnimations || - ((key): boolean => - key === "zIndex" || key === "x" || key === "y"), - }, - ); - - const overTile = tiles.find( - (t) => - cursorX >= t.x && - cursorX < t.x + t.width && - cursorY >= t.y && - cursorY < t.y + t.height, - ); - - if (overTile !== undefined) - dragTile( - tile.item, - overTile.item, - (cursorX - tileX) / tile.width, - (cursorY - tileY) / tile.height, - (cursorX - overTile.x) / overTile.width, - (cursorY - overTile.y) / overTile.height, - ); - }; - - const lastTap = useRef(null); - - // Callback for useDrag. We could call useDrag here, but the default - // pattern of spreading {...bind()} across the children to bind the gesture - // ends up breaking memoization and ruining this component's performance. - // Instead, we pass this callback to each tile via a ref, to let them bind the - // gesture using the much more sensible ref-based method. - const onTileDrag = ( - tileId: string, - - { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - tap, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - initial: [initialX, initialY], - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - delta: [dx, dy], - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - last, - }: Parameters>[0], - ): void => { - if (tap) { - const now = Date.now(); - - if ( - tileId === lastTap.current?.tileId && - now - lastTap.current.ts < 500 - ) { - toggleFocus?.(items.find((i) => i.id === tileId)!); - lastTap.current = null; - } else { - lastTap.current = { tileId, ts: now }; - } - } else { - const tileController = springRef.current.find( - (c) => (c.item as Tile).item.id === tileId, - )!; - - if (canDragTile((tileController.item as Tile).item)) { - if (dragState.current === null) { - const tileSpring = tileController.get(); - dragState.current = { - tileId, - tileX: tileSpring.x, - tileY: tileSpring.y, - cursorX: initialX - gridBounds.x, - cursorY: initialY - gridBounds.y + scrollOffset.current, - }; - } - - dragState.current.tileX += dx; - dragState.current.tileY += dy; - dragState.current.cursorX += dx; - dragState.current.cursorY += dy; - - animateDraggedTile(last); - - if (last) dragState.current = null; - } - } - }; - - const onTileDragRef = useRef(onTileDrag); - onTileDragRef.current = onTileDrag; - - const scrollOffset = useRef(0); - - useScroll( - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - ({ xy: [, y], delta: [, dy] }) => { - scrollOffset.current = y; - - if (dragState.current !== null) { - dragState.current.tileY += dy; - dragState.current.cursorY += dy; - animateDraggedTile(false); - } - }, - { target: gridRef2 }, - ); - - // Render nothing if the grid has yet to be generated - if (grid === null) { - return
; - } - - return ( -
-
- {slots} -
- {tileTransitions((spring, tile) => ( - - {children as (props: ChildrenProperties) => ReactNode} - - ))} -
- ); -} diff --git a/test/video-grid/VideoGrid-test.ts b/test/grid/LegacyGrid-test.ts similarity index 96% rename from test/video-grid/VideoGrid-test.ts rename to test/grid/LegacyGrid-test.ts index cf15c022..44f82d42 100644 --- a/test/video-grid/VideoGrid-test.ts +++ b/test/grid/LegacyGrid-test.ts @@ -15,7 +15,7 @@ limitations under the License. */ import { TileDescriptor } from "../../src/state/CallViewModel"; -import { Tile, reorderTiles } from "../../src/video-grid/VideoGrid"; +import { Tile, reorderTiles } from "../../src/grid/LegacyGrid"; const alice: Tile = { key: "alice", diff --git a/test/video-grid/BigGrid-test.ts b/test/video-grid/BigGrid-test.ts deleted file mode 100644 index 3d29db6c..00000000 --- a/test/video-grid/BigGrid-test.ts +++ /dev/null @@ -1,493 +0,0 @@ -/* -Copyright 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { TileDescriptor } from "../../src/state/CallViewModel"; -import { - addItems, - column, - cycleTileSize, - fillGaps, - forEachCellInArea, - Grid, - SparseGrid, - resize, - row, - moveTile, -} from "../../src/video-grid/BigGrid"; - -/** - * Builds a grid from a string specifying the contents of each cell as a letter. - */ -function mkGrid(spec: string): Grid { - const secondNewline = spec.indexOf("\n", 1); - const columns = secondNewline === -1 ? spec.length : secondNewline - 1; - const cells = spec.match(/[a-z ]/g) ?? ([] as string[]); - const areas = new Set(cells); - areas.delete(" "); // Space represents an empty cell, not an area - const grid: Grid = { columns, cells: new Array(cells.length) }; - - for (const area of areas) { - const start = cells.indexOf(area); - const end = cells.lastIndexOf(area); - const rows = row(end, grid) - row(start, grid) + 1; - const columns = column(end, grid) - column(start, grid) + 1; - - forEachCellInArea(start, end, grid, (_c, i) => { - grid.cells[i] = { - item: { id: area } as unknown as TileDescriptor, - origin: i === start, - rows, - columns, - }; - }); - } - - return grid; -} - -/** - * Turns a grid into a string showing the contents of each cell as a letter. - */ -function showGrid(g: Grid): string { - let result = "\n"; - for (let i = 0; i < g.cells.length; i++) { - if (i > 0 && i % g.columns == 0) result += "\n"; - result += g.cells[i]?.item.id ?? " "; - } - return result; -} - -function testFillGaps(title: string, input: string, output: string): void { - test(`fillGaps ${title}`, () => { - expect(showGrid(fillGaps(mkGrid(input)))).toBe(output); - }); -} - -testFillGaps( - "does nothing on an empty grid", - ` -`, - ` -`, -); - -testFillGaps( - "does nothing if there are no gaps", - ` -ab -cd -ef`, - ` -ab -cd -ef`, -); - -testFillGaps( - "fills a gap", - ` -a b -cde -f`, - ` -cab -fde`, -); - -testFillGaps( - "fills multiple gaps", - ` -a bc -defgh - ijkl -mno`, - ` -aebch -difgl -mjnok`, -); - -testFillGaps( - "fills a big gap with 1×1 tiles", - ` -abcd -e f -g h -ijkl`, - ` -abcd -ehkf -glji`, -); - -testFillGaps( - "fills a big gap with a large tile", - ` - -aa -bc`, - ` -aa -cb`, -); - -testFillGaps( - "prefers moving around large tiles", - ` -a bc -ddde -dddf -ghij -k`, - ` -abce -dddf -dddj -kghi`, -); - -testFillGaps( - "moves through large tiles if necessary", - ` -a bc -dddd -efgh -i`, - ` -afbc -dddd -iegh`, -); - -testFillGaps( - "keeps a large tile from hanging off the bottom", - ` -abcd -efgh - -ii -ii`, - ` -abcd -iigh -iief`, -); - -testFillGaps( - "collapses large tiles trapped at the bottom", - ` -abcd -e fg -hh -hh - ii - ii`, - ` -abcd -hhfg -hhie`, -); - -testFillGaps( - "gives up on pushing large tiles upwards when not possible", - ` -aa -aa -bccd -eccf -ghij`, - ` -aadf -aaji -bcch -eccg`, -); - -function testCycleTileSize( - title: string, - tileId: string, - input: string, - output: string, -): void { - test(`cycleTileSize ${title}`, () => { - const grid = mkGrid(input); - const tile = grid.cells.find((c) => c?.item.id === tileId)!.item; - expect(showGrid(cycleTileSize(grid, tile))).toBe(output); - }); -} - -testCycleTileSize( - "expands a tile to 2×2 in a 3 column layout", - "c", - ` -abc -def -ghi`, - ` -acc -dcc -gbe -ifh`, -); - -testCycleTileSize( - "expands a tile to 3×3 in a 4 column layout", - "g", - ` -abcd -efgh`, - ` -acdh -bggg -fggg -e`, -); - -testCycleTileSize( - "restores a tile to 1×1", - "b", - ` -abbc -dbbe -fghi -jk`, - ` -abhc -djge -fik`, -); - -testCycleTileSize( - "expands a tile even in a crowded grid", - "c", - ` -abb -cbb -dde -ddf -ghi -klm`, - ` -abb -gbb -dde -ddf -ccm -cch -lik`, -); - -testCycleTileSize( - "does nothing if the tile has no room to expand", - "c", - ` -abb -cbb -dde -ddf`, - ` -abb -cbb -dde -ddf`, -); - -test("cycleTileSize is its own inverse", () => { - const input = ` -abc -def -ghi -jk`; - - const grid = mkGrid(input); - let gridAfter = grid; - - const toggle = (tileId: string): void => { - const tile = grid.cells.find((c) => c?.item.id === tileId)!.item; - gridAfter = cycleTileSize(gridAfter, tile); - }; - - // Toggle a series of tiles - toggle("j"); - toggle("h"); - toggle("a"); - // Now do the same thing in reverse - toggle("a"); - toggle("h"); - toggle("j"); - - // The grid should be back to its original state - expect(showGrid(gridAfter)).toBe(input); -}); - -function testAddItems( - title: string, - items: TileDescriptor[], - input: string, - output: string, -): void { - test(`addItems ${title}`, () => { - expect(showGrid(addItems(items, mkGrid(input) as SparseGrid) as Grid)).toBe( - output, - ); - }); -} - -testAddItems( - "appends 1×1 tiles", - ["e", "f"].map((i) => ({ id: i }) as unknown as TileDescriptor), - ` -aab -aac -d`, - ` -aab -aac -def`, -); - -testAddItems( - "places one tile near another on request", - [{ id: "g", placeNear: "b" } as unknown as TileDescriptor], - ` -abc -def`, - ` -abc - g -def`, -); - -testAddItems( - "places items with a large base size", - [{ id: "g", largeBaseSize: true } as unknown as TileDescriptor], - ` -abc -def`, - ` -abc -ggf -gge -d`, -); - -function testMoveTile( - title: string, - from: number, - to: number, - input: string, - output: string, -): void { - test(`moveTile ${title}`, () => { - expect(showGrid(moveTile(mkGrid(input), from, to))).toBe(output); - }); -} - -testMoveTile( - "refuses to move a tile too far to the left", - 1, - -1, - ` -abc`, - ` -abc`, -); - -testMoveTile( - "refuses to move a tile too far to the right", - 1, - 3, - ` -abc`, - ` -abc`, -); - -testMoveTile( - "moves a large tile to an unoccupied space", - 3, - 1, - ` -a b -ccd -cce`, - ` -acc -bcc -d e`, -); - -testMoveTile( - "refuses to move a large tile to an occupied space", - 3, - 1, - ` -abb -ccd -cce`, - ` -abb -ccd -cce`, -); - -function testResize( - title: string, - columns: number, - input: string, - output: string, -): void { - test(`resize ${title}`, () => { - expect(showGrid(resize(mkGrid(input), columns))).toBe(output); - }); -} - -testResize( - "contracts the grid", - 2, - ` -abbb -cbbb -ddde -dddf -gh`, - ` -af -bb -bb -dd -dd -ch -eg`, -); - -testResize( - "expands the grid", - 4, - ` -af -bb -bb -ch -dd -dd -eg`, - ` -afcd -bbbg -bbbe -h`, -); From 41083c0f9e95f15070fddb791393fc715a394612 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 May 2024 15:29:39 -0400 Subject: [PATCH 02/31] Refactor settings to use observables Also removing some unused settings along the way. --- package.json | 1 + public/locales/en-GB/app.json | 1 - src/analytics/PosthogAnalytics.ts | 31 +++------ src/home/RegisteredView.tsx | 7 +- src/home/UnauthenticatedView.tsx | 7 +- src/livekit/MediaDevicesContext.tsx | 42 +++++------ src/room/RoomPage.tsx | 7 +- src/settings/SettingsModal.tsx | 33 +++------ src/settings/settings.ts | 98 ++++++++++++++++++++++++++ src/settings/useSetting.ts | 104 ---------------------------- yarn.lock | 5 ++ 11 files changed, 157 insertions(+), 179 deletions(-) create mode 100644 src/settings/settings.ts delete mode 100644 src/settings/useSetting.ts diff --git a/package.json b/package.json index 7dfbc462..19ca2fa2 100644 --- a/package.json +++ b/package.json @@ -66,6 +66,7 @@ "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#238eea0ef5c82d0a11b8d5cc5c04104d6c94c4c1", "matrix-widget-api": "^1.3.1", "normalize.css": "^8.0.1", + "observable-hooks": "^4.2.3", "pako": "^2.0.4", "postcss-preset-env": "^9.0.0", "posthog-js": "^1.29.0", diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 7a804a5d..4279bbb5 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -140,7 +140,6 @@ "feedback_tab_title": "Feedback", "more_tab_title": "More", "opt_in_description": "<0><1>You may withdraw consent by unchecking this box. If you are currently in a call, this setting will take effect at the end of the call.", - "show_connection_stats_label": "Show connection stats", "speaker_device_selection_label": "Speaker" }, "star_rating_input_label_one": "{{count}} stars", diff --git a/src/analytics/PosthogAnalytics.ts b/src/analytics/PosthogAnalytics.ts index cfeb1e7a..5cbc552b 100644 --- a/src/analytics/PosthogAnalytics.ts +++ b/src/analytics/PosthogAnalytics.ts @@ -20,7 +20,6 @@ import { MatrixClient } from "matrix-js-sdk"; import { Buffer } from "buffer"; import { widget } from "../widget"; -import { getSetting, setSetting, getSettingKey } from "../settings/useSetting"; import { CallEndedTracker, CallStartedTracker, @@ -35,7 +34,7 @@ import { } from "./PosthogEvents"; import { Config } from "../config/Config"; import { getUrlParams } from "../UrlParams"; -import { localStorageBus } from "../useLocalStorage"; +import { optInAnalytics } from "../settings/settings"; /* Posthog analytics tracking. * @@ -131,7 +130,7 @@ export class PosthogAnalytics { const { analyticsID } = getUrlParams(); // if the embedding platform (element web) already got approval to communicating with posthog // element call can also send events to posthog - setSetting("opt-in-analytics", Boolean(analyticsID)); + optInAnalytics.setValue(Boolean(analyticsID)); } this.posthog.init(posthogConfig.project_api_key, { @@ -151,9 +150,7 @@ export class PosthogAnalytics { ); this.enabled = false; } - this.startListeningToSettingsChanges(); - const optInAnalytics = getSetting("opt-in-analytics", false); - this.updateAnonymityAndIdentifyUser(optInAnalytics); + this.startListeningToSettingsChanges(); // Triggers maybeIdentifyUser } private sanitizeProperties = ( @@ -336,8 +333,7 @@ export class PosthogAnalytics { } public onLoginStatusChanged(): void { - const optInAnalytics = getSetting("opt-in-analytics", false); - this.updateAnonymityAndIdentifyUser(optInAnalytics); + this.maybeIdentifyUser(); } private updateSuperProperties(): void { @@ -360,20 +356,12 @@ export class PosthogAnalytics { return this.eventSignup.getSignupEndTime() > new Date(0); } - private async updateAnonymityAndIdentifyUser( - pseudonymousOptIn: boolean, - ): Promise { - // Update this.anonymity based on the user's analytics opt-in settings - const anonymity = pseudonymousOptIn - ? Anonymity.Pseudonymous - : Anonymity.Disabled; - this.setAnonymity(anonymity); - + private async maybeIdentifyUser(): Promise { // We may not yet have a Matrix client at this point, if not, bail. This should get // triggered again by onLoginStatusChanged once we do have a client. if (!window.matrixclient) return; - if (anonymity === Anonymity.Pseudonymous) { + if (this.anonymity === Anonymity.Pseudonymous) { this.setRegistrationType( window.matrixclient.isGuest() || window.passwordlessUser ? RegistrationType.Guest @@ -389,7 +377,7 @@ export class PosthogAnalytics { } } - if (anonymity !== Anonymity.Disabled) { + if (this.anonymity !== Anonymity.Disabled) { this.updateSuperProperties(); } } @@ -419,8 +407,9 @@ export class PosthogAnalytics { // * When the user changes their preferences on this device // Note that for new accounts, pseudonymousAnalyticsOptIn won't be set, so updateAnonymityFromSettings // won't be called (i.e. this.anonymity will be left as the default, until the setting changes) - localStorageBus.on(getSettingKey("opt-in-analytics"), (optInAnalytics) => { - this.updateAnonymityAndIdentifyUser(optInAnalytics); + optInAnalytics.value.subscribe((optIn) => { + this.setAnonymity(optIn ? Anonymity.Pseudonymous : Anonymity.Disabled); + this.maybeIdentifyUser(); }); } diff --git a/src/home/RegisteredView.tsx b/src/home/RegisteredView.tsx index 35e958ab..1c1b0d3a 100644 --- a/src/home/RegisteredView.tsx +++ b/src/home/RegisteredView.tsx @@ -38,9 +38,12 @@ import { UserMenuContainer } from "../UserMenuContainer"; import { JoinExistingCallModal } from "./JoinExistingCallModal"; import { Caption } from "../typography/Typography"; import { Form } from "../form/Form"; -import { useOptInAnalytics } from "../settings/useSetting"; import { AnalyticsNotice } from "../analytics/AnalyticsNotice"; import { E2eeType } from "../e2ee/e2eeType"; +import { + useSetting, + optInAnalytics as optInAnalyticsSetting, +} from "../settings/settings"; interface Props { client: MatrixClient; @@ -49,7 +52,7 @@ interface Props { export const RegisteredView: FC = ({ client }) => { const [loading, setLoading] = useState(false); const [error, setError] = useState(); - const [optInAnalytics] = useOptInAnalytics(); + const [optInAnalytics] = useSetting(optInAnalyticsSetting); const history = useHistory(); const { t } = useTranslation(); const [joinExistingCallModalOpen, setJoinExistingCallModalOpen] = diff --git a/src/home/UnauthenticatedView.tsx b/src/home/UnauthenticatedView.tsx index d5f00fea..35cc832e 100644 --- a/src/home/UnauthenticatedView.tsx +++ b/src/home/UnauthenticatedView.tsx @@ -41,15 +41,18 @@ import styles from "./UnauthenticatedView.module.css"; import commonStyles from "./common.module.css"; import { generateRandomName } from "../auth/generateRandomName"; import { AnalyticsNotice } from "../analytics/AnalyticsNotice"; -import { useOptInAnalytics } from "../settings/useSetting"; import { Config } from "../config/Config"; import { E2eeType } from "../e2ee/e2eeType"; +import { + useSetting, + optInAnalytics as optInAnalyticsSetting, +} from "../settings/settings"; export const UnauthenticatedView: FC = () => { const { setClient } = useClient(); const [loading, setLoading] = useState(false); const [error, setError] = useState(); - const [optInAnalytics] = useOptInAnalytics(); + const [optInAnalytics] = useSetting(optInAnalyticsSetting); const { recaptchaKey, register } = useInteractiveRegistration(); const { execute, reset, recaptchaId } = useRecaptcha(recaptchaKey); diff --git a/src/livekit/MediaDevicesContext.tsx b/src/livekit/MediaDevicesContext.tsx index b42a0afa..31d98a54 100644 --- a/src/livekit/MediaDevicesContext.tsx +++ b/src/livekit/MediaDevicesContext.tsx @@ -29,11 +29,12 @@ import { Observable } from "rxjs"; import { logger } from "matrix-js-sdk/src/logger"; import { + useSetting, + audioInput as audioInputSetting, + audioOutput as audioOutputSetting, + videoInput as videoInputSetting, isFirefox, - useAudioInput, - useAudioOutput, - useVideoInput, -} from "../settings/useSetting"; +} from "../settings/settings"; export interface MediaDevice { available: MediaDeviceInfo[]; @@ -145,43 +146,36 @@ export const MediaDevicesProvider: FC = ({ children }) => { // for ouput devices because the selector wont be shown on FF. const useOutputNames = usingNames && !isFirefox(); - const [audioInputSetting, setAudioInputSetting] = useAudioInput(); - const [audioOutputSetting, setAudioOutputSetting] = useAudioOutput(); - const [videoInputSetting, setVideoInputSetting] = useVideoInput(); + const [storedAudioInput, setStoredAudioInput] = useSetting(audioInputSetting); + const [storedAudioOutput, setStoredAudioOutput] = + useSetting(audioOutputSetting); + const [storedVideoInput, setStoredVideoInput] = useSetting(videoInputSetting); - const audioInput = useMediaDevice( - "audioinput", - audioInputSetting, - usingNames, - ); + const audioInput = useMediaDevice("audioinput", storedAudioInput, usingNames); const audioOutput = useMediaDevice( "audiooutput", - audioOutputSetting, + storedAudioOutput, useOutputNames, alwaysUseDefaultAudio, ); - const videoInput = useMediaDevice( - "videoinput", - videoInputSetting, - usingNames, - ); + const videoInput = useMediaDevice("videoinput", storedVideoInput, usingNames); useEffect(() => { if (audioInput.selectedId !== undefined) - setAudioInputSetting(audioInput.selectedId); - }, [setAudioInputSetting, audioInput.selectedId]); + setStoredAudioInput(audioInput.selectedId); + }, [setStoredAudioInput, audioInput.selectedId]); useEffect(() => { // Skip setting state for ff output. Redundent since it is set to always return 'undefined' // but makes it clear while debugging that this is not happening on FF. + perf ;) if (audioOutput.selectedId !== undefined && !isFirefox()) - setAudioOutputSetting(audioOutput.selectedId); - }, [setAudioOutputSetting, audioOutput.selectedId]); + setStoredAudioOutput(audioOutput.selectedId); + }, [setStoredAudioOutput, audioOutput.selectedId]); useEffect(() => { if (videoInput.selectedId !== undefined) - setVideoInputSetting(videoInput.selectedId); - }, [setVideoInputSetting, videoInput.selectedId]); + setStoredVideoInput(videoInput.selectedId); + }, [setStoredVideoInput, videoInput.selectedId]); const startUsingDeviceNames = useCallback( () => setNumCallersUsingNames((n) => n + 1), diff --git a/src/room/RoomPage.tsx b/src/room/RoomPage.tsx index 2f18cb63..1a14fb7e 100644 --- a/src/room/RoomPage.tsx +++ b/src/room/RoomPage.tsx @@ -26,7 +26,6 @@ import { GroupCallLoader } from "./GroupCallLoader"; import { GroupCallView } from "./GroupCallView"; import { useRoomIdentifier, useUrlParams } from "../UrlParams"; import { useRegisterPasswordlessUser } from "../auth/useRegisterPasswordlessUser"; -import { useOptInAnalytics } from "../settings/useSetting"; import { HomePage } from "../home/HomePage"; import { platform } from "../Platform"; import { AppSelectionModal } from "./AppSelectionModal"; @@ -36,6 +35,10 @@ import { LobbyView } from "./LobbyView"; import { E2eeType } from "../e2ee/e2eeType"; import { useProfile } from "../profile/useProfile"; import { useMuteStates } from "./MuteStates"; +import { + useSetting, + optInAnalytics as optInAnalyticsSetting, +} from "../settings/settings"; export const RoomPage: FC = () => { const { @@ -80,7 +83,7 @@ export const RoomPage: FC = () => { registerPasswordlessUser, ]); - const [optInAnalytics, setOptInAnalytics] = useOptInAnalytics(); + const [optInAnalytics, setOptInAnalytics] = useSetting(optInAnalyticsSetting); useEffect(() => { // During the beta, opt into analytics by default if (optInAnalytics === null && setOptInAnalytics) setOptInAnalytics(true); diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index a86f20c5..05784d15 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -29,12 +29,6 @@ import OverflowIcon from "../icons/Overflow.svg?react"; import UserIcon from "../icons/User.svg?react"; import FeedbackIcon from "../icons/Feedback.svg?react"; import { SelectInput } from "../input/SelectInput"; -import { - useOptInAnalytics, - useDeveloperSettingsTab, - useShowConnectionStats, - isFirefox, -} from "./useSetting"; import { FieldRow, InputField } from "../input/Input"; import { Body, Caption } from "../typography/Typography"; import { AnalyticsNotice } from "../analytics/AnalyticsNotice"; @@ -46,6 +40,12 @@ import { useMediaDeviceNames, } from "../livekit/MediaDevicesContext"; import { widget } from "../widget"; +import { + useSetting, + optInAnalytics as optInAnalyticsSetting, + developerSettingsTab as developerSettingsTabSetting, + isFirefox, +} from "./settings"; type SettingsTab = | "audio" @@ -76,11 +76,10 @@ export const SettingsModal: FC = ({ }) => { const { t } = useTranslation(); - const [optInAnalytics, setOptInAnalytics] = useOptInAnalytics(); - const [developerSettingsTab, setDeveloperSettingsTab] = - useDeveloperSettingsTab(); - const [showConnectionStats, setShowConnectionStats] = - useShowConnectionStats(); + const [optInAnalytics, setOptInAnalytics] = useSetting(optInAnalyticsSetting); + const [developerSettingsTab, setDeveloperSettingsTab] = useSetting( + developerSettingsTabSetting, + ); // Generate a `SelectInput` with a list of devices for a given device kind. const generateDeviceSelection = ( @@ -245,18 +244,6 @@ export const SettingsModal: FC = ({ })} - - ): void => - setShowConnectionStats(e.target.checked) - } - /> - ); diff --git a/src/settings/settings.ts b/src/settings/settings.ts new file mode 100644 index 00000000..73242b56 --- /dev/null +++ b/src/settings/settings.ts @@ -0,0 +1,98 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { logger } from "matrix-js-sdk/src/logger"; +import { BehaviorSubject, Observable } from "rxjs"; +import { useObservableEagerState } from "observable-hooks"; + +import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; + +export class Setting { + public constructor(key: string, defaultValue: T) { + this.key = `matrix-setting-${key}`; + + const storedValue = localStorage.getItem(this.key); + let initialValue = defaultValue; + if (storedValue !== null) { + try { + initialValue = JSON.parse(storedValue); + } catch (e) { + logger.warn(`Invalid value stored for setting ${key}: ${storedValue}`); + } + } + + this._value = new BehaviorSubject(initialValue); + this.value = this._value; + } + + private readonly key: string; + + private readonly _value: BehaviorSubject; + public readonly value: Observable; + + public readonly setValue = (value: T): void => { + this._value.next(value); + localStorage.setItem(this.key, JSON.stringify(value)); + }; +} + +/** + * React hook that returns a settings's current value and a setter. + */ +export function useSetting(setting: Setting): [T, (value: T) => void] { + return [useObservableEagerState(setting.value), setting.setValue]; +} + +// TODO: This doesn't belong here +export const isFirefox = (): boolean => { + const { userAgent } = navigator; + return userAgent.includes("Firefox"); +}; + +// null = undecided +export const optInAnalytics = new Setting( + "opt-in-analytics", + null, +); +// TODO: This setting can be disabled. Work out an approach to disableable +// settings thats works for Observables in addition to React. +export const useOptInAnalytics = (): [ + boolean | null, + ((value: boolean | null) => void) | null, +] => { + const setting = useSetting(optInAnalytics); + if (PosthogAnalytics.instance.isEnabled()) return setting; + + return [false, null]; +}; + +export const developerSettingsTab = new Setting( + "developer-settings-tab", + false, +); + +export const audioInput = new Setting( + "audio-input", + undefined, +); +export const audioOutput = new Setting( + "audio-output", + undefined, +); +export const videoInput = new Setting( + "video-input", + undefined, +); diff --git a/src/settings/useSetting.ts b/src/settings/useSetting.ts deleted file mode 100644 index a2733b98..00000000 --- a/src/settings/useSetting.ts +++ /dev/null @@ -1,104 +0,0 @@ -/* -Copyright 2022 - 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { useCallback, useMemo } from "react"; - -import { PosthogAnalytics } from "../analytics/PosthogAnalytics"; -import { - getLocalStorageItem, - setLocalStorageItem, - useLocalStorage, -} from "../useLocalStorage"; - -type Setting = [T, (value: T) => void]; -type DisableableSetting = [T, ((value: T) => void) | null]; - -export const getSettingKey = (name: string): string => { - return `matrix-setting-${name}`; -}; -// Like useState, but reads from and persists the value to localStorage -export const useSetting = (name: string, defaultValue: T): Setting => { - const key = useMemo(() => getSettingKey(name), [name]); - - const [item, setItem] = useLocalStorage(key); - - const value = useMemo( - () => (item == null ? defaultValue : JSON.parse(item)), - [item, defaultValue], - ); - const setValue = useCallback( - (value: T) => { - setItem(JSON.stringify(value)); - }, - [setItem], - ); - - return [value, setValue]; -}; - -export const getSetting = (name: string, defaultValue: T): T => { - const item = getLocalStorageItem(getSettingKey(name)); - return item === null ? defaultValue : JSON.parse(item); -}; - -export const setSetting = (name: string, newValue: T): void => - setLocalStorageItem(getSettingKey(name), JSON.stringify(newValue)); - -export const isFirefox = (): boolean => { - const { userAgent } = navigator; - return userAgent.includes("Firefox"); -}; - -const canEnableSpatialAudio = (): boolean => { - // Spatial audio means routing audio through audio contexts. On Chrome, - // this bypasses the AEC processor and so breaks echo cancellation. - // We only allow spatial audio to be enabled on Firefox which we know - // passes audio context audio through the AEC algorithm. - // https://bugs.chromium.org/p/chromium/issues/detail?id=687574 is the - // chrome bug for this: once this is fixed and the updated version is deployed - // widely enough, we can allow spatial audio everywhere. It's currently in a - // chrome flag, so we could enable this in Electron if we enabled the chrome flag - // in the Electron wrapper. - return isFirefox(); -}; - -export const useSpatialAudio = (): DisableableSetting => { - const settingVal = useSetting("spatial-audio", false); - if (canEnableSpatialAudio()) return settingVal; - - return [false, null]; -}; - -// null = undecided -export const useOptInAnalytics = (): DisableableSetting => { - const settingVal = useSetting("opt-in-analytics", null); - if (PosthogAnalytics.instance.isEnabled()) return settingVal; - - return [false, null]; -}; - -export const useDeveloperSettingsTab = (): Setting => - useSetting("developer-settings-tab", false); - -export const useShowConnectionStats = (): Setting => - useSetting("show-connection-stats", false); - -export const useAudioInput = (): Setting => - useSetting("audio-input", undefined); -export const useAudioOutput = (): Setting => - useSetting("audio-output", undefined); -export const useVideoInput = (): Setting => - useSetting("video-input", undefined); diff --git a/yarn.lock b/yarn.lock index c02b3bb2..92638203 100644 --- a/yarn.lock +++ b/yarn.lock @@ -7104,6 +7104,11 @@ object.values@^1.1.7: define-properties "^1.2.0" es-abstract "^1.22.1" +observable-hooks@^4.2.3: + version "4.2.3" + resolved "https://registry.yarnpkg.com/observable-hooks/-/observable-hooks-4.2.3.tgz#69e3353caafd7887ad9030bd440b053304e8d2d1" + integrity sha512-d6fYTIU+9sg1V+CT0GhgoE/ntjIqcy9DGaYGE6ELGVP4ojaWIEsaLvL/05hLOM+AL7aySN4DCTLvj6dDF9T8XA== + oidc-client-ts@^3.0.1: version "3.0.1" resolved "https://registry.yarnpkg.com/oidc-client-ts/-/oidc-client-ts-3.0.1.tgz#be264fb87c89f74f73863646431c32cd06f5ceb7" From 14fc1481f37579758f8a913005e0cb0c290bfb0b Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 12 Jul 2024 14:01:32 -0400 Subject: [PATCH 03/31] Address some review feedback --- src/Header.module.css | 2 +- src/Header.tsx | 2 +- src/grid/LegacyGrid.module.css | 2 +- src/grid/LegacyGrid.tsx | 2 +- src/grid/TileWrapper.tsx | 2 +- src/room/InCallView.tsx | 4 ++-- src/room/VideoPreview.tsx | 2 +- src/useReactiveState.ts | 3 ++- test/grid/LegacyGrid-test.ts | 2 +- 9 files changed, 11 insertions(+), 10 deletions(-) diff --git a/src/Header.module.css b/src/Header.module.css index 6aa609f7..4e54009d 100644 --- a/src/Header.module.css +++ b/src/Header.module.css @@ -1,5 +1,5 @@ /* -Copyright 2022 New Vector Ltd +Copyright 2022-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/Header.tsx b/src/Header.tsx index ffb4731e..e0fb9297 100644 --- a/src/Header.tsx +++ b/src/Header.tsx @@ -1,5 +1,5 @@ /* -Copyright 2022 New Vector Ltd +Copyright 2022-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/grid/LegacyGrid.module.css b/src/grid/LegacyGrid.module.css index 6e59e66e..cad3e3c4 100644 --- a/src/grid/LegacyGrid.module.css +++ b/src/grid/LegacyGrid.module.css @@ -1,5 +1,5 @@ /* -Copyright 2022 New Vector Ltd +Copyright 2022-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/grid/LegacyGrid.tsx b/src/grid/LegacyGrid.tsx index 9e1fb876..f04cde78 100644 --- a/src/grid/LegacyGrid.tsx +++ b/src/grid/LegacyGrid.tsx @@ -1,5 +1,5 @@ /* -Copyright 2022-2023 New Vector Ltd +Copyright 2022-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/grid/TileWrapper.tsx b/src/grid/TileWrapper.tsx index ded2be28..dcb8e908 100644 --- a/src/grid/TileWrapper.tsx +++ b/src/grid/TileWrapper.tsx @@ -1,5 +1,5 @@ /* -Copyright 2023 New Vector Ltd +Copyright 2023-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 4407de38..1ccdb2ec 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -1,5 +1,5 @@ /* -Copyright 2022 - 2023 New Vector Ltd +Copyright 2022 - 2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -266,7 +266,7 @@ export const InCallView: FC = subscribe( // useFullscreen so that we can control the fullscreen state of the // spotlight tile in the new layouts with this same hook. const fullscreenItems = useMemo( - () => [...items, ...(hasSpotlight ? [dummySpotlightItem] : [])], + () => (hasSpotlight ? [...items, dummySpotlightItem] : items), [items, hasSpotlight], ); const { fullscreenItem, toggleFullscreen, exitFullscreen } = diff --git a/src/room/VideoPreview.tsx b/src/room/VideoPreview.tsx index dd98421e..3be88f1f 100644 --- a/src/room/VideoPreview.tsx +++ b/src/room/VideoPreview.tsx @@ -1,5 +1,5 @@ /* -Copyright 2022 - 2023 New Vector Ltd +Copyright 2022 - 2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/src/useReactiveState.ts b/src/useReactiveState.ts index af18e84b..afd509fb 100644 --- a/src/useReactiveState.ts +++ b/src/useReactiveState.ts @@ -1,5 +1,5 @@ /* -Copyright 2023 New Vector Ltd +Copyright 2023-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -44,6 +44,7 @@ export const useReactiveState = ( if ( prevDeps.current === undefined || deps.length !== prevDeps.current.length || + // Deps might be NaN, so we compare with Object.is rather than === deps.some((d, i) => !Object.is(d, prevDeps.current![i])) ) { state.current = updateFn(state.current); diff --git a/test/grid/LegacyGrid-test.ts b/test/grid/LegacyGrid-test.ts index 44f82d42..e57adf9d 100644 --- a/test/grid/LegacyGrid-test.ts +++ b/test/grid/LegacyGrid-test.ts @@ -1,5 +1,5 @@ /* -Copyright 2023 New Vector Ltd +Copyright 2023-2024 New Vector Ltd Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. From 599d6fd0074a9fcd84c83707b7bf2d63bfa3b50d Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 12 Jul 2024 14:15:27 -0400 Subject: [PATCH 04/31] Address review feedback --- src/Platform.ts | 5 +++++ src/livekit/MediaDevicesContext.tsx | 2 +- src/settings/SettingsModal.tsx | 2 +- src/settings/settings.ts | 10 +--------- 4 files changed, 8 insertions(+), 11 deletions(-) diff --git a/src/Platform.ts b/src/Platform.ts index 0e5b71f1..86f1bc57 100644 --- a/src/Platform.ts +++ b/src/Platform.ts @@ -36,3 +36,8 @@ if (/android/i.test(navigator.userAgent)) { } else { platform = "desktop"; } + +export const isFirefox = (): boolean => { + const { userAgent } = navigator; + return userAgent.includes("Firefox"); +}; diff --git a/src/livekit/MediaDevicesContext.tsx b/src/livekit/MediaDevicesContext.tsx index 31d98a54..2f654946 100644 --- a/src/livekit/MediaDevicesContext.tsx +++ b/src/livekit/MediaDevicesContext.tsx @@ -33,8 +33,8 @@ import { audioInput as audioInputSetting, audioOutput as audioOutputSetting, videoInput as videoInputSetting, - isFirefox, } from "../settings/settings"; +import { isFirefox } from "../Platform"; export interface MediaDevice { available: MediaDeviceInfo[]; diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index 05784d15..fd0b6295 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -44,8 +44,8 @@ import { useSetting, optInAnalytics as optInAnalyticsSetting, developerSettingsTab as developerSettingsTabSetting, - isFirefox, } from "./settings"; +import { isFirefox } from "../Platform"; type SettingsTab = | "audio" diff --git a/src/settings/settings.ts b/src/settings/settings.ts index 73242b56..307a557e 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -56,12 +56,6 @@ export function useSetting(setting: Setting): [T, (value: T) => void] { return [useObservableEagerState(setting.value), setting.setValue]; } -// TODO: This doesn't belong here -export const isFirefox = (): boolean => { - const { userAgent } = navigator; - return userAgent.includes("Firefox"); -}; - // null = undecided export const optInAnalytics = new Setting( "opt-in-analytics", @@ -74,9 +68,7 @@ export const useOptInAnalytics = (): [ ((value: boolean | null) => void) | null, ] => { const setting = useSetting(optInAnalytics); - if (PosthogAnalytics.instance.isEnabled()) return setting; - - return [false, null]; + return PosthogAnalytics.instance.isEnabled() ? setting : [false, null]; }; export const developerSettingsTab = new Setting( From fdc6d4a1b6245f12c59118167f025f2f63b526d1 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 8 May 2024 16:00:42 -0400 Subject: [PATCH 05/31] Add a developer option to duplicate tiles This is useful for testing how the UI behaves with different numbers of participants. --- public/locales/en-GB/app.json | 1 + src/settings/SettingsModal.tsx | 18 ++++++++++++++- src/settings/settings.ts | 2 ++ src/state/CallViewModel.ts | 41 +++++++++++++++++++++++----------- 4 files changed, 48 insertions(+), 14 deletions(-) diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 4279bbb5..97f2be93 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -132,6 +132,7 @@ "developer_settings_label": "Developer Settings", "developer_settings_label_description": "Expose developer settings in the settings window.", "developer_tab_title": "Developer", + "duplicate_tiles_label": "Number of duplicate tiles", "feedback_tab_body": "If you are experiencing issues or simply would like to provide some feedback, please send us a short description below.", "feedback_tab_description_label": "Your feedback", "feedback_tab_h4": "Submit feedback", diff --git a/src/settings/SettingsModal.tsx b/src/settings/SettingsModal.tsx index fd0b6295..d8430022 100644 --- a/src/settings/SettingsModal.tsx +++ b/src/settings/SettingsModal.tsx @@ -14,7 +14,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { ChangeEvent, FC, Key, ReactNode } from "react"; +import { ChangeEvent, FC, Key, ReactNode, useCallback } from "react"; import { Item } from "@react-stately/collections"; import { Trans, useTranslation } from "react-i18next"; import { MatrixClient } from "matrix-js-sdk"; @@ -44,6 +44,7 @@ import { useSetting, optInAnalytics as optInAnalyticsSetting, developerSettingsTab as developerSettingsTabSetting, + duplicateTiles as duplicateTilesSetting, } from "./settings"; import { isFirefox } from "../Platform"; @@ -80,6 +81,7 @@ export const SettingsModal: FC = ({ const [developerSettingsTab, setDeveloperSettingsTab] = useSetting( developerSettingsTabSetting, ); + const [duplicateTiles, setDuplicateTiles] = useSetting(duplicateTilesSetting); // Generate a `SelectInput` with a list of devices for a given device kind. const generateDeviceSelection = ( @@ -244,6 +246,20 @@ export const SettingsModal: FC = ({ })} + + ): void => { + setDuplicateTiles(event.target.valueAsNumber); + }, + [setDuplicateTiles], + )} + /> + ); diff --git a/src/settings/settings.ts b/src/settings/settings.ts index 307a557e..5f2cc529 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -76,6 +76,8 @@ export const developerSettingsTab = new Setting( false, ); +export const duplicateTiles = new Setting("duplicate-tiles", 0); + export const audioInput = new Setting( "audio-input", undefined, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index e6d55b72..59f0f5dc 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -67,6 +67,7 @@ import { } from "./MediaViewModel"; import { finalizeValue } from "../observable-utils"; import { ObservableScope } from "./ObservableScope"; +import { duplicateTiles } from "../settings/settings"; // How long we wait after a focus switch before showing the real participant // list again @@ -308,11 +309,16 @@ export class CallViewModel extends ViewModel { combineLatest([ this.remoteParticipants, observeParticipantMedia(this.livekitRoom.localParticipant), + duplicateTiles.value, ]).pipe( scan( ( prevItems, - [remoteParticipants, { participant: localParticipant }], + [ + remoteParticipants, + { participant: localParticipant }, + duplicateTiles, + ], ) => { let allGhosts = true; @@ -330,20 +336,29 @@ export class CallViewModel extends ViewModel { ); } - const userMediaId = p.identity; - yield [ - userMediaId, - prevItems.get(userMediaId) ?? - new UserMedia(userMediaId, member, p, this.encrypted), - ]; - - if (p.isScreenShareEnabled) { - const screenShareId = `${userMediaId}:screen-share`; + // Create as many tiles for this participant as called for by + // the duplicateTiles option + for (let i = 0; i < 1 + duplicateTiles; i++) { + const userMediaId = `${p.identity}:${i}`; yield [ - screenShareId, - prevItems.get(screenShareId) ?? - new ScreenShare(screenShareId, member, p, this.encrypted), + userMediaId, + prevItems.get(userMediaId) ?? + new UserMedia(userMediaId, member, p, this.encrypted), ]; + + if (p.isScreenShareEnabled) { + const screenShareId = `${userMediaId}:screen-share`; + yield [ + screenShareId, + prevItems.get(screenShareId) ?? + new ScreenShare( + screenShareId, + member, + p, + this.encrypted, + ), + ]; + } } } }.bind(this)(), From e33fbd77d14daeec4b506da4a2d68f97409e6128 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 16 May 2024 12:32:18 -0400 Subject: [PATCH 06/31] Split local and remote user media into different classes --- src/room/InCallView.tsx | 4 +- src/state/CallViewModel.ts | 23 ++- src/state/MediaViewModel.ts | 122 ++++++++----- src/tile/GridTile.tsx | 332 ++++++++++++++++++++---------------- src/tile/SpotlightTile.tsx | 21 ++- 5 files changed, 292 insertions(+), 210 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 1ccdb2ec..f2e08e5c 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -384,7 +384,7 @@ export const InCallView: FC = subscribe( targetHeight={targetHeight} className={className} style={style} - showSpeakingIndicator={showSpeakingIndicators} + showSpeakingIndicators={showSpeakingIndicators} /> ); }, @@ -424,7 +424,7 @@ export const InCallView: FC = subscribe( targetHeight={gridBounds.height} targetWidth={gridBounds.width} key={maximisedParticipant.id} - showSpeakingIndicator={false} + showSpeakingIndicators={false} onOpenProfile={openProfile} /> ); diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index e6d55b72..df3ca42a 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -61,9 +61,11 @@ import { } from "../livekit/useECConnectionState"; import { usePrevious } from "../usePrevious"; import { + LocalUserMediaViewModel, MediaViewModel, - UserMediaViewModel, + RemoteUserMediaViewModel, ScreenShareViewModel, + UserMediaViewModel, } from "./MediaViewModel"; import { finalizeValue } from "../observable-utils"; import { ObservableScope } from "./ObservableScope"; @@ -151,7 +153,10 @@ class UserMedia { participant: LocalParticipant | RemoteParticipant, callEncrypted: boolean, ) { - this.vm = new UserMediaViewModel(id, member, participant, callEncrypted); + this.vm = + participant instanceof LocalParticipant + ? new LocalUserMediaViewModel(id, member, participant, callEncrypted) + : new RemoteUserMediaViewModel(id, member, participant, callEncrypted); this.speaker = this.vm.speaking.pipeState( // Require 1 s of continuous speaking to become a speaker, and 60 s of @@ -520,7 +525,19 @@ export class CallViewModel extends ViewModel { const userMediaVm = tilesById.get(userMediaId)?.data ?? - new UserMediaViewModel(userMediaId, member, p, this.encrypted); + (p instanceof LocalParticipant + ? new LocalUserMediaViewModel( + userMediaId, + member, + p, + this.encrypted, + ) + : new RemoteUserMediaViewModel( + userMediaId, + member, + p, + this.encrypted, + )); tilesById.delete(userMediaId); const userMediaTile: TileDescriptor = { diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index f1e772da..7f7307f4 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -153,29 +153,14 @@ abstract class BaseMediaViewModel extends ViewModel { * Some participant's media. */ export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel; +export type UserMediaViewModel = + | LocalUserMediaViewModel + | RemoteUserMediaViewModel; /** * Some participant's user media. */ -export class UserMediaViewModel extends BaseMediaViewModel { - /** - * Whether the video should be mirrored. - */ - public readonly mirror = state( - this.video.pipe( - switchMap((v) => { - const track = v.publication?.track; - if (!(track instanceof LocalTrack)) return of(false); - // Watch for track restarts, because they indicate a camera switch - return fromEvent(track, TrackEvent.Restarted).pipe( - startWith(null), - // Mirror only front-facing cameras (those that face the user) - map(() => facingModeFromLocalTrack(track).facingMode === "user"), - ); - }), - ), - ); - +abstract class BaseUserMediaViewModel extends BaseMediaViewModel { /** * Whether the participant is speaking. */ @@ -186,19 +171,6 @@ export class UserMediaViewModel extends BaseMediaViewModel { ).pipe(map((p) => p.isSpeaking)), ); - private readonly _locallyMuted = new BehaviorSubject(false); - /** - * Whether we've disabled this participant's audio. - */ - public readonly locallyMuted = state(this._locallyMuted); - - private readonly _localVolume = new BehaviorSubject(1); - /** - * The volume to which we've set this participant's audio, as a scalar - * multiplier. - */ - public readonly localVolume = state(this._localVolume); - /** * Whether this participant is sending audio (i.e. is unmuted on their side). */ @@ -236,25 +208,83 @@ export class UserMediaViewModel extends BaseMediaViewModel { this.videoEnabled = state( media.pipe(map((m) => m.cameraTrack?.isMuted === false)), ); - - // Sync the local mute state and volume with LiveKit - if (!this.local) - combineLatest([this._locallyMuted, this._localVolume], (muted, volume) => - muted ? 0 : volume, - ) - .pipe(this.scope.bind()) - .subscribe((volume) => { - (this.participant as RemoteParticipant).setVolume(volume); - }); - } - - public toggleLocallyMuted(): void { - this._locallyMuted.next(!this._locallyMuted.value); } public toggleFitContain(): void { this._cropVideo.next(!this._cropVideo.value); } +} + +/** + * The local participant's user media. + */ +export class LocalUserMediaViewModel extends BaseUserMediaViewModel { + /** + * Whether the video should be mirrored. + */ + public readonly mirror = state( + this.video.pipe( + switchMap((v) => { + const track = v.publication?.track; + if (!(track instanceof LocalTrack)) return of(false); + // Watch for track restarts, because they indicate a camera switch + return fromEvent(track, TrackEvent.Restarted).pipe( + startWith(null), + // Mirror only front-facing cameras (those that face the user) + map(() => facingModeFromLocalTrack(track).facingMode === "user"), + ); + }), + ), + ); + + public constructor( + id: string, + member: RoomMember | undefined, + participant: LocalParticipant, + callEncrypted: boolean, + ) { + super(id, member, participant, callEncrypted); + } +} + +/** + * A remote participant's user media. + */ +export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { + private readonly _locallyMuted = new BehaviorSubject(false); + /** + * Whether we've disabled this participant's audio. + */ + public readonly locallyMuted = state(this._locallyMuted); + + private readonly _localVolume = new BehaviorSubject(1); + /** + * The volume to which we've set this participant's audio, as a scalar + * multiplier. + */ + public readonly localVolume = state(this._localVolume); + + public constructor( + id: string, + member: RoomMember | undefined, + participant: RemoteParticipant, + callEncrypted: boolean, + ) { + super(id, member, participant, callEncrypted); + + // Sync the local mute state and volume with LiveKit + combineLatest([this._locallyMuted, this._localVolume], (muted, volume) => + muted ? 0 : volume, + ) + .pipe(this.scope.bind()) + .subscribe((volume) => { + (this.participant as RemoteParticipant).setVolume(volume); + }); + } + + public toggleLocallyMuted(): void { + this._locallyMuted.next(!this._locallyMuted.value); + } public setLocalVolume(value: number): void { this._localVolume.next(value); diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index d88b189f..b0753aca 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -14,7 +14,13 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { ComponentProps, forwardRef, useCallback, useState } from "react"; +import { + ComponentProps, + ReactNode, + forwardRef, + useCallback, + useState, +} from "react"; import { animated } from "@react-spring/web"; import classNames from "classnames"; import { useTranslation } from "react-i18next"; @@ -33,7 +39,7 @@ import { ToggleMenuItem, Menu, } from "@vector-im/compound-web"; -import { useStateObservable } from "@react-rxjs/core"; +import { useObservableEagerState } from "observable-hooks"; import styles from "./GridTile.module.css"; import { @@ -41,71 +47,92 @@ import { MediaViewModel, UserMediaViewModel, useNameData, + LocalUserMediaViewModel, + RemoteUserMediaViewModel, } from "../state/MediaViewModel"; -import { subscribe } from "../state/subscribe"; import { Slider } from "../Slider"; import { MediaView } from "./MediaView"; -interface UserMediaTileProps { - vm: UserMediaViewModel; +interface TileProps { className?: string; style?: ComponentProps["style"]; targetWidth: number; targetHeight: number; maximised: boolean; - onOpenProfile: () => void; - showSpeakingIndicator: boolean; + displayName: string; + nameTag: string; } -const UserMediaTile = subscribe( +interface MediaTileProps + extends TileProps, + Omit, "className"> { + vm: MediaViewModel; + videoEnabled: boolean; + videoFit: "contain" | "cover"; + nameTagLeadingIcon?: ReactNode; + primaryButton: ReactNode; + secondaryButton?: ReactNode; +} + +const MediaTile = forwardRef( + ({ vm, className, maximised, ...props }, ref) => { + const video = useObservableEagerState(vm.video); + const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); + + return ( + + ); + }, +); + +MediaTile.displayName = "MediaTile"; + +interface UserMediaTileProps extends TileProps { + vm: UserMediaViewModel; + showSpeakingIndicators: boolean; + menuStart?: ReactNode; + menuEnd?: ReactNode; +} + +const UserMediaTile = forwardRef( ( { vm, + showSpeakingIndicators, + menuStart, + menuEnd, className, - style, - targetWidth, - targetHeight, - maximised, - onOpenProfile, - showSpeakingIndicator, + nameTag, + ...props }, ref, ) => { const { t } = useTranslation(); - const { displayName, nameTag } = useNameData(vm); - const video = useStateObservable(vm.video); - const audioEnabled = useStateObservable(vm.audioEnabled); - const videoEnabled = useStateObservable(vm.videoEnabled); - const unencryptedWarning = useStateObservable(vm.unencryptedWarning); - const mirror = useStateObservable(vm.mirror); - const speaking = useStateObservable(vm.speaking); - const locallyMuted = useStateObservable(vm.locallyMuted); - const cropVideo = useStateObservable(vm.cropVideo); - const localVolume = useStateObservable(vm.localVolume); - const onChangeMute = useCallback(() => vm.toggleLocallyMuted(), [vm]); + const audioEnabled = useObservableEagerState(vm.audioEnabled); + const videoEnabled = useObservableEagerState(vm.videoEnabled); + const speaking = useObservableEagerState(vm.speaking); + const cropVideo = useObservableEagerState(vm.cropVideo); const onChangeFitContain = useCallback(() => vm.toggleFitContain(), [vm]); - const onSelectMute = useCallback((e: Event) => e.preventDefault(), []); const onSelectFitContain = useCallback( (e: Event) => e.preventDefault(), [], ); - const onChangeLocalVolume = useCallback( - (v: number) => vm.setLocalVolume(v), - [vm], - ); - const MicIcon = audioEnabled ? MicOnSolidIcon : MicOffSolidIcon; - const VolumeIcon = locallyMuted ? VolumeOffIcon : VolumeOnIcon; const [menuOpen, setMenuOpen] = useState(false); - const menu = vm.local ? ( + const menu = ( <> - + {menuStart} ( onChange={onChangeFitContain} onSelect={onSelectFitContain} /> - - ) : ( - <> - - - {/* TODO: Figure out how to make this slider keyboard accessible */} - - - + {menuEnd} ); const tile = ( - ( /> } nameTag={nameTag} - displayName={displayName} primaryButton={ ( {menu} } + {...props} /> ); @@ -202,35 +193,102 @@ const UserMediaTile = subscribe( UserMediaTile.displayName = "UserMediaTile"; -interface ScreenShareTileProps { +interface LocalUserMediaTileProps extends TileProps { + vm: LocalUserMediaViewModel; + onOpenProfile: () => void; + showSpeakingIndicators: boolean; +} + +const LocalUserMediaTile = forwardRef( + ({ vm, onOpenProfile, className, ...props }, ref) => { + const { t } = useTranslation(); + const mirror = useObservableEagerState(vm.mirror); + + return ( + + } + className={classNames(className, { [styles.mirror]: mirror })} + {...props} + /> + ); + }, +); + +LocalUserMediaTile.displayName = "LocalUserMediaTile"; + +interface RemoteUserMediaTileProps extends TileProps { + vm: RemoteUserMediaViewModel; + showSpeakingIndicators: boolean; +} + +const RemoteUserMediaTile = forwardRef< + HTMLDivElement, + RemoteUserMediaTileProps +>(({ vm, ...props }, ref) => { + const { t } = useTranslation(); + const locallyMuted = useObservableEagerState(vm.locallyMuted); + const localVolume = useObservableEagerState(vm.localVolume); + const onChangeMute = useCallback(() => vm.toggleLocallyMuted(), [vm]); + const onSelectMute = useCallback((e: Event) => e.preventDefault(), []); + const onChangeLocalVolume = useCallback( + (v: number) => vm.setLocalVolume(v), + [vm], + ); + + const VolumeIcon = locallyMuted ? VolumeOffIcon : VolumeOnIcon; + + return ( + + + {/* TODO: Figure out how to make this slider keyboard accessible */} + + + + + } + {...props} + /> + ); +}); + +RemoteUserMediaTile.displayName = "RemoteUserMediaTile"; + +interface ScreenShareTileProps extends TileProps { vm: ScreenShareViewModel; - className?: string; - style?: ComponentProps["style"]; - targetWidth: number; - targetHeight: number; - maximised: boolean; fullscreen: boolean; onToggleFullscreen: (itemId: string) => void; } -const ScreenShareTile = subscribe( - ( - { - vm, - className, - style, - targetWidth, - targetHeight, - maximised, - fullscreen, - onToggleFullscreen, - }, - ref, - ) => { +const ScreenShareTile = forwardRef( + ({ vm, fullscreen, onToggleFullscreen, ...props }, ref) => { const { t } = useTranslation(); - const { displayName, nameTag } = useNameData(vm); - const video = useStateObservable(vm.video); - const unencryptedWarning = useStateObservable(vm.unencryptedWarning); const onClickFullScreen = useCallback( () => onToggleFullscreen(vm.id), [onToggleFullscreen, vm], @@ -239,23 +297,10 @@ const ScreenShareTile = subscribe( const FullScreenIcon = fullscreen ? CollapseIcon : ExpandIcon; return ( - ( ) } + videoEnabled + {...props} /> ); }, @@ -277,7 +324,7 @@ const ScreenShareTile = subscribe( ScreenShareTile.displayName = "ScreenShareTile"; -interface Props { +interface GridTileProps { vm: MediaViewModel; maximised: boolean; fullscreen: boolean; @@ -287,51 +334,34 @@ interface Props { targetHeight: number; className?: string; style?: ComponentProps["style"]; - showSpeakingIndicator: boolean; + showSpeakingIndicators: boolean; } -export const GridTile = forwardRef( - ( - { - vm, - maximised, - fullscreen, - onToggleFullscreen, - onOpenProfile, - className, - style, - targetWidth, - targetHeight, - showSpeakingIndicator, - }, - ref, - ) => { - if (vm instanceof UserMediaViewModel) { +export const GridTile = forwardRef( + ({ vm, fullscreen, onToggleFullscreen, onOpenProfile, ...props }, ref) => { + const nameData = useNameData(vm); + + if (vm instanceof LocalUserMediaViewModel) { return ( - ); + } else if (vm instanceof RemoteUserMediaViewModel) { + return ; } else { return ( ); } diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 6abf0cdd..e4bb085d 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -37,8 +37,9 @@ import { MediaView } from "./MediaView"; import styles from "./SpotlightTile.module.css"; import { subscribe } from "../state/subscribe"; import { + LocalUserMediaViewModel, MediaViewModel, - UserMediaViewModel, + RemoteUserMediaViewModel, useNameData, } from "../state/MediaViewModel"; import { useInitial } from "../useInitial"; @@ -48,11 +49,11 @@ import { useReactiveState } from "../useReactiveState"; import { useLatest } from "../useLatest"; // Screen share video is always enabled -const screenShareVideoEnabled = state(of(true)); +const videoEnabledDefault = state(of(true)); // Never mirror screen share video -const screenShareMirror = state(of(false)); +const mirrorDefault = state(of(false)); // Never crop screen share video -const screenShareCropVideo = state(of(false)); +const cropVideoDefault = state(of(false)); interface SpotlightItemProps { vm: MediaViewModel; @@ -72,15 +73,19 @@ const SpotlightItem = subscribe( const { displayName, nameTag } = useNameData(vm); const video = useStateObservable(vm.video); const videoEnabled = useStateObservable( - vm instanceof UserMediaViewModel + vm instanceof LocalUserMediaViewModel || + vm instanceof RemoteUserMediaViewModel ? vm.videoEnabled - : screenShareVideoEnabled, + : videoEnabledDefault, ); const mirror = useStateObservable( - vm instanceof UserMediaViewModel ? vm.mirror : screenShareMirror, + vm instanceof LocalUserMediaViewModel ? vm.mirror : mirrorDefault, ); const cropVideo = useStateObservable( - vm instanceof UserMediaViewModel ? vm.cropVideo : screenShareCropVideo, + vm instanceof LocalUserMediaViewModel || + vm instanceof RemoteUserMediaViewModel + ? vm.cropVideo + : cropVideoDefault, ); const unencryptedWarning = useStateObservable(vm.unencryptedWarning); From 8a414012a03571cb886588b008dfc6da065df282 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 16 May 2024 13:33:02 -0400 Subject: [PATCH 07/31] Add always show flag to view model --- src/settings/settings.ts | 2 ++ src/state/MediaViewModel.ts | 8 ++++++++ 2 files changed, 10 insertions(+) diff --git a/src/settings/settings.ts b/src/settings/settings.ts index 307a557e..4ccc78b8 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -88,3 +88,5 @@ export const videoInput = new Setting( "video-input", undefined, ); + +export const alwaysShowSelf = new Setting("always-show-self", true); diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index 7f7307f4..c61b5255 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -49,6 +49,7 @@ import { useEffect } from "react"; import { ViewModel } from "./ViewModel"; import { useReactiveState } from "../useReactiveState"; +import { alwaysShowSelf } from "../settings/settings"; export interface NameData { /** @@ -237,6 +238,13 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { ), ); + /** + * Whether to show this tile in a highly visible location near the start of + * the grid. + */ + public readonly alwaysShow = alwaysShowSelf.value; + public readonly setAlwaysShow = alwaysShowSelf.setValue; + public constructor( id: string, member: RoomMember | undefined, From 5647619b366902d8dbc27a7ba56b6281d73f2c26 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 20 Jun 2024 10:37:42 -0400 Subject: [PATCH 08/31] Add always show toggle to the UI --- public/locales/en-GB/app.json | 1 + src/tile/GridTile.tsx | 21 +++++++++++++++++++++ 2 files changed, 22 insertions(+) diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 4279bbb5..922a4c79 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -155,6 +155,7 @@ "unmute_microphone_button_label": "Unmute microphone", "version": "Version: {{version}}", "video_tile": { + "always_show": "Always show", "change_fit_contain": "Fit to frame", "exit_full_screen": "Exit full screen", "full_screen": "Full screen", diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index b0753aca..ba615953 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -30,6 +30,7 @@ import MicOffIcon from "@vector-im/compound-design-tokens/icons/mic-off.svg?reac import OverflowHorizontalIcon from "@vector-im/compound-design-tokens/icons/overflow-horizontal.svg?react"; import VolumeOnIcon from "@vector-im/compound-design-tokens/icons/volume-on.svg?react"; import VolumeOffIcon from "@vector-im/compound-design-tokens/icons/volume-off.svg?react"; +import VisibilityOnIcon from "@vector-im/compound-design-tokens/icons/visibility-on.svg?react"; import UserProfileIcon from "@vector-im/compound-design-tokens/icons/user-profile.svg?react"; import ExpandIcon from "@vector-im/compound-design-tokens/icons/expand.svg?react"; import CollapseIcon from "@vector-im/compound-design-tokens/icons/collapse.svg?react"; @@ -52,6 +53,7 @@ import { } from "../state/MediaViewModel"; import { Slider } from "../Slider"; import { MediaView } from "./MediaView"; +import { useLatest } from "../useLatest"; interface TileProps { className?: string; @@ -203,12 +205,31 @@ const LocalUserMediaTile = forwardRef( ({ vm, onOpenProfile, className, ...props }, ref) => { const { t } = useTranslation(); const mirror = useObservableEagerState(vm.mirror); + const alwaysShow = useObservableEagerState(vm.alwaysShow); + const latestAlwaysShow = useLatest(alwaysShow); + const onSelectAlwaysShow = useCallback( + (e: Event) => e.preventDefault(), + [], + ); + const onChangeAlwaysShow = useCallback( + () => vm.setAlwaysShow(!latestAlwaysShow.current), + [vm, latestAlwaysShow], + ); return ( + } + menuEnd={ Date: Thu, 16 May 2024 13:55:31 -0400 Subject: [PATCH 09/31] Use always show flag in importance ordering --- src/state/CallViewModel.ts | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index df3ca42a..f2656166 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -131,14 +131,14 @@ export type WindowMode = "normal" | "full screen" | "pip"; * Sorting bins defining the order in which media tiles appear in the layout. */ enum SortingBin { - SelfStart, + SelfAlwaysShown, Presenters, Speakers, VideoAndAudio, Video, Audio, NoMedia, - SelfEnd, + SelfNotAlwaysShown, } class UserMedia { @@ -410,10 +410,21 @@ export class CallViewModel extends ViewModel { switchMap((ms) => { const bins = ms.map((m) => combineLatest( - [m.speaker, m.presenter, m.vm.audioEnabled, m.vm.videoEnabled], - (speaker, presenter, audio, video) => { + [ + m.speaker, + m.presenter, + m.vm.audioEnabled, + m.vm.videoEnabled, + m.vm instanceof LocalUserMediaViewModel + ? m.vm.alwaysShow + : of(false), + ], + (speaker, presenter, audio, video, alwaysShow) => { let bin: SortingBin; - if (m.vm.local) bin = SortingBin.SelfStart; + if (m.vm.local) + bin = alwaysShow + ? SortingBin.SelfAlwaysShown + : SortingBin.SelfNotAlwaysShown; else if (presenter) bin = SortingBin.Presenters; else if (speaker) bin = SortingBin.Speakers; else if (video) From af0bd795b592b10a6ddcbf9d229744d81676532e Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 16 May 2024 15:23:10 -0400 Subject: [PATCH 10/31] Replace react-rxjs with observable-hooks MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit react-rxjs is the library we've been using to connect our React components to view models and consume observables. However, after spending some time with react-rxjs, I feel that it's a very heavy-handed solution. It requires us to sprinkle and components all throughout the code, and makes React go through an extra render cycle whenever we mount a component that binds to a view model. What I really want is a lightweight React hook that just gets the current value out of a plain observable, without any extra setup. Luckily the observable-hooks library with its useObservableEagerState hook seems to do just that—and it's more actively maintained, too! --- .eslintrc.cjs | 9 - package.json | 1 - src/grid/GridLayout.tsx | 29 +- src/room/InCallView.tsx | 828 ++++++++++++++++++------------------ src/state/CallViewModel.ts | 295 +++++++------ src/state/MediaViewModel.ts | 98 +++-- src/state/subscribe.tsx | 49 --- src/tile/SpotlightTile.tsx | 23 +- yarn.lock | 18 - 9 files changed, 627 insertions(+), 723 deletions(-) delete mode 100644 src/state/subscribe.tsx diff --git a/.eslintrc.cjs b/.eslintrc.cjs index f6a2e569..5970790f 100644 --- a/.eslintrc.cjs +++ b/.eslintrc.cjs @@ -38,15 +38,6 @@ module.exports = { "jsx-a11y/media-has-caption": "off", // We should use the js-sdk logger, never console directly. "no-console": ["error"], - "no-restricted-imports": [ - "error", - { - name: "@react-rxjs/core", - importNames: ["Subscribe", "RemoveSubscribe"], - message: - "These components are easy to misuse, please use the 'subscribe' component wrapper instead", - }, - ], "react/display-name": "error", }, settings: { diff --git a/package.json b/package.json index 19ca2fa2..19b8b937 100644 --- a/package.json +++ b/package.json @@ -41,7 +41,6 @@ "@react-aria/tabs": "^3.1.0", "@react-aria/tooltip": "^3.1.3", "@react-aria/utils": "^3.10.0", - "@react-rxjs/core": "^0.10.7", "@react-spring/web": "^9.4.4", "@react-stately/collections": "^3.3.4", "@react-stately/select": "^3.1.3", diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 8df7753b..6b673e64 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -14,16 +14,15 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { CSSProperties, useMemo } from "react"; -import { StateObservable, state, useStateObservable } from "@react-rxjs/core"; -import { BehaviorSubject, distinctUntilChanged } from "rxjs"; +import { CSSProperties, forwardRef, useMemo } from "react"; +import { BehaviorSubject, Observable, distinctUntilChanged } from "rxjs"; +import { useObservableEagerState } from "observable-hooks"; import { GridLayout as GridLayoutModel } from "../state/CallViewModel"; import { MediaViewModel } from "../state/MediaViewModel"; import { LayoutSystem, Slot } from "./Grid"; import styles from "./GridLayout.module.css"; import { useReactiveState } from "../useReactiveState"; -import { subscribe } from "../state/subscribe"; import { Alignment } from "../room/InCallView"; import { useInitial } from "../useInitial"; @@ -48,7 +47,7 @@ const slotMaxAspectRatio = 17 / 9; const slotMinAspectRatio = 4 / 3; export const gridLayoutSystems = ( - minBounds: StateObservable, + minBounds: Observable, floatingAlignment: BehaviorSubject, ): GridLayoutSystems => ({ // The "fixed" (non-scrolling) part of the layout is where the spotlight tile @@ -58,15 +57,13 @@ export const gridLayoutSystems = ( new Map( model.spotlight === undefined ? [] : [["spotlight", model.spotlight]], ), - Layout: subscribe(function GridLayoutFixed({ model }, ref) { - const { width, height } = useStateObservable(minBounds); - const alignment = useStateObservable( - useInitial>(() => - state( - floatingAlignment.pipe( - distinctUntilChanged( - (a1, a2) => a1.block === a2.block && a1.inline === a2.inline, - ), + Layout: forwardRef(function GridLayoutFixed({ model }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const alignment = useObservableEagerState( + useInitial(() => + floatingAlignment.pipe( + distinctUntilChanged( + (a1, a2) => a1.block === a2.block && a1.inline === a2.inline, ), ), ), @@ -106,8 +103,8 @@ export const gridLayoutSystems = ( // The scrolling part of the layout is where all the grid tiles live scrolling: { tiles: (model) => new Map(model.grid.map((tile) => [tile.id, tile])), - Layout: subscribe(function GridLayout({ model }, ref) { - const { width, height: minHeight } = useStateObservable(minBounds); + Layout: forwardRef(function GridLayout({ model }, ref) { + const { width, height: minHeight } = useObservableEagerState(minBounds); // The goal here is to determine the grid size and padding that maximizes // use of screen space for n tiles without making those tiles too small or diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index f2e08e5c..dba01e9d 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -35,8 +35,8 @@ import { import useMeasure from "react-use-measure"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import classNames from "classnames"; -import { state, useStateObservable } from "@react-rxjs/core"; import { BehaviorSubject } from "rxjs"; +import { useObservableEagerState } from "observable-hooks"; import { useTranslation } from "react-i18next"; import LogoMark from "../icons/LogoMark.svg?react"; @@ -76,7 +76,6 @@ import { TileDescriptor, useCallViewModel, } from "../state/CallViewModel"; -import { subscribe } from "../state/subscribe"; import { Grid, TileProps } from "../grid/Grid"; import { MediaViewModel } from "../state/MediaViewModel"; import { gridLayoutSystems } from "../grid/GridLayout"; @@ -143,458 +142,449 @@ export interface InCallViewProps { onShareClick: (() => void) | null; } -export const InCallView: FC = subscribe( - ({ - client, - matrixInfo, - rtcSession, - livekitRoom, - muteStates, - participantCount, - onLeave, - hideHeader, - otelGroupCallMembership, - connState, - onShareClick, - }) => { - const { t } = useTranslation(); - usePreventScroll(); - useWakeLock(); +export const InCallView: FC = ({ + client, + matrixInfo, + rtcSession, + livekitRoom, + muteStates, + participantCount, + onLeave, + hideHeader, + otelGroupCallMembership, + connState, + onShareClick, +}) => { + const { t } = useTranslation(); + usePreventScroll(); + useWakeLock(); - useEffect(() => { - if (connState === ConnectionState.Disconnected) { - // annoyingly we don't get the disconnection reason this way, - // only by listening for the emitted event - onLeave(new Error("Disconnected from call server")); - } - }, [connState, onLeave]); + useEffect(() => { + if (connState === ConnectionState.Disconnected) { + // annoyingly we don't get the disconnection reason this way, + // only by listening for the emitted event + onLeave(new Error("Disconnected from call server")); + } + }, [connState, onLeave]); - const containerRef1 = useRef(null); - const [containerRef2, bounds] = useMeasure(); - const boundsValid = bounds.height > 0; - // Merge the refs so they can attach to the same element - const containerRef = useMergedRefs(containerRef1, containerRef2); + const containerRef1 = useRef(null); + const [containerRef2, bounds] = useMeasure(); + const boundsValid = bounds.height > 0; + // Merge the refs so they can attach to the same element + const containerRef = useMergedRefs(containerRef1, containerRef2); - const screenSharingTracks = useTracks( - [{ source: Track.Source.ScreenShare, withPlaceholder: false }], - { - room: livekitRoom, - }, - ); - const { layout: legacyLayout, setLayout: setLegacyLayout } = - useLegacyGridLayout(screenSharingTracks.length > 0); - - const { hideScreensharing, showControls } = useUrlParams(); - - const { isScreenShareEnabled, localParticipant } = useLocalParticipant({ + const screenSharingTracks = useTracks( + [{ source: Track.Source.ScreenShare, withPlaceholder: false }], + { room: livekitRoom, - }); + }, + ); + const { layout: legacyLayout, setLayout: setLegacyLayout } = + useLegacyGridLayout(screenSharingTracks.length > 0); - const toggleMicrophone = useCallback( - () => muteStates.audio.setEnabled?.((e) => !e), - [muteStates], - ); - const toggleCamera = useCallback( - () => muteStates.video.setEnabled?.((e) => !e), - [muteStates], - ); + const { hideScreensharing, showControls } = useUrlParams(); - // This function incorrectly assumes that there is a camera and microphone, which is not always the case. - // TODO: Make sure that this module is resilient when it comes to camera/microphone availability! - useCallViewKeyboardShortcuts( - containerRef1, - toggleMicrophone, - toggleCamera, - (muted) => muteStates.audio.setEnabled?.(!muted), - ); + const { isScreenShareEnabled, localParticipant } = useLocalParticipant({ + room: livekitRoom, + }); - useEffect(() => { - widget?.api.transport.send( - legacyLayout === "grid" - ? ElementWidgetActions.TileLayout - : ElementWidgetActions.SpotlightLayout, - {}, + const toggleMicrophone = useCallback( + () => muteStates.audio.setEnabled?.((e) => !e), + [muteStates], + ); + const toggleCamera = useCallback( + () => muteStates.video.setEnabled?.((e) => !e), + [muteStates], + ); + + // This function incorrectly assumes that there is a camera and microphone, which is not always the case. + // TODO: Make sure that this module is resilient when it comes to camera/microphone availability! + useCallViewKeyboardShortcuts( + containerRef1, + toggleMicrophone, + toggleCamera, + (muted) => muteStates.audio.setEnabled?.(!muted), + ); + + useEffect(() => { + widget?.api.transport.send( + legacyLayout === "grid" + ? ElementWidgetActions.TileLayout + : ElementWidgetActions.SpotlightLayout, + {}, + ); + }, [legacyLayout]); + + useEffect(() => { + if (widget) { + const onTileLayout = (ev: CustomEvent): void => { + setLegacyLayout("grid"); + widget!.api.transport.reply(ev.detail, {}); + }; + const onSpotlightLayout = (ev: CustomEvent): void => { + setLegacyLayout("spotlight"); + widget!.api.transport.reply(ev.detail, {}); + }; + + widget.lazyActions.on(ElementWidgetActions.TileLayout, onTileLayout); + widget.lazyActions.on( + ElementWidgetActions.SpotlightLayout, + onSpotlightLayout, ); - }, [legacyLayout]); - useEffect(() => { - if (widget) { - const onTileLayout = (ev: CustomEvent): void => { - setLegacyLayout("grid"); - widget!.api.transport.reply(ev.detail, {}); - }; - const onSpotlightLayout = ( - ev: CustomEvent, - ): void => { - setLegacyLayout("spotlight"); - widget!.api.transport.reply(ev.detail, {}); - }; - - widget.lazyActions.on(ElementWidgetActions.TileLayout, onTileLayout); - widget.lazyActions.on( + return (): void => { + widget!.lazyActions.off(ElementWidgetActions.TileLayout, onTileLayout); + widget!.lazyActions.off( ElementWidgetActions.SpotlightLayout, onSpotlightLayout, ); + }; + } + }, [setLegacyLayout]); - return (): void => { - widget!.lazyActions.off( - ElementWidgetActions.TileLayout, - onTileLayout, - ); - widget!.lazyActions.off( - ElementWidgetActions.SpotlightLayout, - onSpotlightLayout, - ); - }; - } - }, [setLegacyLayout]); + const mobile = boundsValid && bounds.width <= 660; + const reducedControls = boundsValid && bounds.width <= 340; + const noControls = reducedControls && bounds.height <= 400; - const mobile = boundsValid && bounds.width <= 660; - const reducedControls = boundsValid && bounds.width <= 340; - const noControls = reducedControls && bounds.height <= 400; + const vm = useCallViewModel( + rtcSession.room, + livekitRoom, + matrixInfo.e2eeSystem.kind !== E2eeType.NONE, + connState, + ); + const items = useObservableEagerState(vm.tiles); + const layout = useObservableEagerState(vm.layout); + const hasSpotlight = layout.spotlight !== undefined; + // Hack: We insert a dummy "spotlight" tile into the tiles we pass to + // useFullscreen so that we can control the fullscreen state of the + // spotlight tile in the new layouts with this same hook. + const fullscreenItems = useMemo( + () => (hasSpotlight ? [...items, dummySpotlightItem] : items), + [items, hasSpotlight], + ); + const { fullscreenItem, toggleFullscreen, exitFullscreen } = + useFullscreen(fullscreenItems); + const toggleSpotlightFullscreen = useCallback( + () => toggleFullscreen("spotlight"), + [toggleFullscreen], + ); - const vm = useCallViewModel( - rtcSession.room, - livekitRoom, - matrixInfo.e2eeSystem.kind !== E2eeType.NONE, - connState, - ); - const items = useStateObservable(vm.tiles); - const layout = useStateObservable(vm.layout); - const hasSpotlight = layout.spotlight !== undefined; - // Hack: We insert a dummy "spotlight" tile into the tiles we pass to - // useFullscreen so that we can control the fullscreen state of the - // spotlight tile in the new layouts with this same hook. - const fullscreenItems = useMemo( - () => (hasSpotlight ? [...items, dummySpotlightItem] : items), - [items, hasSpotlight], - ); - const { fullscreenItem, toggleFullscreen, exitFullscreen } = - useFullscreen(fullscreenItems); - const toggleSpotlightFullscreen = useCallback( - () => toggleFullscreen("spotlight"), - [toggleFullscreen], - ); + // The maximised participant: either the participant that the user has + // manually put in fullscreen, or the focused (active) participant if the + // window is too small to show everyone + const maximisedParticipant = useMemo( + () => + fullscreenItem ?? + (noControls + ? items.find((item) => item.isSpeaker) ?? items.at(0) ?? null + : null), + [fullscreenItem, noControls, items], + ); - // The maximised participant: either the participant that the user has - // manually put in fullscreen, or the focused (active) participant if the - // window is too small to show everyone - const maximisedParticipant = useMemo( - () => - fullscreenItem ?? - (noControls - ? items.find((item) => item.isSpeaker) ?? items.at(0) ?? null - : null), - [fullscreenItem, noControls, items], - ); + const prefersReducedMotion = usePrefersReducedMotion(); - const prefersReducedMotion = usePrefersReducedMotion(); + const [settingsModalOpen, setSettingsModalOpen] = useState(false); + const [settingsTab, setSettingsTab] = useState(defaultSettingsTab); - const [settingsModalOpen, setSettingsModalOpen] = useState(false); - const [settingsTab, setSettingsTab] = useState(defaultSettingsTab); + const openSettings = useCallback( + () => setSettingsModalOpen(true), + [setSettingsModalOpen], + ); + const closeSettings = useCallback( + () => setSettingsModalOpen(false), + [setSettingsModalOpen], + ); - const openSettings = useCallback( - () => setSettingsModalOpen(true), - [setSettingsModalOpen], - ); - const closeSettings = useCallback( - () => setSettingsModalOpen(false), - [setSettingsModalOpen], - ); + const openProfile = useCallback(() => { + setSettingsTab("profile"); + setSettingsModalOpen(true); + }, [setSettingsTab, setSettingsModalOpen]); - const openProfile = useCallback(() => { - setSettingsTab("profile"); - setSettingsModalOpen(true); - }, [setSettingsTab, setSettingsModalOpen]); + const [headerRef, headerBounds] = useMeasure(); + const [footerRef, footerBounds] = useMeasure(); + const gridBounds = useMemo( + () => ({ + width: footerBounds.width, + height: bounds.height - headerBounds.height - footerBounds.height, + }), + [ + footerBounds.width, + bounds.height, + headerBounds.height, + footerBounds.height, + ], + ); + const gridBoundsObservable = useObservable(gridBounds); + const floatingAlignment = useInitial( + () => new BehaviorSubject(defaultAlignment), + ); + const { fixed, scrolling } = useInitial(() => + gridLayoutSystems(gridBoundsObservable, floatingAlignment), + ); - const [headerRef, headerBounds] = useMeasure(); - const [footerRef, footerBounds] = useMeasure(); - const gridBounds = useMemo( - () => ({ - width: footerBounds.width, - height: bounds.height - headerBounds.height - footerBounds.height, - }), - [ - footerBounds.width, - bounds.height, - headerBounds.height, - footerBounds.height, - ], - ); - const gridBoundsObservable = useObservable(gridBounds); - const floatingAlignment = useInitial( - () => new BehaviorSubject(defaultAlignment), - ); - const { fixed, scrolling } = useInitial(() => - gridLayoutSystems(state(gridBoundsObservable), floatingAlignment), - ); + const setGridMode = useCallback( + (mode: GridMode) => { + setLegacyLayout(mode); + vm.setGridMode(mode); + }, + [setLegacyLayout, vm], + ); - const setGridMode = useCallback( - (mode: GridMode) => { - setLegacyLayout(mode); - vm.setGridMode(mode); - }, - [setLegacyLayout, vm], - ); + const showSpeakingIndicators = + layout.type === "spotlight" || + (layout.type === "grid" && layout.grid.length > 2); - const showSpeakingIndicators = - layout.type === "spotlight" || - (layout.type === "grid" && layout.grid.length > 2); - - const SpotlightTileView = useMemo( - () => - forwardRef>( - function SpotlightTileView( - { className, style, targetWidth, targetHeight, model }, - ref, - ) { - return ( - - ); - }, - ), - [toggleSpotlightFullscreen], - ); - const GridTileView = useMemo( - () => - forwardRef>( - function GridTileView( - { className, style, targetWidth, targetHeight, model }, - ref, - ) { - return ( - - ); - }, - ), - [toggleFullscreen, openProfile, showSpeakingIndicators], - ); - - const renderContent = (): JSX.Element => { - if (items.length === 0) { - return ( -
-

{t("waiting_for_participants")}

-
- ); - } - - if (maximisedParticipant !== null) { - const fullscreen = maximisedParticipant === fullscreenItem; - if (maximisedParticipant.id === "spotlight") { + const SpotlightTileView = useMemo( + () => + forwardRef>( + function SpotlightTileView( + { className, style, targetWidth, targetHeight, model }, + ref, + ) { return ( ); - } - return ( - - ); - } - - // The only new layout we've implemented so far is grid layout for non-1:1 - // calls. All other layouts use the legacy grid system for now. - if ( - legacyLayout === "grid" && - layout.type === "grid" && - !(layout.grid.length === 2 && layout.spotlight === undefined) - ) { - return ( - <> - + forwardRef>( + function GridTileView( + { className, style, targetWidth, targetHeight, model }, + ref, + ) { + return ( + - - - ); - } else { - return ( - - ); - } - }; - - const rageshakeRequestModalProps = useRageshakeRequestModal( - rtcSession.room.roomId, - ); - - const toggleScreensharing = useCallback(async () => { - exitFullscreen(); - await localParticipant.setScreenShareEnabled(!isScreenShareEnabled, { - audio: true, - selfBrowserSurface: "include", - surfaceSwitching: "include", - systemAudio: "include", - }); - }, [localParticipant, isScreenShareEnabled, exitFullscreen]); - - let footer: JSX.Element | null; - - if (noControls) { - footer = null; - } else { - const buttons: JSX.Element[] = []; - - buttons.push( - , - , - ); - - if (!reducedControls) { - if (canScreenshare && !hideScreensharing) { - buttons.push( - , ); - } - buttons.push(); - } + }, + ), + [toggleFullscreen, openProfile, showSpeakingIndicators], + ); - buttons.push( - , - ); - footer = ( -
- {!mobile && !hideHeader && ( -
- - -
- )} - {showControls &&
{buttons}
} - {!mobile && !hideHeader && showControls && ( - - )} + const renderContent = (): JSX.Element => { + if (items.length === 0) { + return ( +
+

{t("waiting_for_participants")}

); } - return ( -
- {!hideHeader && maximisedParticipant === null && ( -
- - - - - {!reducedControls && showControls && onShareClick !== null && ( - - )} - -
- )} - - {renderContent()} - {footer} - {!noControls && ( - - )} - + ); + } + return ( + + ); + } + + // The only new layout we've implemented so far is grid layout for non-1:1 + // calls. All other layouts use the legacy grid system for now. + if ( + legacyLayout === "grid" && + layout.type === "grid" && + !(layout.grid.length === 2 && layout.spotlight === undefined) + ) { + return ( + <> + + + + ); + } else { + return ( + + ); + } + }; + + const rageshakeRequestModalProps = useRageshakeRequestModal( + rtcSession.room.roomId, + ); + + const toggleScreensharing = useCallback(async () => { + exitFullscreen(); + await localParticipant.setScreenShareEnabled(!isScreenShareEnabled, { + audio: true, + selfBrowserSurface: "include", + surfaceSwitching: "include", + systemAudio: "include", + }); + }, [localParticipant, isScreenShareEnabled, exitFullscreen]); + + let footer: JSX.Element | null; + + if (noControls) { + footer = null; + } else { + const buttons: JSX.Element[] = []; + + buttons.push( + , + , + ); + + if (!reducedControls) { + if (canScreenshare && !hideScreensharing) { + buttons.push( + , + ); + } + buttons.push(); + } + + buttons.push( + , + ); + footer = ( +
+ {!mobile && !hideHeader && ( +
+ + +
+ )} + {showControls &&
{buttons}
} + {!mobile && !hideHeader && showControls && ( + + )}
); - }, -); + } + + return ( +
+ {!hideHeader && maximisedParticipant === null && ( +
+ + + + + {!reducedControls && showControls && onShareClick !== null && ( + + )} + +
+ )} + + {renderContent()} + {footer} + {!noControls && } + +
+ ); +}; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index f2656166..b0816dc2 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -50,7 +50,6 @@ import { timer, zip, } from "rxjs"; -import { StateObservable, state } from "@react-rxjs/core"; import { logger } from "matrix-js-sdk/src/logger"; import { ViewModel } from "./ViewModel"; @@ -158,7 +157,7 @@ class UserMedia { ? new LocalUserMediaViewModel(id, member, participant, callEncrypted) : new RemoteUserMediaViewModel(id, member, participant, callEncrypted); - this.speaker = this.vm.speaking.pipeState( + this.speaker = this.vm.speaking.pipe( // Require 1 s of continuous speaking to become a speaker, and 60 s of // continuous silence to stop being considered a speaker audit((s) => @@ -234,9 +233,9 @@ function findMatrixMember( // TODO: Move wayyyy more business logic from the call and lobby views into here export class CallViewModel extends ViewModel { - private readonly rawRemoteParticipants = state( - connectedParticipantsObserver(this.livekitRoom), - ); + private readonly rawRemoteParticipants = connectedParticipantsObserver( + this.livekitRoom, + ).pipe(shareReplay(1)); // Lists of participants to "hold" on display, even if LiveKit claims that // they've left @@ -309,64 +308,60 @@ export class CallViewModel extends ViewModel { }, ); - private readonly mediaItems: StateObservable = state( - combineLatest([ - this.remoteParticipants, - observeParticipantMedia(this.livekitRoom.localParticipant), - ]).pipe( - scan( - ( - prevItems, - [remoteParticipants, { participant: localParticipant }], - ) => { - let allGhosts = true; + private readonly mediaItems: Observable = combineLatest([ + this.remoteParticipants, + observeParticipantMedia(this.livekitRoom.localParticipant), + ]).pipe( + scan( + (prevItems, [remoteParticipants, { participant: localParticipant }]) => { + let allGhosts = true; - const newItems = new Map( - function* (this: CallViewModel): Iterable<[string, MediaItem]> { - for (const p of [localParticipant, ...remoteParticipants]) { - const member = findMatrixMember(this.matrixRoom, p.identity); - allGhosts &&= member === undefined; - // We always start with a local participant with the empty string as - // their ID before we're connected, this is fine and we'll be in - // "all ghosts" mode. - if (p.identity !== "" && member === undefined) { - logger.warn( - `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`, - ); - } - - const userMediaId = p.identity; - yield [ - userMediaId, - prevItems.get(userMediaId) ?? - new UserMedia(userMediaId, member, p, this.encrypted), - ]; - - if (p.isScreenShareEnabled) { - const screenShareId = `${userMediaId}:screen-share`; - yield [ - screenShareId, - prevItems.get(screenShareId) ?? - new ScreenShare(screenShareId, member, p, this.encrypted), - ]; - } + const newItems = new Map( + function* (this: CallViewModel): Iterable<[string, MediaItem]> { + for (const p of [localParticipant, ...remoteParticipants]) { + const member = findMatrixMember(this.matrixRoom, p.identity); + allGhosts &&= member === undefined; + // We always start with a local participant with the empty string as + // their ID before we're connected, this is fine and we'll be in + // "all ghosts" mode. + if (p.identity !== "" && member === undefined) { + logger.warn( + `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`, + ); } - }.bind(this)(), - ); - for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy(); + const userMediaId = p.identity; + yield [ + userMediaId, + prevItems.get(userMediaId) ?? + new UserMedia(userMediaId, member, p, this.encrypted), + ]; - // If every item is a ghost, that probably means we're still connecting - // and shouldn't bother showing anything yet - return allGhosts ? new Map() : newItems; - }, - new Map(), - ), - map((ms) => [...ms.values()]), - finalizeValue((ts) => { - for (const t of ts) t.destroy(); - }), + if (p.isScreenShareEnabled) { + const screenShareId = `${userMediaId}:screen-share`; + yield [ + screenShareId, + prevItems.get(screenShareId) ?? + new ScreenShare(screenShareId, member, p, this.encrypted), + ]; + } + } + }.bind(this)(), + ); + + for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy(); + + // If every item is a ghost, that probably means we're still connecting + // and shouldn't bother showing anything yet + return allGhosts ? new Map() : newItems; + }, + new Map(), ), + map((ms) => [...ms.values()]), + finalizeValue((ts) => { + for (const t of ts) t.destroy(); + }), + shareReplay(1), ); private readonly userMedia: Observable = this.mediaItems.pipe( @@ -462,14 +457,15 @@ export class CallViewModel extends ViewModel { /** * The layout mode of the media tile grid. */ - public readonly gridMode = state(this._gridMode); + public readonly gridMode: Observable = this._gridMode; public setGridMode(value: GridMode): void { this._gridMode.next(value); } - public readonly layout: StateObservable = state( - combineLatest([this._gridMode, this.windowMode], (gridMode, windowMode) => { + public readonly layout: Observable = combineLatest( + [this._gridMode, this.windowMode], + (gridMode, windowMode) => { switch (windowMode) { case "full screen": throw new Error("unimplemented"); @@ -498,110 +494,109 @@ export class CallViewModel extends ViewModel { } } } - }).pipe(switchAll()), - ); + }, + ).pipe(switchAll(), shareReplay(1)); /** * The media tiles to be displayed in the call view. */ // TODO: Get rid of this field, replacing it with the 'layout' field above // which keeps more details of the layout order internal to the view model - public readonly tiles: StateObservable[]> = - state( - combineLatest([ - this.remoteParticipants, - observeParticipantMedia(this.livekitRoom.localParticipant), - ]).pipe( - scan((ts, [remoteParticipants, { participant: localParticipant }]) => { - const ps = [localParticipant, ...remoteParticipants]; - const tilesById = new Map(ts.map((t) => [t.id, t])); - const now = Date.now(); - let allGhosts = true; + public readonly tiles: Observable[]> = + combineLatest([ + this.remoteParticipants, + observeParticipantMedia(this.livekitRoom.localParticipant), + ]).pipe( + scan((ts, [remoteParticipants, { participant: localParticipant }]) => { + const ps = [localParticipant, ...remoteParticipants]; + const tilesById = new Map(ts.map((t) => [t.id, t])); + const now = Date.now(); + let allGhosts = true; - const newTiles = ps.flatMap((p) => { - const userMediaId = p.identity; - const member = findMatrixMember(this.matrixRoom, userMediaId); - allGhosts &&= member === undefined; - const spokeRecently = - p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000; + const newTiles = ps.flatMap((p) => { + const userMediaId = p.identity; + const member = findMatrixMember(this.matrixRoom, userMediaId); + allGhosts &&= member === undefined; + const spokeRecently = + p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000; - // We always start with a local participant with the empty string as - // their ID before we're connected, this is fine and we'll be in - // "all ghosts" mode. - if (userMediaId !== "" && member === undefined) { - logger.warn( - `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`, - ); - } + // We always start with a local participant with the empty string as + // their ID before we're connected, this is fine and we'll be in + // "all ghosts" mode. + if (userMediaId !== "" && member === undefined) { + logger.warn( + `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`, + ); + } - const userMediaVm = - tilesById.get(userMediaId)?.data ?? - (p instanceof LocalParticipant - ? new LocalUserMediaViewModel( - userMediaId, - member, - p, - this.encrypted, - ) - : new RemoteUserMediaViewModel( - userMediaId, - member, - p, - this.encrypted, - )); - tilesById.delete(userMediaId); - - const userMediaTile: TileDescriptor = { - id: userMediaId, - focused: false, - isPresenter: p.isScreenShareEnabled, - isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal, - hasVideo: p.isCameraEnabled, - local: p.isLocal, - largeBaseSize: false, - data: userMediaVm, - }; - - if (p.isScreenShareEnabled) { - const screenShareId = `${userMediaId}:screen-share`; - const screenShareVm = - tilesById.get(screenShareId)?.data ?? - new ScreenShareViewModel( - screenShareId, + const userMediaVm = + tilesById.get(userMediaId)?.data ?? + (p instanceof LocalParticipant + ? new LocalUserMediaViewModel( + userMediaId, member, p, this.encrypted, - ); - tilesById.delete(screenShareId); + ) + : new RemoteUserMediaViewModel( + userMediaId, + member, + p, + this.encrypted, + )); + tilesById.delete(userMediaId); - const screenShareTile: TileDescriptor = { - id: screenShareId, - focused: true, - isPresenter: false, - isSpeaker: false, - hasVideo: true, - local: p.isLocal, - largeBaseSize: true, - placeNear: userMediaId, - data: screenShareVm, - }; - return [userMediaTile, screenShareTile]; - } else { - return [userMediaTile]; - } - }); + const userMediaTile: TileDescriptor = { + id: userMediaId, + focused: false, + isPresenter: p.isScreenShareEnabled, + isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal, + hasVideo: p.isCameraEnabled, + local: p.isLocal, + largeBaseSize: false, + data: userMediaVm, + }; - // Any tiles left in the map are unused and should be destroyed - for (const t of tilesById.values()) t.data.destroy(); + if (p.isScreenShareEnabled) { + const screenShareId = `${userMediaId}:screen-share`; + const screenShareVm = + tilesById.get(screenShareId)?.data ?? + new ScreenShareViewModel( + screenShareId, + member, + p, + this.encrypted, + ); + tilesById.delete(screenShareId); - // If every item is a ghost, that probably means we're still connecting - // and shouldn't bother showing anything yet - return allGhosts ? [] : newTiles; - }, [] as TileDescriptor[]), - finalizeValue((ts) => { - for (const t of ts) t.data.destroy(); - }), - ), + const screenShareTile: TileDescriptor = { + id: screenShareId, + focused: true, + isPresenter: false, + isSpeaker: false, + hasVideo: true, + local: p.isLocal, + largeBaseSize: true, + placeNear: userMediaId, + data: screenShareVm, + }; + return [userMediaTile, screenShareTile]; + } else { + return [userMediaTile]; + } + }); + + // Any tiles left in the map are unused and should be destroyed + for (const t of tilesById.values()) t.data.destroy(); + + // If every item is a ghost, that probably means we're still connecting + // and shouldn't bother showing anything yet + return allGhosts ? [] : newTiles; + }, [] as TileDescriptor[]), + finalizeValue((ts) => { + for (const t of ts) t.data.destroy(); + }), + shareReplay(1), ); public constructor( diff --git a/src/state/MediaViewModel.ts b/src/state/MediaViewModel.ts index c61b5255..8ad565e4 100644 --- a/src/state/MediaViewModel.ts +++ b/src/state/MediaViewModel.ts @@ -21,7 +21,6 @@ import { observeParticipantEvents, observeParticipantMedia, } from "@livekit/components-core"; -import { StateObservable, state } from "@react-rxjs/core"; import { LocalParticipant, LocalTrack, @@ -35,12 +34,14 @@ import { import { RoomMember, RoomMemberEvent } from "matrix-js-sdk/src/matrix"; import { BehaviorSubject, + Observable, combineLatest, distinctUntilChanged, distinctUntilKeyChanged, fromEvent, map, of, + shareReplay, startWith, switchMap, } from "rxjs"; @@ -92,16 +93,15 @@ export function useNameData(vm: MediaViewModel): NameData { function observeTrackReference( participant: Participant, source: Track.Source, -): StateObservable { - return state( - observeParticipantMedia(participant).pipe( - map(() => ({ - participant, - publication: participant.getTrackPublication(source), - source, - })), - distinctUntilKeyChanged("publication"), - ), +): Observable { + return observeParticipantMedia(participant).pipe( + map(() => ({ + participant, + publication: participant.getTrackPublication(source), + source, + })), + distinctUntilKeyChanged("publication"), + shareReplay(1), ); } @@ -113,11 +113,11 @@ abstract class BaseMediaViewModel extends ViewModel { /** * The LiveKit video track for this media. */ - public readonly video: StateObservable; + public readonly video: Observable; /** * Whether there should be a warning that this media is unencrypted. */ - public readonly unencryptedWarning: StateObservable; + public readonly unencryptedWarning: Observable; public constructor( /** @@ -138,15 +138,13 @@ abstract class BaseMediaViewModel extends ViewModel { super(); const audio = observeTrackReference(participant, audioSource); this.video = observeTrackReference(participant, videoSource); - this.unencryptedWarning = state( - combineLatest( - [audio, this.video], - (a, v) => - callEncrypted && - (a.publication?.isEncrypted === false || - v.publication?.isEncrypted === false), - ).pipe(distinctUntilChanged()), - ); + this.unencryptedWarning = combineLatest( + [audio, this.video], + (a, v) => + callEncrypted && + (a.publication?.isEncrypted === false || + v.publication?.isEncrypted === false), + ).pipe(distinctUntilChanged(), shareReplay(1)); } } @@ -165,27 +163,28 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { /** * Whether the participant is speaking. */ - public readonly speaking = state( - observeParticipantEvents( - this.participant, - ParticipantEvent.IsSpeakingChanged, - ).pipe(map((p) => p.isSpeaking)), + public readonly speaking = observeParticipantEvents( + this.participant, + ParticipantEvent.IsSpeakingChanged, + ).pipe( + map((p) => p.isSpeaking), + shareReplay(1), ); /** * Whether this participant is sending audio (i.e. is unmuted on their side). */ - public readonly audioEnabled: StateObservable; + public readonly audioEnabled: Observable; /** * Whether this participant is sending video. */ - public readonly videoEnabled: StateObservable; + public readonly videoEnabled: Observable; private readonly _cropVideo = new BehaviorSubject(true); /** * Whether the tile video should be contained inside the tile or be cropped to fit. */ - public readonly cropVideo = state(this._cropVideo); + public readonly cropVideo: Observable = this._cropVideo; public constructor( id: string, @@ -202,12 +201,12 @@ abstract class BaseUserMediaViewModel extends BaseMediaViewModel { Track.Source.Camera, ); - const media = observeParticipantMedia(participant); - this.audioEnabled = state( - media.pipe(map((m) => m.microphoneTrack?.isMuted === false)), + const media = observeParticipantMedia(participant).pipe(shareReplay(1)); + this.audioEnabled = media.pipe( + map((m) => m.microphoneTrack?.isMuted === false), ); - this.videoEnabled = state( - media.pipe(map((m) => m.cameraTrack?.isMuted === false)), + this.videoEnabled = media.pipe( + map((m) => m.cameraTrack?.isMuted === false), ); } @@ -223,19 +222,18 @@ export class LocalUserMediaViewModel extends BaseUserMediaViewModel { /** * Whether the video should be mirrored. */ - public readonly mirror = state( - this.video.pipe( - switchMap((v) => { - const track = v.publication?.track; - if (!(track instanceof LocalTrack)) return of(false); - // Watch for track restarts, because they indicate a camera switch - return fromEvent(track, TrackEvent.Restarted).pipe( - startWith(null), - // Mirror only front-facing cameras (those that face the user) - map(() => facingModeFromLocalTrack(track).facingMode === "user"), - ); - }), - ), + public readonly mirror = this.video.pipe( + switchMap((v) => { + const track = v.publication?.track; + if (!(track instanceof LocalTrack)) return of(false); + // Watch for track restarts, because they indicate a camera switch + return fromEvent(track, TrackEvent.Restarted).pipe( + startWith(null), + // Mirror only front-facing cameras (those that face the user) + map(() => facingModeFromLocalTrack(track).facingMode === "user"), + ); + }), + shareReplay(1), ); /** @@ -263,14 +261,14 @@ export class RemoteUserMediaViewModel extends BaseUserMediaViewModel { /** * Whether we've disabled this participant's audio. */ - public readonly locallyMuted = state(this._locallyMuted); + public readonly locallyMuted: Observable = this._locallyMuted; private readonly _localVolume = new BehaviorSubject(1); /** * The volume to which we've set this participant's audio, as a scalar * multiplier. */ - public readonly localVolume = state(this._localVolume); + public readonly localVolume: Observable = this._localVolume; public constructor( id: string, diff --git a/src/state/subscribe.tsx b/src/state/subscribe.tsx deleted file mode 100644 index e0441aeb..00000000 --- a/src/state/subscribe.tsx +++ /dev/null @@ -1,49 +0,0 @@ -/* -Copyright 2023 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { - ForwardRefExoticComponent, - ForwardRefRenderFunction, - PropsWithoutRef, - RefAttributes, - forwardRef, -} from "react"; -// eslint-disable-next-line no-restricted-imports -import { Subscribe, RemoveSubscribe } from "@react-rxjs/core"; - -/** - * Wraps a React component that consumes Observables, resulting in a component - * that safely subscribes to its Observables before rendering. The component - * will return null until the subscriptions are created. - */ -export function subscribe( - render: ForwardRefRenderFunction, -): ForwardRefExoticComponent & RefAttributes> { - const Subscriber = forwardRef(({ p }, ref) => ( - {render(p, ref)} - )); - Subscriber.displayName = "Subscriber"; - - // eslint-disable-next-line react/display-name - const OuterComponent = forwardRef((p, ref) => ( - - - - )); - // Copy over the component's display name, default props, etc. - Object.assign(OuterComponent, render); - return OuterComponent; -} diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index e4bb085d..77a3526f 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -28,14 +28,13 @@ import CollapseIcon from "@vector-im/compound-design-tokens/icons/collapse.svg?r import ChevronLeftIcon from "@vector-im/compound-design-tokens/icons/chevron-left.svg?react"; import ChevronRightIcon from "@vector-im/compound-design-tokens/icons/chevron-right.svg?react"; import { animated } from "@react-spring/web"; -import { state, useStateObservable } from "@react-rxjs/core"; import { Observable, map, of } from "rxjs"; +import { useObservableEagerState } from "observable-hooks"; import { useTranslation } from "react-i18next"; import classNames from "classnames"; import { MediaView } from "./MediaView"; import styles from "./SpotlightTile.module.css"; -import { subscribe } from "../state/subscribe"; import { LocalUserMediaViewModel, MediaViewModel, @@ -49,11 +48,11 @@ import { useReactiveState } from "../useReactiveState"; import { useLatest } from "../useLatest"; // Screen share video is always enabled -const videoEnabledDefault = state(of(true)); +const videoEnabledDefault = of(true); // Never mirror screen share video -const mirrorDefault = state(of(false)); +const mirrorDefault = of(false); // Never crop screen share video -const cropVideoDefault = state(of(false)); +const cropVideoDefault = of(false); interface SpotlightItemProps { vm: MediaViewModel; @@ -66,28 +65,28 @@ interface SpotlightItemProps { snap: boolean; } -const SpotlightItem = subscribe( +const SpotlightItem = forwardRef( ({ vm, targetWidth, targetHeight, intersectionObserver, snap }, theirRef) => { const ourRef = useRef(null); const ref = useMergedRefs(ourRef, theirRef); const { displayName, nameTag } = useNameData(vm); - const video = useStateObservable(vm.video); - const videoEnabled = useStateObservable( + const video = useObservableEagerState(vm.video); + const videoEnabled = useObservableEagerState( vm instanceof LocalUserMediaViewModel || vm instanceof RemoteUserMediaViewModel ? vm.videoEnabled : videoEnabledDefault, ); - const mirror = useStateObservable( + const mirror = useObservableEagerState( vm instanceof LocalUserMediaViewModel ? vm.mirror : mirrorDefault, ); - const cropVideo = useStateObservable( + const cropVideo = useObservableEagerState( vm instanceof LocalUserMediaViewModel || vm instanceof RemoteUserMediaViewModel ? vm.cropVideo : cropVideoDefault, ); - const unencryptedWarning = useStateObservable(vm.unencryptedWarning); + const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); // Hook this item up to the intersection observer useEffect(() => { @@ -124,6 +123,8 @@ const SpotlightItem = subscribe( }, ); +SpotlightItem.displayName = "SpotlightItem"; + interface Props { vms: MediaViewModel[]; maximised: boolean; diff --git a/yarn.lock b/yarn.lock index 92638203..387afeb3 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2847,14 +2847,6 @@ resolved "https://registry.yarnpkg.com/@react-hook/latest/-/latest-1.0.3.tgz#c2d1d0b0af8b69ec6e2b3a2412ba0768ac82db80" integrity sha512-dy6duzl+JnAZcDbNTfmaP3xHiKtbXYOaz3G51MGVljh548Y8MWzTr+PHLOfvpypEVW9zwvl+VyKjbWKEVbV1Rg== -"@react-rxjs/core@^0.10.7": - version "0.10.7" - resolved "https://registry.yarnpkg.com/@react-rxjs/core/-/core-0.10.7.tgz#09951f43a6c80892526ac13d51859098b0e74993" - integrity sha512-dornp8pUs9OcdqFKKRh9+I2FVe21gWufNun6RYU1ddts7kUy9i4Thvl0iqcPFbGY61cJQMAJF7dxixWMSD/A/A== - dependencies: - "@rx-state/core" "0.1.4" - use-sync-external-store "^1.0.0" - "@react-spring/animated@~9.7.3": version "9.7.3" resolved "https://registry.yarnpkg.com/@react-spring/animated/-/animated-9.7.3.tgz#4211b1a6d48da0ff474a125e93c0f460ff816e0f" @@ -3194,11 +3186,6 @@ resolved "https://registry.yarnpkg.com/@rollup/rollup-win32-x64-msvc/-/rollup-win32-x64-msvc-4.18.0.tgz#5d694d345ce36b6ecf657349e03eb87297e68da4" integrity sha512-UOo5FdvOL0+eIVTgS4tIdbW+TtnBLWg1YBCcU2KWM7nuNwRz9bksDX1bekJJCpu25N1DVWaCwnT39dVQxzqS8g== -"@rx-state/core@0.1.4": - version "0.1.4" - resolved "https://registry.yarnpkg.com/@rx-state/core/-/core-0.1.4.tgz#586dde80be9dbdac31844006a0dcaa2bc7f35a5c" - integrity sha512-Z+3hjU2xh1HisLxt+W5hlYX/eGSDaXXP+ns82gq/PLZpkXLu0uwcNUh9RLY3Clq4zT+hSsA3vcpIGt6+UAb8rQ== - "@sentry-internal/browser-utils@8.13.0": version "8.13.0" resolved "https://registry.yarnpkg.com/@sentry-internal/browser-utils/-/browser-utils-8.13.0.tgz#b7c3bdd49d2382f60dde31745716d29dd419b6ba" @@ -8988,11 +8975,6 @@ use-sidecar@^1.1.2: detect-node-es "^1.1.0" tslib "^2.0.0" -use-sync-external-store@^1.0.0: - version "1.2.0" - resolved "https://registry.yarnpkg.com/use-sync-external-store/-/use-sync-external-store-1.2.0.tgz#7dbefd6ef3fe4e767a0cf5d7287aacfb5846928a" - integrity sha512-eEgnFxGQ1Ife9bzYs6VLi8/4X6CObHMw9Qr9tPY43iKwsPw8xE8+EFsf/2cFZ5S3esXgpWgtSCtLNS41F+sKPA== - usehooks-ts@2.16.0: version "2.16.0" resolved "https://registry.yarnpkg.com/usehooks-ts/-/usehooks-ts-2.16.0.tgz#31deaa2f1147f65666aae925bd890b54e63b0d3f" From 34c45cb5e244052ac8857e346be38ac697c950dd Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 21 May 2024 17:05:37 -0400 Subject: [PATCH 11/31] Get the right grid offset even when offsetParent is a layout element --- src/grid/Grid.tsx | 31 ++++++++++++++++++++++++++----- 1 file changed, 26 insertions(+), 5 deletions(-) diff --git a/src/grid/Grid.tsx b/src/grid/Grid.tsx index 9aa7f95f..b6cf8fcd 100644 --- a/src/grid/Grid.tsx +++ b/src/grid/Grid.tsx @@ -91,6 +91,28 @@ export const Slot: FC = ({ tile, style, className, ...props }) => ( /> ); +interface Offset { + x: number; + y: number; +} + +/** + * Gets the offset of one element relative to an ancestor. + */ +function offset(element: HTMLElement, relativeTo: Element): Offset { + if ( + !(element.offsetParent instanceof HTMLElement) || + element.offsetParent === relativeTo + ) { + return { x: element.offsetLeft, y: element.offsetTop }; + } else { + const o = offset(element.offsetParent, relativeTo); + o.x += element.offsetLeft; + o.y += element.offsetTop; + return o; + } +} + export interface LayoutProps { ref: LegacyRef; model: Model; @@ -228,24 +250,23 @@ export function Grid< const slotRects = useMemo(() => { const rects = new Map(); - if (layoutRoot !== null) { + if (gridRoot !== null && layoutRoot !== null) { const slots = layoutRoot.getElementsByClassName( styles.slot, ) as HTMLCollectionOf; for (const slot of slots) rects.set(slot.getAttribute("data-tile")!, { - x: slot.offsetLeft, - y: slot.offsetTop, + ...offset(slot, gridRoot), width: slot.offsetWidth, height: slot.offsetHeight, }); } return rects; - // The rects may change due to the grid being resized or rerendered, but + // The rects may change due to the grid updating to a new generation, but // eslint can't statically verify this // eslint-disable-next-line react-hooks/exhaustive-deps - }, [layoutRoot, generation]); + }, [gridRoot, layoutRoot, generation]); const tileModels = useMemo( () => getTileModels(model), From ffbbc74a9654f8703a502760abd7db9b859e3e68 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 17 May 2024 16:38:00 -0400 Subject: [PATCH 12/31] Implement the new spotlight layout --- src/grid/CallLayout.ts | 55 ++++++++++++++++++ src/grid/GridLayout.module.css | 6 +- src/grid/GridLayout.tsx | 25 +++----- src/grid/SpotlightLayout.module.css | 89 +++++++++++++++++++++++++++++ src/grid/SpotlightLayout.tsx | 89 +++++++++++++++++++++++++++++ src/room/InCallView.module.css | 4 +- src/room/InCallView.tsx | 76 ++++++++++++++---------- src/state/CallViewModel.ts | 47 +++++++++++---- src/tile/GridTile.tsx | 11 ++-- 9 files changed, 337 insertions(+), 65 deletions(-) create mode 100644 src/grid/CallLayout.ts create mode 100644 src/grid/SpotlightLayout.module.css create mode 100644 src/grid/SpotlightLayout.tsx diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts new file mode 100644 index 00000000..287f116d --- /dev/null +++ b/src/grid/CallLayout.ts @@ -0,0 +1,55 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { BehaviorSubject, Observable } from "rxjs"; + +import { MediaViewModel } from "../state/MediaViewModel"; +import { LayoutSystem } from "./Grid"; +import { Alignment } from "../room/InCallView"; + +export interface Bounds { + width: number; + height: number; +} + +export interface CallLayoutInputs { + /** + * The minimum bounds of the layout area. + */ + minBounds: Observable; + /** + * The alignment of the floating tile, if any. + */ + floatingAlignment: BehaviorSubject; +} + +export interface CallLayoutOutputs { + /** + * The visually fixed (non-scrolling) layer of the layout. + */ + fixed: LayoutSystem; + /** + * The layer of the layout that can overflow and be scrolled. + */ + scrolling: LayoutSystem; +} + +/** + * A layout system for media tiles. + */ +export type CallLayout = ( + inputs: CallLayoutInputs, +) => CallLayoutOutputs; diff --git a/src/grid/GridLayout.module.css b/src/grid/GridLayout.module.css index ef234b33..084b56bd 100644 --- a/src/grid/GridLayout.module.css +++ b/src/grid/GridLayout.module.css @@ -14,6 +14,11 @@ See the License for the specific language governing permissions and limitations under the License. */ +.fixed, +.scrolling { + margin-inline: var(--inline-content-inset); +} + .scrolling { box-sizing: border-box; block-size: 100%; @@ -22,7 +27,6 @@ limitations under the License. justify-content: center; align-content: center; gap: var(--gap); - box-sizing: border-box; } .scrolling > .slot { diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 6b673e64..75f1e726 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -15,21 +15,15 @@ limitations under the License. */ import { CSSProperties, forwardRef, useMemo } from "react"; -import { BehaviorSubject, Observable, distinctUntilChanged } from "rxjs"; +import { distinctUntilChanged } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import { GridLayout as GridLayoutModel } from "../state/CallViewModel"; -import { MediaViewModel } from "../state/MediaViewModel"; -import { LayoutSystem, Slot } from "./Grid"; +import { Slot } from "./Grid"; import styles from "./GridLayout.module.css"; import { useReactiveState } from "../useReactiveState"; -import { Alignment } from "../room/InCallView"; import { useInitial } from "../useInitial"; - -export interface Bounds { - width: number; - height: number; -} +import { CallLayout } from "./CallLayout"; interface GridCSSProperties extends CSSProperties { "--gap": string; @@ -37,19 +31,14 @@ interface GridCSSProperties extends CSSProperties { "--height": string; } -interface GridLayoutSystems { - scrolling: LayoutSystem; - fixed: LayoutSystem; -} - const slotMinHeight = 130; const slotMaxAspectRatio = 17 / 9; const slotMinAspectRatio = 4 / 3; -export const gridLayoutSystems = ( - minBounds: Observable, - floatingAlignment: BehaviorSubject, -): GridLayoutSystems => ({ +export const makeGridLayout: CallLayout = ({ + minBounds, + floatingAlignment, +}) => ({ // The "fixed" (non-scrolling) part of the layout is where the spotlight tile // lives fixed: { diff --git a/src/grid/SpotlightLayout.module.css b/src/grid/SpotlightLayout.module.css new file mode 100644 index 00000000..bbce45cf --- /dev/null +++ b/src/grid/SpotlightLayout.module.css @@ -0,0 +1,89 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.fixed, +.scrolling { + margin-inline: var(--inline-content-inset); + display: grid; + --grid-slot-width: 180px; + --grid-gap: 20px; + grid-template-columns: 1fr calc( + var(--grid-columns) * var(--grid-slot-width) + (var(--grid-columns) - 1) * + var(--grid-gap) + ); + grid-template-rows: minmax(1fr, auto); + gap: 30px; +} + +.scrolling { + block-size: 100%; +} + +.spotlight { + container: spotlight / size; + display: grid; + place-items: center; +} + +/* CSS makes us put a condition here, even though all we want to do is +unconditionally select the container so we can use cq units */ +@container spotlight (width > 0) { + .spotlight > .slot { + inline-size: min(100cqi, 100cqb * (17 / 9)); + block-size: min(100cqb, 100cqi / (4 / 3)); + } +} + +.grid { + display: flex; + flex-wrap: wrap; + gap: var(--grid-gap); + justify-content: center; + align-content: center; +} + +.grid > .slot { + inline-size: var(--grid-slot-width); + block-size: 135px; +} + +@media (max-width: 600px) { + .fixed, + .scrolling { + margin-inline: 0; + display: block; + } + + .spotlight { + inline-size: 100%; + aspect-ratio: 16 / 9; + margin-block-end: var(--cpd-space-4x); + } + + .grid { + margin-inline: var(--inline-content-inset); + align-content: start; + } + + .grid > .slot { + --grid-columns: 2; + --grid-slot-width: calc( + (100% - (var(--grid-columns) - 1) * var(--grid-gap)) / var(--grid-columns) + ); + block-size: unset; + aspect-ratio: 4 / 3; + } +} diff --git a/src/grid/SpotlightLayout.tsx b/src/grid/SpotlightLayout.tsx new file mode 100644 index 00000000..38bc6e37 --- /dev/null +++ b/src/grid/SpotlightLayout.tsx @@ -0,0 +1,89 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { CSSProperties, forwardRef } from "react"; +import { useObservableEagerState } from "observable-hooks"; + +import { CallLayout } from "./CallLayout"; +import { SpotlightLayout as SpotlightLayoutModel } from "../state/CallViewModel"; +import { useReactiveState } from "../useReactiveState"; +import styles from "./SpotlightLayout.module.css"; +import { Slot } from "./Grid"; + +interface GridCSSProperties extends CSSProperties { + "--grid-columns": number; +} + +const getGridColumns = (gridLength: number): number => + gridLength > 20 ? 2 : 1; + +export const makeSpotlightLayout: CallLayout = ({ + minBounds, +}) => ({ + fixed: { + tiles: (model) => new Map([["spotlight", model.spotlight]]), + Layout: forwardRef(function SpotlightLayoutFixed({ model }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const gridColumns = getGridColumns(model.grid.length); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid.length, width, height], + ); + + return ( +
+
+ +
+
+
+ ); + }), + }, + + scrolling: { + tiles: (model) => new Map(model.grid.map((tile) => [tile.id, tile])), + Layout: forwardRef(function SpotlightLayoutScrolling({ model }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const gridColumns = getGridColumns(model.grid.length); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid, width, height], + ); + + return ( +
+
+
+ {model.grid.map((tile) => ( + + ))} +
+
+ ); + }), + }, +}); diff --git a/src/room/InCallView.module.css b/src/room/InCallView.module.css index f53ba025..76dc9bae 100644 --- a/src/room/InCallView.module.css +++ b/src/room/InCallView.module.css @@ -125,7 +125,7 @@ limitations under the License. .fixedGrid { position: absolute; - inline-size: calc(100% - 2 * var(--inline-content-inset)); + inline-size: 100%; align-self: center; /* Disable pointer events so the overlay doesn't block interaction with elements behind it */ @@ -139,6 +139,6 @@ limitations under the License. .scrollingGrid { position: relative; flex-grow: 1; - inline-size: calc(100% - 2 * var(--inline-content-inset)); + inline-size: 100%; align-self: center; } diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index dba01e9d..b2aeb9d2 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -35,7 +35,7 @@ import { import useMeasure from "react-use-measure"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import classNames from "classnames"; -import { BehaviorSubject } from "rxjs"; +import { BehaviorSubject, map } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import { useTranslation } from "react-i18next"; @@ -73,17 +73,20 @@ import { ECConnectionState } from "../livekit/useECConnectionState"; import { useOpenIDSFU } from "../livekit/openIDSFU"; import { GridMode, + Layout, TileDescriptor, useCallViewModel, } from "../state/CallViewModel"; import { Grid, TileProps } from "../grid/Grid"; import { MediaViewModel } from "../state/MediaViewModel"; -import { gridLayoutSystems } from "../grid/GridLayout"; import { useObservable } from "../state/useObservable"; import { useInitial } from "../useInitial"; import { SpotlightTile } from "../tile/SpotlightTile"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; +import { makeGridLayout } from "../grid/GridLayout"; +import { makeSpotlightLayout } from "../grid/SpotlightLayout"; +import { CallLayout } from "../grid/CallLayout"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -302,6 +305,7 @@ export const InCallView: FC = ({ const [headerRef, headerBounds] = useMeasure(); const [footerRef, footerBounds] = useMeasure(); + const gridBounds = useMemo( () => ({ width: footerBounds.width, @@ -315,11 +319,29 @@ export const InCallView: FC = ({ ], ); const gridBoundsObservable = useObservable(gridBounds); + const floatingAlignment = useInitial( () => new BehaviorSubject(defaultAlignment), ); - const { fixed, scrolling } = useInitial(() => - gridLayoutSystems(gridBoundsObservable, floatingAlignment), + + const layoutSystem = useObservableEagerState( + useInitial(() => + vm.layout.pipe( + map((l) => { + let makeLayout: CallLayout; + if (l.type === "grid" && l.grid.length !== 2) + makeLayout = makeGridLayout as CallLayout; + else if (l.type === "spotlight") + makeLayout = makeSpotlightLayout as CallLayout; + else return null; // Not yet implemented + + return makeLayout({ + minBounds: gridBoundsObservable, + floatingAlignment, + }); + }), + ), + ), ); const setGridMode = useCallback( @@ -423,31 +445,9 @@ export const InCallView: FC = ({ ); } - // The only new layout we've implemented so far is grid layout for non-1:1 - // calls. All other layouts use the legacy grid system for now. - if ( - legacyLayout === "grid" && - layout.type === "grid" && - !(layout.grid.length === 2 && layout.spotlight === undefined) - ) { - return ( - <> - - - - ); - } else { + if (layoutSystem === null) { + // This new layout doesn't yet have an implemented layout system, so fall + // back to the legacy grid system return ( = ({ Tile={GridTileView} /> ); + } else { + return ( + <> + + + + ); } }; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index b0816dc2..cc5afc46 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -28,12 +28,13 @@ import { import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix"; import { useEffect, useRef } from "react"; import { - BehaviorSubject, EMPTY, Observable, + Subject, audit, combineLatest, concat, + concatMap, distinctUntilChanged, filter, map, @@ -48,6 +49,7 @@ import { switchMap, throttleTime, timer, + withLatestFrom, zip, } from "rxjs"; import { logger } from "matrix-js-sdk/src/logger"; @@ -371,6 +373,13 @@ export class CallViewModel extends ViewModel { private readonly screenShares: Observable = this.mediaItems.pipe( map((ms) => ms.filter((m): m is ScreenShare => m instanceof ScreenShare)), + shareReplay(1), + ); + + private readonly hasScreenShares: Observable = + this.screenShares.pipe( + map((ms) => ms.length > 0), + distinctUntilChanged(), ); private readonly spotlightSpeaker: Observable = @@ -385,11 +394,13 @@ export class CallViewModel extends ViewModel { scan<(readonly [UserMedia, boolean])[], UserMedia | null, null>( (prev, ms) => // Decide who to spotlight: - // If the previous speaker is still speaking, stick with them rather - // than switching eagerly to someone else - ms.find(([m, s]) => m === prev && s)?.[0] ?? - // Otherwise, select anyone who is speaking - ms.find(([, s]) => s)?.[0] ?? + // If the previous speaker (not the local user) is still speaking, + // stick with them rather than switching eagerly to someone else + (prev === null || prev.vm.local + ? null + : ms.find(([m, s]) => m === prev && s)?.[0]) ?? + // Otherwise, select any remote user who is speaking + ms.find(([m, s]) => !m.vm.local && s)?.[0] ?? // Otherwise, stick with the person who was last speaking prev ?? // Otherwise, spotlight the local user @@ -398,7 +409,8 @@ export class CallViewModel extends ViewModel { null, ), distinctUntilChanged(), - throttleTime(800, undefined, { leading: true, trailing: true }), + shareReplay(1), + throttleTime(1600, undefined, { leading: true, trailing: true }), ); private readonly grid: Observable = this.userMedia.pipe( @@ -453,18 +465,31 @@ export class CallViewModel extends ViewModel { // orientation private readonly windowMode = of("normal"); - private readonly _gridMode = new BehaviorSubject("grid"); + private readonly gridModeUserSelection = new Subject(); /** * The layout mode of the media tile grid. */ - public readonly gridMode: Observable = this._gridMode; + public readonly gridMode: Observable = merge( + // Always honor a manual user selection + this.gridModeUserSelection, + // If the user hasn't selected spotlight and somebody starts screen sharing, + // automatically switch to spotlight mode and reset when screen sharing ends + this.hasScreenShares.pipe( + withLatestFrom(this.gridModeUserSelection.pipe(startWith(null))), + concatMap(([hasScreenShares, userSelection]) => + userSelection === "spotlight" + ? EMPTY + : of(hasScreenShares ? "spotlight" : "grid"), + ), + ), + ).pipe(distinctUntilChanged(), shareReplay(1)); public setGridMode(value: GridMode): void { - this._gridMode.next(value); + this.gridModeUserSelection.next(value); } public readonly layout: Observable = combineLatest( - [this._gridMode, this.windowMode], + [this.gridMode, this.windowMode], (gridMode, windowMode) => { switch (windowMode) { case "full screen": diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index ba615953..14f85831 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -71,6 +71,7 @@ interface MediaTileProps vm: MediaViewModel; videoEnabled: boolean; videoFit: "contain" | "cover"; + mirror: boolean; nameTagLeadingIcon?: ReactNode; primaryButton: ReactNode; secondaryButton?: ReactNode; @@ -87,7 +88,6 @@ const MediaTile = forwardRef( className={classNames(className, styles.tile)} data-maximised={maximised} video={video} - mirror={false} member={vm.member} unencryptedWarning={unencryptedWarning} {...props} @@ -100,6 +100,7 @@ MediaTile.displayName = "MediaTile"; interface UserMediaTileProps extends TileProps { vm: UserMediaViewModel; + mirror: boolean; showSpeakingIndicators: boolean; menuStart?: ReactNode; menuEnd?: ReactNode; @@ -202,7 +203,7 @@ interface LocalUserMediaTileProps extends TileProps { } const LocalUserMediaTile = forwardRef( - ({ vm, onOpenProfile, className, ...props }, ref) => { + ({ vm, onOpenProfile, ...props }, ref) => { const { t } = useTranslation(); const mirror = useObservableEagerState(vm.mirror); const alwaysShow = useObservableEagerState(vm.alwaysShow); @@ -220,6 +221,7 @@ const LocalUserMediaTile = forwardRef( ( onSelect={onOpenProfile} /> } - className={classNames(className, { [styles.mirror]: mirror })} {...props} /> ); @@ -270,6 +271,7 @@ const RemoteUserMediaTile = forwardRef< ( ( ) } - videoEnabled {...props} /> ); From 54c22f4ab28c987a994ba901045179e4f3807883 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 28 May 2024 13:57:23 -0400 Subject: [PATCH 13/31] Clean up spotlight tile code --- src/tile/SpotlightTile.tsx | 120 +++++++++++++++++++++++++------------ 1 file changed, 83 insertions(+), 37 deletions(-) diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index 77a3526f..fccc5235 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -16,6 +16,7 @@ limitations under the License. import { ComponentProps, + RefAttributes, forwardRef, useCallback, useEffect, @@ -28,17 +29,20 @@ import CollapseIcon from "@vector-im/compound-design-tokens/icons/collapse.svg?r import ChevronLeftIcon from "@vector-im/compound-design-tokens/icons/chevron-left.svg?react"; import ChevronRightIcon from "@vector-im/compound-design-tokens/icons/chevron-right.svg?react"; import { animated } from "@react-spring/web"; -import { Observable, map, of } from "rxjs"; +import { Observable, map } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import { useTranslation } from "react-i18next"; import classNames from "classnames"; +import { TrackReferenceOrPlaceholder } from "@livekit/components-core"; +import { RoomMember } from "matrix-js-sdk"; import { MediaView } from "./MediaView"; import styles from "./SpotlightTile.module.css"; import { LocalUserMediaViewModel, MediaViewModel, - RemoteUserMediaViewModel, + ScreenShareViewModel, + UserMediaViewModel, useNameData, } from "../state/MediaViewModel"; import { useInitial } from "../useInitial"; @@ -47,12 +51,63 @@ import { useObservableRef } from "../state/useObservable"; import { useReactiveState } from "../useReactiveState"; import { useLatest } from "../useLatest"; -// Screen share video is always enabled -const videoEnabledDefault = of(true); -// Never mirror screen share video -const mirrorDefault = of(false); -// Never crop screen share video -const cropVideoDefault = of(false); +interface SpotlightItemBaseProps { + className?: string; + "data-id": string; + targetWidth: number; + targetHeight: number; + video: TrackReferenceOrPlaceholder; + member: RoomMember | undefined; + unencryptedWarning: boolean; + nameTag: string; + displayName: string; +} + +interface SpotlightUserMediaItemBaseProps extends SpotlightItemBaseProps { + videoEnabled: boolean; + videoFit: "contain" | "cover"; +} + +interface SpotlightLocalUserMediaItemProps + extends SpotlightUserMediaItemBaseProps { + vm: LocalUserMediaViewModel; +} + +const SpotlightLocalUserMediaItem = forwardRef< + HTMLDivElement, + SpotlightLocalUserMediaItemProps +>(({ vm, ...props }, ref) => { + const mirror = useObservableEagerState(vm.mirror); + return ; +}); + +SpotlightLocalUserMediaItem.displayName = "SpotlightLocalUserMediaItem"; + +interface SpotlightUserMediaItemProps extends SpotlightItemBaseProps { + vm: UserMediaViewModel; +} + +const SpotlightUserMediaItem = forwardRef< + HTMLDivElement, + SpotlightUserMediaItemProps +>(({ vm, ...props }, ref) => { + const videoEnabled = useObservableEagerState(vm.videoEnabled); + const cropVideo = useObservableEagerState(vm.cropVideo); + + const baseProps: SpotlightUserMediaItemBaseProps = { + videoEnabled, + videoFit: cropVideo ? "cover" : "contain", + ...props, + }; + + return vm instanceof LocalUserMediaViewModel ? ( + + ) : ( + + ); +}); + +SpotlightUserMediaItem.displayName = "SpotlightUserMediaItem"; interface SpotlightItemProps { vm: MediaViewModel; @@ -71,21 +126,6 @@ const SpotlightItem = forwardRef( const ref = useMergedRefs(ourRef, theirRef); const { displayName, nameTag } = useNameData(vm); const video = useObservableEagerState(vm.video); - const videoEnabled = useObservableEagerState( - vm instanceof LocalUserMediaViewModel || - vm instanceof RemoteUserMediaViewModel - ? vm.videoEnabled - : videoEnabledDefault, - ); - const mirror = useObservableEagerState( - vm instanceof LocalUserMediaViewModel ? vm.mirror : mirrorDefault, - ); - const cropVideo = useObservableEagerState( - vm instanceof LocalUserMediaViewModel || - vm instanceof RemoteUserMediaViewModel - ? vm.cropVideo - : cropVideoDefault, - ); const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); // Hook this item up to the intersection observer @@ -103,22 +143,28 @@ const SpotlightItem = forwardRef( }; }, [intersectionObserver]); - return ( + const baseProps: SpotlightItemBaseProps & RefAttributes = { + ref, + "data-id": vm.id, + className: classNames(styles.item, { [styles.snap]: snap }), + targetWidth, + targetHeight, + video, + member: vm.member, + unencryptedWarning, + nameTag, + displayName, + }; + + return vm instanceof ScreenShareViewModel ? ( + ) : ( + ); }, ); From ec1b020d4eaf5eb6adac96802f1208b8d381979d Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 30 May 2024 13:06:24 -0400 Subject: [PATCH 14/31] Add indicators to spotlight tile and make spotlight layout responsive --- src/grid/CallLayout.ts | 20 ++- src/grid/Grid.tsx | 152 ++++++++--------- src/grid/GridLayout.module.css | 4 + src/grid/GridLayout.tsx | 245 ++++++++++++++-------------- src/grid/SpotlightLayout.module.css | 76 +++++---- src/grid/SpotlightLayout.tsx | 143 ++++++++++------ src/room/InCallView.module.css | 10 ++ src/room/InCallView.tsx | 143 +++++++++------- src/tile/GridTile.module.css | 4 - src/tile/SpotlightTile.module.css | 43 ++++- src/tile/SpotlightTile.tsx | 11 ++ 11 files changed, 495 insertions(+), 356 deletions(-) diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts index 287f116d..c4412677 100644 --- a/src/grid/CallLayout.ts +++ b/src/grid/CallLayout.ts @@ -15,9 +15,10 @@ limitations under the License. */ import { BehaviorSubject, Observable } from "rxjs"; +import { ComponentType } from "react"; import { MediaViewModel } from "../state/MediaViewModel"; -import { LayoutSystem } from "./Grid"; +import { LayoutProps } from "./Grid"; import { Alignment } from "../room/InCallView"; export interface Bounds { @@ -36,15 +37,28 @@ export interface CallLayoutInputs { floatingAlignment: BehaviorSubject; } +export interface GridTileModel { + type: "grid"; + vm: MediaViewModel; +} + +export interface SpotlightTileModel { + type: "spotlight"; + vms: MediaViewModel[]; + maximised: boolean; +} + +export type TileModel = GridTileModel | SpotlightTileModel; + export interface CallLayoutOutputs { /** * The visually fixed (non-scrolling) layer of the layout. */ - fixed: LayoutSystem; + fixed: ComponentType>; /** * The layer of the layout that can overflow and be scrolled. */ - scrolling: LayoutSystem; + scrolling: ComponentType>; } /** diff --git a/src/grid/Grid.tsx b/src/grid/Grid.tsx index b6cf8fcd..2e6a48ae 100644 --- a/src/grid/Grid.tsx +++ b/src/grid/Grid.tsx @@ -42,6 +42,7 @@ import { useMergedRefs } from "../useMergedRefs"; import { TileWrapper } from "./TileWrapper"; import { usePrefersReducedMotion } from "../usePrefersReducedMotion"; import { TileSpringUpdate } from "./LegacyGrid"; +import { useInitial } from "../useInitial"; interface Rect { x: number; @@ -50,11 +51,14 @@ interface Rect { height: number; } -interface Tile extends Rect { +interface Tile { id: string; model: Model; + onDrag: DragCallback | undefined; } +type PlacedTile = Tile & Rect; + interface TileSpring { opacity: number; scale: number; @@ -73,24 +77,14 @@ interface DragState { cursorY: number; } -interface SlotProps extends ComponentProps<"div"> { - tile: string; +interface SlotProps extends Omit, "onDrag"> { + id: string; + model: Model; + onDrag?: DragCallback; style?: CSSProperties; className?: string; } -/** - * An invisible "slot" for a tile to go in. - */ -export const Slot: FC = ({ tile, style, className, ...props }) => ( -
-); - interface Offset { x: number; y: number; @@ -113,9 +107,13 @@ function offset(element: HTMLElement, relativeTo: Element): Offset { } } -export interface LayoutProps { +export interface LayoutProps { ref: LegacyRef; - model: Model; + model: LayoutModel; + /** + * Component creating an invisible "slot" for a tile to go in. + */ + Slot: ComponentType>; } export interface TileProps { @@ -152,25 +150,7 @@ interface Drag { yRatio: number; } -type DragCallback = (drag: Drag) => void; - -export interface LayoutSystem { - /** - * Defines the ID and model of each tile present in the layout. - */ - tiles: (model: LayoutModel) => Map; - /** - * A component which creates an invisible layout grid of "slots" for tiles to - * go in. The root element must have a data-generation attribute which - * increments whenever the layout may have changed. - */ - Layout: ComponentType>; - /** - * Gets a drag callback for the tile with the given ID. If this is not - * provided or it returns null, the tile is not draggable. - */ - onDrag?: (model: LayoutModel, tile: string) => DragCallback | null; -} +export type DragCallback = (drag: Drag) => void; interface Props< LayoutModel, @@ -183,9 +163,11 @@ interface Props< */ model: LayoutModel; /** - * The system by which to arrange the layout and respond to interactions. + * A component which creates an invisible layout grid of "slots" for tiles to + * go in. The root element must have a data-generation attribute which + * increments whenever the layout may have changed. */ - system: LayoutSystem; + Layout: ComponentType>; /** * The component used to render each tile in the layout. */ @@ -204,7 +186,7 @@ export function Grid< TileRef extends HTMLElement, >({ model, - system: { tiles: getTileModels, Layout, onDrag }, + Layout, Tile, className, style, @@ -223,8 +205,31 @@ export function Grid< const [layoutRoot, setLayoutRoot] = useState(null); const [generation, setGeneration] = useState(null); + const tiles = useInitial(() => new Map>()); const prefersReducedMotion = usePrefersReducedMotion(); + const Slot: FC> = useMemo( + () => + function Slot({ id, model, onDrag, style, className, ...props }) { + const ref = useRef(null); + useEffect(() => { + tiles.set(id, { id, model, onDrag }); + return (): void => void tiles.delete(id); + }, [id, model, onDrag]); + + return ( +
+ ); + }, + [tiles], + ); + const layoutRef = useCallback( (e: HTMLElement | null) => { setLayoutRoot(e); @@ -247,62 +252,45 @@ export function Grid< } }, [layoutRoot, setGeneration]); - const slotRects = useMemo(() => { - const rects = new Map(); + // Combine the tile definitions and slots together to create placed tiles + const placedTiles = useMemo(() => { + const result: PlacedTile[] = []; if (gridRoot !== null && layoutRoot !== null) { const slots = layoutRoot.getElementsByClassName( styles.slot, ) as HTMLCollectionOf; - for (const slot of slots) - rects.set(slot.getAttribute("data-tile")!, { + for (const slot of slots) { + const id = slot.getAttribute("data-id")!; + result.push({ + ...tiles.get(id)!, ...offset(slot, gridRoot), width: slot.offsetWidth, height: slot.offsetHeight, }); + } } - return rects; + return result; // The rects may change due to the grid updating to a new generation, but // eslint can't statically verify this // eslint-disable-next-line react-hooks/exhaustive-deps - }, [gridRoot, layoutRoot, generation]); - - const tileModels = useMemo( - () => getTileModels(model), - [getTileModels, model], - ); - - // Combine the tile models and slots together to create placed tiles - const tiles = useMemo[]>(() => { - const items: Tile[] = []; - for (const [id, model] of tileModels) { - const rect = slotRects.get(id); - if (rect !== undefined) items.push({ id, model, ...rect }); - } - return items; - }, [slotRects, tileModels]); - - const dragCallbacks = useMemo( - () => - new Map( - (function* (): Iterable<[string, DragCallback | null]> { - if (onDrag !== undefined) - for (const id of tileModels.keys()) yield [id, onDrag(model, id)]; - })(), - ), - [onDrag, tileModels, model], - ); + }, [gridRoot, layoutRoot, tiles, generation]); // Drag state is stored in a ref rather than component state, because we use // react-spring's imperative API during gestures to improve responsiveness const dragState = useRef(null); const [tileTransitions, springRef] = useTransition( - tiles, + placedTiles, () => ({ key: ({ id }: Tile): string => id, - from: ({ x, y, width, height }: Tile): TileSpringUpdate => ({ + from: ({ + x, + y, + width, + height, + }: PlacedTile): TileSpringUpdate => ({ opacity: 0, scale: 0, zIndex: 1, @@ -319,7 +307,7 @@ export function Grid< y, width, height, - }: Tile): TileSpringUpdate | null => + }: PlacedTile): TileSpringUpdate | null => id === dragState.current?.tileId ? null : { @@ -334,7 +322,7 @@ export function Grid< }), // react-spring's types are bugged and can't infer the spring type ) as unknown as [ - TransitionFn, TileSpring>, + TransitionFn, TileSpring>, SpringRef, ]; @@ -342,14 +330,14 @@ export function Grid< // firing animations manually whenever the tiles array updates useEffect(() => { springRef.start(); - }, [tiles, springRef]); + }, [placedTiles, springRef]); const animateDraggedTile = ( endOfGesture: boolean, callback: DragCallback, ): void => { const { tileId, tileX, tileY } = dragState.current!; - const tile = tiles.find((t) => t.id === tileId)!; + const tile = placedTiles.find((t) => t.id === tileId)!; springRef.current .find((c) => (c.item as Tile).id === tileId) @@ -416,7 +404,7 @@ export function Grid< const tileController = springRef.current.find( (c) => (c.item as Tile).id === tileId, )!; - const callback = dragCallbacks.get(tileController.item.id); + const callback = tiles.get(tileController.item.id)!.onDrag; if (callback != null) { if (dragState.current === null) { @@ -456,7 +444,7 @@ export function Grid< if (dragState.current !== null) { dragState.current.tileY += dy; dragState.current.cursorY += dy; - animateDraggedTile(false, onDrag!(model, dragState.current.tileId)!); + animateDraggedTile(false, tiles.get(dragState.current.tileId)!.onDrag!); } }, { target: gridRoot ?? undefined }, @@ -468,12 +456,12 @@ export function Grid< className={classNames(className, styles.grid)} style={style} > - - {tileTransitions((spring, { id, model, width, height }) => ( + + {tileTransitions((spring, { id, model, onDrag, width, height }) => ( .slot { position: absolute; inline-size: 404px; diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 75f1e726..3861457e 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -14,16 +14,16 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { CSSProperties, forwardRef, useMemo } from "react"; +import { CSSProperties, forwardRef, useCallback, useMemo } from "react"; import { distinctUntilChanged } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import { GridLayout as GridLayoutModel } from "../state/CallViewModel"; -import { Slot } from "./Grid"; import styles from "./GridLayout.module.css"; import { useReactiveState } from "../useReactiveState"; import { useInitial } from "../useInitial"; -import { CallLayout } from "./CallLayout"; +import { CallLayout, GridTileModel, TileModel } from "./CallLayout"; +import { DragCallback } from "./Grid"; interface GridCSSProperties extends CSSProperties { "--gap": string; @@ -41,135 +41,144 @@ export const makeGridLayout: CallLayout = ({ }) => ({ // The "fixed" (non-scrolling) part of the layout is where the spotlight tile // lives - fixed: { - tiles: (model) => - new Map( - model.spotlight === undefined ? [] : [["spotlight", model.spotlight]], - ), - Layout: forwardRef(function GridLayoutFixed({ model }, ref) { - const { width, height } = useObservableEagerState(minBounds); - const alignment = useObservableEagerState( - useInitial(() => - floatingAlignment.pipe( - distinctUntilChanged( - (a1, a2) => a1.block === a2.block && a1.inline === a2.inline, - ), + fixed: forwardRef(function GridLayoutFixed({ model, Slot }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const alignment = useObservableEagerState( + useInitial(() => + floatingAlignment.pipe( + distinctUntilChanged( + (a1, a2) => a1.block === a2.block && a1.inline === a2.inline, ), ), - ); - const [generation] = useReactiveState( - (prev) => (prev === undefined ? 0 : prev + 1), - [model.spotlight === undefined, width, height, alignment], - ); - - return ( -
- {model.spotlight && ( - - )} -
- ); - }), - onDrag: + ), + ); + const tileModel: TileModel | undefined = useMemo( () => + model.spotlight && { + type: "spotlight", + vms: model.spotlight, + maximised: false, + }, + [model.spotlight], + ); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.spotlight === undefined, width, height, alignment], + ); + + const onDragSpotlight: DragCallback = useCallback( ({ xRatio, yRatio }) => floatingAlignment.next({ block: yRatio < 0.5 ? "start" : "end", inline: xRatio < 0.5 ? "start" : "end", }), - }, + [], + ); + + return ( +
+ {tileModel && ( + + )} +
+ ); + }), // The scrolling part of the layout is where all the grid tiles live - scrolling: { - tiles: (model) => new Map(model.grid.map((tile) => [tile.id, tile])), - Layout: forwardRef(function GridLayout({ model }, ref) { - const { width, height: minHeight } = useObservableEagerState(minBounds); + scrolling: forwardRef(function GridLayout({ model, Slot }, ref) { + const { width, height: minHeight } = useObservableEagerState(minBounds); - // The goal here is to determine the grid size and padding that maximizes - // use of screen space for n tiles without making those tiles too small or - // too cropped (having an extreme aspect ratio) - const [gap, slotWidth, slotHeight] = useMemo(() => { - const gap = width < 800 ? 16 : 20; - const slotMinWidth = width < 500 ? 150 : 180; + // The goal here is to determine the grid size and padding that maximizes + // use of screen space for n tiles without making those tiles too small or + // too cropped (having an extreme aspect ratio) + const [gap, slotWidth, slotHeight] = useMemo(() => { + const gap = width < 800 ? 16 : 20; + const slotMinWidth = width < 500 ? 150 : 180; - let columns = Math.min( - // Don't create more columns than we have items for - model.grid.length, - // The ideal number of columns is given by a packing of equally-sized - // squares into a grid. - // width / column = height / row. - // columns * rows = number of squares. - // ∴ columns = sqrt(width / height * number of squares). - // Except we actually want 16:9-ish slots rather than squares, so we - // divide the width-to-height ratio by the target aspect ratio. - Math.ceil( - Math.sqrt( - (width / minHeight / slotMaxAspectRatio) * model.grid.length, - ), + let columns = Math.min( + // Don't create more columns than we have items for + model.grid.length, + // The ideal number of columns is given by a packing of equally-sized + // squares into a grid. + // width / column = height / row. + // columns * rows = number of squares. + // ∴ columns = sqrt(width / height * number of squares). + // Except we actually want 16:9-ish slots rather than squares, so we + // divide the width-to-height ratio by the target aspect ratio. + Math.ceil( + Math.sqrt( + (width / minHeight / slotMaxAspectRatio) * model.grid.length, ), - ); - let rows = Math.ceil(model.grid.length / columns); + ), + ); + let rows = Math.ceil(model.grid.length / columns); - let slotWidth = (width - (columns - 1) * gap) / columns; - let slotHeight = (minHeight - (rows - 1) * gap) / rows; + let slotWidth = (width - (columns - 1) * gap) / columns; + let slotHeight = (minHeight - (rows - 1) * gap) / rows; - // Impose a minimum width and height on the slots - if (slotWidth < slotMinWidth) { - // In this case we want the slot width to determine the number of columns, - // not the other way around. If we take the above equation for the slot - // width (w = (W - (c - 1) * g) / c) and solve for c, we get - // c = (W + g) / (w + g). - columns = Math.floor((width + gap) / (slotMinWidth + gap)); - rows = Math.ceil(model.grid.length / columns); - slotWidth = (width - (columns - 1) * gap) / columns; - slotHeight = (minHeight - (rows - 1) * gap) / rows; + // Impose a minimum width and height on the slots + if (slotWidth < slotMinWidth) { + // In this case we want the slot width to determine the number of columns, + // not the other way around. If we take the above equation for the slot + // width (w = (W - (c - 1) * g) / c) and solve for c, we get + // c = (W + g) / (w + g). + columns = Math.floor((width + gap) / (slotMinWidth + gap)); + rows = Math.ceil(model.grid.length / columns); + slotWidth = (width - (columns - 1) * gap) / columns; + slotHeight = (minHeight - (rows - 1) * gap) / rows; + } + if (slotHeight < slotMinHeight) slotHeight = slotMinHeight; + // Impose a minimum and maximum aspect ratio on the slots + const slotAspectRatio = slotWidth / slotHeight; + if (slotAspectRatio > slotMaxAspectRatio) + slotWidth = slotHeight * slotMaxAspectRatio; + else if (slotAspectRatio < slotMinAspectRatio) + slotHeight = slotWidth / slotMinAspectRatio; + // TODO: We might now be hitting the minimum height or width limit again + + return [gap, slotWidth, slotHeight]; + }, [width, minHeight, model.grid.length]); + + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid, width, minHeight], + ); + + const tileModels: GridTileModel[] = useMemo( + () => model.grid.map((vm) => ({ type: "grid", vm })), + [model.grid], + ); + + return ( +
slotMaxAspectRatio) - slotWidth = slotHeight * slotMaxAspectRatio; - else if (slotAspectRatio < slotMinAspectRatio) - slotHeight = slotWidth / slotMinAspectRatio; - // TODO: We might now be hitting the minimum height or width limit again - - return [gap, slotWidth, slotHeight]; - }, [width, minHeight, model.grid.length]); - - const [generation] = useReactiveState( - (prev) => (prev === undefined ? 0 : prev + 1), - [model.grid, width, minHeight], - ); - - return ( -
- {model.grid.map((tile) => ( - - ))} -
- ); - }), - }, + > + {tileModels.map((m) => ( + + ))} +
+ ); + }), }); diff --git a/src/grid/SpotlightLayout.module.css b/src/grid/SpotlightLayout.module.css index bbce45cf..af43216c 100644 --- a/src/grid/SpotlightLayout.module.css +++ b/src/grid/SpotlightLayout.module.css @@ -14,18 +14,20 @@ See the License for the specific language governing permissions and limitations under the License. */ -.fixed, -.scrolling { +.layer { margin-inline: var(--inline-content-inset); display: grid; - --grid-slot-width: 180px; --grid-gap: 20px; + gap: 30px; +} + +.layer[data-orientation="landscape"] { + --grid-slot-width: 180px; grid-template-columns: 1fr calc( var(--grid-columns) * var(--grid-slot-width) + (var(--grid-columns) - 1) * var(--grid-gap) ); grid-template-rows: minmax(1fr, auto); - gap: 30px; } .scrolling { @@ -41,7 +43,7 @@ limitations under the License. /* CSS makes us put a condition here, even though all we want to do is unconditionally select the container so we can use cq units */ @container spotlight (width > 0) { - .spotlight > .slot { + .layer[data-orientation="landscape"] > .spotlight > .slot { inline-size: min(100cqi, 100cqb * (17 / 9)); block-size: min(100cqb, 100cqi / (4 / 3)); } @@ -52,38 +54,48 @@ unconditionally select the container so we can use cq units */ flex-wrap: wrap; gap: var(--grid-gap); justify-content: center; +} + +.layer[data-orientation="landscape"] > .grid { align-content: center; } -.grid > .slot { +.layer > .grid > .slot { inline-size: var(--grid-slot-width); +} + +.layer[data-orientation="landscape"] > .grid > .slot { block-size: 135px; } -@media (max-width: 600px) { - .fixed, - .scrolling { - margin-inline: 0; - display: block; - } - - .spotlight { - inline-size: 100%; - aspect-ratio: 16 / 9; - margin-block-end: var(--cpd-space-4x); - } - - .grid { - margin-inline: var(--inline-content-inset); - align-content: start; - } - - .grid > .slot { - --grid-columns: 2; - --grid-slot-width: calc( - (100% - (var(--grid-columns) - 1) * var(--grid-gap)) / var(--grid-columns) - ); - block-size: unset; - aspect-ratio: 4 / 3; - } +.layer[data-orientation="portrait"] { + margin-inline: 0; + display: block; +} + +.layer[data-orientation="portrait"] > .spotlight { + inline-size: 100%; + aspect-ratio: 16 / 9; + margin-block-end: var(--cpd-space-4x); +} + +.layer[data-orientation="portrait"] > .spotlight.withIndicators { + margin-block-end: calc(2 * var(--cpd-space-4x) + 2px); +} + +.layer[data-orientation="portrait"] > .spotlight > .slot { + inline-size: 100%; + block-size: 100%; +} + +.layer[data-orientation="portrait"] > .grid { + margin-inline: var(--inline-content-inset); + align-content: start; +} + +.layer[data-orientation="portrait"] > .grid > .slot { + --grid-slot-width: calc( + (100% - (var(--grid-columns) - 1) * var(--grid-gap)) / var(--grid-columns) + ); + aspect-ratio: 4 / 3; } diff --git a/src/grid/SpotlightLayout.tsx b/src/grid/SpotlightLayout.tsx index 38bc6e37..3e07a0b2 100644 --- a/src/grid/SpotlightLayout.tsx +++ b/src/grid/SpotlightLayout.tsx @@ -14,76 +14,113 @@ See the License for the specific language governing permissions and limitations under the License. */ -import { CSSProperties, forwardRef } from "react"; +import { CSSProperties, forwardRef, useMemo } from "react"; import { useObservableEagerState } from "observable-hooks"; +import classNames from "classnames"; -import { CallLayout } from "./CallLayout"; +import { CallLayout, GridTileModel, TileModel } from "./CallLayout"; import { SpotlightLayout as SpotlightLayoutModel } from "../state/CallViewModel"; -import { useReactiveState } from "../useReactiveState"; import styles from "./SpotlightLayout.module.css"; -import { Slot } from "./Grid"; +import { useReactiveState } from "../useReactiveState"; interface GridCSSProperties extends CSSProperties { "--grid-columns": number; } -const getGridColumns = (gridLength: number): number => - gridLength > 20 ? 2 : 1; +interface Layout { + orientation: "portrait" | "landscape"; + gridColumns: number; +} + +function getLayout(gridLength: number, width: number): Layout { + const orientation = width < 800 ? "portrait" : "landscape"; + return { + orientation, + gridColumns: + orientation === "portrait" + ? Math.floor(width / 190) + : gridLength > 20 + ? 2 + : 1, + }; +} export const makeSpotlightLayout: CallLayout = ({ minBounds, }) => ({ - fixed: { - tiles: (model) => new Map([["spotlight", model.spotlight]]), - Layout: forwardRef(function SpotlightLayoutFixed({ model }, ref) { - const { width, height } = useObservableEagerState(minBounds); - const gridColumns = getGridColumns(model.grid.length); - const [generation] = useReactiveState( - (prev) => (prev === undefined ? 0 : prev + 1), - [model.grid.length, width, height], - ); + fixed: forwardRef(function SpotlightLayoutFixed({ model, Slot }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const layout = getLayout(model.grid.length, width); + const tileModel: TileModel = useMemo( + () => ({ + type: "spotlight", + vms: model.spotlight, + maximised: layout.orientation === "portrait", + }), + [model.spotlight, layout.orientation], + ); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid.length, width, height], + ); - return ( -
-
- -
-
+ return ( +
+
+
- ); - }), - }, +
+
+ ); + }), - scrolling: { - tiles: (model) => new Map(model.grid.map((tile) => [tile.id, tile])), - Layout: forwardRef(function SpotlightLayoutScrolling({ model }, ref) { - const { width, height } = useObservableEagerState(minBounds); - const gridColumns = getGridColumns(model.grid.length); - const [generation] = useReactiveState( - (prev) => (prev === undefined ? 0 : prev + 1), - [model.grid, width, height], - ); + scrolling: forwardRef(function SpotlightLayoutScrolling( + { model, Slot }, + ref, + ) { + const { width, height } = useObservableEagerState(minBounds); + const layout = getLayout(model.grid.length, width); + const tileModels: GridTileModel[] = useMemo( + () => model.grid.map((vm) => ({ type: "grid", vm })), + [model.grid], + ); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.spotlight.length, model.grid, width, height], + ); - return ( + return ( +
-
-
- {model.grid.map((tile) => ( - - ))} -
+ className={classNames(styles.spotlight, { + [styles.withIndicators]: model.spotlight.length > 1, + })} + /> +
+ {tileModels.map((m) => ( + + ))}
- ); - }), - }, +
+ ); + }), }); diff --git a/src/room/InCallView.module.css b/src/room/InCallView.module.css index 76dc9bae..60c46aa6 100644 --- a/src/room/InCallView.module.css +++ b/src/room/InCallView.module.css @@ -142,3 +142,13 @@ limitations under the License. inline-size: 100%; align-self: center; } + +.tile { + position: absolute; + inset-block-start: 0; +} + +.tile.maximised { + position: relative; + flex-grow: 1; +} diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index b2aeb9d2..3a16e1c6 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -25,6 +25,7 @@ import { ConnectionState, Room, Track } from "livekit-client"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { FC, + PropsWithoutRef, forwardRef, useCallback, useEffect, @@ -86,7 +87,7 @@ import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; import { makeGridLayout } from "../grid/GridLayout"; import { makeSpotlightLayout } from "../grid/SpotlightLayout"; -import { CallLayout } from "../grid/CallLayout"; +import { CallLayout, GridTileModel, TileModel } from "../grid/CallLayout"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -329,7 +330,10 @@ export const InCallView: FC = ({ vm.layout.pipe( map((l) => { let makeLayout: CallLayout; - if (l.type === "grid" && l.grid.length !== 2) + if ( + l.type === "grid" && + !(l.grid.length === 2 && l.spotlight === undefined) + ) makeLayout = makeGridLayout as CallLayout; else if (l.type === "spotlight") makeLayout = makeSpotlightLayout as CallLayout; @@ -352,59 +356,79 @@ export const InCallView: FC = ({ [setLegacyLayout, vm], ); - const showSpeakingIndicators = + const showSpotlightIndicators = useObservable(layout.type === "spotlight"); + const showSpeakingIndicators = useObservable( layout.type === "spotlight" || - (layout.type === "grid" && layout.grid.length > 2); - - const SpotlightTileView = useMemo( - () => - forwardRef>( - function SpotlightTileView( - { className, style, targetWidth, targetHeight, model }, - ref, - ) { - return ( - - ); - }, - ), - [toggleSpotlightFullscreen], + (layout.type === "grid" && layout.grid.length > 2), ); - const GridTileView = useMemo( + + const Tile = useMemo( () => - forwardRef>( - function GridTileView( - { className, style, targetWidth, targetHeight, model }, - ref, - ) { - return ( - - ); - }, - ), - [toggleFullscreen, openProfile, showSpeakingIndicators], + forwardRef< + HTMLDivElement, + PropsWithoutRef> + >(function Tile( + { className, style, targetWidth, targetHeight, model }, + ref, + ) { + const showSpeakingIndicatorsValue = useObservableEagerState( + showSpeakingIndicators, + ); + const showSpotlightIndicatorsValue = useObservableEagerState( + showSpotlightIndicators, + ); + + return model.type === "grid" ? ( + + ) : ( + + ); + }), + [ + toggleFullscreen, + toggleSpotlightFullscreen, + openProfile, + showSpeakingIndicators, + showSpotlightIndicators, + ], + ); + + const LegacyTile = useMemo( + () => + forwardRef< + HTMLDivElement, + PropsWithoutRef> + >(function LegacyTile({ model: legacyModel, ...props }, ref) { + const model: GridTileModel = useMemo( + () => ({ type: "grid", vm: legacyModel }), + [legacyModel], + ); + return ; + }), + [Tile], ); const renderContent = (): JSX.Element => { @@ -421,17 +445,20 @@ export const InCallView: FC = ({ if (maximisedParticipant.id === "spotlight") { return ( ); } return ( = ({ items={items} layout={legacyLayout} disableAnimations={prefersReducedMotion} - Tile={GridTileView} + Tile={LegacyTile} /> ); } else { @@ -462,15 +489,15 @@ export const InCallView: FC = ({ ); diff --git a/src/tile/GridTile.module.css b/src/tile/GridTile.module.css index 923c7633..7ef66d8d 100644 --- a/src/tile/GridTile.module.css +++ b/src/tile/GridTile.module.css @@ -15,8 +15,6 @@ limitations under the License. */ .tile { - position: absolute; - top: 0; --media-view-border-radius: var(--cpd-space-4x); transition: outline-color ease 0.15s; outline: var(--cpd-border-width-2) solid rgb(0 0 0 / 0); @@ -62,8 +60,6 @@ borders don't support gradients */ } .tile[data-maximised="true"] { - position: relative; - flex-grow: 1; --media-view-border-radius: 0; --media-view-fg-inset: 10px; } diff --git a/src/tile/SpotlightTile.module.css b/src/tile/SpotlightTile.module.css index 9d772c1d..cc591fee 100644 --- a/src/tile/SpotlightTile.module.css +++ b/src/tile/SpotlightTile.module.css @@ -15,14 +15,10 @@ limitations under the License. */ .tile { - position: absolute; - top: 0; --border-width: var(--cpd-space-3x); } .tile.maximised { - position: relative; - flex-grow: 1; --border-width: 0px; } @@ -54,14 +50,14 @@ limitations under the License. border-radius: 0; } -.item { +.contents > .item { height: 100%; flex-basis: 100%; flex-shrink: 0; --media-view-fg-inset: 10px; } -.item.snap { +.contents > .item.snap { scroll-snap-align: start; } @@ -151,3 +147,38 @@ limitations under the License. .tile:has(:focus-visible) > button { opacity: 1; } + +.indicators { + display: flex; + gap: var(--cpd-space-2x); + position: absolute; + inset-inline-start: 0; + inset-block-end: calc(-1 * var(--cpd-space-6x)); + width: 100%; + justify-content: start; + transition: opacity ease 0.15s; + opacity: 0; +} + +.indicators.show { + opacity: 1; +} + +.maximised .indicators { + inset-block-end: calc(-1 * var(--cpd-space-4x) - 2px); + justify-content: center; +} + +.indicators > .item { + inline-size: 32px; + block-size: 2px; + transition: background-color ease 0.15s; +} + +.indicators > .item[data-visible="false"] { + background: var(--cpd-color-alpha-gray-600); +} + +.indicators > .item[data-visible="true"] { + background: var(--cpd-color-gray-1400); +} diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index fccc5235..f77ce4cf 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -178,6 +178,7 @@ interface Props { onToggleFullscreen: () => void; targetWidth: number; targetHeight: number; + showIndicators: boolean; className?: string; style?: ComponentProps["style"]; } @@ -191,6 +192,7 @@ export const SpotlightTile = forwardRef( onToggleFullscreen, targetWidth, targetHeight, + showIndicators, className, style, }, @@ -307,6 +309,15 @@ export const SpotlightTile = forwardRef( )} +
1, + })} + > + {vms.map((vm) => ( +
+ ))} +
); }, From 7f40ce8dde4d6b5a915eaf6a349a4b0482419a06 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 31 May 2024 10:54:17 -0400 Subject: [PATCH 15/31] Fix advance buttons showing up for the spotlight speaker --- src/tile/SpotlightTile.tsx | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index f77ce4cf..a171fe4f 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -204,8 +204,9 @@ export const SpotlightTile = forwardRef( const [visibleId, setVisibleId] = useState(vms[0].id); const latestVms = useLatest(vms); const latestVisibleId = useLatest(visibleId); - const canGoBack = visibleId !== vms[0].id; - const canGoToNext = visibleId !== vms[vms.length - 1].id; + const visibleIndex = vms.findIndex((vm) => vm.id === visibleId); + const canGoBack = visibleIndex > 0; + const canGoToNext = visibleIndex !== -1 && visibleIndex < vms.length - 1; // To keep track of which item is visible, we need an intersection observer // hooked up to the root element and the items. Because the items will run From dfda7539d6b592b8ae7eef5edcf35d0cdbaaff61 Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 4 Jun 2024 16:06:40 -0400 Subject: [PATCH 16/31] Only switch to spotlight for remote screen shares --- src/state/CallViewModel.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index cc5afc46..3bbc0a2d 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -376,9 +376,9 @@ export class CallViewModel extends ViewModel { shareReplay(1), ); - private readonly hasScreenShares: Observable = + private readonly hasRemoteScreenShares: Observable = this.screenShares.pipe( - map((ms) => ms.length > 0), + map((ms) => ms.find((m) => !m.vm.local) !== undefined), distinctUntilChanged(), ); @@ -474,7 +474,7 @@ export class CallViewModel extends ViewModel { this.gridModeUserSelection, // If the user hasn't selected spotlight and somebody starts screen sharing, // automatically switch to spotlight mode and reset when screen sharing ends - this.hasScreenShares.pipe( + this.hasRemoteScreenShares.pipe( withLatestFrom(this.gridModeUserSelection.pipe(startWith(null))), concatMap(([hasScreenShares, userSelection]) => userSelection === "spotlight" From 12b719da95ac8221f5aa9e8f44f8a0797af4041e Mon Sep 17 00:00:00 2001 From: Robin Date: Tue, 4 Jun 2024 16:07:07 -0400 Subject: [PATCH 17/31] Make layout reactivity a little more fine-grained --- src/state/CallViewModel.ts | 59 ++++++++++++++++++++------------------ 1 file changed, 31 insertions(+), 28 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 3bbc0a2d..31a1fdb3 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -45,7 +45,6 @@ import { scan, shareReplay, startWith, - switchAll, switchMap, throttleTime, timer, @@ -488,39 +487,43 @@ export class CallViewModel extends ViewModel { this.gridModeUserSelection.next(value); } - public readonly layout: Observable = combineLatest( - [this.gridMode, this.windowMode], - (gridMode, windowMode) => { + public readonly layout: Observable = this.windowMode.pipe( + switchMap((windowMode) => { switch (windowMode) { case "full screen": throw new Error("unimplemented"); case "pip": throw new Error("unimplemented"); - case "normal": { - switch (gridMode) { - case "grid": - return combineLatest( - [this.grid, this.spotlight, this.screenShares], - (grid, spotlight, screenShares): Layout => ({ - type: "grid", - spotlight: screenShares.length > 0 ? spotlight : undefined, - grid, - }), - ); - case "spotlight": - return combineLatest( - [this.grid, this.spotlight], - (grid, spotlight): Layout => ({ - type: "spotlight", - spotlight, - grid, - }), - ); - } - } + case "normal": + return this.gridMode.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + return combineLatest( + [this.grid, this.spotlight, this.screenShares], + (grid, spotlight, screenShares): Layout => ({ + type: "grid", + spotlight: + screenShares.length > 0 ? spotlight : undefined, + grid, + }), + ); + case "spotlight": + return combineLatest( + [this.grid, this.spotlight], + (grid, spotlight): Layout => ({ + type: "spotlight", + spotlight, + grid, + }), + ); + } + }), + ); } - }, - ).pipe(switchAll(), shareReplay(1)); + }), + shareReplay(1), + ); /** * The media tiles to be displayed in the call view. From 183d2d9050ad9cac4ffc48588253c20ae8cd4158 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 7 Jun 2024 11:48:58 -0400 Subject: [PATCH 18/31] Show speaker in the spotlight in large grids --- src/state/CallViewModel.ts | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 31a1fdb3..3d48da22 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -504,7 +504,9 @@ export class CallViewModel extends ViewModel { (grid, spotlight, screenShares): Layout => ({ type: "grid", spotlight: - screenShares.length > 0 ? spotlight : undefined, + screenShares.length > 0 || grid.length > 20 + ? spotlight + : undefined, grid, }), ); From e0b10d89b5c78ec5d11f1757c77ce722f610fdd9 Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 7 Jun 2024 12:27:13 -0400 Subject: [PATCH 19/31] Add model for one-on-one layout --- src/room/InCallView.tsx | 5 +---- src/state/CallViewModel.ts | 37 +++++++++++++++++++++++++++++-------- 2 files changed, 30 insertions(+), 12 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 3a16e1c6..bab24991 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -330,10 +330,7 @@ export const InCallView: FC = ({ vm.layout.pipe( map((l) => { let makeLayout: CallLayout; - if ( - l.type === "grid" && - !(l.grid.length === 2 && l.spotlight === undefined) - ) + if (l.type === "grid") makeLayout = makeGridLayout as CallLayout; else if (l.type === "spotlight") makeLayout = makeSpotlightLayout as CallLayout; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 3d48da22..62c35a8e 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -102,6 +102,13 @@ export interface SpotlightLayout { grid: UserMediaViewModel[]; } +export interface OneOnOneLayout { + type: "one-on-one"; + spotlight?: ScreenShareViewModel[]; + local: LocalUserMediaViewModel; + remote: RemoteUserMediaViewModel; +} + export interface FullScreenLayout { type: "full screen"; spotlight: MediaViewModel[]; @@ -120,6 +127,7 @@ export interface PipLayout { export type Layout = | GridLayout | SpotlightLayout + | OneOnOneLayout | FullScreenLayout | PipLayout; @@ -501,14 +509,27 @@ export class CallViewModel extends ViewModel { case "grid": return combineLatest( [this.grid, this.spotlight, this.screenShares], - (grid, spotlight, screenShares): Layout => ({ - type: "grid", - spotlight: - screenShares.length > 0 || grid.length > 20 - ? spotlight - : undefined, - grid, - }), + (grid, spotlight, screenShares): Layout => + grid.length == 2 + ? { + type: "one-on-one", + spotlight: + screenShares.length > 0 ? spotlight : undefined, + local: grid.find( + (vm) => vm.local, + ) as LocalUserMediaViewModel, + remote: grid.find( + (vm) => !vm.local, + ) as RemoteUserMediaViewModel, + } + : { + type: "grid", + spotlight: + screenShares.length > 0 || grid.length > 20 + ? spotlight + : undefined, + grid, + }, ); case "spotlight": return combineLatest( From 7979493371a149ac2017c809b9703a395f61b19f Mon Sep 17 00:00:00 2001 From: Robin Date: Fri, 7 Jun 2024 16:59:56 -0400 Subject: [PATCH 20/31] Implement the new one-on-one layout --- src/grid/CallLayout.ts | 85 +++++++++++++++++- src/grid/GridLayout.module.css | 3 +- src/grid/GridLayout.tsx | 83 ++++------------- src/grid/OneOnOneLayout.module.css | 56 ++++++++++++ src/grid/OneOnOneLayout.tsx | 132 ++++++++++++++++++++++++++++ src/grid/SpotlightLayout.module.css | 5 +- src/grid/SpotlightLayout.tsx | 8 +- src/room/InCallView.tsx | 33 ++++--- 8 files changed, 312 insertions(+), 93 deletions(-) create mode 100644 src/grid/OneOnOneLayout.module.css create mode 100644 src/grid/OneOnOneLayout.tsx diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts index c4412677..8f8106de 100644 --- a/src/grid/CallLayout.ts +++ b/src/grid/CallLayout.ts @@ -19,22 +19,36 @@ import { ComponentType } from "react"; import { MediaViewModel } from "../state/MediaViewModel"; import { LayoutProps } from "./Grid"; -import { Alignment } from "../room/InCallView"; export interface Bounds { width: number; height: number; } +export interface Alignment { + inline: "start" | "end"; + block: "start" | "end"; +} + +export const defaultSpotlightAlignment: Alignment = { + inline: "end", + block: "end", +}; +export const defaultPipAlignment: Alignment = { inline: "end", block: "start" }; + export interface CallLayoutInputs { /** * The minimum bounds of the layout area. */ minBounds: Observable; /** - * The alignment of the floating tile, if any. + * The alignment of the floating spotlight tile, if present. */ - floatingAlignment: BehaviorSubject; + spotlightAlignment: BehaviorSubject; + /** + * The alignment of the small picture-in-picture tile, if present. + */ + pipAlignment: BehaviorSubject; } export interface GridTileModel { @@ -67,3 +81,68 @@ export interface CallLayoutOutputs { export type CallLayout = ( inputs: CallLayoutInputs, ) => CallLayoutOutputs; + +export interface GridArrangement { + tileWidth: number; + tileHeight: number; + gap: number; + columns: number; +} + +const tileMinHeight = 130; +const tileMaxAspectRatio = 17 / 9; +const tileMinAspectRatio = 4 / 3; + +/** + * Determine the ideal arrangement of tiles into a grid of a particular size. + */ +export function arrangeTiles( + width: number, + minHeight: number, + tileCount: number, +): GridArrangement { + // The goal here is to determine the grid size and padding that maximizes + // use of screen space for n tiles without making those tiles too small or + // too cropped (having an extreme aspect ratio) + const gap = width < 800 ? 16 : 20; + const tileMinWidth = width < 500 ? 150 : 180; + + let columns = Math.min( + // Don't create more columns than we have items for + tileCount, + // The ideal number of columns is given by a packing of equally-sized + // squares into a grid. + // width / column = height / row. + // columns * rows = number of squares. + // ∴ columns = sqrt(width / height * number of squares). + // Except we actually want 16:9-ish tiles rather than squares, so we + // divide the width-to-height ratio by the target aspect ratio. + Math.ceil(Math.sqrt((width / minHeight / tileMaxAspectRatio) * tileCount)), + ); + let rows = Math.ceil(tileCount / columns); + + let tileWidth = (width - (columns - 1) * gap) / columns; + let tileHeight = (minHeight - (rows - 1) * gap) / rows; + + // Impose a minimum width and height on the tiles + if (tileWidth < tileMinWidth) { + // In this case we want the tile width to determine the number of columns, + // not the other way around. If we take the above equation for the tile + // width (w = (W - (c - 1) * g) / c) and solve for c, we get + // c = (W + g) / (w + g). + columns = Math.floor((width + gap) / (tileMinWidth + gap)); + rows = Math.ceil(tileCount / columns); + tileWidth = (width - (columns - 1) * gap) / columns; + tileHeight = (minHeight - (rows - 1) * gap) / rows; + } + if (tileHeight < tileMinHeight) tileHeight = tileMinHeight; + // Impose a minimum and maximum aspect ratio on the tiles + const tileAspectRatio = tileWidth / tileHeight; + if (tileAspectRatio > tileMaxAspectRatio) + tileWidth = tileHeight * tileMaxAspectRatio; + else if (tileAspectRatio < tileMinAspectRatio) + tileHeight = tileWidth / tileMinAspectRatio; + // TODO: We might now be hitting the minimum height or width limit again + + return { tileWidth, tileHeight, gap, columns }; +} diff --git a/src/grid/GridLayout.module.css b/src/grid/GridLayout.module.css index 5e6aa9e1..33edc3be 100644 --- a/src/grid/GridLayout.module.css +++ b/src/grid/GridLayout.module.css @@ -17,11 +17,10 @@ limitations under the License. .fixed, .scrolling { margin-inline: var(--inline-content-inset); + block-size: 100%; } .scrolling { - box-sizing: border-box; - block-size: 100%; display: flex; flex-wrap: wrap; justify-content: center; diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 3861457e..4d499eed 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -22,7 +22,12 @@ import { GridLayout as GridLayoutModel } from "../state/CallViewModel"; import styles from "./GridLayout.module.css"; import { useReactiveState } from "../useReactiveState"; import { useInitial } from "../useInitial"; -import { CallLayout, GridTileModel, TileModel } from "./CallLayout"; +import { + CallLayout, + GridTileModel, + TileModel, + arrangeTiles, +} from "./CallLayout"; import { DragCallback } from "./Grid"; interface GridCSSProperties extends CSSProperties { @@ -31,13 +36,9 @@ interface GridCSSProperties extends CSSProperties { "--height": string; } -const slotMinHeight = 130; -const slotMaxAspectRatio = 17 / 9; -const slotMinAspectRatio = 4 / 3; - export const makeGridLayout: CallLayout = ({ minBounds, - floatingAlignment, + spotlightAlignment, }) => ({ // The "fixed" (non-scrolling) part of the layout is where the spotlight tile // lives @@ -45,7 +46,7 @@ export const makeGridLayout: CallLayout = ({ const { width, height } = useObservableEagerState(minBounds); const alignment = useObservableEagerState( useInitial(() => - floatingAlignment.pipe( + spotlightAlignment.pipe( distinctUntilChanged( (a1, a2) => a1.block === a2.block && a1.inline === a2.inline, ), @@ -68,7 +69,7 @@ export const makeGridLayout: CallLayout = ({ const onDragSpotlight: DragCallback = useCallback( ({ xRatio, yRatio }) => - floatingAlignment.next({ + spotlightAlignment.next({ block: yRatio < 0.5 ? "start" : "end", inline: xRatio < 0.5 ? "start" : "end", }), @@ -76,12 +77,7 @@ export const makeGridLayout: CallLayout = ({ ); return ( -
+
{tileModel && ( = ({ // The scrolling part of the layout is where all the grid tiles live scrolling: forwardRef(function GridLayout({ model, Slot }, ref) { const { width, height: minHeight } = useObservableEagerState(minBounds); - - // The goal here is to determine the grid size and padding that maximizes - // use of screen space for n tiles without making those tiles too small or - // too cropped (having an extreme aspect ratio) - const [gap, slotWidth, slotHeight] = useMemo(() => { - const gap = width < 800 ? 16 : 20; - const slotMinWidth = width < 500 ? 150 : 180; - - let columns = Math.min( - // Don't create more columns than we have items for - model.grid.length, - // The ideal number of columns is given by a packing of equally-sized - // squares into a grid. - // width / column = height / row. - // columns * rows = number of squares. - // ∴ columns = sqrt(width / height * number of squares). - // Except we actually want 16:9-ish slots rather than squares, so we - // divide the width-to-height ratio by the target aspect ratio. - Math.ceil( - Math.sqrt( - (width / minHeight / slotMaxAspectRatio) * model.grid.length, - ), - ), - ); - let rows = Math.ceil(model.grid.length / columns); - - let slotWidth = (width - (columns - 1) * gap) / columns; - let slotHeight = (minHeight - (rows - 1) * gap) / rows; - - // Impose a minimum width and height on the slots - if (slotWidth < slotMinWidth) { - // In this case we want the slot width to determine the number of columns, - // not the other way around. If we take the above equation for the slot - // width (w = (W - (c - 1) * g) / c) and solve for c, we get - // c = (W + g) / (w + g). - columns = Math.floor((width + gap) / (slotMinWidth + gap)); - rows = Math.ceil(model.grid.length / columns); - slotWidth = (width - (columns - 1) * gap) / columns; - slotHeight = (minHeight - (rows - 1) * gap) / rows; - } - if (slotHeight < slotMinHeight) slotHeight = slotMinHeight; - // Impose a minimum and maximum aspect ratio on the slots - const slotAspectRatio = slotWidth / slotHeight; - if (slotAspectRatio > slotMaxAspectRatio) - slotWidth = slotHeight * slotMaxAspectRatio; - else if (slotAspectRatio < slotMinAspectRatio) - slotHeight = slotWidth / slotMinAspectRatio; - // TODO: We might now be hitting the minimum height or width limit again - - return [gap, slotWidth, slotHeight]; - }, [width, minHeight, model.grid.length]); + const { gap, tileWidth, tileHeight } = useMemo( + () => arrangeTiles(width, minHeight, model.grid.length), + [width, minHeight, model.grid.length], + ); const [generation] = useReactiveState( (prev) => (prev === undefined ? 0 : prev + 1), @@ -170,8 +119,8 @@ export const makeGridLayout: CallLayout = ({ { width, "--gap": `${gap}px`, - "--width": `${Math.floor(slotWidth)}px`, - "--height": `${Math.floor(slotHeight)}px`, + "--width": `${Math.floor(tileWidth)}px`, + "--height": `${Math.floor(tileHeight)}px`, } as GridCSSProperties } > diff --git a/src/grid/OneOnOneLayout.module.css b/src/grid/OneOnOneLayout.module.css new file mode 100644 index 00000000..0d2ad4ff --- /dev/null +++ b/src/grid/OneOnOneLayout.module.css @@ -0,0 +1,56 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.layer { + margin-inline: var(--inline-content-inset); + block-size: 100%; + display: grid; + place-items: center; +} + +.container { + position: relative; +} + +.local { + position: absolute; + inline-size: 180px; + block-size: 135px; + inset: var(--cpd-space-4x); +} + +.spotlight { + position: absolute; + inline-size: 404px; + block-size: 233px; + inset: -12px; +} + +.slot[data-block-alignment="start"] { + inset-block-end: unset; +} + +.slot[data-block-alignment="end"] { + inset-block-start: unset; +} + +.slot[data-inline-alignment="start"] { + inset-inline-end: unset; +} + +.slot[data-inline-alignment="end"] { + inset-inline-start: unset; +} diff --git a/src/grid/OneOnOneLayout.tsx b/src/grid/OneOnOneLayout.tsx new file mode 100644 index 00000000..2eac1b7e --- /dev/null +++ b/src/grid/OneOnOneLayout.tsx @@ -0,0 +1,132 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { forwardRef, useCallback, useMemo } from "react"; +import { useObservableEagerState } from "observable-hooks"; +import classNames from "classnames"; + +import { OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewModel"; +import { + CallLayout, + GridTileModel, + SpotlightTileModel, + arrangeTiles, +} from "./CallLayout"; +import { useReactiveState } from "../useReactiveState"; +import styles from "./OneOnOneLayout.module.css"; +import { DragCallback } from "./Grid"; + +export const makeOneOnOneLayout: CallLayout = ({ + minBounds, + spotlightAlignment, + pipAlignment, +}) => ({ + fixed: forwardRef(function OneOnOneLayoutFixed({ model, Slot }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const spotlightAlignmentValue = useObservableEagerState(spotlightAlignment); + + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [width, height, model.spotlight === undefined, spotlightAlignmentValue], + ); + + const spotlightTileModel: SpotlightTileModel | undefined = useMemo( + () => + model.spotlight && { + type: "spotlight", + vms: model.spotlight, + maximised: false, + }, + [model.spotlight], + ); + + const onDragSpotlight: DragCallback = useCallback( + ({ xRatio, yRatio }) => + spotlightAlignment.next({ + block: yRatio < 0.5 ? "start" : "end", + inline: xRatio < 0.5 ? "start" : "end", + }), + [], + ); + + return ( +
+ {spotlightTileModel && ( + + )} +
+ ); + }), + + scrolling: forwardRef(function OneOnOneLayoutScrolling({ model, Slot }, ref) { + const { width, height } = useObservableEagerState(minBounds); + const pipAlignmentValue = useObservableEagerState(pipAlignment); + const { tileWidth, tileHeight } = useMemo( + () => arrangeTiles(width, height, 1), + [width, height], + ); + + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [width, height, pipAlignmentValue], + ); + + const remoteTileModel: GridTileModel = useMemo( + () => ({ type: "grid", vm: model.remote }), + [model.remote], + ); + const localTileModel: GridTileModel = useMemo( + () => ({ type: "grid", vm: model.local }), + [model.local], + ); + + const onDragLocalTile: DragCallback = useCallback( + ({ xRatio, yRatio }) => + pipAlignment.next({ + block: yRatio < 0.5 ? "start" : "end", + inline: xRatio < 0.5 ? "start" : "end", + }), + [], + ); + + return ( +
+ + + +
+ ); + }), +}); diff --git a/src/grid/SpotlightLayout.module.css b/src/grid/SpotlightLayout.module.css index af43216c..d58a95a1 100644 --- a/src/grid/SpotlightLayout.module.css +++ b/src/grid/SpotlightLayout.module.css @@ -16,6 +16,7 @@ limitations under the License. .layer { margin-inline: var(--inline-content-inset); + block-size: 100%; display: grid; --grid-gap: 20px; gap: 30px; @@ -30,10 +31,6 @@ limitations under the License. grid-template-rows: minmax(1fr, auto); } -.scrolling { - block-size: 100%; -} - .spotlight { container: spotlight / size; display: grid; diff --git a/src/grid/SpotlightLayout.tsx b/src/grid/SpotlightLayout.tsx index 3e07a0b2..9ddbce10 100644 --- a/src/grid/SpotlightLayout.tsx +++ b/src/grid/SpotlightLayout.tsx @@ -69,10 +69,8 @@ export const makeSpotlightLayout: CallLayout = ({ ref={ref} data-generation={generation} data-orientation={layout.orientation} - className={classNames(styles.layer, styles.fixed)} - style={ - { "--grid-columns": layout.gridColumns, height } as GridCSSProperties - } + className={styles.layer} + style={{ "--grid-columns": layout.gridColumns } as GridCSSProperties} >
@@ -102,7 +100,7 @@ export const makeSpotlightLayout: CallLayout = ({ ref={ref} data-generation={generation} data-orientation={layout.orientation} - className={classNames(styles.layer, styles.scrolling)} + className={styles.layer} style={{ "--grid-columns": layout.gridColumns } as GridCSSProperties} >
; @@ -321,8 +321,11 @@ export const InCallView: FC = ({ ); const gridBoundsObservable = useObservable(gridBounds); - const floatingAlignment = useInitial( - () => new BehaviorSubject(defaultAlignment), + const spotlightAlignment = useInitial( + () => new BehaviorSubject(defaultSpotlightAlignment), + ); + const pipAlignment = useInitial( + () => new BehaviorSubject(defaultPipAlignment), ); const layoutSystem = useObservableEagerState( @@ -334,11 +337,14 @@ export const InCallView: FC = ({ makeLayout = makeGridLayout as CallLayout; else if (l.type === "spotlight") makeLayout = makeSpotlightLayout as CallLayout; + else if (l.type === "one-on-one") + makeLayout = makeOneOnOneLayout as CallLayout; else return null; // Not yet implemented return makeLayout({ minBounds: gridBoundsObservable, - floatingAlignment, + spotlightAlignment, + pipAlignment, }); }), ), @@ -491,7 +497,10 @@ export const InCallView: FC = ({ /> Date: Fri, 7 Jun 2024 17:29:48 -0400 Subject: [PATCH 21/31] Delete the legacy grid system --- public/locales/en-GB/app.json | 3 +- src/grid/CallLayout.ts | 4 +- src/grid/Grid.tsx | 8 +- src/grid/LegacyGrid.module.css | 22 - src/grid/LegacyGrid.tsx | 1405 -------------------------------- src/room/InCallView.tsx | 227 ++---- src/room/useFullscreen.ts | 26 +- src/state/CallViewModel.ts | 118 --- src/tile/GridTile.tsx | 112 +-- test/grid/LegacyGrid-test.ts | 69 -- 10 files changed, 102 insertions(+), 1892 deletions(-) delete mode 100644 src/grid/LegacyGrid.module.css delete mode 100644 src/grid/LegacyGrid.tsx delete mode 100644 test/grid/LegacyGrid-test.ts diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 922a4c79..b72e0bcb 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -162,6 +162,5 @@ "mute_for_me": "Mute for me", "sfu_participant_local": "You", "volume": "Volume" - }, - "waiting_for_participants": "Waiting for other participants…" + } } diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts index 8f8106de..e97b18a2 100644 --- a/src/grid/CallLayout.ts +++ b/src/grid/CallLayout.ts @@ -17,7 +17,7 @@ limitations under the License. import { BehaviorSubject, Observable } from "rxjs"; import { ComponentType } from "react"; -import { MediaViewModel } from "../state/MediaViewModel"; +import { MediaViewModel, UserMediaViewModel } from "../state/MediaViewModel"; import { LayoutProps } from "./Grid"; export interface Bounds { @@ -53,7 +53,7 @@ export interface CallLayoutInputs { export interface GridTileModel { type: "grid"; - vm: MediaViewModel; + vm: UserMediaViewModel; } export interface SpotlightTileModel { diff --git a/src/grid/Grid.tsx b/src/grid/Grid.tsx index 2e6a48ae..6c1bab9b 100644 --- a/src/grid/Grid.tsx +++ b/src/grid/Grid.tsx @@ -41,7 +41,6 @@ import styles from "./Grid.module.css"; import { useMergedRefs } from "../useMergedRefs"; import { TileWrapper } from "./TileWrapper"; import { usePrefersReducedMotion } from "../usePrefersReducedMotion"; -import { TileSpringUpdate } from "./LegacyGrid"; import { useInitial } from "../useInitial"; interface Rect { @@ -69,6 +68,13 @@ interface TileSpring { height: number; } +interface TileSpringUpdate extends Partial { + from?: Partial; + reset?: boolean; + immediate?: boolean | ((key: string) => boolean); + delay?: (key: string) => number; +} + interface DragState { tileId: string; tileX: number; diff --git a/src/grid/LegacyGrid.module.css b/src/grid/LegacyGrid.module.css deleted file mode 100644 index cad3e3c4..00000000 --- a/src/grid/LegacyGrid.module.css +++ /dev/null @@ -1,22 +0,0 @@ -/* -Copyright 2022-2024 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - -http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -.grid { - position: relative; - overflow: hidden; - flex: 1; - touch-action: none; -} diff --git a/src/grid/LegacyGrid.tsx b/src/grid/LegacyGrid.tsx deleted file mode 100644 index f04cde78..00000000 --- a/src/grid/LegacyGrid.tsx +++ /dev/null @@ -1,1405 +0,0 @@ -/* -Copyright 2022-2024 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { - ComponentProps, - ComponentType, - MutableRefObject, - ReactNode, - Ref, - useCallback, - useEffect, - useRef, - useState, -} from "react"; -import { - EventTypes, - FullGestureState, - Handler, - useGesture, -} from "@use-gesture/react"; -import { - animated, - SpringRef, - SpringValues, - useSprings, -} from "@react-spring/web"; -import useMeasure from "react-use-measure"; -import { ResizeObserver as JuggleResizeObserver } from "@juggle/resize-observer"; -import { logger } from "matrix-js-sdk/src/logger"; - -import styles from "./LegacyGrid.module.css"; -import { Layout } from "../room/LayoutToggle"; -import { TileWrapper } from "./TileWrapper"; -import { TileDescriptor } from "../state/CallViewModel"; -import { TileProps } from "./Grid"; - -interface TilePosition { - x: number; - y: number; - width: number; - height: number; - zIndex: number; -} - -export interface Tile { - key: string; - order: number; - item: TileDescriptor; - remove: boolean; - focused: boolean; - isPresenter: boolean; - isSpeaker: boolean; - hasVideo: boolean; -} - -export interface TileSpring { - opacity: number; - scale: number; - shadow: number; - shadowSpread: number; - zIndex: number; - x: number; - y: number; - width: number; - height: number; -} - -export interface TileSpringUpdate extends Partial { - from?: Partial; - reset?: boolean; - immediate?: boolean | ((key: string) => boolean); - delay?: (key: string) => number; -} - -type LayoutDirection = "vertical" | "horizontal"; - -export function useLegacyGridLayout(hasScreenshareFeeds: boolean): { - layout: Layout; - setLayout: (layout: Layout) => void; -} { - const layoutRef = useRef("grid"); - const revertLayoutRef = useRef("grid"); - const prevHasScreenshareFeeds = useRef(hasScreenshareFeeds); - const [, forceUpdate] = useState({}); - - const setLayout = useCallback((layout: Layout) => { - // Store the user's set layout to revert to after a screenshare is finished - revertLayoutRef.current = layout; - layoutRef.current = layout; - forceUpdate({}); - }, []); - - // Note: We need the returned layout to update synchronously with a change in hasScreenshareFeeds - // so use refs and avoid useEffect. - if (prevHasScreenshareFeeds.current !== hasScreenshareFeeds) { - if (hasScreenshareFeeds) { - // Automatically switch to spotlight layout when there's a screenshare - layoutRef.current = "spotlight"; - } else { - // When the screenshares have ended, revert to the previous layout - layoutRef.current = revertLayoutRef.current; - } - } - - prevHasScreenshareFeeds.current = hasScreenshareFeeds; - - return { layout: layoutRef.current, setLayout }; -} - -const GAP = 20; - -function useIsMounted(): MutableRefObject { - const isMountedRef = useRef(false); - - useEffect(() => { - isMountedRef.current = true; - - return (): void => { - isMountedRef.current = false; - }; - }, []); - - return isMountedRef; -} - -function isInside([x, y]: number[], targetTile: TilePosition): boolean { - const left = targetTile.x; - const top = targetTile.y; - const bottom = targetTile.y + targetTile.height; - const right = targetTile.x + targetTile.width; - - if (x < left || x > right || y < top || y > bottom) { - return false; - } - - return true; -} - -const getPipGap = (gridAspectRatio: number, gridWidth: number): number => - gridAspectRatio < 1 || gridWidth < 700 ? 12 : 24; - -function getTilePositions( - tileCount: number, - focusedTileCount: number, - gridWidth: number, - gridHeight: number, - pipXRatio: number, - pipYRatio: number, - layout: Layout, -): TilePosition[] { - if (layout === "grid") { - if (tileCount === 2 && focusedTileCount === 0) { - return getOneOnOneLayoutTilePositions( - gridWidth, - gridHeight, - pipXRatio, - pipYRatio, - ); - } - - return getFreedomLayoutTilePositions( - tileCount, - focusedTileCount, - gridWidth, - gridHeight, - ); - } else { - return getSpotlightLayoutTilePositions(tileCount, gridWidth, gridHeight); - } -} - -function getOneOnOneLayoutTilePositions( - gridWidth: number, - gridHeight: number, - pipXRatio: number, - pipYRatio: number, -): TilePosition[] { - const [remotePosition] = getFreedomLayoutTilePositions( - 1, - 0, - gridWidth, - gridHeight, - ); - - const gridAspectRatio = gridWidth / gridHeight; - - const smallPip = gridAspectRatio < 1 || gridWidth < 700; - const maxPipWidth = smallPip ? 114 : 230; - const maxPipHeight = smallPip ? 163 : 155; - // Cap the PiP size at 1/3 the remote tile size, preserving aspect ratio - const pipScaleFactor = Math.min( - 1, - remotePosition.width / 3 / maxPipWidth, - remotePosition.height / 3 / maxPipHeight, - ); - const pipWidth = maxPipWidth * pipScaleFactor; - const pipHeight = maxPipHeight * pipScaleFactor; - const pipGap = getPipGap(gridAspectRatio, gridWidth); - - const pipMinX = remotePosition.x + pipGap; - const pipMinY = remotePosition.y + pipGap; - const pipMaxX = remotePosition.x + remotePosition.width - pipWidth - pipGap; - const pipMaxY = remotePosition.y + remotePosition.height - pipHeight - pipGap; - - return [ - { - // Apply the PiP position as a proportion of the available space - x: pipMinX + pipXRatio * (pipMaxX - pipMinX), - y: pipMinY + pipYRatio * (pipMaxY - pipMinY), - width: pipWidth, - height: pipHeight, - zIndex: 1, - }, - remotePosition, - ]; -} - -function getSpotlightLayoutTilePositions( - tileCount: number, - gridWidth: number, - gridHeight: number, -): TilePosition[] { - const tilePositions: TilePosition[] = []; - - const gridAspectRatio = gridWidth / gridHeight; - - if (gridAspectRatio < 1) { - // Vertical layout (mobile) - const spotlightTileHeight = - tileCount > 1 ? (gridHeight - GAP * 3) * (4 / 5) : gridHeight - GAP * 2; - const spectatorTileSize = - tileCount > 1 ? gridHeight - GAP * 3 - spotlightTileHeight : 0; - - for (let i = 0; i < tileCount; i++) { - if (i === 0) { - // Spotlight tile - tilePositions.push({ - x: GAP, - y: GAP, - width: gridWidth - GAP * 2, - height: spotlightTileHeight, - zIndex: 0, - }); - } else { - // Spectator tile - tilePositions.push({ - x: (GAP + spectatorTileSize) * (i - 1) + GAP, - y: spotlightTileHeight + GAP * 2, - width: spectatorTileSize, - height: spectatorTileSize, - zIndex: 0, - }); - } - } - } else { - // Horizontal layout (desktop) - const spotlightTileWidth = - tileCount > 1 ? ((gridWidth - GAP * 3) * 4) / 5 : gridWidth - GAP * 2; - const spectatorTileWidth = - tileCount > 1 ? gridWidth - GAP * 3 - spotlightTileWidth : 0; - const spectatorTileHeight = spectatorTileWidth * (9 / 16); - - for (let i = 0; i < tileCount; i++) { - if (i === 0) { - tilePositions.push({ - x: GAP, - y: GAP, - width: spotlightTileWidth, - height: gridHeight - GAP * 2, - zIndex: 0, - }); - } else { - tilePositions.push({ - x: GAP * 2 + spotlightTileWidth, - y: (GAP + spectatorTileHeight) * (i - 1) + GAP, - width: spectatorTileWidth, - height: spectatorTileHeight, - zIndex: 0, - }); - } - } - } - - return tilePositions; -} - -function getFreedomLayoutTilePositions( - tileCount: number, - focusedTileCount: number, - gridWidth: number, - gridHeight: number, -): TilePosition[] { - if (tileCount === 0) { - return []; - } - - if (tileCount > 12) { - logger.warn("Over 12 tiles is not currently supported"); - } - - const { layoutDirection, itemGridRatio } = getGridLayout( - tileCount, - focusedTileCount, - gridWidth, - gridHeight, - ); - - let itemGridWidth; - let itemGridHeight; - - if (layoutDirection === "vertical") { - itemGridWidth = gridWidth; - itemGridHeight = Math.round(gridHeight * itemGridRatio); - } else { - itemGridWidth = Math.round(gridWidth * itemGridRatio); - itemGridHeight = gridHeight; - } - - const itemTileCount = tileCount - focusedTileCount; - - const { - columnCount: itemColumnCount, - rowCount: itemRowCount, - tileAspectRatio: itemTileAspectRatio, - } = getSubGridLayout(itemTileCount, itemGridWidth, itemGridHeight); - - const itemGridPositions = getSubGridPositions( - itemTileCount, - itemColumnCount, - itemRowCount, - itemTileAspectRatio, - itemGridWidth, - itemGridHeight, - ); - const itemGridBounds = getSubGridBoundingBox(itemGridPositions); - - let focusedGridWidth: number; - let focusedGridHeight: number; - - if (focusedTileCount === 0) { - focusedGridWidth = 0; - focusedGridHeight = 0; - } else if (layoutDirection === "vertical") { - focusedGridWidth = gridWidth; - focusedGridHeight = - gridHeight - (itemGridBounds.height + (itemTileCount ? GAP * 2 : 0)); - } else { - focusedGridWidth = - gridWidth - (itemGridBounds.width + (itemTileCount ? GAP * 2 : 0)); - focusedGridHeight = gridHeight; - } - - const { - columnCount: focusedColumnCount, - rowCount: focusedRowCount, - tileAspectRatio: focusedTileAspectRatio, - } = getSubGridLayout(focusedTileCount, focusedGridWidth, focusedGridHeight); - - const focusedGridPositions = getSubGridPositions( - focusedTileCount, - focusedColumnCount, - focusedRowCount, - focusedTileAspectRatio, - focusedGridWidth, - focusedGridHeight, - ); - - const tilePositions = [...focusedGridPositions, ...itemGridPositions]; - - centerTiles(focusedGridPositions, focusedGridWidth, focusedGridHeight, 0, 0); - - if (layoutDirection === "vertical") { - centerTiles( - itemGridPositions, - gridWidth, - gridHeight - focusedGridHeight, - 0, - focusedGridHeight, - ); - } else { - centerTiles( - itemGridPositions, - gridWidth - focusedGridWidth, - gridHeight, - focusedGridWidth, - 0, - ); - } - - return tilePositions; -} - -function getSubGridBoundingBox(positions: TilePosition[]): { - left: number; - right: number; - top: number; - bottom: number; - width: number; - height: number; -} { - let left = 0; - let right = 0; - let top = 0; - let bottom = 0; - - for (let i = 0; i < positions.length; i++) { - const { x, y, width, height } = positions[i]; - - if (i === 0) { - left = x; - right = x + width; - top = y; - bottom = y + height; - } else { - if (x < left) { - left = x; - } - - if (y < top) { - top = y; - } - - if (x + width > right) { - right = x + width; - } - - if (y + height > bottom) { - bottom = y + height; - } - } - } - - return { - left, - right, - top, - bottom, - width: right - left, - height: bottom - top, - }; -} - -function isMobileBreakpoint(gridWidth: number, gridHeight: number): boolean { - const gridAspectRatio = gridWidth / gridHeight; - return gridAspectRatio < 1; -} - -function getGridLayout( - tileCount: number, - focusedTileCount: number, - gridWidth: number, - gridHeight: number, -): { itemGridRatio: number; layoutDirection: LayoutDirection } { - let layoutDirection: LayoutDirection = "horizontal"; - let itemGridRatio = 1; - - if (focusedTileCount === 0) { - return { itemGridRatio, layoutDirection }; - } - - if (isMobileBreakpoint(gridWidth, gridHeight)) { - layoutDirection = "vertical"; - itemGridRatio = 1 / 3; - } else { - layoutDirection = "horizontal"; - itemGridRatio = 1 / 3; - } - - return { itemGridRatio, layoutDirection }; -} - -function centerTiles( - positions: TilePosition[], - gridWidth: number, - gridHeight: number, - offsetLeft: number, - offsetTop: number, -): TilePosition[] { - const bounds = getSubGridBoundingBox(positions); - - const leftOffset = Math.round((gridWidth - bounds.width) / 2) + offsetLeft; - const topOffset = Math.round((gridHeight - bounds.height) / 2) + offsetTop; - - applyTileOffsets(positions, leftOffset, topOffset); - - return positions; -} - -function applyTileOffsets( - positions: TilePosition[], - leftOffset: number, - topOffset: number, -): TilePosition[] { - for (const position of positions) { - position.x += leftOffset; - position.y += topOffset; - } - - return positions; -} - -function getSubGridLayout( - tileCount: number, - gridWidth: number, - gridHeight: number, -): { columnCount: number; rowCount: number; tileAspectRatio: number } { - const gridAspectRatio = gridWidth / gridHeight; - - let columnCount: number; - let rowCount: number; - let tileAspectRatio: number = 16 / 9; - - if (gridAspectRatio < 3 / 4) { - // Phone - if (tileCount === 1) { - columnCount = 1; - rowCount = 1; - tileAspectRatio = 0; - } else if (tileCount <= 4) { - columnCount = 1; - rowCount = tileCount; - } else if (tileCount <= 12) { - columnCount = 2; - rowCount = Math.ceil(tileCount / columnCount); - tileAspectRatio = 0; - } else { - // Unsupported - columnCount = 3; - rowCount = Math.ceil(tileCount / columnCount); - tileAspectRatio = 1; - } - } else if (gridAspectRatio < 1) { - // Tablet - if (tileCount === 1) { - columnCount = 1; - rowCount = 1; - tileAspectRatio = 0; - } else if (tileCount <= 4) { - columnCount = 1; - rowCount = tileCount; - } else if (tileCount <= 12) { - columnCount = 2; - rowCount = Math.ceil(tileCount / columnCount); - } else { - // Unsupported - columnCount = 3; - rowCount = Math.ceil(tileCount / columnCount); - tileAspectRatio = 1; - } - } else if (gridAspectRatio < 17 / 9) { - // Computer - if (tileCount === 1) { - columnCount = 1; - rowCount = 1; - } else if (tileCount === 2) { - columnCount = 2; - rowCount = 1; - } else if (tileCount <= 4) { - columnCount = 2; - rowCount = 2; - } else if (tileCount <= 6) { - columnCount = 3; - rowCount = 2; - } else if (tileCount <= 8) { - columnCount = 4; - rowCount = 2; - tileAspectRatio = 1; - } else if (tileCount <= 12) { - columnCount = 4; - rowCount = 3; - tileAspectRatio = 1; - } else { - // Unsupported - columnCount = 4; - rowCount = 4; - } - } else if (gridAspectRatio <= 32 / 9) { - // Ultrawide - if (tileCount === 1) { - columnCount = 1; - rowCount = 1; - } else if (tileCount === 2) { - columnCount = 2; - rowCount = 1; - } else if (tileCount <= 4) { - columnCount = 2; - rowCount = 2; - } else if (tileCount <= 6) { - columnCount = 3; - rowCount = 2; - } else if (tileCount <= 8) { - columnCount = 4; - rowCount = 2; - } else if (tileCount <= 12) { - columnCount = 4; - rowCount = 3; - } else { - // Unsupported - columnCount = 4; - rowCount = 4; - } - } else { - // Super Ultrawide - if (tileCount <= 6) { - columnCount = tileCount; - rowCount = 1; - } else { - columnCount = Math.ceil(tileCount / 2); - rowCount = 2; - } - } - - return { columnCount, rowCount, tileAspectRatio }; -} - -function getSubGridPositions( - tileCount: number, - columnCount: number, - rowCount: number, - tileAspectRatio: number, - gridWidth: number, - gridHeight: number, -): TilePosition[] { - if (tileCount === 0) { - return []; - } - - const newTilePositions: TilePosition[] = []; - - const boxWidth = Math.round( - (gridWidth - GAP * (columnCount + 1)) / columnCount, - ); - const boxHeight = Math.round((gridHeight - GAP * (rowCount + 1)) / rowCount); - - let tileWidth: number; - let tileHeight: number; - - if (tileAspectRatio) { - const boxAspectRatio = boxWidth / boxHeight; - - if (boxAspectRatio > tileAspectRatio) { - tileWidth = boxHeight * tileAspectRatio; - tileHeight = boxHeight; - } else { - tileWidth = boxWidth; - tileHeight = boxWidth / tileAspectRatio; - } - } else { - tileWidth = boxWidth; - tileHeight = boxHeight; - } - - for (let i = 0; i < tileCount; i++) { - const verticalIndex = Math.floor(i / columnCount); - const top = verticalIndex * GAP + verticalIndex * tileHeight; - - let rowItemCount: number; - - if (verticalIndex + 1 === rowCount && tileCount % columnCount !== 0) { - rowItemCount = tileCount % columnCount; - } else { - rowItemCount = columnCount; - } - - const horizontalIndex = i % columnCount; - - let centeringPadding = 0; - - if (rowItemCount < columnCount) { - const subgridWidth = tileWidth * columnCount + (GAP * columnCount - 1); - centeringPadding = Math.round( - (subgridWidth - (tileWidth * rowItemCount + (GAP * rowItemCount - 1))) / - 2, - ); - } - - const left = - centeringPadding + GAP * horizontalIndex + tileWidth * horizontalIndex; - - newTilePositions.push({ - width: tileWidth, - height: tileHeight, - x: left, - y: top, - zIndex: 0, - }); - } - - return newTilePositions; -} - -// Calculates the number of possible tiles that can be displayed -function displayedTileCount( - layout: Layout, - tileCount: number, - gridWidth: number, - gridHeight: number, -): number { - let displayedTile = -1; - if (layout === "grid") { - return displayedTile; - } - if (tileCount < 2) { - return displayedTile; - } - - const gridAspectRatio = gridWidth / gridHeight; - - if (gridAspectRatio < 1) { - // Vertical layout (mobile) - const spotlightTileHeight = (gridHeight - GAP * 3) * (4 / 5); - const spectatorTileSize = gridHeight - GAP * 3 - spotlightTileHeight; - displayedTile = Math.round(gridWidth / spectatorTileSize); - } else { - const spotlightTileWidth = ((gridWidth - GAP * 3) * 4) / 5; - const spectatorTileWidth = gridWidth - GAP * 3 - spotlightTileWidth; - const spectatorTileHeight = spectatorTileWidth * (9 / 16); - displayedTile = Math.round(gridHeight / spectatorTileHeight); - } - - return displayedTile; -} - -// Sets the 'order' property on tiles based on the layout param and -// other properties of the tiles, eg. 'focused' and 'presenter' -export function reorderTiles( - tiles: Tile[], - layout: Layout, - displayedTile = -1, -): void { - // We use a special layout for 1:1 to always put the local tile first. - // We only do this if there are two tiles (obviously) and exactly one - // of them is local: during startup we can have tiles from other users - // but not our own, due to the order they're added, so without this we - // can assign multiple remote tiles order '1' and this persists through - // subsequent reorders because we preserve the order of the tiles. - if ( - layout === "grid" && - tiles.length === 2 && - tiles.filter((t) => t.item.local).length === 1 && - !tiles.some((t) => t.focused) - ) { - // 1:1 layout - tiles.forEach((tile) => (tile.order = tile.item.local ? 0 : 1)); - } else { - const focusedTiles: Tile[] = []; - const presenterTiles: Tile[] = []; - const onlyVideoTiles: Tile[] = []; - const otherTiles: Tile[] = []; - - const orderedTiles: Tile[] = new Array(tiles.length); - tiles.forEach((tile) => (orderedTiles[tile.order] = tile)); - - let firstLocalTile: Tile | undefined; - orderedTiles.forEach((tile) => { - if (tile.focused) { - focusedTiles.push(tile); - } else if (tile.isPresenter) { - presenterTiles.push(tile); - } else if (tile.hasVideo) { - if (tile.order === 0 && tile.item.local) { - firstLocalTile = tile; - } else { - onlyVideoTiles.push(tile); - } - } else { - if (tile.order === 0 && tile.item.local) { - firstLocalTile = tile; - } else { - otherTiles.push(tile); - } - } - }); - - if (firstLocalTile) { - if (firstLocalTile.hasVideo) { - onlyVideoTiles.push(firstLocalTile); - } else { - otherTiles.push(firstLocalTile); - } - } - - const reorderedTiles = [ - ...focusedTiles, - ...presenterTiles, - ...onlyVideoTiles, - ...otherTiles, - ]; - let nextSpeakerTileIndex = focusedTiles.length + presenterTiles.length; - - reorderedTiles.forEach((tile, i) => { - // If a speaker's natural ordering would place it outside the default - // visible area, promote them to the section dedicated to speakers - if (tile.isSpeaker && displayedTile <= i && nextSpeakerTileIndex < i) { - // Remove the tile from its current section - reorderedTiles.splice(i, 1); - // Insert it into the speaker section - reorderedTiles.splice(nextSpeakerTileIndex, 0, tile); - nextSpeakerTileIndex++; - } - }); - - reorderedTiles.forEach((tile, i) => (tile.order = i)); - } -} - -interface DragTileData { - offsetX: number; - offsetY: number; - key: string; - x: number; - y: number; -} - -export interface ChildrenProperties { - ref: Ref; - style: ComponentProps["style"]; - /** - * The width this tile will have once its animations have settled. - */ - targetWidth: number; - /** - * The height this tile will have once its animations have settled. - */ - targetHeight: number; - data: T; -} - -export interface LegacyGridProps { - items: TileDescriptor[]; - layout: Layout; - disableAnimations: boolean; - Tile: ComponentType>; -} - -export function LegacyGrid({ - items, - layout, - disableAnimations, - Tile, -}: LegacyGridProps): ReactNode { - // Place the PiP in the bottom right corner by default - const [pipXRatio, setPipXRatio] = useState(1); - const [pipYRatio, setPipYRatio] = useState(1); - - const [{ tiles, tilePositions }, setTileState] = useState<{ - tiles: Tile[]; - tilePositions: TilePosition[]; - }>({ - tiles: [], - tilePositions: [], - }); - const [scrollPosition, setScrollPosition] = useState(0); - const draggingTileRef = useRef(null); - const lastTappedRef = useRef<{ [index: string]: number }>({}); - const lastLayoutRef = useRef(layout); - const isMounted = useIsMounted(); - - // The 'polyfill' argument to useMeasure is not a polyfill at all but is the impl that is always used - // if passed, whether the browser has native support or not, so pass in either the browser native - // version or the ponyfill (yes, pony) because Juggle's resizeobserver ponyfill is being weirdly - // buggy for me on my dev env my never updating the size until the window resizes. - const [gridRef, gridBounds] = useMeasure({ - polyfill: window.ResizeObserver ?? JuggleResizeObserver, - }); - - useEffect(() => { - setTileState(({ tiles, ...rest }) => { - const newTiles: Tile[] = []; - const removedTileKeys: Set = new Set(); - - for (const tile of tiles) { - let item = items.find((item) => item.id === tile.key); - - let remove = false; - - if (!item) { - remove = true; - item = tile.item; - removedTileKeys.add(tile.key); - } - - let focused: boolean; - let isSpeaker: boolean; - let isPresenter: boolean; - let hasVideo: boolean; - if (layout === "spotlight") { - focused = item.focused; - isPresenter = item.isPresenter; - isSpeaker = item.isSpeaker; - hasVideo = item.hasVideo; - } else { - focused = layout === lastLayoutRef.current ? tile.focused : false; - isPresenter = false; - isSpeaker = false; - hasVideo = false; - } - - newTiles.push({ - key: item.id, - order: tile.order, - item, - remove, - focused, - isSpeaker: isSpeaker, - isPresenter: isPresenter, - hasVideo: hasVideo, - }); - } - - for (const item of items) { - const existingTileIndex = newTiles.findIndex( - ({ key }) => item.id === key, - ); - - const existingTile = newTiles[existingTileIndex]; - - if (existingTile && !existingTile.remove) { - continue; - } - - const newTile: Tile = { - key: item.id, - order: existingTile?.order ?? newTiles.length, - item, - remove: false, - focused: layout === "spotlight" && item.focused, - isPresenter: item.isPresenter, - isSpeaker: item.isSpeaker, - hasVideo: item.hasVideo, - }; - - if (existingTile) { - // Replace an existing tile - newTiles.splice(existingTileIndex, 1, newTile); - } else { - // Added tiles - newTiles.push(newTile); - } - } - - const presenter = newTiles.find((t) => t.isPresenter); - let displayedTile = -1; - // Only on screen share we will not move active displayed speaker - if (presenter !== undefined) { - displayedTile = displayedTileCount( - layout, - newTiles.length, - gridBounds.width, - gridBounds.height, - ); - } - - reorderTiles(newTiles, layout, displayedTile); - - if (removedTileKeys.size > 0) { - setTimeout(() => { - if (!isMounted.current) { - return; - } - - setTileState(({ tiles, ...rest }) => { - const newTiles: Tile[] = tiles - .filter((tile) => !removedTileKeys.has(tile.key)) - .map((tile) => ({ ...tile })); // clone before reordering - reorderTiles(newTiles, layout); - - const focusedTileCount = newTiles.reduce( - (count, tile) => count + (tile.focused ? 1 : 0), - 0, - ); - - return { - ...rest, - tiles: newTiles, - tilePositions: getTilePositions( - newTiles.length, - focusedTileCount, - gridBounds.width, - gridBounds.height, - pipXRatio, - pipYRatio, - layout, - ), - }; - }); - }, 250); - } - - const focusedTileCount = newTiles.reduce( - (count, tile) => count + (tile.focused ? 1 : 0), - 0, - ); - - lastLayoutRef.current = layout; - - return { - ...rest, - tiles: newTiles, - tilePositions: getTilePositions( - newTiles.length, - focusedTileCount, - gridBounds.width, - gridBounds.height, - pipXRatio, - pipYRatio, - layout, - ), - }; - }); - }, [items, gridBounds, layout, isMounted, pipXRatio, pipYRatio]); - - const tilePositionsValid = useRef(false); - - const animate = useCallback( - (tiles: Tile[]) => { - // Whether the tile positions were valid at the time of the previous - // animation - const tilePositionsWereValid = tilePositionsValid.current; - const oneOnOneLayout = - tiles.length === 2 && !tiles.some((t) => t.focused); - - return (tileIndex: number): TileSpringUpdate => { - const tile = tiles[tileIndex]; - const tilePosition = tilePositions[tile.order]; - const draggingTile = draggingTileRef.current; - const dragging = draggingTile && tile.key === draggingTile.key; - const remove = tile.remove; - tilePositionsValid.current = tilePosition.height > 0; - - if (dragging) { - return { - width: tilePosition.width, - height: tilePosition.height, - x: draggingTile.offsetX + draggingTile.x, - y: draggingTile.offsetY + draggingTile.y, - scale: 1.1, - opacity: 1, - zIndex: 2, - shadow: 15, - shadowSpread: 0, - immediate: (key: string): boolean => - disableAnimations || - key === "zIndex" || - key === "x" || - key === "y" || - key === "shadow" || - key === "shadowSpread", - from: { - shadow: 0, - scale: 0, - opacity: 0, - zIndex: 0, - }, - reset: false, - }; - } else { - const isMobile = isMobileBreakpoint( - gridBounds.width, - gridBounds.height, - ); - - const x = - tilePosition.x + - (layout === "spotlight" && tile.order !== 0 && isMobile - ? scrollPosition - : 0); - const y = - tilePosition.y + - (layout === "spotlight" && tile.order !== 0 && !isMobile - ? scrollPosition - : 0); - const from: { - shadow: number; - scale: number; - opacity: number; - zIndex?: number; - x?: number; - y?: number; - width?: number; - height?: number; - } = { shadow: 0, scale: 0, opacity: 0 }; - let reset = false; - - if (!tilePositionsWereValid) { - // This indicates that the component just mounted. We discard the - // previous keyframe by resetting the tile's position, so that it - // animates in from the right place on screen, rather than wherever - // the zero-height grid placed it. - from.x = x; - from.y = y; - from.width = tilePosition.width; - from.height = tilePosition.height; - reset = true; - } - - return { - x, - y, - width: tilePosition.width, - height: tilePosition.height, - scale: remove ? 0 : 1, - opacity: remove ? 0 : 1, - zIndex: tilePosition.zIndex, - shadow: oneOnOneLayout && tile.item.local ? 1 : 0, - shadowSpread: oneOnOneLayout && tile.item.local ? 1 : 0, - from, - reset, - immediate: (key: string): boolean => - disableAnimations || - key === "zIndex" || - key === "shadow" || - key === "shadowSpread", - // If we just stopped dragging a tile, give it time for the - // animation to settle before pushing its z-index back down - delay: (key: string): number => (key === "zIndex" ? 500 : 0), - }; - } - }; - }, - [tilePositions, disableAnimations, scrollPosition, layout, gridBounds], - ); - - const [springs, api] = useSprings(tiles.length, animate(tiles), [ - tilePositions, - tiles, - scrollPosition, - // react-spring's types are bugged and can't infer the spring type - ]) as unknown as [SpringValues[], SpringRef]; - - const onTap = useCallback( - (tileKey: string) => { - const lastTapped = lastTappedRef.current[tileKey]; - - if (!lastTapped || Date.now() - lastTapped > 500) { - lastTappedRef.current[tileKey] = Date.now(); - return; - } - - lastTappedRef.current[tileKey] = 0; - - const tile = tiles.find((tile) => tile.key === tileKey); - if (!tile || layout !== "grid") return; - const item = tile.item; - - setTileState(({ tiles, ...state }) => { - let focusedTileCount = 0; - const newTiles = tiles.map((tile) => { - const newTile = { ...tile }; // clone before reordering - - if (tile.item === item) { - newTile.focused = !tile.focused; - } - if (newTile.focused) { - focusedTileCount++; - } - - return newTile; - }); - - reorderTiles(newTiles, layout); - - return { - ...state, - tiles: newTiles, - tilePositions: getTilePositions( - newTiles.length, - focusedTileCount, - gridBounds.width, - gridBounds.height, - pipXRatio, - pipYRatio, - layout, - ), - }; - }); - }, - [tiles, layout, gridBounds.width, gridBounds.height, pipXRatio, pipYRatio], - ); - - // Callback for useDrag. We could call useDrag here, but the default - // pattern of spreading {...bind()} across the children to bind the gesture - // ends up breaking memoization and ruining this component's performance. - // Instead, we pass this callback to each tile via a ref, to let them bind the - // gesture using the much more sensible ref-based method. - const onTileDrag = ( - tileId: string, - { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - active, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - xy, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - movement, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - tap, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - last, - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - event, - }: Parameters>[0], - ): void => { - event.preventDefault(); - - if (tap) { - onTap(tileId); - return; - } - - if (layout !== "grid") return; - - const dragTileIndex = tiles.findIndex((tile) => tile.key === tileId); - const dragTile = tiles[dragTileIndex]; - const dragTilePosition = tilePositions[dragTile.order]; - - const cursorPosition = [xy[0] - gridBounds.left, xy[1] - gridBounds.top]; - - let newTiles = tiles; - - if (tiles.length === 2 && !tiles.some((t) => t.focused)) { - // We're in 1:1 mode, so only the local tile should be draggable - if (!dragTile.item.local) return; - - // Position should only update on the very last event, to avoid - // compounding the offset on every drag event - if (last) { - const remotePosition = tilePositions[1]; - - const pipGap = getPipGap( - gridBounds.width / gridBounds.height, - gridBounds.width, - ); - const pipMinX = remotePosition.x + pipGap; - const pipMinY = remotePosition.y + pipGap; - const pipMaxX = - remotePosition.x + - remotePosition.width - - dragTilePosition.width - - pipGap; - const pipMaxY = - remotePosition.y + - remotePosition.height - - dragTilePosition.height - - pipGap; - - const newPipXRatio = - (dragTilePosition.x + movement[0] - pipMinX) / (pipMaxX - pipMinX); - const newPipYRatio = - (dragTilePosition.y + movement[1] - pipMinY) / (pipMaxY - pipMinY); - - setPipXRatio(Math.max(0, Math.min(1, newPipXRatio))); - setPipYRatio(Math.max(0, Math.min(1, newPipYRatio))); - } - } else { - const hoverTile = tiles.find( - (tile) => - tile.key !== tileId && - isInside(cursorPosition, tilePositions[tile.order]), - ); - - if (hoverTile) { - // Shift the tiles into their new order - newTiles = newTiles.map((tile) => { - let order = tile.order; - if (order < dragTile.order) { - if (order >= hoverTile.order) order++; - } else if (order > dragTile.order) { - if (order <= hoverTile.order) order--; - } else { - order = hoverTile.order; - } - - let focused; - if (tile === hoverTile) { - focused = dragTile.focused; - } else if (tile === dragTile) { - focused = hoverTile.focused; - } else { - focused = tile.focused; - } - - return { ...tile, order, focused }; - }); - - reorderTiles(newTiles, layout); - - setTileState((state) => ({ ...state, tiles: newTiles })); - } - } - - if (active) { - if (!draggingTileRef.current) { - draggingTileRef.current = { - key: dragTile.key, - offsetX: dragTilePosition.x, - offsetY: dragTilePosition.y, - x: movement[0], - y: movement[1], - }; - } else { - draggingTileRef.current.x = movement[0]; - draggingTileRef.current.y = movement[1]; - } - } else { - draggingTileRef.current = null; - } - - api.start(animate(newTiles)); - }; - - const onTileDragRef = useRef(onTileDrag); - onTileDragRef.current = onTileDrag; - - const onGridGesture = useCallback( - ( - e: - | Omit, "event"> - | Omit, "event">, - isWheel: boolean, - ) => { - if (layout !== "spotlight") { - return; - } - - const isMobile = isMobileBreakpoint(gridBounds.width, gridBounds.height); - - let movement = e.delta[isMobile ? 0 : 1]; - - if (isWheel) { - movement = -movement; - } - - let min = 0; - - if (tilePositions.length > 1) { - const lastTile = tilePositions[tilePositions.length - 1]; - min = isMobile - ? gridBounds.width - lastTile.x - lastTile.width - GAP - : gridBounds.height - lastTile.y - lastTile.height - GAP; - } - - setScrollPosition((scrollPosition) => - Math.min(Math.max(movement + scrollPosition, min), 0), - ); - }, - [layout, gridBounds, tilePositions], - ); - - const bindGrid = useGesture( - { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - onWheel: (e) => onGridGesture(e, true), - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-ignore - onDrag: (e) => onGridGesture(e, false), - }, - {}, - ); - - return ( -
- {springs.map((spring, i) => { - const tile = tiles[i]; - const tilePosition = tilePositions[tile.order]; - - return ( - - ); - })} -
- ); -} - -LegacyGrid.defaultProps = { - layout: "grid", -}; diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 90791de6..a1b3f4cc 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -18,10 +18,9 @@ import { RoomAudioRenderer, RoomContext, useLocalParticipant, - useTracks, } from "@livekit/components-react"; import { usePreventScroll } from "@react-aria/overlays"; -import { ConnectionState, Room, Track } from "livekit-client"; +import { ConnectionState, Room } from "livekit-client"; import { MatrixClient } from "matrix-js-sdk/src/client"; import { FC, @@ -38,7 +37,6 @@ import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import classNames from "classnames"; import { BehaviorSubject, map } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; -import { useTranslation } from "react-i18next"; import LogoMark from "../icons/LogoMark.svg?react"; import LogoType from "../icons/LogoType.svg?react"; @@ -51,10 +49,8 @@ import { SettingsButton, } from "../button"; import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header"; -import { LegacyGrid, useLegacyGridLayout } from "../grid/LegacyGrid"; import { useUrlParams } from "../UrlParams"; import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts"; -import { usePrefersReducedMotion } from "../usePrefersReducedMotion"; import { ElementWidgetActions, widget } from "../widget"; import styles from "./InCallView.module.css"; import { GridTile } from "../tile/GridTile"; @@ -72,14 +68,8 @@ import { InviteButton } from "../button/InviteButton"; import { LayoutToggle } from "./LayoutToggle"; import { ECConnectionState } from "../livekit/useECConnectionState"; import { useOpenIDSFU } from "../livekit/openIDSFU"; -import { - GridMode, - Layout, - TileDescriptor, - useCallViewModel, -} from "../state/CallViewModel"; +import { GridMode, Layout, useCallViewModel } from "../state/CallViewModel"; import { Grid, TileProps } from "../grid/Grid"; -import { MediaViewModel } from "../state/MediaViewModel"; import { useObservable } from "../state/useObservable"; import { useInitial } from "../useInitial"; import { SpotlightTile } from "../tile/SpotlightTile"; @@ -89,7 +79,6 @@ import { makeGridLayout } from "../grid/GridLayout"; import { makeSpotlightLayout } from "../grid/SpotlightLayout"; import { CallLayout, - GridTileModel, TileModel, defaultPipAlignment, defaultSpotlightAlignment, @@ -98,10 +87,6 @@ import { makeOneOnOneLayout } from "../grid/OneOnOneLayout"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); -const dummySpotlightItem = { - id: "spotlight", -} as TileDescriptor; - export interface ActiveCallProps extends Omit { e2eeSystem: EncryptionSystem; @@ -155,11 +140,9 @@ export const InCallView: FC = ({ participantCount, onLeave, hideHeader, - otelGroupCallMembership, connState, onShareClick, }) => { - const { t } = useTranslation(); usePreventScroll(); useWakeLock(); @@ -177,15 +160,6 @@ export const InCallView: FC = ({ // Merge the refs so they can attach to the same element const containerRef = useMergedRefs(containerRef1, containerRef2); - const screenSharingTracks = useTracks( - [{ source: Track.Source.ScreenShare, withPlaceholder: false }], - { - room: livekitRoom, - }, - ); - const { layout: legacyLayout, setLayout: setLegacyLayout } = - useLegacyGridLayout(screenSharingTracks.length > 0); - const { hideScreensharing, showControls } = useUrlParams(); const { isScreenShareEnabled, localParticipant } = useLocalParticipant({ @@ -210,42 +184,6 @@ export const InCallView: FC = ({ (muted) => muteStates.audio.setEnabled?.(!muted), ); - useEffect(() => { - widget?.api.transport.send( - legacyLayout === "grid" - ? ElementWidgetActions.TileLayout - : ElementWidgetActions.SpotlightLayout, - {}, - ); - }, [legacyLayout]); - - useEffect(() => { - if (widget) { - const onTileLayout = (ev: CustomEvent): void => { - setLegacyLayout("grid"); - widget!.api.transport.reply(ev.detail, {}); - }; - const onSpotlightLayout = (ev: CustomEvent): void => { - setLegacyLayout("spotlight"); - widget!.api.transport.reply(ev.detail, {}); - }; - - widget.lazyActions.on(ElementWidgetActions.TileLayout, onTileLayout); - widget.lazyActions.on( - ElementWidgetActions.SpotlightLayout, - onSpotlightLayout, - ); - - return (): void => { - widget!.lazyActions.off(ElementWidgetActions.TileLayout, onTileLayout); - widget!.lazyActions.off( - ElementWidgetActions.SpotlightLayout, - onSpotlightLayout, - ); - }; - } - }, [setLegacyLayout]); - const mobile = boundsValid && bounds.width <= 660; const reducedControls = boundsValid && bounds.width <= 340; const noControls = reducedControls && bounds.height <= 400; @@ -256,15 +194,12 @@ export const InCallView: FC = ({ matrixInfo.e2eeSystem.kind !== E2eeType.NONE, connState, ); - const items = useObservableEagerState(vm.tiles); const layout = useObservableEagerState(vm.layout); + const gridMode = useObservableEagerState(vm.gridMode); const hasSpotlight = layout.spotlight !== undefined; - // Hack: We insert a dummy "spotlight" tile into the tiles we pass to - // useFullscreen so that we can control the fullscreen state of the - // spotlight tile in the new layouts with this same hook. const fullscreenItems = useMemo( - () => (hasSpotlight ? [...items, dummySpotlightItem] : items), - [items, hasSpotlight], + () => (hasSpotlight ? ["spotlight"] : []), + [hasSpotlight], ); const { fullscreenItem, toggleFullscreen, exitFullscreen } = useFullscreen(fullscreenItems); @@ -274,18 +209,9 @@ export const InCallView: FC = ({ ); // The maximised participant: either the participant that the user has - // manually put in fullscreen, or the focused (active) participant if the - // window is too small to show everyone - const maximisedParticipant = useMemo( - () => - fullscreenItem ?? - (noControls - ? items.find((item) => item.isSpeaker) ?? items.at(0) ?? null - : null), - [fullscreenItem, noControls, items], - ); - - const prefersReducedMotion = usePrefersReducedMotion(); + // manually put in fullscreen, or (TODO) the spotlight if the window is too + // small to show everyone + const maximisedParticipant = fullscreenItem; const [settingsModalOpen, setSettingsModalOpen] = useState(false); const [settingsTab, setSettingsTab] = useState(defaultSettingsTab); @@ -339,7 +265,7 @@ export const InCallView: FC = ({ makeLayout = makeSpotlightLayout as CallLayout; else if (l.type === "one-on-one") makeLayout = makeOneOnOneLayout as CallLayout; - else return null; // Not yet implemented + else throw new Error(`Unimplemented layout: ${l.type}`); return makeLayout({ minBounds: gridBoundsObservable, @@ -352,13 +278,46 @@ export const InCallView: FC = ({ ); const setGridMode = useCallback( - (mode: GridMode) => { - setLegacyLayout(mode); - vm.setGridMode(mode); - }, - [setLegacyLayout, vm], + (mode: GridMode) => vm.setGridMode(mode), + [vm], ); + useEffect(() => { + widget?.api.transport.send( + gridMode === "grid" + ? ElementWidgetActions.TileLayout + : ElementWidgetActions.SpotlightLayout, + {}, + ); + }, [gridMode]); + + useEffect(() => { + if (widget) { + const onTileLayout = (ev: CustomEvent): void => { + setGridMode("grid"); + widget!.api.transport.reply(ev.detail, {}); + }; + const onSpotlightLayout = (ev: CustomEvent): void => { + setGridMode("spotlight"); + widget!.api.transport.reply(ev.detail, {}); + }; + + widget.lazyActions.on(ElementWidgetActions.TileLayout, onTileLayout); + widget.lazyActions.on( + ElementWidgetActions.SpotlightLayout, + onSpotlightLayout, + ); + + return (): void => { + widget!.lazyActions.off(ElementWidgetActions.TileLayout, onTileLayout); + widget!.lazyActions.off( + ElementWidgetActions.SpotlightLayout, + onSpotlightLayout, + ); + }; + } + }, [setGridMode]); + const showSpotlightIndicators = useObservable(layout.type === "spotlight"); const showSpeakingIndicators = useObservable( layout.type === "spotlight" || @@ -419,33 +378,10 @@ export const InCallView: FC = ({ ], ); - const LegacyTile = useMemo( - () => - forwardRef< - HTMLDivElement, - PropsWithoutRef> - >(function LegacyTile({ model: legacyModel, ...props }, ref) { - const model: GridTileModel = useMemo( - () => ({ type: "grid", vm: legacyModel }), - [legacyModel], - ); - return ; - }), - [Tile], - ); - const renderContent = (): JSX.Element => { - if (items.length === 0) { - return ( -
-

{t("waiting_for_participants")}

-
- ); - } - if (maximisedParticipant !== null) { const fullscreen = maximisedParticipant === fullscreenItem; - if (maximisedParticipant.id === "spotlight") { + if (maximisedParticipant === "spotlight") { return ( = ({ /> ); } - return ( - - ); } - if (layoutSystem === null) { - // This new layout doesn't yet have an implemented layout system, so fall - // back to the legacy grid system - return ( - + - ); - } else { - return ( - <> - - - - ); - } + + + ); }; const rageshakeRequestModalProps = useRageshakeRequestModal( @@ -596,7 +505,7 @@ export const InCallView: FC = ({ {!mobile && !hideHeader && showControls && ( )} diff --git a/src/room/useFullscreen.ts b/src/room/useFullscreen.ts index db769fa5..650cc6ea 100644 --- a/src/room/useFullscreen.ts +++ b/src/room/useFullscreen.ts @@ -20,7 +20,6 @@ import { useCallback, useLayoutEffect, useRef } from "react"; import { useReactiveState } from "../useReactiveState"; import { useEventTarget } from "../useEvents"; -import { TileDescriptor } from "../state/CallViewModel"; const isFullscreen = (): boolean => Boolean(document.fullscreenElement) || @@ -55,31 +54,30 @@ function useFullscreenChange(onFullscreenChange: () => void): void { * Provides callbacks for controlling the full-screen view, which can hold one * item at a time. */ -export function useFullscreen(items: TileDescriptor[]): { - fullscreenItem: TileDescriptor | null; +// TODO: Simplify this. Nowadays we only allow the spotlight to be fullscreen, +// so we don't need to bother with multiple items. +export function useFullscreen(items: string[]): { + fullscreenItem: string | null; toggleFullscreen: (itemId: string) => void; exitFullscreen: () => void; } { - const [fullscreenItem, setFullscreenItem] = - useReactiveState | null>( - (prevItem) => - prevItem == null - ? null - : items.find((i) => i.id === prevItem.id) ?? null, - [items], - ); + const [fullscreenItem, setFullscreenItem] = useReactiveState( + (prevItem) => + prevItem == null ? null : items.find((i) => i === prevItem) ?? null, + [items], + ); - const latestItems = useRef[]>(items); + const latestItems = useRef(items); latestItems.current = items; - const latestFullscreenItem = useRef | null>(fullscreenItem); + const latestFullscreenItem = useRef(fullscreenItem); latestFullscreenItem.current = fullscreenItem; const toggleFullscreen = useCallback( (itemId: string) => { setFullscreenItem( latestFullscreenItem.current === null - ? latestItems.current.find((i) => i.id === itemId) ?? null + ? latestItems.current.find((i) => i === itemId) ?? null : null, ); }, diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 62c35a8e..975d069b 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -74,22 +74,6 @@ import { ObservableScope } from "./ObservableScope"; // list again const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; -// Represents something that should get a tile on the layout, -// ie. a user's video feed or a screen share feed. -// TODO: This exposes too much information to the view layer, let's keep this -// information internal to the view model and switch to using Tile instead -export interface TileDescriptor { - id: string; - focused: boolean; - isPresenter: boolean; - isSpeaker: boolean; - hasVideo: boolean; - local: boolean; - largeBaseSize: boolean; - placeNear?: string; - data: T; -} - export interface GridLayout { type: "grid"; spotlight?: MediaViewModel[]; @@ -548,108 +532,6 @@ export class CallViewModel extends ViewModel { shareReplay(1), ); - /** - * The media tiles to be displayed in the call view. - */ - // TODO: Get rid of this field, replacing it with the 'layout' field above - // which keeps more details of the layout order internal to the view model - public readonly tiles: Observable[]> = - combineLatest([ - this.remoteParticipants, - observeParticipantMedia(this.livekitRoom.localParticipant), - ]).pipe( - scan((ts, [remoteParticipants, { participant: localParticipant }]) => { - const ps = [localParticipant, ...remoteParticipants]; - const tilesById = new Map(ts.map((t) => [t.id, t])); - const now = Date.now(); - let allGhosts = true; - - const newTiles = ps.flatMap((p) => { - const userMediaId = p.identity; - const member = findMatrixMember(this.matrixRoom, userMediaId); - allGhosts &&= member === undefined; - const spokeRecently = - p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000; - - // We always start with a local participant with the empty string as - // their ID before we're connected, this is fine and we'll be in - // "all ghosts" mode. - if (userMediaId !== "" && member === undefined) { - logger.warn( - `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`, - ); - } - - const userMediaVm = - tilesById.get(userMediaId)?.data ?? - (p instanceof LocalParticipant - ? new LocalUserMediaViewModel( - userMediaId, - member, - p, - this.encrypted, - ) - : new RemoteUserMediaViewModel( - userMediaId, - member, - p, - this.encrypted, - )); - tilesById.delete(userMediaId); - - const userMediaTile: TileDescriptor = { - id: userMediaId, - focused: false, - isPresenter: p.isScreenShareEnabled, - isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal, - hasVideo: p.isCameraEnabled, - local: p.isLocal, - largeBaseSize: false, - data: userMediaVm, - }; - - if (p.isScreenShareEnabled) { - const screenShareId = `${userMediaId}:screen-share`; - const screenShareVm = - tilesById.get(screenShareId)?.data ?? - new ScreenShareViewModel( - screenShareId, - member, - p, - this.encrypted, - ); - tilesById.delete(screenShareId); - - const screenShareTile: TileDescriptor = { - id: screenShareId, - focused: true, - isPresenter: false, - isSpeaker: false, - hasVideo: true, - local: p.isLocal, - largeBaseSize: true, - placeNear: userMediaId, - data: screenShareVm, - }; - return [userMediaTile, screenShareTile]; - } else { - return [userMediaTile]; - } - }); - - // Any tiles left in the map are unused and should be destroyed - for (const t of tilesById.values()) t.data.destroy(); - - // If every item is a ghost, that probably means we're still connecting - // and shouldn't bother showing anything yet - return allGhosts ? [] : newTiles; - }, [] as TileDescriptor[]), - finalizeValue((ts) => { - for (const t of ts) t.data.destroy(); - }), - shareReplay(1), - ); - public constructor( // A call is permanently tied to a single Matrix room and LiveKit room private readonly matrixRoom: MatrixRoom, diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 14f85831..4dd83567 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -33,7 +33,6 @@ import VolumeOffIcon from "@vector-im/compound-design-tokens/icons/volume-off.sv import VisibilityOnIcon from "@vector-im/compound-design-tokens/icons/visibility-on.svg?react"; import UserProfileIcon from "@vector-im/compound-design-tokens/icons/user-profile.svg?react"; import ExpandIcon from "@vector-im/compound-design-tokens/icons/expand.svg?react"; -import CollapseIcon from "@vector-im/compound-design-tokens/icons/collapse.svg?react"; import { ContextMenu, MenuItem, @@ -44,8 +43,6 @@ import { useObservableEagerState } from "observable-hooks"; import styles from "./GridTile.module.css"; import { - ScreenShareViewModel, - MediaViewModel, UserMediaViewModel, useNameData, LocalUserMediaViewModel, @@ -63,45 +60,12 @@ interface TileProps { maximised: boolean; displayName: string; nameTag: string; + showSpeakingIndicators: boolean; } -interface MediaTileProps - extends TileProps, - Omit, "className"> { - vm: MediaViewModel; - videoEnabled: boolean; - videoFit: "contain" | "cover"; - mirror: boolean; - nameTagLeadingIcon?: ReactNode; - primaryButton: ReactNode; - secondaryButton?: ReactNode; -} - -const MediaTile = forwardRef( - ({ vm, className, maximised, ...props }, ref) => { - const video = useObservableEagerState(vm.video); - const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); - - return ( - - ); - }, -); - -MediaTile.displayName = "MediaTile"; - interface UserMediaTileProps extends TileProps { vm: UserMediaViewModel; mirror: boolean; - showSpeakingIndicators: boolean; menuStart?: ReactNode; menuEnd?: ReactNode; } @@ -115,11 +79,14 @@ const UserMediaTile = forwardRef( menuEnd, className, nameTag, + maximised, ...props }, ref, ) => { const { t } = useTranslation(); + const video = useObservableEagerState(vm.video); + const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning); const audioEnabled = useObservableEagerState(vm.audioEnabled); const videoEnabled = useObservableEagerState(vm.videoEnabled); const speaking = useObservableEagerState(vm.speaking); @@ -148,12 +115,14 @@ const UserMediaTile = forwardRef( ); const tile = ( - ( {menu} } + data-maximised={maximised} {...props} /> ); @@ -199,7 +169,6 @@ UserMediaTile.displayName = "UserMediaTile"; interface LocalUserMediaTileProps extends TileProps { vm: LocalUserMediaViewModel; onOpenProfile: () => void; - showSpeakingIndicators: boolean; } const LocalUserMediaTile = forwardRef( @@ -248,7 +217,6 @@ LocalUserMediaTile.displayName = "LocalUserMediaTile"; interface RemoteUserMediaTileProps extends TileProps { vm: RemoteUserMediaViewModel; - showSpeakingIndicators: boolean; } const RemoteUserMediaTile = forwardRef< @@ -303,53 +271,8 @@ const RemoteUserMediaTile = forwardRef< RemoteUserMediaTile.displayName = "RemoteUserMediaTile"; -interface ScreenShareTileProps extends TileProps { - vm: ScreenShareViewModel; - fullscreen: boolean; - onToggleFullscreen: (itemId: string) => void; -} - -const ScreenShareTile = forwardRef( - ({ vm, fullscreen, onToggleFullscreen, ...props }, ref) => { - const { t } = useTranslation(); - const onClickFullScreen = useCallback( - () => onToggleFullscreen(vm.id), - [onToggleFullscreen, vm], - ); - - const FullScreenIcon = fullscreen ? CollapseIcon : ExpandIcon; - - return ( - - - - ) - } - {...props} - /> - ); - }, -); - -ScreenShareTile.displayName = "ScreenShareTile"; - interface GridTileProps { - vm: MediaViewModel; + vm: UserMediaViewModel; maximised: boolean; fullscreen: boolean; onToggleFullscreen: (itemId: string) => void; @@ -375,19 +298,8 @@ export const GridTile = forwardRef( {...nameData} /> ); - } else if (vm instanceof RemoteUserMediaViewModel) { - return ; } else { - return ( - - ); + return ; } }, ); diff --git a/test/grid/LegacyGrid-test.ts b/test/grid/LegacyGrid-test.ts deleted file mode 100644 index e57adf9d..00000000 --- a/test/grid/LegacyGrid-test.ts +++ /dev/null @@ -1,69 +0,0 @@ -/* -Copyright 2023-2024 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -import { TileDescriptor } from "../../src/state/CallViewModel"; -import { Tile, reorderTiles } from "../../src/grid/LegacyGrid"; - -const alice: Tile = { - key: "alice", - order: 0, - item: { local: false } as unknown as TileDescriptor, - remove: false, - focused: false, - isPresenter: false, - isSpeaker: false, - hasVideo: true, -}; -const bob: Tile = { - key: "bob", - order: 1, - item: { local: false } as unknown as TileDescriptor, - remove: false, - focused: false, - isPresenter: false, - isSpeaker: false, - hasVideo: false, -}; - -test("reorderTiles does not promote a non-speaker", () => { - const tiles = [{ ...alice }, { ...bob }]; - reorderTiles(tiles, "spotlight", 1); - expect(tiles).toEqual([ - expect.objectContaining({ key: "alice", order: 0 }), - expect.objectContaining({ key: "bob", order: 1 }), - ]); -}); - -test("reorderTiles promotes a speaker into the visible area", () => { - const tiles = [{ ...alice }, { ...bob, isSpeaker: true }]; - reorderTiles(tiles, "spotlight", 1); - expect(tiles).toEqual([ - expect.objectContaining({ key: "alice", order: 1 }), - expect.objectContaining({ key: "bob", order: 0 }), - ]); -}); - -test("reorderTiles keeps a promoted speaker in the visible area", () => { - const tiles = [ - { ...alice, order: 1 }, - { ...bob, isSpeaker: true, order: 0 }, - ]; - reorderTiles(tiles, "spotlight", 1); - expect(tiles).toEqual([ - expect.objectContaining({ key: "alice", order: 1 }), - expect.objectContaining({ key: "bob", order: 0 }), - ]); -}); From a16f2352779e6b9016b89c70b9331e510a18ea33 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 12 Jun 2024 15:26:00 -0400 Subject: [PATCH 22/31] Fix crash in spotlight mode while connecting Because we were hiding even the local participant during initial connection, there would be no participants, and therefore nothing to put in the spotlight. The designs don't really tell us what the connecting state should look like, so I've taken the liberty of restoring it to its former glory of showing the local participant immediately. --- src/grid/Grid.tsx | 13 +++++++------ src/state/CallViewModel.ts | 21 ++++++--------------- 2 files changed, 13 insertions(+), 21 deletions(-) diff --git a/src/grid/Grid.tsx b/src/grid/Grid.tsx index 6c1bab9b..ea33a32d 100644 --- a/src/grid/Grid.tsx +++ b/src/grid/Grid.tsx @@ -268,12 +268,13 @@ export function Grid< ) as HTMLCollectionOf; for (const slot of slots) { const id = slot.getAttribute("data-id")!; - result.push({ - ...tiles.get(id)!, - ...offset(slot, gridRoot), - width: slot.offsetWidth, - height: slot.offsetHeight, - }); + if (slot.offsetWidth > 0 && slot.offsetHeight > 0) + result.push({ + ...tiles.get(id)!, + ...offset(slot, gridRoot), + width: slot.offsetWidth, + height: slot.offsetHeight, + }); } } diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 975d069b..6ed21a59 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -207,7 +207,8 @@ function findMatrixMember( room: MatrixRoom, id: string, ): RoomMember | undefined { - if (!id) return undefined; + if (id === "local") + return room.getMember(room.client.getUserId()!) ?? undefined; const parts = id.split(":"); // must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon @@ -307,23 +308,16 @@ export class CallViewModel extends ViewModel { ]).pipe( scan( (prevItems, [remoteParticipants, { participant: localParticipant }]) => { - let allGhosts = true; - const newItems = new Map( function* (this: CallViewModel): Iterable<[string, MediaItem]> { for (const p of [localParticipant, ...remoteParticipants]) { - const member = findMatrixMember(this.matrixRoom, p.identity); - allGhosts &&= member === undefined; - // We always start with a local participant with the empty string as - // their ID before we're connected, this is fine and we'll be in - // "all ghosts" mode. - if (p.identity !== "" && member === undefined) { + const userMediaId = p === localParticipant ? "local" : p.identity; + const member = findMatrixMember(this.matrixRoom, userMediaId); + if (member === undefined) logger.warn( `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`, ); - } - const userMediaId = p.identity; yield [ userMediaId, prevItems.get(userMediaId) ?? @@ -343,10 +337,7 @@ export class CallViewModel extends ViewModel { ); for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy(); - - // If every item is a ghost, that probably means we're still connecting - // and shouldn't bother showing anything yet - return allGhosts ? new Map() : newItems; + return newItems; }, new Map(), ), From 244003763953db033b5e5350142b3f00b7c8910a Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 3 Jul 2024 15:08:30 -0400 Subject: [PATCH 23/31] Implement most of the remaining layout changes Includes the mobile UX optimizations and the tweaks we've made to cut down on wasted space, but does not yet include the change to embed the spotlight tile within the grid. --- src/Header.module.css | 1 - src/grid/CallLayout.ts | 8 +- src/grid/GridLayout.module.css | 3 +- src/grid/GridLayout.tsx | 2 + src/grid/OneOnOneLayout.module.css | 2 - src/grid/OneOnOneLayout.tsx | 52 +--- src/grid/SpotlightExpandedLayout.module.css | 47 ++++ src/grid/SpotlightExpandedLayout.tsx | 99 ++++++++ src/grid/SpotlightLandscapeLayout.module.css | 54 ++++ src/grid/SpotlightLandscapeLayout.tsx | 93 +++++++ src/grid/SpotlightLayout.module.css | 98 -------- src/grid/SpotlightPortraitLayout.module.css | 56 +++++ ...Layout.tsx => SpotlightPortraitLayout.tsx} | 75 +++--- src/observable-utils.ts | 14 +- src/room/InCallView.module.css | 27 +- src/room/InCallView.tsx | 194 +++++++-------- src/room/VideoPreview.module.css | 30 +-- src/room/VideoPreview.tsx | 21 +- src/state/CallViewModel.ts | 231 +++++++++++++----- src/tile/GridTile.module.css | 13 +- src/tile/GridTile.tsx | 8 +- src/tile/MediaView.module.css | 8 +- src/tile/MediaView.tsx | 3 - src/tile/SpotlightTile.module.css | 37 +-- src/tile/SpotlightTile.tsx | 82 +++---- 25 files changed, 761 insertions(+), 497 deletions(-) create mode 100644 src/grid/SpotlightExpandedLayout.module.css create mode 100644 src/grid/SpotlightExpandedLayout.tsx create mode 100644 src/grid/SpotlightLandscapeLayout.module.css create mode 100644 src/grid/SpotlightLandscapeLayout.tsx delete mode 100644 src/grid/SpotlightLayout.module.css create mode 100644 src/grid/SpotlightPortraitLayout.module.css rename src/grid/{SpotlightLayout.tsx => SpotlightPortraitLayout.tsx} (61%) diff --git a/src/Header.module.css b/src/Header.module.css index 4e54009d..5a408bd3 100644 --- a/src/Header.module.css +++ b/src/Header.module.css @@ -22,7 +22,6 @@ limitations under the License. user-select: none; flex-shrink: 0; padding-inline: var(--inline-content-inset); - padding-block-end: var(--cpd-space-4x); } .nav { diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts index e97b18a2..e1dd7338 100644 --- a/src/grid/CallLayout.ts +++ b/src/grid/CallLayout.ts @@ -65,6 +65,10 @@ export interface SpotlightTileModel { export type TileModel = GridTileModel | SpotlightTileModel; export interface CallLayoutOutputs { + /** + * Whether the scrolling layer of the layout should appear on top. + */ + scrollingOnTop: boolean; /** * The visually fixed (non-scrolling) layer of the layout. */ @@ -121,7 +125,7 @@ export function arrangeTiles( ); let rows = Math.ceil(tileCount / columns); - let tileWidth = (width - (columns - 1) * gap) / columns; + let tileWidth = (width - (columns + 1) * gap) / columns; let tileHeight = (minHeight - (rows - 1) * gap) / rows; // Impose a minimum width and height on the tiles @@ -132,7 +136,7 @@ export function arrangeTiles( // c = (W + g) / (w + g). columns = Math.floor((width + gap) / (tileMinWidth + gap)); rows = Math.ceil(tileCount / columns); - tileWidth = (width - (columns - 1) * gap) / columns; + tileWidth = (width - (columns + 1) * gap) / columns; tileHeight = (minHeight - (rows - 1) * gap) / rows; } if (tileHeight < tileMinHeight) tileHeight = tileMinHeight; diff --git a/src/grid/GridLayout.module.css b/src/grid/GridLayout.module.css index 33edc3be..6838ae91 100644 --- a/src/grid/GridLayout.module.css +++ b/src/grid/GridLayout.module.css @@ -16,7 +16,6 @@ limitations under the License. .fixed, .scrolling { - margin-inline: var(--inline-content-inset); block-size: 100%; } @@ -41,7 +40,7 @@ limitations under the License. position: absolute; inline-size: 404px; block-size: 233px; - inset: -12px; + inset: 0; } .fixed > .slot[data-block-alignment="start"] { diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 4d499eed..b49bb32a 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -40,6 +40,8 @@ export const makeGridLayout: CallLayout = ({ minBounds, spotlightAlignment, }) => ({ + scrollingOnTop: false, + // The "fixed" (non-scrolling) part of the layout is where the spotlight tile // lives fixed: forwardRef(function GridLayoutFixed({ model, Slot }, ref) { diff --git a/src/grid/OneOnOneLayout.module.css b/src/grid/OneOnOneLayout.module.css index 0d2ad4ff..5bdeb2c8 100644 --- a/src/grid/OneOnOneLayout.module.css +++ b/src/grid/OneOnOneLayout.module.css @@ -15,7 +15,6 @@ limitations under the License. */ .layer { - margin-inline: var(--inline-content-inset); block-size: 100%; display: grid; place-items: center; @@ -36,7 +35,6 @@ limitations under the License. position: absolute; inline-size: 404px; block-size: 233px; - inset: -12px; } .slot[data-block-alignment="start"] { diff --git a/src/grid/OneOnOneLayout.tsx b/src/grid/OneOnOneLayout.tsx index 2eac1b7e..1f9a39e7 100644 --- a/src/grid/OneOnOneLayout.tsx +++ b/src/grid/OneOnOneLayout.tsx @@ -19,63 +19,19 @@ import { useObservableEagerState } from "observable-hooks"; import classNames from "classnames"; import { OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewModel"; -import { - CallLayout, - GridTileModel, - SpotlightTileModel, - arrangeTiles, -} from "./CallLayout"; +import { CallLayout, GridTileModel, arrangeTiles } from "./CallLayout"; import { useReactiveState } from "../useReactiveState"; import styles from "./OneOnOneLayout.module.css"; import { DragCallback } from "./Grid"; export const makeOneOnOneLayout: CallLayout = ({ minBounds, - spotlightAlignment, pipAlignment, }) => ({ - fixed: forwardRef(function OneOnOneLayoutFixed({ model, Slot }, ref) { - const { width, height } = useObservableEagerState(minBounds); - const spotlightAlignmentValue = useObservableEagerState(spotlightAlignment); + scrollingOnTop: false, - const [generation] = useReactiveState( - (prev) => (prev === undefined ? 0 : prev + 1), - [width, height, model.spotlight === undefined, spotlightAlignmentValue], - ); - - const spotlightTileModel: SpotlightTileModel | undefined = useMemo( - () => - model.spotlight && { - type: "spotlight", - vms: model.spotlight, - maximised: false, - }, - [model.spotlight], - ); - - const onDragSpotlight: DragCallback = useCallback( - ({ xRatio, yRatio }) => - spotlightAlignment.next({ - block: yRatio < 0.5 ? "start" : "end", - inline: xRatio < 0.5 ? "start" : "end", - }), - [], - ); - - return ( -
- {spotlightTileModel && ( - - )} -
- ); + fixed: forwardRef(function OneOnOneLayoutFixed(_props, ref) { + return
; }), scrolling: forwardRef(function OneOnOneLayoutScrolling({ model, Slot }, ref) { diff --git a/src/grid/SpotlightExpandedLayout.module.css b/src/grid/SpotlightExpandedLayout.module.css new file mode 100644 index 00000000..6556110e --- /dev/null +++ b/src/grid/SpotlightExpandedLayout.module.css @@ -0,0 +1,47 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.layer { + block-size: 100%; +} + +.spotlight { + block-size: 100%; + inline-size: 100%; +} + +.pip { + position: absolute; + inline-size: 180px; + block-size: 135px; + inset: var(--cpd-space-4x); +} + +.pip[data-block-alignment="start"] { + inset-block-end: unset; +} + +.pip[data-block-alignment="end"] { + inset-block-start: unset; +} + +.pip[data-inline-alignment="start"] { + inset-inline-end: unset; +} + +.pip[data-inline-alignment="end"] { + inset-inline-start: unset; +} diff --git a/src/grid/SpotlightExpandedLayout.tsx b/src/grid/SpotlightExpandedLayout.tsx new file mode 100644 index 00000000..40f77ca9 --- /dev/null +++ b/src/grid/SpotlightExpandedLayout.tsx @@ -0,0 +1,99 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { forwardRef, useCallback, useMemo } from "react"; +import { useObservableEagerState } from "observable-hooks"; + +import { SpotlightExpandedLayout as SpotlightExpandedLayoutModel } from "../state/CallViewModel"; +import { CallLayout, GridTileModel, SpotlightTileModel } from "./CallLayout"; +import { DragCallback } from "./Grid"; +import styles from "./SpotlightExpandedLayout.module.css"; +import { useReactiveState } from "../useReactiveState"; + +export const makeSpotlightExpandedLayout: CallLayout< + SpotlightExpandedLayoutModel +> = ({ minBounds, pipAlignment }) => ({ + scrollingOnTop: true, + + fixed: forwardRef(function SpotlightExpandedLayoutFixed( + { model, Slot }, + ref, + ) { + const { width, height } = useObservableEagerState(minBounds); + + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [width, height], + ); + + const spotlightTileModel: SpotlightTileModel = useMemo( + () => ({ type: "spotlight", vms: model.spotlight, maximised: true }), + [model.spotlight], + ); + + return ( +
+ +
+ ); + }), + + scrolling: forwardRef(function SpotlightExpandedLayoutScrolling( + { model, Slot }, + ref, + ) { + const { width, height } = useObservableEagerState(minBounds); + const pipAlignmentValue = useObservableEagerState(pipAlignment); + + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [width, height, model.pip === undefined, pipAlignmentValue], + ); + + const pipTileModel: GridTileModel | undefined = useMemo( + () => model.pip && { type: "grid", vm: model.pip }, + [model.pip], + ); + + const onDragPip: DragCallback = useCallback( + ({ xRatio, yRatio }) => + pipAlignment.next({ + block: yRatio < 0.5 ? "start" : "end", + inline: xRatio < 0.5 ? "start" : "end", + }), + [], + ); + + return ( +
+ {pipTileModel && ( + + )} +
+ ); + }), +}); diff --git a/src/grid/SpotlightLandscapeLayout.module.css b/src/grid/SpotlightLandscapeLayout.module.css new file mode 100644 index 00000000..8ca91e10 --- /dev/null +++ b/src/grid/SpotlightLandscapeLayout.module.css @@ -0,0 +1,54 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.layer { + block-size: 100%; + display: grid; + --gap: 20px; + gap: var(--gap); + --grid-slot-width: 180px; + grid-template-columns: 1fr var(--grid-slot-width); + grid-template-rows: minmax(1fr, auto); + padding-inline: var(--gap); +} + +.spotlight { + container: spotlight / size; + display: grid; + place-items: center; +} + +/* CSS makes us put a condition here, even though all we want to do is +unconditionally select the container so we can use cq units */ +@container spotlight (width > 0) { + .spotlight > .slot { + inline-size: min(100cqi, 100cqb * (17 / 9)); + block-size: min(100cqb, 100cqi / (4 / 3)); + } +} + +.grid { + display: flex; + flex-wrap: wrap; + gap: var(--gap); + justify-content: center; + align-content: center; +} + +.grid > .slot { + inline-size: 180px; + block-size: 135px; +} diff --git a/src/grid/SpotlightLandscapeLayout.tsx b/src/grid/SpotlightLandscapeLayout.tsx new file mode 100644 index 00000000..40b02f9a --- /dev/null +++ b/src/grid/SpotlightLandscapeLayout.tsx @@ -0,0 +1,93 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { forwardRef, useMemo } from "react"; +import { useObservableEagerState } from "observable-hooks"; +import classNames from "classnames"; + +import { CallLayout, GridTileModel, TileModel } from "./CallLayout"; +import { SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../state/CallViewModel"; +import styles from "./SpotlightLandscapeLayout.module.css"; +import { useReactiveState } from "../useReactiveState"; + +export const makeSpotlightLandscapeLayout: CallLayout< + SpotlightLandscapeLayoutModel +> = ({ minBounds }) => ({ + scrollingOnTop: false, + + fixed: forwardRef(function SpotlightLandscapeLayoutFixed( + { model, Slot }, + ref, + ) { + const { width, height } = useObservableEagerState(minBounds); + const tileModel: TileModel = useMemo( + () => ({ + type: "spotlight", + vms: model.spotlight, + maximised: false, + }), + [model.spotlight], + ); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.grid.length, width, height], + ); + + return ( +
+
+ +
+
+
+ ); + }), + + scrolling: forwardRef(function SpotlightLandscapeLayoutScrolling( + { model, Slot }, + ref, + ) { + const { width, height } = useObservableEagerState(minBounds); + const tileModels: GridTileModel[] = useMemo( + () => model.grid.map((vm) => ({ type: "grid", vm })), + [model.grid], + ); + const [generation] = useReactiveState( + (prev) => (prev === undefined ? 0 : prev + 1), + [model.spotlight.length, model.grid, width, height], + ); + + return ( +
+
1, + })} + /> +
+ {tileModels.map((m) => ( + + ))} +
+
+ ); + }), +}); diff --git a/src/grid/SpotlightLayout.module.css b/src/grid/SpotlightLayout.module.css deleted file mode 100644 index d58a95a1..00000000 --- a/src/grid/SpotlightLayout.module.css +++ /dev/null @@ -1,98 +0,0 @@ -/* -Copyright 2024 New Vector Ltd - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -.layer { - margin-inline: var(--inline-content-inset); - block-size: 100%; - display: grid; - --grid-gap: 20px; - gap: 30px; -} - -.layer[data-orientation="landscape"] { - --grid-slot-width: 180px; - grid-template-columns: 1fr calc( - var(--grid-columns) * var(--grid-slot-width) + (var(--grid-columns) - 1) * - var(--grid-gap) - ); - grid-template-rows: minmax(1fr, auto); -} - -.spotlight { - container: spotlight / size; - display: grid; - place-items: center; -} - -/* CSS makes us put a condition here, even though all we want to do is -unconditionally select the container so we can use cq units */ -@container spotlight (width > 0) { - .layer[data-orientation="landscape"] > .spotlight > .slot { - inline-size: min(100cqi, 100cqb * (17 / 9)); - block-size: min(100cqb, 100cqi / (4 / 3)); - } -} - -.grid { - display: flex; - flex-wrap: wrap; - gap: var(--grid-gap); - justify-content: center; -} - -.layer[data-orientation="landscape"] > .grid { - align-content: center; -} - -.layer > .grid > .slot { - inline-size: var(--grid-slot-width); -} - -.layer[data-orientation="landscape"] > .grid > .slot { - block-size: 135px; -} - -.layer[data-orientation="portrait"] { - margin-inline: 0; - display: block; -} - -.layer[data-orientation="portrait"] > .spotlight { - inline-size: 100%; - aspect-ratio: 16 / 9; - margin-block-end: var(--cpd-space-4x); -} - -.layer[data-orientation="portrait"] > .spotlight.withIndicators { - margin-block-end: calc(2 * var(--cpd-space-4x) + 2px); -} - -.layer[data-orientation="portrait"] > .spotlight > .slot { - inline-size: 100%; - block-size: 100%; -} - -.layer[data-orientation="portrait"] > .grid { - margin-inline: var(--inline-content-inset); - align-content: start; -} - -.layer[data-orientation="portrait"] > .grid > .slot { - --grid-slot-width: calc( - (100% - (var(--grid-columns) - 1) * var(--grid-gap)) / var(--grid-columns) - ); - aspect-ratio: 4 / 3; -} diff --git a/src/grid/SpotlightPortraitLayout.module.css b/src/grid/SpotlightPortraitLayout.module.css new file mode 100644 index 00000000..1ee91334 --- /dev/null +++ b/src/grid/SpotlightPortraitLayout.module.css @@ -0,0 +1,56 @@ +/* +Copyright 2024 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +.layer { + block-size: 100%; + display: grid; + --gap: 20px; + gap: var(--gap); + margin-inline: 0; + display: block; +} + +.spotlight { + container: spotlight / size; + display: grid; + place-items: center; + inline-size: 100%; + aspect-ratio: 16 / 9; + margin-block-end: var(--cpd-space-4x); +} + +.spotlight.withIndicators { + margin-block-end: calc(2 * var(--cpd-space-4x) + 2px); +} + +.spotlight > .slot { + inline-size: 100%; + block-size: 100%; +} + +.grid { + display: flex; + flex-wrap: wrap; + gap: var(--grid-gap); + justify-content: center; + align-content: start; + padding-inline: var(--grid-gap); +} + +.grid > .slot { + inline-size: var(--grid-tile-width); + block-size: var(--grid-tile-height); +} diff --git a/src/grid/SpotlightLayout.tsx b/src/grid/SpotlightPortraitLayout.tsx similarity index 61% rename from src/grid/SpotlightLayout.tsx rename to src/grid/SpotlightPortraitLayout.tsx index 9ddbce10..5c0cb0a8 100644 --- a/src/grid/SpotlightLayout.tsx +++ b/src/grid/SpotlightPortraitLayout.tsx @@ -18,46 +18,39 @@ import { CSSProperties, forwardRef, useMemo } from "react"; import { useObservableEagerState } from "observable-hooks"; import classNames from "classnames"; -import { CallLayout, GridTileModel, TileModel } from "./CallLayout"; -import { SpotlightLayout as SpotlightLayoutModel } from "../state/CallViewModel"; -import styles from "./SpotlightLayout.module.css"; +import { + CallLayout, + GridTileModel, + TileModel, + arrangeTiles, +} from "./CallLayout"; +import { SpotlightPortraitLayout as SpotlightPortraitLayoutModel } from "../state/CallViewModel"; +import styles from "./SpotlightPortraitLayout.module.css"; import { useReactiveState } from "../useReactiveState"; interface GridCSSProperties extends CSSProperties { - "--grid-columns": number; + "--grid-gap": string; + "--grid-tile-width": string; + "--grid-tile-height": string; } -interface Layout { - orientation: "portrait" | "landscape"; - gridColumns: number; -} +export const makeSpotlightPortraitLayout: CallLayout< + SpotlightPortraitLayoutModel +> = ({ minBounds }) => ({ + scrollingOnTop: false, -function getLayout(gridLength: number, width: number): Layout { - const orientation = width < 800 ? "portrait" : "landscape"; - return { - orientation, - gridColumns: - orientation === "portrait" - ? Math.floor(width / 190) - : gridLength > 20 - ? 2 - : 1, - }; -} - -export const makeSpotlightLayout: CallLayout = ({ - minBounds, -}) => ({ - fixed: forwardRef(function SpotlightLayoutFixed({ model, Slot }, ref) { + fixed: forwardRef(function SpotlightPortraitLayoutFixed( + { model, Slot }, + ref, + ) { const { width, height } = useObservableEagerState(minBounds); - const layout = getLayout(model.grid.length, width); const tileModel: TileModel = useMemo( () => ({ type: "spotlight", vms: model.spotlight, - maximised: layout.orientation === "portrait", + maximised: true, }), - [model.spotlight, layout.orientation], + [model.spotlight], ); const [generation] = useReactiveState( (prev) => (prev === undefined ? 0 : prev + 1), @@ -65,27 +58,24 @@ export const makeSpotlightLayout: CallLayout = ({ ); return ( -
+
-
); }), - scrolling: forwardRef(function SpotlightLayoutScrolling( + scrolling: forwardRef(function SpotlightPortraitLayoutScrolling( { model, Slot }, ref, ) { const { width, height } = useObservableEagerState(minBounds); - const layout = getLayout(model.grid.length, width); + const { gap, tileWidth, tileHeight } = arrangeTiles( + width, + 0, + model.grid.length, + ); const tileModels: GridTileModel[] = useMemo( () => model.grid.map((vm) => ({ type: "grid", vm })), [model.grid], @@ -99,9 +89,14 @@ export const makeSpotlightLayout: CallLayout = ({
(callback: (finalValue: T) => void) { ); }); } + +/** + * RxJS operator that accumulates a state from a source of events. This is like + * scan, except it emits an initial value immediately before any events arrive. + */ +export function accumulate( + initial: State, + update: (state: State, event: Event) => State, +) { + return (events: Observable): Observable => + events.pipe(scan(update, initial), startWith(initial)); +} diff --git a/src/room/InCallView.module.css b/src/room/InCallView.module.css index 60c46aa6..b8cf9f5e 100644 --- a/src/room/InCallView.module.css +++ b/src/room/InCallView.module.css @@ -68,7 +68,7 @@ limitations under the License. align-items: center; gap: var(--cpd-space-3x); padding-block: var(--cpd-space-4x); - margin-inline: var(--inline-content-inset); + padding-inline: var(--inline-content-inset); background: linear-gradient( 180deg, rgba(0, 0, 0, 0) 0%, @@ -123,17 +123,16 @@ limitations under the License. display: none; } +.footer.overlay { + position: absolute; + inset-block-end: 0; + inset-inline: 0; +} + .fixedGrid { position: absolute; inline-size: 100%; align-self: center; - /* Disable pointer events so the overlay doesn't block interaction with - elements behind it */ - pointer-events: none; -} - -.fixedGrid > :not(:first-child) { - pointer-events: initial; } .scrollingGrid { @@ -143,6 +142,18 @@ limitations under the License. align-self: center; } +.fixedGrid, +.scrollingGrid { + /* Disable pointer events so the overlay doesn't block interaction with + elements behind it */ + pointer-events: none; +} + +.fixedGrid > :not(:first-child), +.scrollingGrid > :not(:first-child) { + pointer-events: initial; +} + .tile { position: absolute; inset-block-start: 0; diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index a1b3f4cc..a3e02869 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -35,7 +35,7 @@ import { import useMeasure from "react-use-measure"; import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession"; import classNames from "classnames"; -import { BehaviorSubject, map } from "rxjs"; +import { BehaviorSubject } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; import LogoMark from "../icons/LogoMark.svg?react"; @@ -59,7 +59,6 @@ import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal"; import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { useLiveKit } from "../livekit/useLiveKit"; -import { useFullscreen } from "./useFullscreen"; import { useWakeLock } from "../useWakeLock"; import { useMergedRefs } from "../useMergedRefs"; import { MuteStates } from "./MuteStates"; @@ -76,14 +75,16 @@ import { SpotlightTile } from "../tile/SpotlightTile"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; import { E2eeType } from "../e2ee/e2eeType"; import { makeGridLayout } from "../grid/GridLayout"; -import { makeSpotlightLayout } from "../grid/SpotlightLayout"; import { - CallLayout, + CallLayoutOutputs, TileModel, defaultPipAlignment, defaultSpotlightAlignment, } from "../grid/CallLayout"; import { makeOneOnOneLayout } from "../grid/OneOnOneLayout"; +import { makeSpotlightExpandedLayout } from "../grid/SpotlightExpandedLayout"; +import { makeSpotlightLandscapeLayout } from "../grid/SpotlightLandscapeLayout"; +import { makeSpotlightPortraitLayout } from "../grid/SpotlightPortraitLayout"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -194,24 +195,9 @@ export const InCallView: FC = ({ matrixInfo.e2eeSystem.kind !== E2eeType.NONE, connState, ); + const windowMode = useObservableEagerState(vm.windowMode); const layout = useObservableEagerState(vm.layout); const gridMode = useObservableEagerState(vm.gridMode); - const hasSpotlight = layout.spotlight !== undefined; - const fullscreenItems = useMemo( - () => (hasSpotlight ? ["spotlight"] : []), - [hasSpotlight], - ); - const { fullscreenItem, toggleFullscreen, exitFullscreen } = - useFullscreen(fullscreenItems); - const toggleSpotlightFullscreen = useCallback( - () => toggleFullscreen("spotlight"), - [toggleFullscreen], - ); - - // The maximised participant: either the participant that the user has - // manually put in fullscreen, or (TODO) the spotlight if the window is too - // small to show everyone - const maximisedParticipant = fullscreenItem; const [settingsModalOpen, setSettingsModalOpen] = useState(false); const [settingsTab, setSettingsTab] = useState(defaultSettingsTab); @@ -235,14 +221,18 @@ export const InCallView: FC = ({ const gridBounds = useMemo( () => ({ - width: footerBounds.width, - height: bounds.height - headerBounds.height - footerBounds.height, + width: bounds.width, + height: + bounds.height - + headerBounds.height - + (windowMode === "flat" ? 0 : footerBounds.height), }), [ - footerBounds.width, + bounds.width, bounds.height, headerBounds.height, footerBounds.height, + windowMode, ], ); const gridBoundsObservable = useObservable(gridBounds); @@ -254,29 +244,6 @@ export const InCallView: FC = ({ () => new BehaviorSubject(defaultPipAlignment), ); - const layoutSystem = useObservableEagerState( - useInitial(() => - vm.layout.pipe( - map((l) => { - let makeLayout: CallLayout; - if (l.type === "grid") - makeLayout = makeGridLayout as CallLayout; - else if (l.type === "spotlight") - makeLayout = makeSpotlightLayout as CallLayout; - else if (l.type === "one-on-one") - makeLayout = makeOneOnOneLayout as CallLayout; - else throw new Error(`Unimplemented layout: ${l.type}`); - - return makeLayout({ - minBounds: gridBoundsObservable, - spotlightAlignment, - pipAlignment, - }); - }), - ), - ), - ); - const setGridMode = useCallback( (mode: GridMode) => vm.setGridMode(mode), [vm], @@ -318,10 +285,9 @@ export const InCallView: FC = ({ } }, [setGridMode]); - const showSpotlightIndicators = useObservable(layout.type === "spotlight"); - const showSpeakingIndicators = useObservable( - layout.type === "spotlight" || - (layout.type === "grid" && layout.grid.length > 2), + const toggleSpotlightExpanded = useCallback( + () => vm.toggleSpotlightExpanded(), + [vm], ); const Tile = useMemo( @@ -333,20 +299,18 @@ export const InCallView: FC = ({ { className, style, targetWidth, targetHeight, model }, ref, ) { + const spotlightExpanded = useObservableEagerState(vm.spotlightExpanded); const showSpeakingIndicatorsValue = useObservableEagerState( - showSpeakingIndicators, + vm.showSpeakingIndicators, ); const showSpotlightIndicatorsValue = useObservableEagerState( - showSpotlightIndicators, + vm.showSpotlightIndicators, ); return model.type === "grid" ? ( = ({ ref={ref} vms={model.vms} maximised={model.maximised} - fullscreen={false} - onToggleFullscreen={toggleSpotlightFullscreen} + expanded={spotlightExpanded} + onToggleExpanded={toggleSpotlightExpanded} targetWidth={targetWidth} targetHeight={targetHeight} showIndicators={showSpotlightIndicatorsValue} @@ -369,52 +333,74 @@ export const InCallView: FC = ({ /> ); }), - [ - toggleFullscreen, - toggleSpotlightFullscreen, - openProfile, - showSpeakingIndicators, - showSpotlightIndicators, - ], + [vm, toggleSpotlightExpanded, openProfile], ); + const layouts = useMemo(() => { + const inputs = { + minBounds: gridBoundsObservable, + spotlightAlignment, + pipAlignment, + }; + return { + grid: makeGridLayout(inputs), + "spotlight landscape": makeSpotlightLandscapeLayout(inputs), + "spotlight portrait": makeSpotlightPortraitLayout(inputs), + "spotlight expanded": makeSpotlightExpandedLayout(inputs), + "one-on-one": makeOneOnOneLayout(inputs), + }; + }, [gridBoundsObservable, spotlightAlignment, pipAlignment]); + const renderContent = (): JSX.Element => { - if (maximisedParticipant !== null) { - const fullscreen = maximisedParticipant === fullscreenItem; - if (maximisedParticipant === "spotlight") { - return ( - - ); - } + if (layout.type === "pip") { + return ( + + ); } - return ( + const layers = layouts[layout.type] as CallLayoutOutputs; + const fixedGrid = ( + + ); + const scrollingGrid = ( + + ); + // The grid tiles go *under* the spotlight in the portrait layout, but + // *over* the spotlight in the expanded layout + return layout.type === "spotlight expanded" ? ( <> - - + {fixedGrid} + {scrollingGrid} + + ) : ( + <> + {scrollingGrid} + {fixedGrid} ); }; @@ -424,14 +410,13 @@ export const InCallView: FC = ({ ); const toggleScreensharing = useCallback(async () => { - exitFullscreen(); await localParticipant.setScreenShareEnabled(!isScreenShareEnabled, { audio: true, selfBrowserSurface: "include", surfaceSwitching: "include", systemAudio: "include", }); - }, [localParticipant, isScreenShareEnabled, exitFullscreen]); + }, [localParticipant, isScreenShareEnabled]); let footer: JSX.Element | null; @@ -484,11 +469,10 @@ export const InCallView: FC = ({
{!mobile && !hideHeader && ( @@ -515,7 +499,7 @@ export const InCallView: FC = ({ return (
- {!hideHeader && maximisedParticipant === null && ( + {!hideHeader && windowMode !== "pip" && windowMode !== "flat" && (
video { width: 100%; height: 100%; object-fit: cover; @@ -69,12 +61,20 @@ limitations under the License. ); } -.preview.content .buttonBar { - padding-inline: var(--inline-content-inset); -} - @media (min-aspect-ratio: 1 / 1) { - .preview video { + .preview > video { aspect-ratio: 16 / 9; } } + +@media (max-width: 550px) { + .preview { + margin-inline: 0; + border-radius: 0; + block-size: 100%; + } + + .buttonBar { + padding-inline: var(--inline-content-inset); + } +} diff --git a/src/room/VideoPreview.tsx b/src/room/VideoPreview.tsx index 3be88f1f..5899a8bf 100644 --- a/src/room/VideoPreview.tsx +++ b/src/room/VideoPreview.tsx @@ -21,13 +21,11 @@ import { usePreviewTracks } from "@livekit/components-react"; import { LocalVideoTrack, Track } from "livekit-client"; import classNames from "classnames"; import { logger } from "matrix-js-sdk/src/logger"; -import { Glass } from "@vector-im/compound-web"; import { Avatar } from "../Avatar"; import styles from "./VideoPreview.module.css"; import { useMediaDevices } from "../livekit/MediaDevicesContext"; import { MuteStates } from "./MuteStates"; -import { useMediaQuery } from "../useMediaQuery"; import { useInitial } from "../useInitial"; import { EncryptionSystem } from "../e2ee/sharedKeyManagement"; @@ -116,8 +114,8 @@ export const VideoPreview: FC = ({ }; }, [videoTrack]); - const content = ( - <> + return ( +
)}
{children}
- - ); - - return useMediaQuery("(max-width: 550px)") ? ( -
- {content}
- ) : ( - -
- {content} -
- ); }; diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 6ed21a59..5d50c5c5 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -34,9 +34,9 @@ import { audit, combineLatest, concat, - concatMap, distinctUntilChanged, filter, + fromEvent, map, merge, mergeAll, @@ -44,11 +44,11 @@ import { sample, scan, shareReplay, + skip, startWith, switchMap, throttleTime, timer, - withLatestFrom, zip, } from "rxjs"; import { logger } from "matrix-js-sdk/src/logger"; @@ -67,7 +67,7 @@ import { ScreenShareViewModel, UserMediaViewModel, } from "./MediaViewModel"; -import { finalizeValue } from "../observable-utils"; +import { accumulate, finalizeValue } from "../observable-utils"; import { ObservableScope } from "./ObservableScope"; // How long we wait after a focus switch before showing the real participant @@ -80,25 +80,30 @@ export interface GridLayout { grid: UserMediaViewModel[]; } -export interface SpotlightLayout { - type: "spotlight"; +export interface SpotlightLandscapeLayout { + type: "spotlight landscape"; spotlight: MediaViewModel[]; grid: UserMediaViewModel[]; } -export interface OneOnOneLayout { - type: "one-on-one"; - spotlight?: ScreenShareViewModel[]; - local: LocalUserMediaViewModel; - remote: RemoteUserMediaViewModel; +export interface SpotlightPortraitLayout { + type: "spotlight portrait"; + spotlight: MediaViewModel[]; + grid: UserMediaViewModel[]; } -export interface FullScreenLayout { - type: "full screen"; +export interface SpotlightExpandedLayout { + type: "spotlight expanded"; spotlight: MediaViewModel[]; pip?: UserMediaViewModel; } +export interface OneOnOneLayout { + type: "one-on-one"; + local: LocalUserMediaViewModel; + remote: RemoteUserMediaViewModel; +} + export interface PipLayout { type: "pip"; spotlight: MediaViewModel[]; @@ -110,14 +115,15 @@ export interface PipLayout { */ export type Layout = | GridLayout - | SpotlightLayout + | SpotlightLandscapeLayout + | SpotlightPortraitLayout + | SpotlightExpandedLayout | OneOnOneLayout - | FullScreenLayout | PipLayout; export type GridMode = "grid" | "spotlight"; -export type WindowMode = "normal" | "full screen" | "pip"; +export type WindowMode = "normal" | "narrow" | "flat" | "pip"; /** * Sorting bins defining the order in which media tiles appear in the layout. @@ -269,16 +275,13 @@ export class CallViewModel extends ViewModel { }, ).pipe( mergeAll(), - // Aggregate the hold instructions into a single list showing which + // Accumulate the hold instructions into a single list showing which // participants are being held - scan( - (holds, instruction) => - "hold" in instruction - ? [instruction.hold, ...holds] - : holds.filter((h) => h !== instruction.unhold), - [] as RemoteParticipant[][], + accumulate([] as RemoteParticipant[][], (holds, instruction) => + "hold" in instruction + ? [instruction.hold, ...holds] + : holds.filter((h) => h !== instruction.unhold), ), - startWith([]), ); private readonly remoteParticipants: Observable = @@ -352,6 +355,11 @@ export class CallViewModel extends ViewModel { map((ms) => ms.filter((m): m is UserMedia => m instanceof UserMedia)), ); + private readonly localUserMedia: Observable = + this.mediaItems.pipe( + map((ms) => ms.find((m) => m.vm.local)!.vm as LocalUserMediaViewModel), + ); + private readonly screenShares: Observable = this.mediaItems.pipe( map((ms) => ms.filter((m): m is ScreenShare => m instanceof ScreenShare)), @@ -364,7 +372,7 @@ export class CallViewModel extends ViewModel { distinctUntilChanged(), ); - private readonly spotlightSpeaker: Observable = + private readonly spotlightSpeaker: Observable = this.userMedia.pipe( switchMap((ms) => ms.length === 0 @@ -373,7 +381,7 @@ export class CallViewModel extends ViewModel { ms.map((m) => m.vm.speaking.pipe(map((s) => [m, s] as const))), ), ), - scan<(readonly [UserMedia, boolean])[], UserMedia | null, null>( + scan<(readonly [UserMedia, boolean])[], UserMedia, null>( (prev, ms) => // Decide who to spotlight: // If the previous speaker (not the local user) is still speaking, @@ -386,11 +394,11 @@ export class CallViewModel extends ViewModel { // Otherwise, stick with the person who was last speaking prev ?? // Otherwise, spotlight the local user - ms.find(([m]) => m.vm.local)?.[0] ?? - null, + ms.find(([m]) => m.vm.local)![0], null, ), distinctUntilChanged(), + map((speaker) => speaker.vm), shareReplay(1), throttleTime(1600, undefined, { leading: true, trailing: true }), ); @@ -433,38 +441,91 @@ export class CallViewModel extends ViewModel { }), ); - private readonly spotlight: Observable = combineLatest( - [this.screenShares, this.spotlightSpeaker], - (screenShares, spotlightSpeaker): MediaViewModel[] => + private readonly spotlightAndPip: Observable< + [Observable, Observable] + > = this.screenShares.pipe( + map((screenShares) => screenShares.length > 0 - ? screenShares.map((m) => m.vm) - : spotlightSpeaker === null - ? [] - : [spotlightSpeaker.vm], + ? ([of(screenShares.map((m) => m.vm)), this.spotlightSpeaker] as const) + : ([ + this.spotlightSpeaker.pipe(map((speaker) => [speaker!])), + this.localUserMedia.pipe( + switchMap((vm) => + vm.alwaysShow.pipe( + map((alwaysShow) => (alwaysShow ? vm : null)), + ), + ), + ), + ] as const), + ), ); - // TODO: Make this react to changes in window dimensions and screen - // orientation - private readonly windowMode = of("normal"); + private readonly spotlight: Observable = + this.spotlightAndPip.pipe( + switchMap(([spotlight]) => spotlight), + shareReplay(1), + ); + + private readonly pip: Observable = + this.spotlightAndPip.pipe(switchMap(([, pip]) => pip)); + + /** + * The general shape of the window. + */ + public readonly windowMode: Observable = fromEvent( + window, + "resize", + ).pipe( + startWith(null), + map(() => { + const height = window.innerHeight; + const width = window.innerWidth; + if (height <= 400 && width <= 340) return "pip"; + if (width <= 660) return "narrow"; + if (height <= 660) return "flat"; + return "normal"; + }), + distinctUntilChanged(), + shareReplay(1), + ); + + private readonly spotlightExpandedToggle = new Subject(); + public readonly spotlightExpanded: Observable = + this.spotlightExpandedToggle.pipe( + accumulate(false, (expanded) => !expanded), + shareReplay(1), + ); + + public toggleSpotlightExpanded(): void { + this.spotlightExpandedToggle.next(); + } private readonly gridModeUserSelection = new Subject(); /** * The layout mode of the media tile grid. */ - public readonly gridMode: Observable = merge( - // Always honor a manual user selection - this.gridModeUserSelection, + public readonly gridMode: Observable = // If the user hasn't selected spotlight and somebody starts screen sharing, // automatically switch to spotlight mode and reset when screen sharing ends - this.hasRemoteScreenShares.pipe( - withLatestFrom(this.gridModeUserSelection.pipe(startWith(null))), - concatMap(([hasScreenShares, userSelection]) => - userSelection === "spotlight" + this.gridModeUserSelection.pipe( + startWith(null), + switchMap((userSelection) => + (userSelection === "spotlight" ? EMPTY - : of(hasScreenShares ? "spotlight" : "grid"), + : combineLatest([this.hasRemoteScreenShares, this.windowMode]).pipe( + skip(userSelection === null ? 0 : 1), + map( + ([hasScreenShares, windowMode]): GridMode => + hasScreenShares || windowMode === "flat" + ? "spotlight" + : "grid", + ), + ) + ).pipe(startWith(userSelection ?? "grid")), ), - ), - ).pipe(distinctUntilChanged(), shareReplay(1)); + distinctUntilChanged(), + shareReplay(1), + ); public setGridMode(value: GridMode): void { this.gridModeUserSelection.next(value); @@ -472,11 +533,24 @@ export class CallViewModel extends ViewModel { public readonly layout: Observable = this.windowMode.pipe( switchMap((windowMode) => { + const spotlightLandscapeLayout = combineLatest( + [this.grid, this.spotlight], + (grid, spotlight): Layout => ({ + type: "spotlight landscape", + spotlight, + grid, + }), + ); + const spotlightExpandedLayout = combineLatest( + [this.spotlight, this.pip], + (spotlight, pip): Layout => ({ + type: "spotlight expanded", + spotlight, + pip: pip ?? undefined, + }), + ); + switch (windowMode) { - case "full screen": - throw new Error("unimplemented"); - case "pip": - throw new Error("unimplemented"); case "normal": return this.gridMode.pipe( switchMap((gridMode) => { @@ -485,11 +559,9 @@ export class CallViewModel extends ViewModel { return combineLatest( [this.grid, this.spotlight, this.screenShares], (grid, spotlight, screenShares): Layout => - grid.length == 2 + grid.length == 2 && screenShares.length === 0 ? { type: "one-on-one", - spotlight: - screenShares.length > 0 ? spotlight : undefined, local: grid.find( (vm) => vm.local, ) as LocalUserMediaViewModel, @@ -507,22 +579,59 @@ export class CallViewModel extends ViewModel { }, ); case "spotlight": - return combineLatest( - [this.grid, this.spotlight], - (grid, spotlight): Layout => ({ - type: "spotlight", - spotlight, - grid, - }), + return this.spotlightExpanded.pipe( + switchMap((expanded) => + expanded + ? spotlightExpandedLayout + : spotlightLandscapeLayout, + ), ); } }), ); + case "narrow": + return combineLatest( + [this.grid, this.spotlight], + (grid, spotlight): Layout => ({ + type: "spotlight portrait", + spotlight, + grid, + }), + ); + case "flat": + return this.gridMode.pipe( + switchMap((gridMode) => { + switch (gridMode) { + case "grid": + // Yes, grid mode actually gets you a "spotlight" layout in + // this window mode. + return spotlightLandscapeLayout; + case "spotlight": + return spotlightExpandedLayout; + } + }), + ); + case "pip": + return this.spotlight.pipe( + map((spotlight): Layout => ({ type: "pip", spotlight })), + ); } }), shareReplay(1), ); + public showSpotlightIndicators: Observable = this.layout.pipe( + map((l) => l.type !== "grid"), + distinctUntilChanged(), + shareReplay(1), + ); + + public showSpeakingIndicators: Observable = this.layout.pipe( + map((l) => l.type !== "one-on-one" && l.type !== "spotlight expanded"), + distinctUntilChanged(), + shareReplay(1), + ); + public constructor( // A call is permanently tied to a single Matrix room and LiveKit room private readonly matrixRoom: MatrixRoom, diff --git a/src/tile/GridTile.module.css b/src/tile/GridTile.module.css index 7ef66d8d..ea015f43 100644 --- a/src/tile/GridTile.module.css +++ b/src/tile/GridTile.module.css @@ -22,7 +22,7 @@ limitations under the License. /* Use a pseudo-element to create the expressive speaking border, since CSS borders don't support gradients */ -.tile[data-maximised="false"]::before { +.tile::before { content: ""; position: absolute; z-index: -1; /* Put it below the outline */ @@ -43,27 +43,22 @@ borders don't support gradients */ background-blend-mode: overlay, normal; } -.tile[data-maximised="false"].speaking { +.tile.speaking { /* !important because speaking border should take priority over hover */ outline: var(--cpd-border-width-1) solid var(--cpd-color-bg-canvas-default) !important; } -.tile[data-maximised="false"].speaking::before { +.tile.speaking::before { opacity: 1; } @media (hover: hover) { - .tile[data-maximised="false"]:hover { + .tile:hover { outline: var(--cpd-border-width-2) solid var(--cpd-color-border-interactive-hovered); } } -.tile[data-maximised="true"] { - --media-view-border-radius: 0; - --media-view-fg-inset: 10px; -} - .muteIcon[data-muted="true"] { color: var(--cpd-color-icon-secondary); } diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 4dd83567..eb2625e8 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -57,7 +57,6 @@ interface TileProps { style?: ComponentProps["style"]; targetWidth: number; targetHeight: number; - maximised: boolean; displayName: string; nameTag: string; showSpeakingIndicators: boolean; @@ -79,7 +78,6 @@ const UserMediaTile = forwardRef( menuEnd, className, nameTag, - maximised, ...props }, ref, @@ -151,7 +149,6 @@ const UserMediaTile = forwardRef( {menu} } - data-maximised={maximised} {...props} /> ); @@ -273,9 +270,6 @@ RemoteUserMediaTile.displayName = "RemoteUserMediaTile"; interface GridTileProps { vm: UserMediaViewModel; - maximised: boolean; - fullscreen: boolean; - onToggleFullscreen: (itemId: string) => void; onOpenProfile: () => void; targetWidth: number; targetHeight: number; @@ -285,7 +279,7 @@ interface GridTileProps { } export const GridTile = forwardRef( - ({ vm, fullscreen, onToggleFullscreen, onOpenProfile, ...props }, ref) => { + ({ vm, onOpenProfile, ...props }, ref) => { const nameData = useNameData(vm); if (vm instanceof LocalUserMediaViewModel) { diff --git a/src/tile/MediaView.module.css b/src/tile/MediaView.module.css index 65cf9fc7..e3622f4d 100644 --- a/src/tile/MediaView.module.css +++ b/src/tile/MediaView.module.css @@ -94,7 +94,7 @@ unconditionally select the container so we can use cqmin units */ display: grid; grid-template-columns: 1fr auto; grid-template-rows: 1fr auto; - grid-template-areas: ". button2" "nameTag button1"; + grid-template-areas: ". ." "nameTag button"; gap: var(--cpd-space-1x); place-items: start; } @@ -175,9 +175,5 @@ unconditionally select the container so we can use cqmin units */ } .fg > button:first-of-type { - grid-area: button1; -} - -.fg > button:nth-of-type(2) { - grid-area: button2; + grid-area: button; } diff --git a/src/tile/MediaView.tsx b/src/tile/MediaView.tsx index 69c3591e..e34b4fdd 100644 --- a/src/tile/MediaView.tsx +++ b/src/tile/MediaView.tsx @@ -42,7 +42,6 @@ interface Props extends ComponentProps { nameTag: string; displayName: string; primaryButton?: ReactNode; - secondaryButton?: ReactNode; } export const MediaView = forwardRef( @@ -62,7 +61,6 @@ export const MediaView = forwardRef( nameTag, displayName, primaryButton, - secondaryButton, ...props }, ref, @@ -120,7 +118,6 @@ export const MediaView = forwardRef( )}
{primaryButton} - {secondaryButton}
); diff --git a/src/tile/SpotlightTile.module.css b/src/tile/SpotlightTile.module.css index cc591fee..1aee4589 100644 --- a/src/tile/SpotlightTile.module.css +++ b/src/tile/SpotlightTile.module.css @@ -15,28 +15,11 @@ limitations under the License. */ .tile { - --border-width: var(--cpd-space-3x); -} - -.tile.maximised { - --border-width: 0px; -} - -.border { - box-sizing: border-box; - block-size: 100%; - inline-size: 100%; -} - -.tile.maximised .border { - display: contents; -} - -.contents { display: flex; border-radius: var(--cpd-space-6x); contain: strict; - overflow: auto; + overflow-x: auto; + overflow-y: hidden; scrollbar-width: none; scroll-snap-type: inline mandatory; scroll-snap-stop: always; @@ -46,18 +29,18 @@ limitations under the License. scroll-behavior: smooth; */ } -.tile.maximised .contents { +.tile.maximised { border-radius: 0; } -.contents > .item { +.item { height: 100%; flex-basis: 100%; flex-shrink: 0; --media-view-fg-inset: 10px; } -.contents > .item.snap { +.item.snap { scroll-snap-align: start; } @@ -105,7 +88,7 @@ limitations under the License. inset-inline-end: var(--cpd-space-1x); } -.fullScreen { +.expand { appearance: none; cursor: pointer; opacity: 0; @@ -118,23 +101,23 @@ limitations under the License. transition-property: opacity, background-color; position: absolute; z-index: 1; - --inset: calc(var(--border-width) + 6px); + --inset: 6px; inset-block-end: var(--inset); inset-inline-end: var(--inset); } -.fullScreen > svg { +.expand > svg { display: block; color: var(--cpd-color-icon-on-solid-primary); } @media (hover) { - .fullScreen:hover { + .expand:hover { background: var(--cpd-color-bg-action-primary-hovered); } } -.fullScreen:active { +.expand:active { background: var(--cpd-color-bg-action-primary-pressed); } diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index a171fe4f..5407b1a7 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -23,7 +23,6 @@ import { useRef, useState, } from "react"; -import { Glass } from "@vector-im/compound-web"; import ExpandIcon from "@vector-im/compound-design-tokens/icons/expand.svg?react"; import CollapseIcon from "@vector-im/compound-design-tokens/icons/collapse.svg?react"; import ChevronLeftIcon from "@vector-im/compound-design-tokens/icons/chevron-left.svg?react"; @@ -174,8 +173,8 @@ SpotlightItem.displayName = "SpotlightItem"; interface Props { vms: MediaViewModel[]; maximised: boolean; - fullscreen: boolean; - onToggleFullscreen: () => void; + expanded: boolean; + onToggleExpanded: (() => void) | null; targetWidth: number; targetHeight: number; showIndicators: boolean; @@ -188,8 +187,8 @@ export const SpotlightTile = forwardRef( { vms, maximised, - fullscreen, - onToggleFullscreen, + expanded, + onToggleExpanded, targetWidth, targetHeight, showIndicators, @@ -254,9 +253,8 @@ export const SpotlightTile = forwardRef( setScrollToId(vms[visibleIndex + 1].id); }, [latestVisibleId, latestVms, setScrollToId]); - const FullScreenIcon = fullscreen ? CollapseIcon : ExpandIcon; + const ToggleExpandIcon = expanded ? CollapseIcon : ExpandIcon; - // We need a wrapper element because Glass doesn't provide an animated.div return ( ( )} - - {/* Similarly we need a wrapper element here because Glass expects a - single child */} -
- {vms.map((vm) => ( - - ))} -
- - + {vms.map((vm) => ( + + ))} + {onToggleExpanded && ( + + )} {canGoToNext && ( )} -
1, - })} - > - {vms.map((vm) => ( -
- ))} -
+ {!expanded && ( +
1, + })} + > + {vms.map((vm) => ( +
+ ))} +
+ )} ); }, From 8c21e8f277915c3c38d38fdafbb9ae43b3db343e Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 17 Jul 2024 14:55:45 -0400 Subject: [PATCH 24/31] Use a more descriptive string --- public/locales/en-GB/app.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/public/locales/en-GB/app.json b/public/locales/en-GB/app.json index 97f2be93..95bf334f 100644 --- a/public/locales/en-GB/app.json +++ b/public/locales/en-GB/app.json @@ -132,7 +132,7 @@ "developer_settings_label": "Developer Settings", "developer_settings_label_description": "Expose developer settings in the settings window.", "developer_tab_title": "Developer", - "duplicate_tiles_label": "Number of duplicate tiles", + "duplicate_tiles_label": "Number of additional tile copies per participant", "feedback_tab_body": "If you are experiencing issues or simply would like to provide some feedback, please send us a short description below.", "feedback_tab_description_label": "Your feedback", "feedback_tab_h4": "Submit feedback", From a59875dab5936ed606b0533c486712de8b86d989 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 17 Jul 2024 15:37:41 -0400 Subject: [PATCH 25/31] Explain what each sorting bin means --- src/state/CallViewModel.ts | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index f2656166..74061abe 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -131,13 +131,37 @@ export type WindowMode = "normal" | "full screen" | "pip"; * Sorting bins defining the order in which media tiles appear in the layout. */ enum SortingBin { + /** + * Yourself, when the "always show self" option is on. + */ SelfAlwaysShown, + /** + * Participants that are sharing their screen. + */ Presenters, + /** + * Participants that have been speaking recently. + */ Speakers, + /** + * Participants with both video and audio. + */ VideoAndAudio, + /** + * Participants with video but no audio. + */ Video, + /** + * Participants with audio but no video. + */ Audio, + /** + * Participants not sharing any media. + */ NoMedia, + /** + * Yourself, when the "always show self" option is off. + */ SelfNotAlwaysShown, } From 2bc56dbff2efad277edbde1c80614842d0def8d5 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 17 Jul 2024 15:37:55 -0400 Subject: [PATCH 26/31] Use fewer ML-style variable names --- src/state/CallViewModel.ts | 30 ++++++++++++++++++------------ 1 file changed, 18 insertions(+), 12 deletions(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 74061abe..083cf556 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -386,7 +386,7 @@ export class CallViewModel extends ViewModel { }, new Map(), ), - map((ms) => [...ms.values()]), + map((mediaItems) => [...mediaItems.values()]), finalizeValue((ts) => { for (const t of ts) t.destroy(); }), @@ -394,35 +394,41 @@ export class CallViewModel extends ViewModel { ); private readonly userMedia: Observable = this.mediaItems.pipe( - map((ms) => ms.filter((m): m is UserMedia => m instanceof UserMedia)), + map((mediaItems) => + mediaItems.filter((m): m is UserMedia => m instanceof UserMedia), + ), ); private readonly screenShares: Observable = this.mediaItems.pipe( - map((ms) => ms.filter((m): m is ScreenShare => m instanceof ScreenShare)), + map((mediaItems) => + mediaItems.filter((m): m is ScreenShare => m instanceof ScreenShare), + ), ); private readonly spotlightSpeaker: Observable = this.userMedia.pipe( - switchMap((ms) => - ms.length === 0 + switchMap((mediaItems) => + mediaItems.length === 0 ? of([]) : combineLatest( - ms.map((m) => m.vm.speaking.pipe(map((s) => [m, s] as const))), + mediaItems.map((m) => + m.vm.speaking.pipe(map((s) => [m, s] as const)), + ), ), ), scan<(readonly [UserMedia, boolean])[], UserMedia | null, null>( - (prev, ms) => + (prev, mediaItems) => // Decide who to spotlight: // If the previous speaker is still speaking, stick with them rather // than switching eagerly to someone else - ms.find(([m, s]) => m === prev && s)?.[0] ?? + mediaItems.find(([m, s]) => m === prev && s)?.[0] ?? // Otherwise, select anyone who is speaking - ms.find(([, s]) => s)?.[0] ?? + mediaItems.find(([, s]) => s)?.[0] ?? // Otherwise, stick with the person who was last speaking prev ?? // Otherwise, spotlight the local user - ms.find(([m]) => m.vm.local)?.[0] ?? + mediaItems.find(([m]) => m.vm.local)?.[0] ?? null, null, ), @@ -431,8 +437,8 @@ export class CallViewModel extends ViewModel { ); private readonly grid: Observable = this.userMedia.pipe( - switchMap((ms) => { - const bins = ms.map((m) => + switchMap((mediaItems) => { + const bins = mediaItems.map((m) => combineLatest( [ m.speaker, From 1efa594430d6d64d5507adbf22b3f2559d70b507 Mon Sep 17 00:00:00 2001 From: Robin Date: Wed, 17 Jul 2024 16:06:48 -0400 Subject: [PATCH 27/31] Use Array.some where it's appropriate --- src/state/CallViewModel.ts | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 31a1fdb3..62d71048 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -377,7 +377,7 @@ export class CallViewModel extends ViewModel { private readonly hasRemoteScreenShares: Observable = this.screenShares.pipe( - map((ms) => ms.find((m) => !m.vm.local) !== undefined), + map((ms) => ms.some((m) => !m.vm.local)), distinctUntilChanged(), ); From e04affe93e7e40a82fc47dcc321f95492d66dbb4 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Jul 2024 10:00:26 -0400 Subject: [PATCH 28/31] Justify the use of a participant count threshold --- src/state/CallViewModel.ts | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 3d48da22..c3d038df 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -74,6 +74,13 @@ import { ObservableScope } from "./ObservableScope"; // list again const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000; +// This is the number of participants that we think constitutes a "large" grid. +// The hypothesis is that, after this many participants there's enough cognitive +// load that it makes sense to show the speaker in an easy-to-locate spotlight +// tile. We might change this to a scroll-based condition or do something else +// entirely with the spotlight tile, if we workshop this further. +const largeGridThreshold = 20; + // Represents something that should get a tile on the layout, // ie. a user's video feed or a screen share feed. // TODO: This exposes too much information to the view layer, let's keep this @@ -504,7 +511,8 @@ export class CallViewModel extends ViewModel { (grid, spotlight, screenShares): Layout => ({ type: "grid", spotlight: - screenShares.length > 0 || grid.length > 20 + screenShares.length > 0 || + grid.length > largeGridThreshold ? spotlight : undefined, grid, From 7526826b0c01c6f6e70db4176ece928b92083a85 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Jul 2024 11:01:21 -0400 Subject: [PATCH 29/31] Improve aspect ratios on mobile --- src/grid/CallLayout.ts | 11 +++++++++-- src/grid/OneOnOneLayout.module.css | 11 +++++++++-- 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts index e97b18a2..e0054a1c 100644 --- a/src/grid/CallLayout.ts +++ b/src/grid/CallLayout.ts @@ -92,6 +92,7 @@ export interface GridArrangement { const tileMinHeight = 130; const tileMaxAspectRatio = 17 / 9; const tileMinAspectRatio = 4 / 3; +const tileMobileMinAspectRatio = 2 / 3; /** * Determine the ideal arrangement of tiles into a grid of a particular size. @@ -136,12 +137,18 @@ export function arrangeTiles( tileHeight = (minHeight - (rows - 1) * gap) / rows; } if (tileHeight < tileMinHeight) tileHeight = tileMinHeight; + // Impose a minimum and maximum aspect ratio on the tiles const tileAspectRatio = tileWidth / tileHeight; + // We enforce a different min aspect ratio in 1:1s on mobile + const minAspectRatio = + tileCount === 1 && width < 600 + ? tileMobileMinAspectRatio + : tileMinAspectRatio; if (tileAspectRatio > tileMaxAspectRatio) tileWidth = tileHeight * tileMaxAspectRatio; - else if (tileAspectRatio < tileMinAspectRatio) - tileHeight = tileWidth / tileMinAspectRatio; + else if (tileAspectRatio < minAspectRatio) + tileHeight = tileWidth / minAspectRatio; // TODO: We might now be hitting the minimum height or width limit again return { tileWidth, tileHeight, gap, columns }; diff --git a/src/grid/OneOnOneLayout.module.css b/src/grid/OneOnOneLayout.module.css index 0d2ad4ff..54c39d25 100644 --- a/src/grid/OneOnOneLayout.module.css +++ b/src/grid/OneOnOneLayout.module.css @@ -27,11 +27,18 @@ limitations under the License. .local { position: absolute; - inline-size: 180px; - block-size: 135px; + inline-size: 135px; + block-size: 160px; inset: var(--cpd-space-4x); } +@media (min-width: 600px) { + .local { + inline-size: 170px; + block-size: 110px; + } +} + .spotlight { position: absolute; inline-size: 404px; From 4955535374567995f2039b3f7f9c72e878c6580a Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Jul 2024 11:24:18 -0400 Subject: [PATCH 30/31] Use more consistent names for layout types --- src/room/InCallView.tsx | 8 ++++---- src/state/CallViewModel.ts | 14 +++++++------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index a3e02869..43808b8e 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -344,9 +344,9 @@ export const InCallView: FC = ({ }; return { grid: makeGridLayout(inputs), - "spotlight landscape": makeSpotlightLandscapeLayout(inputs), - "spotlight portrait": makeSpotlightPortraitLayout(inputs), - "spotlight expanded": makeSpotlightExpandedLayout(inputs), + "spotlight-landscape": makeSpotlightLandscapeLayout(inputs), + "spotlight-portrait": makeSpotlightPortraitLayout(inputs), + "spotlight-expanded": makeSpotlightExpandedLayout(inputs), "one-on-one": makeOneOnOneLayout(inputs), }; }, [gridBoundsObservable, spotlightAlignment, pipAlignment]); @@ -392,7 +392,7 @@ export const InCallView: FC = ({ ); // The grid tiles go *under* the spotlight in the portrait layout, but // *over* the spotlight in the expanded layout - return layout.type === "spotlight expanded" ? ( + return layout.type === "spotlight-expanded" ? ( <> {fixedGrid} {scrollingGrid} diff --git a/src/state/CallViewModel.ts b/src/state/CallViewModel.ts index 4bb1aaa3..54029b0b 100644 --- a/src/state/CallViewModel.ts +++ b/src/state/CallViewModel.ts @@ -89,19 +89,19 @@ export interface GridLayout { } export interface SpotlightLandscapeLayout { - type: "spotlight landscape"; + type: "spotlight-landscape"; spotlight: MediaViewModel[]; grid: UserMediaViewModel[]; } export interface SpotlightPortraitLayout { - type: "spotlight portrait"; + type: "spotlight-portrait"; spotlight: MediaViewModel[]; grid: UserMediaViewModel[]; } export interface SpotlightExpandedLayout { - type: "spotlight expanded"; + type: "spotlight-expanded"; spotlight: MediaViewModel[]; pip?: UserMediaViewModel; } @@ -583,7 +583,7 @@ export class CallViewModel extends ViewModel { const spotlightLandscapeLayout = combineLatest( [this.grid, this.spotlight], (grid, spotlight): Layout => ({ - type: "spotlight landscape", + type: "spotlight-landscape", spotlight, grid, }), @@ -591,7 +591,7 @@ export class CallViewModel extends ViewModel { const spotlightExpandedLayout = combineLatest( [this.spotlight, this.pip], (spotlight, pip): Layout => ({ - type: "spotlight expanded", + type: "spotlight-expanded", spotlight, pip: pip ?? undefined, }), @@ -641,7 +641,7 @@ export class CallViewModel extends ViewModel { return combineLatest( [this.grid, this.spotlight], (grid, spotlight): Layout => ({ - type: "spotlight portrait", + type: "spotlight-portrait", spotlight, grid, }), @@ -675,7 +675,7 @@ export class CallViewModel extends ViewModel { ); public showSpeakingIndicators: Observable = this.layout.pipe( - map((l) => l.type !== "one-on-one" && l.type !== "spotlight expanded"), + map((l) => l.type !== "one-on-one" && l.type !== "spotlight-expanded"), distinctUntilChanged(), shareReplay(1), ); From 377b7ff5de0b014e594ef969657918fb3e758af4 Mon Sep 17 00:00:00 2001 From: Robin Date: Thu, 18 Jul 2024 11:33:20 -0400 Subject: [PATCH 31/31] Explain each layout --- src/grid/GridLayout.tsx | 4 ++++ src/grid/OneOnOneLayout.tsx | 4 ++++ src/grid/SpotlightExpandedLayout.tsx | 4 ++++ src/grid/SpotlightLandscapeLayout.tsx | 5 +++++ src/grid/SpotlightPortraitLayout.tsx | 5 +++++ 5 files changed, 22 insertions(+) diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index b49bb32a..18a10cbf 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -36,6 +36,10 @@ interface GridCSSProperties extends CSSProperties { "--height": string; } +/** + * An implementation of the "grid" layout, in which all participants are shown + * together in a scrolling grid. + */ export const makeGridLayout: CallLayout = ({ minBounds, spotlightAlignment, diff --git a/src/grid/OneOnOneLayout.tsx b/src/grid/OneOnOneLayout.tsx index 1f9a39e7..635c7898 100644 --- a/src/grid/OneOnOneLayout.tsx +++ b/src/grid/OneOnOneLayout.tsx @@ -24,6 +24,10 @@ import { useReactiveState } from "../useReactiveState"; import styles from "./OneOnOneLayout.module.css"; import { DragCallback } from "./Grid"; +/** + * An implementation of the "one-on-one" layout, in which the remote participant + * is shown at maximum size, overlaid by a small view of the local participant. + */ export const makeOneOnOneLayout: CallLayout = ({ minBounds, pipAlignment, diff --git a/src/grid/SpotlightExpandedLayout.tsx b/src/grid/SpotlightExpandedLayout.tsx index 40f77ca9..b92460ee 100644 --- a/src/grid/SpotlightExpandedLayout.tsx +++ b/src/grid/SpotlightExpandedLayout.tsx @@ -23,6 +23,10 @@ import { DragCallback } from "./Grid"; import styles from "./SpotlightExpandedLayout.module.css"; import { useReactiveState } from "../useReactiveState"; +/** + * An implementation of the "expanded spotlight" layout, in which the spotlight + * tile stretches edge-to-edge and is overlaid by a picture-in-picture tile. + */ export const makeSpotlightExpandedLayout: CallLayout< SpotlightExpandedLayoutModel > = ({ minBounds, pipAlignment }) => ({ diff --git a/src/grid/SpotlightLandscapeLayout.tsx b/src/grid/SpotlightLandscapeLayout.tsx index 40b02f9a..e3ca1bf9 100644 --- a/src/grid/SpotlightLandscapeLayout.tsx +++ b/src/grid/SpotlightLandscapeLayout.tsx @@ -23,6 +23,11 @@ import { SpotlightLandscapeLayout as SpotlightLandscapeLayoutModel } from "../st import styles from "./SpotlightLandscapeLayout.module.css"; import { useReactiveState } from "../useReactiveState"; +/** + * An implementation of the "spotlight landscape" layout, in which the spotlight + * tile takes up most of the space on the left, and the grid of participants is + * shown as a scrolling rail on the right. + */ export const makeSpotlightLandscapeLayout: CallLayout< SpotlightLandscapeLayoutModel > = ({ minBounds }) => ({ diff --git a/src/grid/SpotlightPortraitLayout.tsx b/src/grid/SpotlightPortraitLayout.tsx index 5c0cb0a8..5b82ca62 100644 --- a/src/grid/SpotlightPortraitLayout.tsx +++ b/src/grid/SpotlightPortraitLayout.tsx @@ -34,6 +34,11 @@ interface GridCSSProperties extends CSSProperties { "--grid-tile-height": string; } +/** + * An implementation of the "spotlight portrait" layout, in which the spotlight + * tile is shown across the top of the screen, and the grid of participants + * scrolls behind it. + */ export const makeSpotlightPortraitLayout: CallLayout< SpotlightPortraitLayoutModel > = ({ minBounds }) => ({