Implement the new spotlight layout

src/grid/CallLayout.ts

@@ -0,0 +1,55 @@
+/*
+Copyright 2024 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { BehaviorSubject, Observable } from "rxjs";
+
+import { MediaViewModel } from "../state/MediaViewModel";
+import { LayoutSystem } from "./Grid";
+import { Alignment } from "../room/InCallView";
+
+export interface Bounds {
+  width: number;
+  height: number;
+}
+
+export interface CallLayoutInputs {
+  /**
+   * The minimum bounds of the layout area.
+   */
+  minBounds: Observable<Bounds>;
+  /**
+   * The alignment of the floating tile, if any.
+   */
+  floatingAlignment: BehaviorSubject<Alignment>;
+}
+
+export interface CallLayoutOutputs<Model> {
+  /**
+   * The visually fixed (non-scrolling) layer of the layout.
+   */
+  fixed: LayoutSystem<Model, MediaViewModel[], HTMLDivElement>;
+  /**
+   * The layer of the layout that can overflow and be scrolled.
+   */
+  scrolling: LayoutSystem<Model, MediaViewModel, HTMLDivElement>;
+}
+
+/**
+ * A layout system for media tiles.
+ */
+export type CallLayout<Model> = (
+  inputs: CallLayoutInputs,
+) => CallLayoutOutputs<Model>;

src/grid/GridLayout.module.css

@@ -14,6 +14,11 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+.fixed,
+.scrolling {
+  margin-inline: var(--inline-content-inset);
+}
+
 .scrolling {
   box-sizing: border-box;
   block-size: 100%;
@@ -22,7 +27,6 @@ limitations under the License.
   justify-content: center;
   align-content: center;
   gap: var(--gap);
-  box-sizing: border-box;
 }
 
 .scrolling > .slot {

src/grid/GridLayout.tsx

@@ -15,21 +15,15 @@ limitations under the License.
 */
 
 import { CSSProperties, forwardRef, useMemo } from "react";
-import { BehaviorSubject, Observable, distinctUntilChanged } from "rxjs";
+import { distinctUntilChanged } from "rxjs";
 import { useObservableEagerState } from "observable-hooks";
 
 import { GridLayout as GridLayoutModel } from "../state/CallViewModel";
-import { MediaViewModel } from "../state/MediaViewModel";
+import { Slot } from "./Grid";
-import { LayoutSystem, Slot } from "./Grid";
 import styles from "./GridLayout.module.css";
 import { useReactiveState } from "../useReactiveState";
-import { Alignment } from "../room/InCallView";
 import { useInitial } from "../useInitial";
-
+import { CallLayout } from "./CallLayout";
-export interface Bounds {
-  width: number;
-  height: number;
-}
 
 interface GridCSSProperties extends CSSProperties {
   "--gap": string;
@@ -37,19 +31,14 @@ interface GridCSSProperties extends CSSProperties {
   "--height": string;
 }
 
-interface GridLayoutSystems {
-  scrolling: LayoutSystem<GridLayoutModel, MediaViewModel, HTMLDivElement>;
-  fixed: LayoutSystem<GridLayoutModel, MediaViewModel[], HTMLDivElement>;
-}
-
 const slotMinHeight = 130;
 const slotMaxAspectRatio = 17 / 9;
 const slotMinAspectRatio = 4 / 3;
 
-export const gridLayoutSystems = (
+export const makeGridLayout: CallLayout<GridLayoutModel> = ({
-  minBounds: Observable<Bounds>,
+  minBounds,
-  floatingAlignment: BehaviorSubject<Alignment>,
+  floatingAlignment,
-): GridLayoutSystems => ({
+}) => ({
   // The "fixed" (non-scrolling) part of the layout is where the spotlight tile
   // lives
   fixed: {

src/grid/SpotlightLayout.module.css

@@ -0,0 +1,89 @@
+/*
+Copyright 2024 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+.fixed,
+.scrolling {
+  margin-inline: var(--inline-content-inset);
+  display: grid;
+  --grid-slot-width: 180px;
+  --grid-gap: 20px;
+  grid-template-columns: 1fr calc(
+      var(--grid-columns) * var(--grid-slot-width) + (var(--grid-columns) - 1) *
+        var(--grid-gap)
+    );
+  grid-template-rows: minmax(1fr, auto);
+  gap: 30px;
+}
+
+.scrolling {
+  block-size: 100%;
+}
+
+.spotlight {
+  container: spotlight / size;
+  display: grid;
+  place-items: center;
+}
+
+/* CSS makes us put a condition here, even though all we want to do is
+unconditionally select the container so we can use cq units */
+@container spotlight (width > 0) {
+  .spotlight > .slot {
+    inline-size: min(100cqi, 100cqb * (17 / 9));
+    block-size: min(100cqb, 100cqi / (4 / 3));
+  }
+}
+
+.grid {
+  display: flex;
+  flex-wrap: wrap;
+  gap: var(--grid-gap);
+  justify-content: center;
+  align-content: center;
+}
+
+.grid > .slot {
+  inline-size: var(--grid-slot-width);
+  block-size: 135px;
+}
+
+@media (max-width: 600px) {
+  .fixed,
+  .scrolling {
+    margin-inline: 0;
+    display: block;
+  }
+
+  .spotlight {
+    inline-size: 100%;
+    aspect-ratio: 16 / 9;
+    margin-block-end: var(--cpd-space-4x);
+  }
+
+  .grid {
+    margin-inline: var(--inline-content-inset);
+    align-content: start;
+  }
+
+  .grid > .slot {
+    --grid-columns: 2;
+    --grid-slot-width: calc(
+      (100% - (var(--grid-columns) - 1) * var(--grid-gap)) / var(--grid-columns)
+    );
+    block-size: unset;
+    aspect-ratio: 4 / 3;
+  }
+}

src/grid/SpotlightLayout.tsx

@@ -0,0 +1,89 @@
+/*
+Copyright 2024 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { CSSProperties, forwardRef } from "react";
+import { useObservableEagerState } from "observable-hooks";
+
+import { CallLayout } from "./CallLayout";
+import { SpotlightLayout as SpotlightLayoutModel } from "../state/CallViewModel";
+import { useReactiveState } from "../useReactiveState";
+import styles from "./SpotlightLayout.module.css";
+import { Slot } from "./Grid";
+
+interface GridCSSProperties extends CSSProperties {
+  "--grid-columns": number;
+}
+
+const getGridColumns = (gridLength: number): number =>
+  gridLength > 20 ? 2 : 1;
+
+export const makeSpotlightLayout: CallLayout<SpotlightLayoutModel> = ({
+  minBounds,
+}) => ({
+  fixed: {
+    tiles: (model) => new Map([["spotlight", model.spotlight]]),
+    Layout: forwardRef(function SpotlightLayoutFixed({ model }, ref) {
+      const { width, height } = useObservableEagerState(minBounds);
+      const gridColumns = getGridColumns(model.grid.length);
+      const [generation] = useReactiveState<number>(
+        (prev) => (prev === undefined ? 0 : prev + 1),
+        [model.grid.length, width, height],
+      );
+
+      return (
+        <div
+          ref={ref}
+          data-generation={generation}
+          className={styles.fixed}
+          style={{ "--grid-columns": gridColumns, height } as GridCSSProperties}
+        >
+          <div className={styles.spotlight}>
+            <Slot className={styles.slot} tile="spotlight" />
+          </div>
+          <div className={styles.grid} />
+        </div>
+      );
+    }),
+  },
+
+  scrolling: {
+    tiles: (model) => new Map(model.grid.map((tile) => [tile.id, tile])),
+    Layout: forwardRef(function SpotlightLayoutScrolling({ model }, ref) {
+      const { width, height } = useObservableEagerState(minBounds);
+      const gridColumns = getGridColumns(model.grid.length);
+      const [generation] = useReactiveState<number>(
+        (prev) => (prev === undefined ? 0 : prev + 1),
+        [model.grid, width, height],
+      );
+
+      return (
+        <div
+          ref={ref}
+          data-generation={generation}
+          className={styles.scrolling}
+          style={{ "--grid-columns": gridColumns } as GridCSSProperties}
+        >
+          <div className={styles.spotlight} />
+          <div className={styles.grid}>
+            {model.grid.map((tile) => (
+              <Slot key={tile.id} className={styles.slot} tile={tile.id} />
+            ))}
+          </div>
+        </div>
+      );
+    }),
+  },
+});

src/room/InCallView.module.css

@@ -125,7 +125,7 @@ limitations under the License.
 
 .fixedGrid {
   position: absolute;
-  inline-size: calc(100% - 2 * var(--inline-content-inset));
+  inline-size: 100%;
   align-self: center;
   /* Disable pointer events so the overlay doesn't block interaction with
   elements behind it */
@@ -139,6 +139,6 @@ limitations under the License.
 .scrollingGrid {
   position: relative;
   flex-grow: 1;
-  inline-size: calc(100% - 2 * var(--inline-content-inset));
+  inline-size: 100%;
   align-self: center;
 }

src/room/InCallView.tsx

@@ -35,7 +35,7 @@ import {
 import useMeasure from "react-use-measure";
 import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
 import classNames from "classnames";
-import { BehaviorSubject } from "rxjs";
+import { BehaviorSubject, map } from "rxjs";
 import { useObservableEagerState } from "observable-hooks";
 import { useTranslation } from "react-i18next";
 
@@ -73,17 +73,20 @@ import { ECConnectionState } from "../livekit/useECConnectionState";
 import { useOpenIDSFU } from "../livekit/openIDSFU";
 import {
   GridMode,
+  Layout,
   TileDescriptor,
   useCallViewModel,
 } from "../state/CallViewModel";
 import { Grid, TileProps } from "../grid/Grid";
 import { MediaViewModel } from "../state/MediaViewModel";
-import { gridLayoutSystems } from "../grid/GridLayout";
 import { useObservable } from "../state/useObservable";
 import { useInitial } from "../useInitial";
 import { SpotlightTile } from "../tile/SpotlightTile";
 import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
 import { E2eeType } from "../e2ee/e2eeType";
+import { makeGridLayout } from "../grid/GridLayout";
+import { makeSpotlightLayout } from "../grid/SpotlightLayout";
+import { CallLayout } from "../grid/CallLayout";
 
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 
@@ -302,6 +305,7 @@ export const InCallView: FC<InCallViewProps> = ({
 
   const [headerRef, headerBounds] = useMeasure();
   const [footerRef, footerBounds] = useMeasure();
+
   const gridBounds = useMemo(
     () => ({
       width: footerBounds.width,
@@ -315,11 +319,29 @@ export const InCallView: FC<InCallViewProps> = ({
     ],
   );
   const gridBoundsObservable = useObservable(gridBounds);
+
   const floatingAlignment = useInitial(
     () => new BehaviorSubject(defaultAlignment),
   );
-  const { fixed, scrolling } = useInitial(() =>
+
-    gridLayoutSystems(gridBoundsObservable, floatingAlignment),
+  const layoutSystem = useObservableEagerState(
+    useInitial(() =>
+      vm.layout.pipe(
+        map((l) => {
+          let makeLayout: CallLayout<Layout>;
+          if (l.type === "grid" && l.grid.length !== 2)
+            makeLayout = makeGridLayout as CallLayout<Layout>;
+          else if (l.type === "spotlight")
+            makeLayout = makeSpotlightLayout as CallLayout<Layout>;
+          else return null; // Not yet implemented
+
+          return makeLayout({
+            minBounds: gridBoundsObservable,
+            floatingAlignment,
+          });
+        }),
+      ),
+    ),
   );
 
   const setGridMode = useCallback(
@@ -423,31 +445,9 @@ export const InCallView: FC<InCallViewProps> = ({
       );
     }
 
-    // The only new layout we've implemented so far is grid layout for non-1:1
+    if (layoutSystem === null) {
-    // calls. All other layouts use the legacy grid system for now.
+      // This new layout doesn't yet have an implemented layout system, so fall
-    if (
+      // back to the legacy grid system
-      legacyLayout === "grid" &&
-      layout.type === "grid" &&
-      !(layout.grid.length === 2 && layout.spotlight === undefined)
-    ) {
-      return (
-        <>
-          <Grid
-            className={styles.scrollingGrid}
-            model={layout}
-            system={scrolling}
-            Tile={GridTileView}
-          />
-          <Grid
-            className={styles.fixedGrid}
-            style={{ insetBlockStart: headerBounds.bottom }}
-            model={layout}
-            system={fixed}
-            Tile={SpotlightTileView}
-          />
-        </>
-      );
-    } else {
       return (
         <LegacyGrid
           items={items}
@@ -456,6 +456,24 @@ export const InCallView: FC<InCallViewProps> = ({
           Tile={GridTileView}
         />
       );
+    } else {
+      return (
+        <>
+          <Grid
+            className={styles.scrollingGrid}
+            model={layout}
+            system={layoutSystem.scrolling}
+            Tile={GridTileView}
+          />
+          <Grid
+            className={styles.fixedGrid}
+            style={{ insetBlockStart: headerBounds.bottom }}
+            model={layout}
+            system={layoutSystem.fixed}
+            Tile={SpotlightTileView}
+          />
+        </>
+      );
     }
   };
 

src/state/CallViewModel.ts

@@ -28,12 +28,13 @@ import {
 import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix";
 import { useEffect, useRef } from "react";
 import {
-  BehaviorSubject,
   EMPTY,
   Observable,
+  Subject,
   audit,
   combineLatest,
   concat,
+  concatMap,
   distinctUntilChanged,
   filter,
   map,
@@ -48,6 +49,7 @@ import {
   switchMap,
   throttleTime,
   timer,
+  withLatestFrom,
   zip,
 } from "rxjs";
 import { logger } from "matrix-js-sdk/src/logger";
@@ -371,6 +373,13 @@ export class CallViewModel extends ViewModel {
   private readonly screenShares: Observable<ScreenShare[]> =
     this.mediaItems.pipe(
       map((ms) => ms.filter((m): m is ScreenShare => m instanceof ScreenShare)),
+      shareReplay(1),
+    );
+
+  private readonly hasScreenShares: Observable<boolean> =
+    this.screenShares.pipe(
+      map((ms) => ms.length > 0),
+      distinctUntilChanged(),
     );
 
   private readonly spotlightSpeaker: Observable<UserMedia | null> =
@@ -385,11 +394,13 @@ export class CallViewModel extends ViewModel {
       scan<(readonly [UserMedia, boolean])[], UserMedia | null, null>(
         (prev, ms) =>
           // Decide who to spotlight:
-          // If the previous speaker is still speaking, stick with them rather
+          // If the previous speaker (not the local user) is still speaking,
-          // than switching eagerly to someone else
+          // stick with them rather than switching eagerly to someone else
-          ms.find(([m, s]) => m === prev && s)?.[0] ??
+          (prev === null || prev.vm.local
-          // Otherwise, select anyone who is speaking
+            ? null
-          ms.find(([, s]) => s)?.[0] ??
+            : ms.find(([m, s]) => m === prev && s)?.[0]) ??
+          // Otherwise, select any remote user who is speaking
+          ms.find(([m, s]) => !m.vm.local && s)?.[0] ??
           // Otherwise, stick with the person who was last speaking
           prev ??
           // Otherwise, spotlight the local user
@@ -398,7 +409,8 @@ export class CallViewModel extends ViewModel {
         null,
       ),
       distinctUntilChanged(),
-      throttleTime(800, undefined, { leading: true, trailing: true }),
+      shareReplay(1),
+      throttleTime(1600, undefined, { leading: true, trailing: true }),
     );
 
   private readonly grid: Observable<UserMediaViewModel[]> = this.userMedia.pipe(
@@ -453,18 +465,31 @@ export class CallViewModel extends ViewModel {
   // orientation
   private readonly windowMode = of<WindowMode>("normal");
 
-  private readonly _gridMode = new BehaviorSubject<GridMode>("grid");
+  private readonly gridModeUserSelection = new Subject<GridMode>();
   /**
    * The layout mode of the media tile grid.
    */
-  public readonly gridMode: Observable<GridMode> = this._gridMode;
+  public readonly gridMode: Observable<GridMode> = merge(
+    // Always honor a manual user selection
+    this.gridModeUserSelection,
+    // If the user hasn't selected spotlight and somebody starts screen sharing,
+    // automatically switch to spotlight mode and reset when screen sharing ends
+    this.hasScreenShares.pipe(
+      withLatestFrom(this.gridModeUserSelection.pipe(startWith(null))),
+      concatMap(([hasScreenShares, userSelection]) =>
+        userSelection === "spotlight"
+          ? EMPTY
+          : of<GridMode>(hasScreenShares ? "spotlight" : "grid"),
+      ),
+    ),
+  ).pipe(distinctUntilChanged(), shareReplay(1));
 
   public setGridMode(value: GridMode): void {
-    this._gridMode.next(value);
+    this.gridModeUserSelection.next(value);
   }
 
   public readonly layout: Observable<Layout> = combineLatest(
-    [this._gridMode, this.windowMode],
+    [this.gridMode, this.windowMode],
     (gridMode, windowMode) => {
       switch (windowMode) {
         case "full screen":

src/tile/GridTile.tsx

@@ -71,6 +71,7 @@ interface MediaTileProps
   vm: MediaViewModel;
   videoEnabled: boolean;
   videoFit: "contain" | "cover";
+  mirror: boolean;
   nameTagLeadingIcon?: ReactNode;
   primaryButton: ReactNode;
   secondaryButton?: ReactNode;
@@ -87,7 +88,6 @@ const MediaTile = forwardRef<HTMLDivElement, MediaTileProps>(
         className={classNames(className, styles.tile)}
         data-maximised={maximised}
         video={video}
-        mirror={false}
         member={vm.member}
         unencryptedWarning={unencryptedWarning}
         {...props}
@@ -100,6 +100,7 @@ MediaTile.displayName = "MediaTile";
 
 interface UserMediaTileProps extends TileProps {
   vm: UserMediaViewModel;
+  mirror: boolean;
   showSpeakingIndicators: boolean;
   menuStart?: ReactNode;
   menuEnd?: ReactNode;
@@ -202,7 +203,7 @@ interface LocalUserMediaTileProps extends TileProps {
 }
 
 const LocalUserMediaTile = forwardRef<HTMLDivElement, LocalUserMediaTileProps>(
-  ({ vm, onOpenProfile, className, ...props }, ref) => {
+  ({ vm, onOpenProfile, ...props }, ref) => {
     const { t } = useTranslation();
     const mirror = useObservableEagerState(vm.mirror);
     const alwaysShow = useObservableEagerState(vm.alwaysShow);
@@ -220,6 +221,7 @@ const LocalUserMediaTile = forwardRef<HTMLDivElement, LocalUserMediaTileProps>(
       <UserMediaTile
         ref={ref}
         vm={vm}
+        mirror={mirror}
         menuStart={
           <ToggleMenuItem
             Icon={VisibilityOnIcon}
@@ -236,7 +238,6 @@ const LocalUserMediaTile = forwardRef<HTMLDivElement, LocalUserMediaTileProps>(
             onSelect={onOpenProfile}
           />
         }
-        className={classNames(className, { [styles.mirror]: mirror })}
         {...props}
       />
     );
@@ -270,6 +271,7 @@ const RemoteUserMediaTile = forwardRef<
     <UserMediaTile
       ref={ref}
       vm={vm}
+      mirror={false}
       menuStart={
         <>
           <ToggleMenuItem
@@ -321,7 +323,9 @@ const ScreenShareTile = forwardRef<HTMLDivElement, ScreenShareTileProps>(
       <MediaTile
         ref={ref}
         vm={vm}
+        videoEnabled
         videoFit="contain"
+        mirror={false}
         primaryButton={
           !vm.local && (
             <button
@@ -336,7 +340,6 @@ const ScreenShareTile = forwardRef<HTMLDivElement, ScreenShareTileProps>(
             </button>
           )
         }
-        videoEnabled
         {...props}
       />
     );