Merge pull request #2086 from robintown/layout-state

Add data models for the new call layouts

src/state/CallViewModel.ts

@@ -1,5 +1,5 @@
 /*
-Copyright 2023 New Vector Ltd
+Copyright 2023-2024 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -16,25 +16,41 @@ limitations under the License.
 
 import {
   connectedParticipantsObserver,
+  observeParticipantEvents,
   observeParticipantMedia,
 } from "@livekit/components-core";
-import { Room as LivekitRoom, RemoteParticipant } from "livekit-client";
+import {
+  Room as LivekitRoom,
+  LocalParticipant,
+  ParticipantEvent,
+  RemoteParticipant,
+} from "livekit-client";
 import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix";
 import { useEffect, useRef } from "react";
 import {
+  BehaviorSubject,
   EMPTY,
   Observable,
+  audit,
   combineLatest,
   concat,
+  distinctUntilChanged,
+  filter,
+  map,
+  merge,
   mergeAll,
   of,
   sample,
   scan,
+  shareReplay,
   startWith,
-  takeUntil,
+  switchAll,
+  switchMap,
+  throttleTime,
+  timer,
   zip,
 } from "rxjs";
-import { state } from "@react-rxjs/core";
+import { StateObservable, state } from "@react-rxjs/core";
 import { logger } from "matrix-js-sdk/src/logger";
 
 import { ViewModel } from "./ViewModel";
@@ -45,14 +61,21 @@ import {
 } from "../livekit/useECConnectionState";
 import { usePrevious } from "../usePrevious";
 import {
-  TileViewModel,
+  MediaViewModel,
-  UserMediaTileViewModel,
+  UserMediaViewModel,
-  ScreenShareTileViewModel,
+  ScreenShareViewModel,
-} from "./TileViewModel";
+} from "./MediaViewModel";
 import { finalizeValue } from "../observable-utils";
+import { ObservableScope } from "./ObservableScope";
+
+// How long we wait after a focus switch before showing the real participant
+// list again
+const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
 
 // Represents something that should get a tile on the layout,
 // ie. a user's video feed or a screen share feed.
+// TODO: This exposes too much information to the view layer, let's keep this
+// information internal to the view model and switch to using Tile<T> instead
 export interface TileDescriptor<T> {
   id: string;
   focused: boolean;
@@ -65,9 +88,123 @@ export interface TileDescriptor<T> {
   data: T;
 }
 
-// How long we wait after a focus switch before showing the real participant
+export interface GridLayout {
-// list again
+  type: "grid";
-const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
+  spotlight?: MediaViewModel[];
+  grid: UserMediaViewModel[];
+}
+
+export interface SpotlightLayout {
+  type: "spotlight";
+  spotlight: MediaViewModel[];
+  grid: UserMediaViewModel[];
+}
+
+export interface FullScreenLayout {
+  type: "full screen";
+  spotlight: MediaViewModel[];
+  pip?: UserMediaViewModel;
+}
+
+export interface PipLayout {
+  type: "pip";
+  spotlight: MediaViewModel[];
+}
+
+/**
+ * A layout defining the media tiles present on screen and their visual
+ * arrangement.
+ */
+export type Layout =
+  | GridLayout
+  | SpotlightLayout
+  | FullScreenLayout
+  | PipLayout;
+
+export type GridMode = "grid" | "spotlight";
+
+export type WindowMode = "normal" | "full screen" | "pip";
+
+/**
+ * Sorting bins defining the order in which media tiles appear in the layout.
+ */
+enum SortingBin {
+  SelfStart,
+  Presenters,
+  Speakers,
+  VideoAndAudio,
+  Video,
+  Audio,
+  NoMedia,
+  SelfEnd,
+}
+
+class UserMedia {
+  private readonly scope = new ObservableScope();
+  public readonly vm: UserMediaViewModel;
+  public readonly speaker: Observable<boolean>;
+  public readonly presenter: Observable<boolean>;
+
+  public constructor(
+    public readonly id: string,
+    member: RoomMember | undefined,
+    participant: LocalParticipant | RemoteParticipant,
+    callEncrypted: boolean,
+  ) {
+    this.vm = new UserMediaViewModel(id, member, participant, callEncrypted);
+
+    this.speaker = this.vm.speaking.pipeState(
+      // Require 1 s of continuous speaking to become a speaker, and 10 s of
+      // continuous silence to stop being considered a speaker
+      audit((s) =>
+        merge(
+          timer(s ? 1000 : 10000),
+          // If the speaking flag resets to its original value during this time,
+          // end the silencing window to stick with that original value
+          this.vm.speaking.pipe(filter((s1) => s1 !== s)),
+        ),
+      ),
+      startWith(false),
+      distinctUntilChanged(),
+      this.scope.bind(),
+      // Make this Observable hot so that the timers don't reset when you
+      // resubscribe
+      shareReplay(1),
+    );
+
+    this.presenter = observeParticipantEvents(
+      participant,
+      ParticipantEvent.TrackPublished,
+      ParticipantEvent.TrackUnpublished,
+      ParticipantEvent.LocalTrackPublished,
+      ParticipantEvent.LocalTrackUnpublished,
+    ).pipe(map((p) => p.isScreenShareEnabled));
+  }
+
+  public destroy(): void {
+    this.scope.end();
+    this.vm.destroy();
+  }
+}
+
+class ScreenShare {
+  public readonly vm: ScreenShareViewModel;
+
+  public constructor(
+    id: string,
+    member: RoomMember | undefined,
+    participant: LocalParticipant | RemoteParticipant,
+    callEncrypted: boolean,
+  ) {
+    this.vm = new ScreenShareViewModel(id, member, participant, callEncrypted);
+  }
+
+  public destroy(): void {
+    this.vm.destroy();
+  }
+}
+
+type MediaItem = UserMedia | ScreenShare;
 
 function findMatrixMember(
   room: MatrixRoom,
@@ -98,7 +235,8 @@ export class CallViewModel extends ViewModel {
 
   // Lists of participants to "hold" on display, even if LiveKit claims that
   // they've left
-  private readonly remoteParticipantHolds = zip(
+  private readonly remoteParticipantHolds: Observable<RemoteParticipant[][]> =
+    zip(
       this.connectionState,
       this.rawRemoteParticipants.pipe(sample(this.connectionState)),
       (s, ps) => {
@@ -116,7 +254,7 @@ export class CallViewModel extends ViewModel {
               ),
               new Promise<void>((resolve) => {
                 const subscription = this.connectionState
-                .pipe(takeUntil(this.destroyed))
+                  .pipe(this.scope.bind())
                   .subscribe((s) => {
                     if (s !== ECAddonConnectionState.ECSwitchingFocus) {
                       resolve();
@@ -145,7 +283,8 @@ export class CallViewModel extends ViewModel {
       startWith([]),
     );
 
-  private readonly remoteParticipants = combineLatest(
+  private readonly remoteParticipants: Observable<RemoteParticipant[]> =
+    combineLatest(
       [this.rawRemoteParticipants, this.remoteParticipantHolds],
       (raw, holds) => {
         const result = [...raw];
@@ -165,10 +304,194 @@ export class CallViewModel extends ViewModel {
       },
     );
 
+  private readonly mediaItems: StateObservable<MediaItem[]> = state(
+    combineLatest([
+      this.remoteParticipants,
+      observeParticipantMedia(this.livekitRoom.localParticipant),
+    ]).pipe(
+      scan(
+        (
+          prevItems,
+          [remoteParticipants, { participant: localParticipant }],
+        ) => {
+          let allGhosts = true;
+
+          const newItems = new Map(
+            function* (this: CallViewModel): Iterable<[string, MediaItem]> {
+              for (const p of [localParticipant, ...remoteParticipants]) {
+                const member = findMatrixMember(this.matrixRoom, p.identity);
+                allGhosts &&= member === undefined;
+                // We always start with a local participant with the empty string as
+                // their ID before we're connected, this is fine and we'll be in
+                // "all ghosts" mode.
+                if (p.identity !== "" && member === undefined) {
+                  logger.warn(
+                    `Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`,
+                  );
+                }
+
+                const userMediaId = p.identity;
+                yield [
+                  userMediaId,
+                  prevItems.get(userMediaId) ??
+                    new UserMedia(userMediaId, member, p, this.encrypted),
+                ];
+
+                if (p.isScreenShareEnabled) {
+                  const screenShareId = `${userMediaId}:screen-share`;
+                  yield [
+                    screenShareId,
+                    prevItems.get(screenShareId) ??
+                      new ScreenShare(screenShareId, member, p, this.encrypted),
+                  ];
+                }
+              }
+            }.bind(this)(),
+          );
+
+          for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy();
+
+          // If every item is a ghost, that probably means we're still connecting
+          // and shouldn't bother showing anything yet
+          return allGhosts ? new Map() : newItems;
+        },
+        new Map<string, MediaItem>(),
+      ),
+      map((ms) => [...ms.values()]),
+      finalizeValue((ts) => {
+        for (const t of ts) t.destroy();
+      }),
+    ),
+  );
+
+  private readonly userMedia: Observable<UserMedia[]> = this.mediaItems.pipe(
+    map((ms) => ms.filter((m): m is UserMedia => m instanceof UserMedia)),
+  );
+
+  private readonly screenShares: Observable<ScreenShare[]> =
+    this.mediaItems.pipe(
+      map((ms) => ms.filter((m): m is ScreenShare => m instanceof ScreenShare)),
+    );
+
+  private readonly spotlightSpeaker: Observable<UserMedia | null> =
+    this.userMedia.pipe(
+      switchMap((ms) =>
+        ms.length === 0
+          ? of([])
+          : combineLatest(
+              ms.map((m) => m.vm.speaking.pipe(map((s) => [m, s] as const))),
+            ),
+      ),
+      scan<(readonly [UserMedia, boolean])[], UserMedia | null, null>(
+        (prev, ms) =>
+          // Decide who to spotlight:
+          // If the previous speaker is still speaking, stick with them rather
+          // than switching eagerly to someone else
+          ms.find(([m, s]) => m === prev && s)?.[0] ??
+          // Otherwise, select anyone who is speaking
+          ms.find(([, s]) => s)?.[0] ??
+          // Otherwise, stick with the person who was last speaking
+          prev ??
+          // Otherwise, spotlight the local user
+          ms.find(([m]) => m.vm.local)?.[0] ??
+          null,
+        null,
+      ),
+      distinctUntilChanged(),
+      throttleTime(800, undefined, { leading: true, trailing: true }),
+    );
+
+  private readonly grid: Observable<UserMediaViewModel[]> = this.userMedia.pipe(
+    switchMap((ms) => {
+      const bins = ms.map((m) =>
+        combineLatest(
+          [m.speaker, m.presenter, m.vm.audioEnabled, m.vm.videoEnabled],
+          (speaker, presenter, audio, video) => {
+            let bin: SortingBin;
+            if (m.vm.local) bin = SortingBin.SelfStart;
+            else if (presenter) bin = SortingBin.Presenters;
+            else if (speaker) bin = SortingBin.Speakers;
+            else if (video)
+              bin = audio ? SortingBin.VideoAndAudio : SortingBin.Video;
+            else bin = audio ? SortingBin.Audio : SortingBin.NoMedia;
+
+            return [m, bin] as const;
+          },
+        ),
+      );
+      // Sort the media by bin order and generate a tile for each one
+      return bins.length === 0
+        ? of([])
+        : combineLatest(bins, (...bins) =>
+            bins.sort(([, bin1], [, bin2]) => bin1 - bin2).map(([m]) => m.vm),
+          );
+    }),
+  );
+
+  private readonly spotlight: Observable<MediaViewModel[]> = combineLatest(
+    [this.screenShares, this.spotlightSpeaker],
+    (screenShares, spotlightSpeaker): MediaViewModel[] =>
+      screenShares.length > 0
+        ? screenShares.map((m) => m.vm)
+        : spotlightSpeaker === null
+          ? []
+          : [spotlightSpeaker.vm],
+  );
+
+  // TODO: Make this react to changes in window dimensions and screen
+  // orientation
+  private readonly windowMode = of<WindowMode>("normal");
+
+  private readonly _gridMode = new BehaviorSubject<GridMode>("grid");
+  /**
+   * The layout mode of the media tile grid.
+   */
+  public readonly gridMode = state(this._gridMode);
+
+  public setGridMode(value: GridMode): void {
+    this._gridMode.next(value);
+  }
+
+  public readonly layout: StateObservable<Layout> = state(
+    combineLatest([this._gridMode, this.windowMode], (gridMode, windowMode) => {
+      switch (windowMode) {
+        case "full screen":
+          throw new Error("unimplemented");
+        case "pip":
+          throw new Error("unimplemented");
+        case "normal": {
+          switch (gridMode) {
+            case "grid":
+              return combineLatest(
+                [this.grid, this.spotlight, this.screenShares],
+                (grid, spotlight, screenShares): Layout => ({
+                  type: "grid",
+                  spotlight: screenShares.length > 0 ? spotlight : undefined,
+                  grid,
+                }),
+              );
+            case "spotlight":
+              return combineLatest(
+                [this.grid, this.spotlight],
+                (grid, spotlight): Layout => ({
+                  type: "spotlight",
+                  spotlight,
+                  grid,
+                }),
+              );
+          }
+        }
+      }
+    }).pipe(switchAll()),
+  );
+
   /**
    * The media tiles to be displayed in the call view.
    */
-  public readonly tiles = state(
+  // TODO: Get rid of this field, replacing it with the 'layout' field above
+  // which keeps more details of the layout order internal to the view model
+  public readonly tiles: StateObservable<TileDescriptor<MediaViewModel>[]> =
+    state(
       combineLatest([
         this.remoteParticipants,
         observeParticipantMedia(this.livekitRoom.localParticipant),
@@ -197,10 +520,10 @@ export class CallViewModel extends ViewModel {
 
             const userMediaVm =
               tilesById.get(userMediaId)?.data ??
-            new UserMediaTileViewModel(userMediaId, member, p, this.encrypted);
+              new UserMediaViewModel(userMediaId, member, p, this.encrypted);
             tilesById.delete(userMediaId);
 
-          const userMediaTile: TileDescriptor<TileViewModel> = {
+            const userMediaTile: TileDescriptor<MediaViewModel> = {
               id: userMediaId,
               focused: false,
               isPresenter: p.isScreenShareEnabled,
@@ -215,7 +538,7 @@ export class CallViewModel extends ViewModel {
               const screenShareId = `${userMediaId}:screen-share`;
               const screenShareVm =
                 tilesById.get(screenShareId)?.data ??
-              new ScreenShareTileViewModel(
+                new ScreenShareViewModel(
                   screenShareId,
                   member,
                   p,
@@ -223,7 +546,7 @@ export class CallViewModel extends ViewModel {
                 );
               tilesById.delete(screenShareId);
 
-            const screenShareTile: TileDescriptor<TileViewModel> = {
+              const screenShareTile: TileDescriptor<MediaViewModel> = {
                 id: screenShareId,
                 focused: true,
                 isPresenter: false,
@@ -246,7 +569,7 @@ export class CallViewModel extends ViewModel {
           // If every item is a ghost, that probably means we're still connecting
           // and shouldn't bother showing anything yet
           return allGhosts ? [] : newTiles;
-      }, [] as TileDescriptor<TileViewModel>[]),
+        }, [] as TileDescriptor<MediaViewModel>[]),
         finalizeValue((ts) => {
           for (const t of ts) t.data.destroy();
         }),

src/state/MediaViewModel.ts

@@ -1,5 +1,5 @@
 /*
-Copyright 2023 New Vector Ltd
+Copyright 2023-2024 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -43,7 +43,6 @@ import {
   of,
   startWith,
   switchMap,
-  takeUntil,
 } from "rxjs";
 
 import { ViewModel } from "./ViewModel";
@@ -64,13 +63,13 @@ function observeTrackReference(
   );
 }
 
-abstract class BaseTileViewModel extends ViewModel {
+abstract class BaseMediaViewModel extends ViewModel {
   /**
-   * Whether the tile belongs to the local user.
+   * Whether the media belongs to the local user.
    */
   public readonly local = this.participant.isLocal;
   /**
-   * The LiveKit video track to be shown on this tile.
+   * The LiveKit video track for this media.
    */
   public readonly video: StateObservable<TrackReferenceOrPlaceholder>;
   /**
@@ -83,7 +82,7 @@ abstract class BaseTileViewModel extends ViewModel {
     // soon as that code is moved into the view models
     public readonly id: string,
     /**
-     * The Matrix room member to which this tile belongs.
+     * The Matrix room member to which this media belongs.
      */
     // TODO: Fully separate the data layer from the UI layer by keeping the
     // member object internal
@@ -109,14 +108,14 @@ abstract class BaseTileViewModel extends ViewModel {
 }
 
 /**
- * A tile displaying some media.
+ * Some participant's media.
  */
-export type TileViewModel = UserMediaTileViewModel | ScreenShareTileViewModel;
+export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel;
 
 /**
- * A tile displaying some participant's user media.
+ * Some participant's user media.
  */
-export class UserMediaTileViewModel extends BaseTileViewModel {
+export class UserMediaViewModel extends BaseMediaViewModel {
   /**
    * Whether the video should be mirrored.
    */
@@ -201,7 +200,7 @@ export class UserMediaTileViewModel extends BaseTileViewModel {
       combineLatest([this._locallyMuted, this._localVolume], (muted, volume) =>
         muted ? 0 : volume,
       )
-        .pipe(takeUntil(this.destroyed))
+        .pipe(this.scope.bind())
         .subscribe((volume) => {
           (this.participant as RemoteParticipant).setVolume(volume);
         });
@@ -221,9 +220,9 @@ export class UserMediaTileViewModel extends BaseTileViewModel {
 }
 
 /**
- * A tile displaying some participant's screen share.
+ * Some participant's screen share media.
  */
-export class ScreenShareTileViewModel extends BaseTileViewModel {
+export class ScreenShareViewModel extends BaseMediaViewModel {
   public constructor(
     id: string,
     member: RoomMember | undefined,

src/state/ObservableScope.ts

@@ -0,0 +1,40 @@
+/*
+Copyright 2024 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { MonoTypeOperatorFunction, Subject, takeUntil } from "rxjs";
+
+/**
+ * A scope which limits the execution lifetime of its bound Observables.
+ */
+export class ObservableScope {
+  private readonly ended = new Subject<void>();
+
+  /**
+   * Binds an Observable to this scope, so that it completes when the scope
+   * ends.
+   */
+  public bind<T>(): MonoTypeOperatorFunction<T> {
+    return takeUntil(this.ended);
+  }
+
+  /**
+   * Ends the scope, causing any bound Observables to complete.
+   */
+  public end(): void {
+    this.ended.next();
+    this.ended.complete();
+  }
+}

src/state/ViewModel.ts

@@ -1,5 +1,5 @@
 /*
-Copyright 2023 New Vector Ltd
+Copyright 2023-2024 New Vector Ltd
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,20 +14,19 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { Subject } from "rxjs";
+import { ObservableScope } from "./ObservableScope";
 
 /**
  * An MVVM view model.
  */
 export abstract class ViewModel {
-  protected readonly destroyed = new Subject<void>();
+  protected readonly scope = new ObservableScope();
 
   /**
    * Instructs the ViewModel to clean up its resources. If you forget to call
    * this, there may be memory leaks!
    */
   public destroy(): void {
-    this.destroyed.next();
+    this.scope.end();
-    this.destroyed.complete();
   }
 }

src/video-grid/VideoTile.tsx

@@ -58,10 +58,10 @@ import { Avatar } from "../Avatar";
 import styles from "./VideoTile.module.css";
 import { useReactiveState } from "../useReactiveState";
 import {
-  ScreenShareTileViewModel,
+  ScreenShareViewModel,
-  TileViewModel,
+  MediaViewModel,
-  UserMediaTileViewModel,
+  UserMediaViewModel,
-} from "../state/TileViewModel";
+} from "../state/MediaViewModel";
 import { subscribe } from "../state/subscribe";
 import { useMergedRefs } from "../useMergedRefs";
 import { Slider } from "../Slider";
@@ -170,7 +170,7 @@ const Tile = forwardRef<HTMLDivElement, TileProps>(
 Tile.displayName = "Tile";
 
 interface UserMediaTileProps {
-  vm: UserMediaTileViewModel;
+  vm: UserMediaViewModel;
   className?: string;
   style?: ComponentProps<typeof animated.div>["style"];
   targetWidth: number;
@@ -329,7 +329,7 @@ const UserMediaTile = subscribe<UserMediaTileProps, HTMLDivElement>(
 UserMediaTile.displayName = "UserMediaTile";
 
 interface ScreenShareTileProps {
-  vm: ScreenShareTileViewModel;
+  vm: ScreenShareViewModel;
   className?: string;
   style?: ComponentProps<typeof animated.div>["style"];
   targetWidth: number;
@@ -403,7 +403,7 @@ const ScreenShareTile = subscribe<ScreenShareTileProps, HTMLDivElement>(
 ScreenShareTile.displayName = "ScreenShareTile";
 
 interface Props {
-  vm: TileViewModel;
+  vm: MediaViewModel;
   maximised: boolean;
   fullscreen: boolean;
   onToggleFullscreen: (itemId: string) => void;
@@ -455,7 +455,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
       ? t("video_tile.sfu_participant_local")
       : displayName;
 
-    if (vm instanceof UserMediaTileViewModel) {
+    if (vm instanceof UserMediaViewModel) {
       return (
         <UserMediaTile
           ref={ref}