Refactor/redesign video tiles

2023-12-01 17:43:09 -05:00
parent 0ab3e0e090
commit b2bc8edcc1
28 changed files with 1086 additions and 709 deletions
--- a/src/state/CallViewModel.ts
+++ b/src/state/CallViewModel.ts
@@ -49,6 +49,7 @@ import {
  UserMediaTileViewModel,
  ScreenShareTileViewModel,
 } from "./TileViewModel";
+import { finalizeValue } from "../observable-utils";

 // Represents something that should get a tile on the layout,
 // ie. a user's video feed or a screen share feed.
@@ -164,6 +165,9 @@ export class CallViewModel extends ViewModel {
    },
  );

+  /**
+   * The media tiles to be displayed in the call view.
+   */
  public readonly tiles = state(
    combineLatest([
      this.remoteParticipants,
@@ -176,8 +180,8 @@ export class CallViewModel extends ViewModel {
        let allGhosts = true;

        const newTiles = ps.flatMap((p) => {
-          const id = p.identity;
-          const member = findMatrixMember(this.matrixRoom, id);
+          const userMediaId = p.identity;
+          const member = findMatrixMember(this.matrixRoom, userMediaId);
          allGhosts &&= member === undefined;
          const spokeRecently =
            p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000;
@@ -185,27 +189,40 @@ export class CallViewModel extends ViewModel {
          // We always start with a local participant with the empty string as
          // their ID before we're connected, this is fine and we'll be in
          // "all ghosts" mode.
-          if (id !== "" && member === undefined) {
+          if (userMediaId !== "" && member === undefined) {
            logger.warn(
-              `Ruh, roh! No matrix member found for SFU participant '${id}': creating g-g-g-ghost!`,
+              `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`,
            );
          }

+          const userMediaVm =
+            tilesById.get(userMediaId)?.data ??
+            new UserMediaTileViewModel(userMediaId, member, p, this.encrypted);
+          tilesById.delete(userMediaId);
+
          const userMediaTile: TileDescriptor<TileViewModel> = {
-            id,
+            id: userMediaId,
            focused: false,
            isPresenter: p.isScreenShareEnabled,
            isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal,
            hasVideo: p.isCameraEnabled,
            local: p.isLocal,
            largeBaseSize: false,
-            data:
-              tilesById.get(id)?.data ??
-              new UserMediaTileViewModel(id, member, p),
+            data: userMediaVm,
          };

          if (p.isScreenShareEnabled) {
-            const screenShareId = `${id}:screen-share`;
+            const screenShareId = `${userMediaId}:screen-share`;
+            const screenShareVm =
+              tilesById.get(screenShareId)?.data ??
+              new ScreenShareTileViewModel(
+                screenShareId,
+                member,
+                p,
+                this.encrypted,
+              );
+            tilesById.delete(screenShareId);
+
            const screenShareTile: TileDescriptor<TileViewModel> = {
              id: screenShareId,
              focused: true,
@@ -214,10 +231,8 @@ export class CallViewModel extends ViewModel {
              hasVideo: true,
              local: p.isLocal,
              largeBaseSize: true,
-              placeNear: id,
-              data:
-                tilesById.get(screenShareId)?.data ??
-                new ScreenShareTileViewModel(screenShareId, member, p),
+              placeNear: userMediaId,
+              data: screenShareVm,
            };
            return [userMediaTile, screenShareTile];
          } else {
@@ -225,10 +240,16 @@ export class CallViewModel extends ViewModel {
          }
        });

+        // Any tiles left in the map are unused and should be destroyed
+        for (const t of tilesById.values()) t.data.destroy();
+
        // If every item is a ghost, that probably means we're still connecting
        // and shouldn't bother showing anything yet
        return allGhosts ? [] : newTiles;
      }, [] as TileDescriptor<TileViewModel>[]),
+      finalizeValue((ts) => {
+        for (const t of ts) t.data.destroy();
+      }),
    ),
  );

@@ -236,6 +257,7 @@ export class CallViewModel extends ViewModel {
    // A call is permanently tied to a single Matrix room and LiveKit room
    private readonly matrixRoom: MatrixRoom,
    private readonly livekitRoom: LivekitRoom,
+    private readonly encrypted: boolean,
    private readonly connectionState: Observable<ECConnectionState>,
  ) {
    super();
@@ -245,18 +267,25 @@ export class CallViewModel extends ViewModel {
 export function useCallViewModel(
  matrixRoom: MatrixRoom,
  livekitRoom: LivekitRoom,
+  encrypted: boolean,
  connectionState: ECConnectionState,
 ): CallViewModel {
  const prevMatrixRoom = usePrevious(matrixRoom);
  const prevLivekitRoom = usePrevious(livekitRoom);
+  const prevEncrypted = usePrevious(encrypted);
  const connectionStateObservable = useObservable(connectionState);

  const vm = useRef<CallViewModel>();
-  if (matrixRoom !== prevMatrixRoom || livekitRoom !== prevLivekitRoom) {
+  if (
+    matrixRoom !== prevMatrixRoom ||
+    livekitRoom !== prevLivekitRoom ||
+    encrypted !== prevEncrypted
+  ) {
    vm.current?.destroy();
    vm.current = new CallViewModel(
      matrixRoom,
      livekitRoom,
+      encrypted,
      connectionStateObservable,
    );
  }
--- a/src/state/TileViewModel.ts
+++ b/src/state/TileViewModel.ts
@@ -14,40 +14,219 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-import { LocalParticipant, RemoteParticipant } from "livekit-client";
+import {
+  AudioSource,
+  TrackReferenceOrPlaceholder,
+  VideoSource,
+  observeParticipantEvents,
+  observeParticipantMedia,
+} from "@livekit/components-core";
+import { StateObservable, state } from "@react-rxjs/core";
+import {
+  LocalParticipant,
+  LocalTrack,
+  Participant,
+  ParticipantEvent,
+  RemoteParticipant,
+  Track,
+  TrackEvent,
+  facingModeFromLocalTrack,
+} from "livekit-client";
 import { RoomMember } from "matrix-js-sdk/src/matrix";
+import {
+  BehaviorSubject,
+  combineLatest,
+  distinctUntilChanged,
+  distinctUntilKeyChanged,
+  fromEvent,
+  map,
+  of,
+  startWith,
+  switchMap,
+  takeUntil,
+} from "rxjs";

-export abstract class TileViewModel {
-  // TODO: Properly separate the data layer from the UI layer by keeping the
-  // member and LiveKit participant objects internal. The only LiveKit-specific
-  // thing we need to expose here is a TrackReference for the video, everything
-  // else should be simple strings, flags, and callbacks.
-  public abstract readonly id: string;
-  public abstract readonly member: RoomMember | undefined;
-  public abstract readonly sfuParticipant: LocalParticipant | RemoteParticipant;
+import { ViewModel } from "./ViewModel";
+
+function observeTrackReference(
+  participant: Participant,
+  source: Track.Source,
+): StateObservable<TrackReferenceOrPlaceholder> {
+  return state(
+    observeParticipantMedia(participant).pipe(
+      map(() => ({
+        participant,
+        publication: participant.getTrack(source),
+        source,
+      })),
+      distinctUntilKeyChanged("publication"),
+    ),
+  );
 }

-// Right now it looks kind of pointless to have user media and screen share be
-// represented by two classes rather than a single flag, but this will come in
-// handy when we go to move more business logic out of VideoTile and into this
-// file
+abstract class BaseTileViewModel extends ViewModel {
+  /**
+   * Whether the tile belongs to the local user.
+   */
+  public readonly local = this.participant.isLocal;
+  /**
+   * The LiveKit video track to be shown on this tile.
+   */
+  public readonly video: StateObservable<TrackReferenceOrPlaceholder>;
+  /**
+   * Whether there should be a warning that this media is unencrypted.
+   */
+  public readonly unencryptedWarning: StateObservable<boolean>;

-export class UserMediaTileViewModel extends TileViewModel {
  public constructor(
+    // TODO: This is only needed for full screen toggling and can be removed as
+    // soon as that code is moved into the view models
    public readonly id: string,
+    /**
+     * The Matrix room member to which this tile belongs.
+     */
+    // TODO: Fully separate the data layer from the UI layer by keeping the
+    // member object internal
    public readonly member: RoomMember | undefined,
-    public readonly sfuParticipant: LocalParticipant | RemoteParticipant,
+    protected readonly participant: LocalParticipant | RemoteParticipant,
+    callEncrypted: boolean,
+    audioSource: AudioSource,
+    videoSource: VideoSource,
  ) {
    super();
+    const audio = observeTrackReference(participant, audioSource);
+    this.video = observeTrackReference(participant, videoSource);
+    this.unencryptedWarning = state(
+      combineLatest(
+        [audio, this.video],
+        (a, v) =>
+          callEncrypted &&
+          (a.publication?.isEncrypted === false ||
+            v.publication?.isEncrypted === false),
+      ).pipe(distinctUntilChanged()),
+    );
  }
 }

-export class ScreenShareTileViewModel extends TileViewModel {
+/**
+ * A tile displaying some media.
+ */
+export type TileViewModel = UserMediaTileViewModel | ScreenShareTileViewModel;
+
+/**
+ * A tile displaying some participant's user media.
+ */
+export class UserMediaTileViewModel extends BaseTileViewModel {
+  /**
+   * Whether the video should be mirrored.
+   */
+  public readonly mirror = state(
+    this.video.pipe(
+      switchMap((v) => {
+        const track = v.publication?.track;
+        if (!(track instanceof LocalTrack)) return of(false);
+        // Watch for track restarts, because they indicate a camera switch
+        return fromEvent(track, TrackEvent.Restarted).pipe(
+          startWith(null),
+          // Mirror only front-facing cameras (those that face the user)
+          map(() => facingModeFromLocalTrack(track).facingMode === "user"),
+        );
+      }),
+    ),
+  );
+
+  /**
+   * Whether the participant is speaking.
+   */
+  public readonly speaking = state(
+    observeParticipantEvents(
+      this.participant,
+      ParticipantEvent.IsSpeakingChanged,
+    ).pipe(map((p) => p.isSpeaking)),
+  );
+
+  private readonly _locallyMuted = new BehaviorSubject(false);
+  /**
+   * Whether we've disabled this participant's audio.
+   */
+  public readonly locallyMuted = state(this._locallyMuted);
+
+  private readonly _localVolume = new BehaviorSubject(1);
+  /**
+   * The volume to which we've set this participant's audio, as a scalar
+   * multiplier.
+   */
+  public readonly localVolume = state(this._localVolume);
+
+  /**
+   * Whether this participant is sending audio (i.e. is unmuted on their side).
+   */
+  public readonly audioEnabled: StateObservable<boolean>;
+  /**
+   * Whether this participant is sending video.
+   */
+  public readonly videoEnabled: StateObservable<boolean>;
+
  public constructor(
-    public readonly id: string,
-    public readonly member: RoomMember | undefined,
-    public readonly sfuParticipant: LocalParticipant | RemoteParticipant,
+    id: string,
+    member: RoomMember | undefined,
+    participant: LocalParticipant | RemoteParticipant,
+    callEncrypted: boolean,
  ) {
-    super();
+    super(
+      id,
+      member,
+      participant,
+      callEncrypted,
+      Track.Source.Microphone,
+      Track.Source.Camera,
+    );
+
+    const media = observeParticipantMedia(participant);
+    this.audioEnabled = state(
+      media.pipe(map((m) => m.microphoneTrack?.isMuted === false)),
+    );
+    this.videoEnabled = state(
+      media.pipe(map((m) => m.cameraTrack?.isMuted === false)),
+    );
+
+    // Sync the local mute state and volume with LiveKit
+    if (!this.local)
+      combineLatest([this._locallyMuted, this._localVolume], (muted, volume) =>
+        muted ? 0 : volume,
+      )
+        .pipe(takeUntil(this.destroyed))
+        .subscribe((volume) => {
+          (this.participant as RemoteParticipant).setVolume(volume);
+        });
+  }
+
+  public toggleLocallyMuted(): void {
+    this._locallyMuted.next(!this._locallyMuted.value);
+  }
+
+  public setLocalVolume(value: number): void {
+    this._localVolume.next(value);
+  }
+}
+
+/**
+ * A tile displaying some participant's screen share.
+ */
+export class ScreenShareTileViewModel extends BaseTileViewModel {
+  public constructor(
+    id: string,
+    member: RoomMember | undefined,
+    participant: LocalParticipant | RemoteParticipant,
+    callEncrypted: boolean,
+  ) {
+    super(
+      id,
+      member,
+      participant,
+      callEncrypted,
+      Track.Source.ScreenShareAudio,
+      Track.Source.ScreenShare,
+    );
  }
 }
--- a/src/state/subscribe.tsx
+++ b/src/state/subscribe.tsx
@@ -14,7 +14,13 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-import { FC } from "react";
+import {
+  ForwardRefExoticComponent,
+  ForwardRefRenderFunction,
+  PropsWithoutRef,
+  RefAttributes,
+  forwardRef,
+} from "react";
 // eslint-disable-next-line no-restricted-imports
 import { Subscribe, RemoveSubscribe } from "@react-rxjs/core";

@@ -23,15 +29,17 @@ import { Subscribe, RemoveSubscribe } from "@react-rxjs/core";
 * that safely subscribes to its Observables before rendering. The component
 * will return null until the subscriptions are created.
 */
-export function subscribe<P>(render: FC<P>): FC<P> {
-  const InnerComponent: FC<{ p: P }> = ({ p }) => (
-    <RemoveSubscribe>{render(p)}</RemoveSubscribe>
-  );
-  const OuterComponent: FC<P> = (p) => (
+export function subscribe<P, R>(
+  render: ForwardRefRenderFunction<R, P>,
+): ForwardRefExoticComponent<PropsWithoutRef<P> & RefAttributes<R>> {
+  const InnerComponent = forwardRef<R, { p: P }>(({ p }, ref) => (
+    <RemoveSubscribe>{render(p, ref)}</RemoveSubscribe>
+  ));
+  const OuterComponent = forwardRef<R, P>((p, ref) => (
    <Subscribe>
-      <InnerComponent p={p} />
+      <InnerComponent ref={ref} p={p} />
    </Subscribe>
-  );
+  ));
  // Copy over the component's display name, default props, etc.
  Object.assign(OuterComponent, render);
  return OuterComponent;