rzerres/element-call
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Add data models for the new call layouts

Browse Source 
This is a start at implementing the call layouts from the new designs. I've added data types to model the contents of each possible layout, and begun implementing the business logic to produce these layouts in the call view model.
This commit is contained in:
Robin
2024-01-20 20:39:12 -05:00
parent 353b243686
commit 6cfc2066c9
5 changed files with 407 additions and 43 deletions
Download Patch File Download Diff File Expand all files Collapse all files

360
src/state/CallViewModel.ts
View File

@@ -1,5 +1,5 @@
/*
Copyright 2023 New Vector Ltd
Copyright 2023-2024 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -16,25 +16,41 @@ limitations under the License.
import {
connectedParticipantsObserver,
observeParticipantEvents,
observeParticipantMedia,
} from "@livekit/components-core";
import { Room as LivekitRoom, RemoteParticipant } from "livekit-client";
import {
Room as LivekitRoom,
LocalParticipant,
ParticipantEvent,
RemoteParticipant,
} from "livekit-client";
import { Room as MatrixRoom, RoomMember } from "matrix-js-sdk/src/matrix";
import { useEffect, useRef } from "react";
import {
BehaviorSubject,
EMPTY,
Observable,
audit,
combineLatest,
concat,
distinctUntilChanged,
filter,
map,
merge,
mergeAll,
of,
sample,
scan,
shareReplay,
startWith,
takeUntil,
switchAll,
switchMap,
throttleTime,
timer,
zip,
} from "rxjs";
import { state } from "@react-rxjs/core";
import { StateObservable, state } from "@react-rxjs/core";
import { logger } from "matrix-js-sdk/src/logger";
import { ViewModel } from "./ViewModel";
@@ -45,14 +61,21 @@ import {
} from "../livekit/useECConnectionState";
import { usePrevious } from "../usePrevious";
import {
TileViewModel,
UserMediaTileViewModel,
ScreenShareTileViewModel,
} from "./TileViewModel";
MediaViewModel,
UserMediaViewModel,
ScreenShareViewModel,
} from "./MediaViewModel";
import { finalizeValue } from "../observable-utils";
import { ObservableScope } from "./ObservableScope";
// How long we wait after a focus switch before showing the real participant
// list again
const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
// Represents something that should get a tile on the layout,
// ie. a user's video feed or a screen share feed.
// TODO: This exposes too much information to the view layer, let's keep this
// information internal to the view model and switch to using Tile<T> instead
export interface TileDescriptor<T> {
id: string;
focused: boolean;
@@ -65,9 +88,128 @@ export interface TileDescriptor<T> {
data: T;
}
// How long we wait after a focus switch before showing the real participant
// list again
const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
/**
* A media tile within the call interface.
*/
export interface Tile<T> {
id: string;
data: T;
}
export interface GridLayout {
type: "grid";
spotlight?: Tile<MediaViewModel[]>;
grid: Tile<UserMediaViewModel>[];
}
export interface SpotlightLayout {
type: "spotlight";
spotlight: Tile<MediaViewModel[]>;
grid: Tile<UserMediaViewModel>[];
}
export interface FullScreenLayout {
type: "full screen";
spotlight: Tile<MediaViewModel[]>;
pip?: Tile<UserMediaViewModel>;
}
export interface PipLayout {
type: "pip";
spotlight: Tile<MediaViewModel>;
}
/**
* A layout defining the media tiles present on screen and their visual
* arrangement.
*/
export type Layout =
| GridLayout
| SpotlightLayout
| FullScreenLayout
| PipLayout;
export type GridMode = "grid" | "spotlight";
export type WindowMode = "normal" | "full screen" | "pip";
/**
* Sorting bins defining the order in which media tiles appear in the grid.
*/
enum GridBin {
SelfStart,
Presenters,
Speakers,
VideoAndAudio,
Video,
Audio,
NoMedia,
SelfEnd,
}
class UserMedia {
private readonly scope = new ObservableScope();
public readonly vm: UserMediaViewModel;
public readonly speaker: Observable<boolean>;
public readonly presenter: Observable<boolean>;
public constructor(
public readonly id: string,
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
callEncrypted: boolean,
) {
this.vm = new UserMediaViewModel(id, member, participant, callEncrypted);
this.speaker = this.vm.speaking.pipeState(
// Require 1 s of continuous speaking to become a speaker, and 10 s of
// continuous silence to stop being considered a speaker
audit((s) =>
merge(
timer(s ? 1000 : 10000),
this.vm.speaking.pipe(filter((s1) => s1 !== s)),
),
),
distinctUntilChanged(),
this.scope.bind(),
// Make this Observable hot so that the timers don't reset when you
// resubscribe
shareReplay(1),
);
this.presenter = observeParticipantEvents(
participant,
ParticipantEvent.TrackPublished,
ParticipantEvent.TrackUnpublished,
ParticipantEvent.LocalTrackPublished,
ParticipantEvent.LocalTrackUnpublished,
).pipe(map((p) => p.isScreenShareEnabled));
}
public destroy(): void {
this.scope.end();
this.vm.destroy();
}
}
class ScreenShare {
public readonly vm: ScreenShareViewModel;
public constructor(
id: string,
member: RoomMember | undefined,
participant: LocalParticipant | RemoteParticipant,
callEncrypted: boolean,
) {
this.vm = new ScreenShareViewModel(id, member, participant, callEncrypted);
}
public destroy(): void {
this.vm.destroy();
}
}
type MediaItem = UserMedia | ScreenShare;
function findMatrixMember(
room: MatrixRoom,
@@ -116,7 +258,7 @@ export class CallViewModel extends ViewModel {
),
new Promise<void>((resolve) => {
const subscription = this.connectionState
.pipe(takeUntil(this.destroyed))
.pipe(this.scope.bind())
.subscribe((s) => {
if (s !== ECAddonConnectionState.ECSwitchingFocus) {
resolve();
@@ -165,9 +307,193 @@ export class CallViewModel extends ViewModel {
},
);
private readonly mediaItems = state(
combineLatest([
this.remoteParticipants,
observeParticipantMedia(this.livekitRoom.localParticipant),
]).pipe(
scan(
(
prevItems,
[remoteParticipants, { participant: localParticipant }],
) => {
let allGhosts = true;
const newItems = new Map(
function* (this: CallViewModel): Iterable<[string, MediaItem]> {
for (const p of [localParticipant, ...remoteParticipants]) {
const member = findMatrixMember(this.matrixRoom, p.identity);
allGhosts &&= member === undefined;
// We always start with a local participant with the empty string as
// their ID before we're connected, this is fine and we'll be in
// "all ghosts" mode.
if (p.identity !== "" && member === undefined) {
logger.warn(
`Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`,
);
}
const userMediaId = p.identity;
yield [
userMediaId,
prevItems.get(userMediaId) ??
new UserMedia(userMediaId, member, p, this.encrypted),
];
if (p.isScreenShareEnabled) {
const screenShareId = `${userMediaId}:screen-share`;
yield [
screenShareId,
prevItems.get(screenShareId) ??
new ScreenShare(screenShareId, member, p, this.encrypted),
];
}
}
}.bind(this)(),
);
for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy();
// If every item is a ghost, that probably means we're still connecting
// and shouldn't bother showing anything yet
return allGhosts ? new Map() : newItems;
},
new Map<string, MediaItem>(),
),
map((ms) => [...ms.values()]),
finalizeValue((ts) => {
for (const t of ts) t.destroy();
}),
),
);
private readonly userMedia = this.mediaItems.pipe(
map((ms) => ms.filter((m): m is UserMedia => m instanceof UserMedia)),
);
private readonly screenShares = this.mediaItems.pipe(
map((ms) => ms.filter((m): m is ScreenShare => m instanceof ScreenShare)),
);
private readonly spotlightSpeaker = this.userMedia.pipe(
switchMap((ms) =>
combineLatest(
ms.map((m) => m.vm.speaking.pipe(map((s) => [m, s] as const))),
),
),
scan<(readonly [UserMedia, boolean])[], UserMedia | null, null>(
(prev, ms) =>
// Decide who to spotlight:
// If the previous speaker is still speaking, stick with them rather
// than switching eagerly to someone else
ms.find(([m, s]) => m === prev && s)?.[0] ??
// Otherwise, select anyone who is speaking
ms.find(([, s]) => s)?.[0] ??
// Otherwise, stick with the person who was last speaking
prev ??
// Otherwise, spotlight the local user
ms.find(([m]) => m.vm.local)?.[0] ??
null,
null,
),
distinctUntilChanged(),
throttleTime(800, undefined, { leading: true, trailing: true }),
);
private readonly grid = this.userMedia.pipe(
switchMap((ms) => {
const bins = ms.map((m) =>
combineLatest(
[m.speaker, m.presenter, m.vm.audioEnabled, m.vm.videoEnabled],
(speaker, presenter, audio, video) =>
[
m,
m.vm.local
? GridBin.SelfStart
: presenter
? GridBin.Presenters
: speaker
? GridBin.Speakers
: video
? audio
? GridBin.VideoAndAudio
: GridBin.Video
: audio
? GridBin.Audio
: GridBin.NoMedia,
] as const,
),
);
// Sort the media by bin order and generate a tile for each one
return combineLatest(bins, (...bins) =>
bins
.sort(([, bin1], [, bin2]) => bin1 - bin2)
.map(([m]) => ({ id: m.id, data: m.vm })),
);
}),
);
private readonly spotlight = combineLatest(
[this.screenShares, this.spotlightSpeaker],
(screenShares, spotlightSpeaker): Tile<MediaViewModel[]> => ({
id: "spotlight",
data:
screenShares.length > 0
? screenShares.map((m) => m.vm)
: spotlightSpeaker === null
? []
: [spotlightSpeaker.vm],
}),
);
private readonly gridMode = new BehaviorSubject<GridMode>("grid");
public setGridMode(value: GridMode): void {
this.gridMode.next(value);
}
// TODO: Make this react to changes in window dimensions and screen
// orientation
private readonly windowMode = of<WindowMode>("normal");
public readonly layout: StateObservable<Layout> = state(
combineLatest([this.gridMode, this.windowMode], (gridMode, windowMode) => {
switch (windowMode) {
case "full screen":
throw new Error("unimplemented");
case "pip":
throw new Error("unimplemented");
case "normal": {
switch (gridMode) {
case "grid":
return combineLatest(
[this.grid, this.spotlight, this.screenShares],
(grid, spotlight, screenShares): Layout => ({
type: "grid",
spotlight: screenShares.length > 0 ? spotlight : undefined,
grid,
}),
);
case "spotlight":
return combineLatest(
[this.grid, this.spotlight],
(grid, spotlight): Layout => ({
type: "spotlight",
spotlight,
grid,
}),
);
}
}
}
}).pipe(switchAll()),
);
/**
* The media tiles to be displayed in the call view.
*/
// TODO: Get rid of this field, replacing it with the 'layout' field above
// which keeps more details of the layout order internal to the view model
public readonly tiles = state(
combineLatest([
this.remoteParticipants,
@@ -197,10 +523,10 @@ export class CallViewModel extends ViewModel {
const userMediaVm =
tilesById.get(userMediaId)?.data ??
new UserMediaTileViewModel(userMediaId, member, p, this.encrypted);
new UserMediaViewModel(userMediaId, member, p, this.encrypted);
tilesById.delete(userMediaId);
const userMediaTile: TileDescriptor<TileViewModel> = {
const userMediaTile: TileDescriptor<MediaViewModel> = {
id: userMediaId,
focused: false,
isPresenter: p.isScreenShareEnabled,
@@ -215,7 +541,7 @@ export class CallViewModel extends ViewModel {
const screenShareId = `${userMediaId}:screen-share`;
const screenShareVm =
tilesById.get(screenShareId)?.data ??
new ScreenShareTileViewModel(
new ScreenShareViewModel(
screenShareId,
member,
p,
@@ -223,7 +549,7 @@ export class CallViewModel extends ViewModel {
);
tilesById.delete(screenShareId);
const screenShareTile: TileDescriptor<TileViewModel> = {
const screenShareTile: TileDescriptor<MediaViewModel> = {
id: screenShareId,
focused: true,
isPresenter: false,
@@ -246,7 +572,7 @@ export class CallViewModel extends ViewModel {
// If every item is a ghost, that probably means we're still connecting
// and shouldn't bother showing anything yet
return allGhosts ? [] : newTiles;
}, [] as TileDescriptor<TileViewModel>[]),
}, [] as TileDescriptor<MediaViewModel>[]),
finalizeValue((ts) => {
for (const t of ts) t.data.destroy();
}),

25
src/state/TileViewModel.ts → src/state/MediaViewModel.ts
View File

@@ -1,5 +1,5 @@
/*
Copyright 2023 New Vector Ltd
Copyright 2023-2024 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -43,7 +43,6 @@ import {
of,
startWith,
switchMap,
takeUntil,
} from "rxjs";
import { ViewModel } from "./ViewModel";
@@ -64,13 +63,13 @@ function observeTrackReference(
);
}
abstract class BaseTileViewModel extends ViewModel {
abstract class BaseMediaViewModel extends ViewModel {
/**
* Whether the tile belongs to the local user.
* Whether the media belongs to the local user.
*/
public readonly local = this.participant.isLocal;
/**
* The LiveKit video track to be shown on this tile.
* The LiveKit video track for this media.
*/
public readonly video: StateObservable<TrackReferenceOrPlaceholder>;
/**
@@ -83,7 +82,7 @@ abstract class BaseTileViewModel extends ViewModel {
// soon as that code is moved into the view models
public readonly id: string,
/**
* The Matrix room member to which this tile belongs.
* The Matrix room member to which this media belongs.
*/
// TODO: Fully separate the data layer from the UI layer by keeping the
// member object internal
@@ -109,14 +108,14 @@ abstract class BaseTileViewModel extends ViewModel {
}
/**
* A tile displaying some media.
* Some participant's media.
*/
export type TileViewModel = UserMediaTileViewModel | ScreenShareTileViewModel;
export type MediaViewModel = UserMediaViewModel | ScreenShareViewModel;
/**
* A tile displaying some participant's user media.
* Some participant's user media.
*/
export class UserMediaTileViewModel extends BaseTileViewModel {
export class UserMediaViewModel extends BaseMediaViewModel {
/**
* Whether the video should be mirrored.
*/
@@ -201,7 +200,7 @@ export class UserMediaTileViewModel extends BaseTileViewModel {
combineLatest([this._locallyMuted, this._localVolume], (muted, volume) =>
muted ? 0 : volume,
)
.pipe(takeUntil(this.destroyed))
.pipe(this.scope.bind())
.subscribe((volume) => {
(this.participant as RemoteParticipant).setVolume(volume);
});
@@ -221,9 +220,9 @@ export class UserMediaTileViewModel extends BaseTileViewModel {
}
/**
* A tile displaying some participant's screen share.
* Some participant's screen share media.
*/
export class ScreenShareTileViewModel extends BaseTileViewModel {
export class ScreenShareViewModel extends BaseMediaViewModel {
public constructor(
id: string,
member: RoomMember | undefined,

40
src/state/ObservableScope.ts Normal file
View File

@@ -0,0 +1,40 @@
/*
Copyright 2024 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { MonoTypeOperatorFunction, Subject, takeUntil } from "rxjs";
/**
* A scope which limits the execution lifetime of its bound Observables.
*/
export class ObservableScope {
private readonly ended = new Subject<void>();
/**
* Binds an Observable to this scope, so that it completes when the scope
* ends.
*/
public bind<T>(): MonoTypeOperatorFunction<T> {
return takeUntil(this.ended);
}
/**
* Ends the scope, causing any bound Observables to complete.
*/
public end(): void {
this.ended.next();
this.ended.complete();
}
}

9
src/state/ViewModel.ts
View File

@@ -1,5 +1,5 @@
/*
Copyright 2023 New Vector Ltd
Copyright 2023-2024 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -14,20 +14,19 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import { Subject } from "rxjs";
import { ObservableScope } from "./ObservableScope";
/**
* An MVVM view model.
*/
export abstract class ViewModel {
protected readonly destroyed = new Subject<void>();
protected readonly scope = new ObservableScope();
/**
* Instructs the ViewModel to clean up its resources. If you forget to call
* this, there may be memory leaks!
*/
public destroy(): void {
this.destroyed.next();
this.destroyed.complete();
this.scope.end();
}
}

16
src/video-grid/VideoTile.tsx
View File

@@ -58,10 +58,10 @@ import { Avatar } from "../Avatar";
import styles from "./VideoTile.module.css";
import { useReactiveState } from "../useReactiveState";
import {
ScreenShareTileViewModel,
TileViewModel,
UserMediaTileViewModel,
} from "../state/TileViewModel";
ScreenShareViewModel,
MediaViewModel,
UserMediaViewModel,
} from "../state/MediaViewModel";
import { subscribe } from "../state/subscribe";
import { useMergedRefs } from "../useMergedRefs";
import { Slider } from "../Slider";
@@ -170,7 +170,7 @@ const Tile = forwardRef<HTMLDivElement, TileProps>(
Tile.displayName = "Tile";
interface UserMediaTileProps {
vm: UserMediaTileViewModel;
vm: UserMediaViewModel;
className?: string;
style?: ComponentProps<typeof animated.div>["style"];
targetWidth: number;
@@ -329,7 +329,7 @@ const UserMediaTile = subscribe<UserMediaTileProps, HTMLDivElement>(
UserMediaTile.displayName = "UserMediaTile";
interface ScreenShareTileProps {
vm: ScreenShareTileViewModel;
vm: ScreenShareViewModel;
className?: string;
style?: ComponentProps<typeof animated.div>["style"];
targetWidth: number;
@@ -403,7 +403,7 @@ const ScreenShareTile = subscribe<ScreenShareTileProps, HTMLDivElement>(
ScreenShareTile.displayName = "ScreenShareTile";
interface Props {
vm: TileViewModel;
vm: MediaViewModel;
maximised: boolean;
fullscreen: boolean;
onToggleFullscreen: (itemId: string) => void;
@@ -455,7 +455,7 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
? t("video_tile.sfu_participant_local")
: displayName;
if (vm instanceof UserMediaTileViewModel) {
if (vm instanceof UserMediaViewModel) {
return (
<UserMediaTile
ref={ref}