rzerres/element-call
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Respond to suggestions from code review

Browse Source
This commit is contained in:
Robin
2024-05-02 16:32:48 -04:00
parent dcb4d10afb
commit e9c98a02f0
1 changed files with 191 additions and 192 deletions
Download Patch File Download Diff File Expand all files Collapse all files

383
src/state/CallViewModel.ts
View File

@@ -126,9 +126,9 @@ export type GridMode = "grid" | "spotlight";
export type WindowMode = "normal" | "full screen" | "pip"; export type WindowMode = "normal" | "full screen" | "pip";
/** /**
* Sorting bins defining the order in which media tiles appear in the grid. * Sorting bins defining the order in which media tiles appear in the layout.
*/ */
enum GridBin { enum SortingBin {
SelfStart, SelfStart,
Presenters, Presenters,
Speakers, Speakers,
@@ -235,74 +235,76 @@ export class CallViewModel extends ViewModel {
// Lists of participants to "hold" on display, even if LiveKit claims that // Lists of participants to "hold" on display, even if LiveKit claims that
// they've left // they've left
private readonly remoteParticipantHolds = zip( private readonly remoteParticipantHolds: Observable<RemoteParticipant[][]> =
this.connectionState, zip(
this.rawRemoteParticipants.pipe(sample(this.connectionState)), this.connectionState,
(s, ps) => { this.rawRemoteParticipants.pipe(sample(this.connectionState)),
// Whenever we switch focuses, we should retain all the previous (s, ps) => {
// participants for at least POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS ms to // Whenever we switch focuses, we should retain all the previous
// give their clients time to switch over and avoid jarring layout shifts // participants for at least POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS ms to
if (s === ECAddonConnectionState.ECSwitchingFocus) { // give their clients time to switch over and avoid jarring layout shifts
return concat( if (s === ECAddonConnectionState.ECSwitchingFocus) {
// Hold these participants return concat(
of({ hold: ps }), // Hold these participants
// Wait for time to pass and the connection state to have changed of({ hold: ps }),
Promise.all([ // Wait for time to pass and the connection state to have changed
new Promise<void>((resolve) => Promise.all([
setTimeout(resolve, POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS), new Promise<void>((resolve) =>
), setTimeout(resolve, POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS),
new Promise<void>((resolve) => { ),
const subscription = this.connectionState new Promise<void>((resolve) => {
.pipe(this.scope.bind()) const subscription = this.connectionState
.subscribe((s) => { .pipe(this.scope.bind())
if (s !== ECAddonConnectionState.ECSwitchingFocus) { .subscribe((s) => {
resolve(); if (s !== ECAddonConnectionState.ECSwitchingFocus) {
subscription.unsubscribe(); resolve();
} subscription.unsubscribe();
}); }
}), });
// Then unhold them }),
]).then(() => Promise.resolve({ unhold: ps })), // Then unhold them
); ]).then(() => Promise.resolve({ unhold: ps })),
} else { );
return EMPTY; } else {
} return EMPTY;
}, }
).pipe( },
mergeAll(), ).pipe(
// Aggregate the hold instructions into a single list showing which mergeAll(),
// participants are being held // Aggregate the hold instructions into a single list showing which
scan( // participants are being held
(holds, instruction) => scan(
"hold" in instruction (holds, instruction) =>
? [instruction.hold, ...holds] "hold" in instruction
: holds.filter((h) => h !== instruction.unhold), ? [instruction.hold, ...holds]
[] as RemoteParticipant[][], : holds.filter((h) => h !== instruction.unhold),
), [] as RemoteParticipant[][],
startWith([]), ),
); startWith([]),
);
private readonly remoteParticipants = combineLatest( private readonly remoteParticipants: Observable<RemoteParticipant[]> =
[this.rawRemoteParticipants, this.remoteParticipantHolds], combineLatest(
(raw, holds) => { [this.rawRemoteParticipants, this.remoteParticipantHolds],
const result = [...raw]; (raw, holds) => {
const resultIds = new Set(result.map((p) => p.identity)); const result = [...raw];
const resultIds = new Set(result.map((p) => p.identity));
// Incorporate the held participants into the list // Incorporate the held participants into the list
for (const hold of holds) { for (const hold of holds) {
for (const p of hold) { for (const p of hold) {
if (!resultIds.has(p.identity)) { if (!resultIds.has(p.identity)) {
result.push(p); result.push(p);
resultIds.add(p.identity); resultIds.add(p.identity);
}
} }
} }
}
return result; return result;
}, },
); );
private readonly mediaItems = state( private readonly mediaItems: StateObservable<MediaItem[]> = state(
combineLatest([ combineLatest([
this.remoteParticipants, this.remoteParticipants,
observeParticipantMedia(this.livekitRoom.localParticipant), observeParticipantMedia(this.livekitRoom.localParticipant),
@@ -362,63 +364,59 @@ export class CallViewModel extends ViewModel {
), ),
); );
private readonly userMedia = this.mediaItems.pipe( private readonly userMedia: Observable<UserMedia[]> = this.mediaItems.pipe(
map((ms) => ms.filter((m): m is UserMedia => m instanceof UserMedia)), map((ms) => ms.filter((m): m is UserMedia => m instanceof UserMedia)),
); );
private readonly screenShares = this.mediaItems.pipe( private readonly screenShares: Observable<ScreenShare[]> =
map((ms) => ms.filter((m): m is ScreenShare => m instanceof ScreenShare)), this.mediaItems.pipe(
); map((ms) => ms.filter((m): m is ScreenShare => m instanceof ScreenShare)),
);
private readonly spotlightSpeaker = this.userMedia.pipe( private readonly spotlightSpeaker: Observable<UserMedia | null> =
switchMap((ms) => this.userMedia.pipe(
ms.length === 0 switchMap((ms) =>
? of([]) ms.length === 0
: combineLatest( ? of([])
ms.map((m) => m.vm.speaking.pipe(map((s) => [m, s] as const))), : combineLatest(
), ms.map((m) => m.vm.speaking.pipe(map((s) => [m, s] as const))),
), ),
scan<(readonly [UserMedia, boolean])[], UserMedia | null, null>( ),
(prev, ms) => scan<(readonly [UserMedia, boolean])[], UserMedia | null, null>(
// Decide who to spotlight: (prev, ms) =>
// If the previous speaker is still speaking, stick with them rather // Decide who to spotlight:
// than switching eagerly to someone else // If the previous speaker is still speaking, stick with them rather
ms.find(([m, s]) => m === prev && s)?.[0] ?? // than switching eagerly to someone else
// Otherwise, select anyone who is speaking ms.find(([m, s]) => m === prev && s)?.[0] ??
ms.find(([, s]) => s)?.[0] ?? // Otherwise, select anyone who is speaking
// Otherwise, stick with the person who was last speaking ms.find(([, s]) => s)?.[0] ??
prev ?? // Otherwise, stick with the person who was last speaking
// Otherwise, spotlight the local user prev ??
ms.find(([m]) => m.vm.local)?.[0] ?? // Otherwise, spotlight the local user
ms.find(([m]) => m.vm.local)?.[0] ??
null,
null, null,
null, ),
), distinctUntilChanged(),
distinctUntilChanged(), throttleTime(800, undefined, { leading: true, trailing: true }),
throttleTime(800, undefined, { leading: true, trailing: true }), );
);
private readonly grid = this.userMedia.pipe( private readonly grid: Observable<UserMediaViewModel[]> = this.userMedia.pipe(
switchMap((ms) => { switchMap((ms) => {
const bins = ms.map((m) => const bins = ms.map((m) =>
combineLatest( combineLatest(
[m.speaker, m.presenter, m.vm.audioEnabled, m.vm.videoEnabled], [m.speaker, m.presenter, m.vm.audioEnabled, m.vm.videoEnabled],
(speaker, presenter, audio, video) => (speaker, presenter, audio, video) => {
[ let bin: SortingBin;
m, if (m.vm.local) bin = SortingBin.SelfStart;
m.vm.local else if (presenter) bin = SortingBin.Presenters;
? GridBin.SelfStart else if (speaker) bin = SortingBin.Speakers;
: presenter else if (video)
? GridBin.Presenters bin = audio ? SortingBin.VideoAndAudio : SortingBin.Video;
: speaker else bin = audio ? SortingBin.Audio : SortingBin.NoMedia;
? GridBin.Speakers
: video return [m, bin] as const;
? audio },
? GridBin.VideoAndAudio
: GridBin.Video
: audio
? GridBin.Audio
: GridBin.NoMedia,
] as const,
), ),
); );
// Sort the media by bin order and generate a tile for each one // Sort the media by bin order and generate a tile for each one
@@ -430,7 +428,7 @@ export class CallViewModel extends ViewModel {
}), }),
); );
private readonly spotlight = combineLatest( private readonly spotlight: Observable<MediaViewModel[]> = combineLatest(
[this.screenShares, this.spotlightSpeaker], [this.screenShares, this.spotlightSpeaker],
(screenShares, spotlightSpeaker): MediaViewModel[] => (screenShares, spotlightSpeaker): MediaViewModel[] =>
screenShares.length > 0 screenShares.length > 0
@@ -440,6 +438,10 @@ export class CallViewModel extends ViewModel {
: [spotlightSpeaker.vm], : [spotlightSpeaker.vm],
); );
// TODO: Make this react to changes in window dimensions and screen
// orientation
private readonly windowMode = of<WindowMode>("normal");
private readonly _gridMode = new BehaviorSubject<GridMode>("grid"); private readonly _gridMode = new BehaviorSubject<GridMode>("grid");
/** /**
* The layout mode of the media tile grid. * The layout mode of the media tile grid.
@@ -450,10 +452,6 @@ export class CallViewModel extends ViewModel {
this._gridMode.next(value); this._gridMode.next(value);
} }
// TODO: Make this react to changes in window dimensions and screen
// orientation
private readonly windowMode = of<WindowMode>("normal");
public readonly layout: StateObservable<Layout> = state( public readonly layout: StateObservable<Layout> = state(
combineLatest([this._gridMode, this.windowMode], (gridMode, windowMode) => { combineLatest([this._gridMode, this.windowMode], (gridMode, windowMode) => {
switch (windowMode) { switch (windowMode) {
@@ -492,90 +490,91 @@ export class CallViewModel extends ViewModel {
*/ */
// TODO: Get rid of this field, replacing it with the 'layout' field above // TODO: Get rid of this field, replacing it with the 'layout' field above
// which keeps more details of the layout order internal to the view model // which keeps more details of the layout order internal to the view model
public readonly tiles = state( public readonly tiles: StateObservable<TileDescriptor<MediaViewModel>[]> =
combineLatest([ state(
this.remoteParticipants, combineLatest([
observeParticipantMedia(this.livekitRoom.localParticipant), this.remoteParticipants,
]).pipe( observeParticipantMedia(this.livekitRoom.localParticipant),
scan((ts, [remoteParticipants, { participant: localParticipant }]) => { ]).pipe(
const ps = [localParticipant, ...remoteParticipants]; scan((ts, [remoteParticipants, { participant: localParticipant }]) => {
const tilesById = new Map(ts.map((t) => [t.id, t])); const ps = [localParticipant, ...remoteParticipants];
const now = Date.now(); const tilesById = new Map(ts.map((t) => [t.id, t]));
let allGhosts = true; const now = Date.now();
let allGhosts = true;
const newTiles = ps.flatMap((p) => { const newTiles = ps.flatMap((p) => {
const userMediaId = p.identity; const userMediaId = p.identity;
const member = findMatrixMember(this.matrixRoom, userMediaId); const member = findMatrixMember(this.matrixRoom, userMediaId);
allGhosts &&= member === undefined; allGhosts &&= member === undefined;
const spokeRecently = const spokeRecently =
p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000; p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000;
// We always start with a local participant with the empty string as // We always start with a local participant with the empty string as
// their ID before we're connected, this is fine and we'll be in // their ID before we're connected, this is fine and we'll be in
// "all ghosts" mode. // "all ghosts" mode.
if (userMediaId !== "" && member === undefined) { if (userMediaId !== "" && member === undefined) {
logger.warn( logger.warn(
`Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`, `Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`,
);
}
const userMediaVm =
tilesById.get(userMediaId)?.data ??
new UserMediaViewModel(userMediaId, member, p, this.encrypted);
tilesById.delete(userMediaId);
const userMediaTile: TileDescriptor<MediaViewModel> = {
id: userMediaId,
focused: false,
isPresenter: p.isScreenShareEnabled,
isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal,
hasVideo: p.isCameraEnabled,
local: p.isLocal,
largeBaseSize: false,
data: userMediaVm,
};
if (p.isScreenShareEnabled) {
const screenShareId = `${userMediaId}:screen-share`;
const screenShareVm =
tilesById.get(screenShareId)?.data ??
new ScreenShareViewModel(
screenShareId,
member,
p,
this.encrypted,
); );
tilesById.delete(screenShareId); }
const screenShareTile: TileDescriptor<MediaViewModel> = { const userMediaVm =
id: screenShareId, tilesById.get(userMediaId)?.data ??
focused: true, new UserMediaViewModel(userMediaId, member, p, this.encrypted);
isPresenter: false, tilesById.delete(userMediaId);
isSpeaker: false,
hasVideo: true, const userMediaTile: TileDescriptor<MediaViewModel> = {
id: userMediaId,
focused: false,
isPresenter: p.isScreenShareEnabled,
isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal,
hasVideo: p.isCameraEnabled,
local: p.isLocal, local: p.isLocal,
largeBaseSize: true, largeBaseSize: false,
placeNear: userMediaId, data: userMediaVm,
data: screenShareVm,
}; };
return [userMediaTile, screenShareTile];
} else {
return [userMediaTile];
}
});
// Any tiles left in the map are unused and should be destroyed if (p.isScreenShareEnabled) {
for (const t of tilesById.values()) t.data.destroy(); const screenShareId = `${userMediaId}:screen-share`;
const screenShareVm =
tilesById.get(screenShareId)?.data ??
new ScreenShareViewModel(
screenShareId,
member,
p,
this.encrypted,
);
tilesById.delete(screenShareId);
// If every item is a ghost, that probably means we're still connecting const screenShareTile: TileDescriptor<MediaViewModel> = {
// and shouldn't bother showing anything yet id: screenShareId,
return allGhosts ? [] : newTiles; focused: true,
}, [] as TileDescriptor<MediaViewModel>[]), isPresenter: false,
finalizeValue((ts) => { isSpeaker: false,
for (const t of ts) t.data.destroy(); hasVideo: true,
}), local: p.isLocal,
), largeBaseSize: true,
); placeNear: userMediaId,
data: screenShareVm,
};
return [userMediaTile, screenShareTile];
} else {
return [userMediaTile];
}
});
// Any tiles left in the map are unused and should be destroyed
for (const t of tilesById.values()) t.data.destroy();
// If every item is a ghost, that probably means we're still connecting
// and shouldn't bother showing anything yet
return allGhosts ? [] : newTiles;
}, [] as TileDescriptor<MediaViewModel>[]),
finalizeValue((ts) => {
for (const t of ts) t.data.destroy();
}),
),
);
public constructor( public constructor(
// A call is permanently tied to a single Matrix room and LiveKit room // A call is permanently tied to a single Matrix room and LiveKit room