rzerres/element-call
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #2417 from robintown/one-on-one-layout

Browse Source 
New one-on-one layout
This commit is contained in:
Robin
2024-07-18 11:09:11 -04:00
committed by GitHub
parent d561a41666 7526826b0c
commit bcc06d86ff
16 changed files with 468 additions and 2012 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
public/locales/en-GB/app.json
View File

@@ -163,6 +163,5 @@
"mute_for_me": "Mute for me",
"sfu_participant_local": "You",
"volume": "Volume"
},
"waiting_for_participants": "Waiting for other participants…"
}
}

96
src/grid/CallLayout.ts
View File

@@ -17,29 +17,43 @@ limitations under the License.
import { BehaviorSubject, Observable } from "rxjs";
import { ComponentType } from "react";
import { MediaViewModel } from "../state/MediaViewModel";
import { MediaViewModel, UserMediaViewModel } from "../state/MediaViewModel";
import { LayoutProps } from "./Grid";
import { Alignment } from "../room/InCallView";
export interface Bounds {
width: number;
height: number;
}
export interface Alignment {
inline: "start" | "end";
block: "start" | "end";
}
export const defaultSpotlightAlignment: Alignment = {
inline: "end",
block: "end",
};
export const defaultPipAlignment: Alignment = { inline: "end", block: "start" };
export interface CallLayoutInputs {
/**
* The minimum bounds of the layout area.
*/
minBounds: Observable<Bounds>;
/**
* The alignment of the floating tile, if any.
* The alignment of the floating spotlight tile, if present.
*/
floatingAlignment: BehaviorSubject<Alignment>;
spotlightAlignment: BehaviorSubject<Alignment>;
/**
* The alignment of the small picture-in-picture tile, if present.
*/
pipAlignment: BehaviorSubject<Alignment>;
}
export interface GridTileModel {
type: "grid";
vm: MediaViewModel;
vm: UserMediaViewModel;
}
export interface SpotlightTileModel {
@@ -67,3 +81,75 @@ export interface CallLayoutOutputs<Model> {
export type CallLayout<Model> = (
inputs: CallLayoutInputs,
) => CallLayoutOutputs<Model>;
export interface GridArrangement {
tileWidth: number;
tileHeight: number;
gap: number;
columns: number;
}
const tileMinHeight = 130;
const tileMaxAspectRatio = 17 / 9;
const tileMinAspectRatio = 4 / 3;
const tileMobileMinAspectRatio = 2 / 3;
/**
* Determine the ideal arrangement of tiles into a grid of a particular size.
*/
export function arrangeTiles(
width: number,
minHeight: number,
tileCount: number,
): GridArrangement {
// The goal here is to determine the grid size and padding that maximizes
// use of screen space for n tiles without making those tiles too small or
// too cropped (having an extreme aspect ratio)
const gap = width < 800 ? 16 : 20;
const tileMinWidth = width < 500 ? 150 : 180;
let columns = Math.min(
// Don't create more columns than we have items for
tileCount,
// The ideal number of columns is given by a packing of equally-sized
// squares into a grid.
// width / column = height / row.
// columns * rows = number of squares.
// ∴ columns = sqrt(width / height * number of squares).
// Except we actually want 16:9-ish tiles rather than squares, so we
// divide the width-to-height ratio by the target aspect ratio.
Math.ceil(Math.sqrt((width / minHeight / tileMaxAspectRatio) * tileCount)),
);
let rows = Math.ceil(tileCount / columns);
let tileWidth = (width - (columns - 1) * gap) / columns;
let tileHeight = (minHeight - (rows - 1) * gap) / rows;
// Impose a minimum width and height on the tiles
if (tileWidth < tileMinWidth) {
// In this case we want the tile width to determine the number of columns,
// not the other way around. If we take the above equation for the tile
// width (w = (W - (c - 1) * g) / c) and solve for c, we get
// c = (W + g) / (w + g).
columns = Math.floor((width + gap) / (tileMinWidth + gap));
rows = Math.ceil(tileCount / columns);
tileWidth = (width - (columns - 1) * gap) / columns;
tileHeight = (minHeight - (rows - 1) * gap) / rows;
}
if (tileHeight < tileMinHeight) tileHeight = tileMinHeight;
// Impose a minimum and maximum aspect ratio on the tiles
const tileAspectRatio = tileWidth / tileHeight;
// We enforce a different min aspect ratio in 1:1s on mobile
const minAspectRatio =
tileCount === 1 && width < 600
? tileMobileMinAspectRatio
: tileMinAspectRatio;
if (tileAspectRatio > tileMaxAspectRatio)
tileWidth = tileHeight * tileMaxAspectRatio;
else if (tileAspectRatio < minAspectRatio)
tileHeight = tileWidth / minAspectRatio;
// TODO: We might now be hitting the minimum height or width limit again
return { tileWidth, tileHeight, gap, columns };
}

21
src/grid/Grid.tsx
View File

@@ -41,7 +41,6 @@ import styles from "./Grid.module.css";
import { useMergedRefs } from "../useMergedRefs";
import { TileWrapper } from "./TileWrapper";
import { usePrefersReducedMotion } from "../usePrefersReducedMotion";
import { TileSpringUpdate } from "./LegacyGrid";
import { useInitial } from "../useInitial";
interface Rect {
@@ -69,6 +68,13 @@ interface TileSpring {
height: number;
}
interface TileSpringUpdate extends Partial<TileSpring> {
from?: Partial<TileSpring>;
reset?: boolean;
immediate?: boolean | ((key: string) => boolean);
delay?: (key: string) => number;
}
interface DragState {
tileId: string;
tileX: number;
@@ -262,12 +268,13 @@ export function Grid<
) as HTMLCollectionOf<HTMLElement>;
for (const slot of slots) {
const id = slot.getAttribute("data-id")!;
result.push({
...tiles.get(id)!,
...offset(slot, gridRoot),
width: slot.offsetWidth,
height: slot.offsetHeight,
});
if (slot.offsetWidth > 0 && slot.offsetHeight > 0)
result.push({
...tiles.get(id)!,
...offset(slot, gridRoot),
width: slot.offsetWidth,
height: slot.offsetHeight,
});
}
}

3
src/grid/GridLayout.module.css
View File

@@ -17,11 +17,10 @@ limitations under the License.
.fixed,
.scrolling {
margin-inline: var(--inline-content-inset);
block-size: 100%;
}
.scrolling {
box-sizing: border-box;
block-size: 100%;
display: flex;
flex-wrap: wrap;
justify-content: center;

83
src/grid/GridLayout.tsx
View File

@@ -22,7 +22,12 @@ import { GridLayout as GridLayoutModel } from "../state/CallViewModel";
import styles from "./GridLayout.module.css";
import { useReactiveState } from "../useReactiveState";
import { useInitial } from "../useInitial";
import { CallLayout, GridTileModel, TileModel } from "./CallLayout";
import {
CallLayout,
GridTileModel,
TileModel,
arrangeTiles,
} from "./CallLayout";
import { DragCallback } from "./Grid";
interface GridCSSProperties extends CSSProperties {
@@ -31,13 +36,9 @@ interface GridCSSProperties extends CSSProperties {
"--height": string;
}
const slotMinHeight = 130;
const slotMaxAspectRatio = 17 / 9;
const slotMinAspectRatio = 4 / 3;
export const makeGridLayout: CallLayout<GridLayoutModel> = ({
minBounds,
floatingAlignment,
spotlightAlignment,
}) => ({
// The "fixed" (non-scrolling) part of the layout is where the spotlight tile
// lives
@@ -45,7 +46,7 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
const { width, height } = useObservableEagerState(minBounds);
const alignment = useObservableEagerState(
useInitial(() =>
floatingAlignment.pipe(
spotlightAlignment.pipe(
distinctUntilChanged(
(a1, a2) => a1.block === a2.block && a1.inline === a2.inline,
),
@@ -68,7 +69,7 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
const onDragSpotlight: DragCallback = useCallback(
({ xRatio, yRatio }) =>
floatingAlignment.next({
spotlightAlignment.next({
block: yRatio < 0.5 ? "start" : "end",
inline: xRatio < 0.5 ? "start" : "end",
}),
@@ -76,12 +77,7 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
);
return (
<div
ref={ref}
className={styles.fixed}
data-generation={generation}
style={{ height }}
>
<div ref={ref} className={styles.fixed} data-generation={generation}>
{tileModel && (
<Slot
className={styles.slot}
@@ -99,57 +95,10 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
// The scrolling part of the layout is where all the grid tiles live
scrolling: forwardRef(function GridLayout({ model, Slot }, ref) {
const { width, height: minHeight } = useObservableEagerState(minBounds);
// The goal here is to determine the grid size and padding that maximizes
// use of screen space for n tiles without making those tiles too small or
// too cropped (having an extreme aspect ratio)
const [gap, slotWidth, slotHeight] = useMemo(() => {
const gap = width < 800 ? 16 : 20;
const slotMinWidth = width < 500 ? 150 : 180;
let columns = Math.min(
// Don't create more columns than we have items for
model.grid.length,
// The ideal number of columns is given by a packing of equally-sized
// squares into a grid.
// width / column = height / row.
// columns * rows = number of squares.
// ∴ columns = sqrt(width / height * number of squares).
// Except we actually want 16:9-ish slots rather than squares, so we
// divide the width-to-height ratio by the target aspect ratio.
Math.ceil(
Math.sqrt(
(width / minHeight / slotMaxAspectRatio) * model.grid.length,
),
),
);
let rows = Math.ceil(model.grid.length / columns);
let slotWidth = (width - (columns - 1) * gap) / columns;
let slotHeight = (minHeight - (rows - 1) * gap) / rows;
// Impose a minimum width and height on the slots
if (slotWidth < slotMinWidth) {
// In this case we want the slot width to determine the number of columns,
// not the other way around. If we take the above equation for the slot
// width (w = (W - (c - 1) * g) / c) and solve for c, we get
// c = (W + g) / (w + g).
columns = Math.floor((width + gap) / (slotMinWidth + gap));
rows = Math.ceil(model.grid.length / columns);
slotWidth = (width - (columns - 1) * gap) / columns;
slotHeight = (minHeight - (rows - 1) * gap) / rows;
}
if (slotHeight < slotMinHeight) slotHeight = slotMinHeight;
// Impose a minimum and maximum aspect ratio on the slots
const slotAspectRatio = slotWidth / slotHeight;
if (slotAspectRatio > slotMaxAspectRatio)
slotWidth = slotHeight * slotMaxAspectRatio;
else if (slotAspectRatio < slotMinAspectRatio)
slotHeight = slotWidth / slotMinAspectRatio;
// TODO: We might now be hitting the minimum height or width limit again
return [gap, slotWidth, slotHeight];
}, [width, minHeight, model.grid.length]);
const { gap, tileWidth, tileHeight } = useMemo(
() => arrangeTiles(width, minHeight, model.grid.length),
[width, minHeight, model.grid.length],
);
const [generation] = useReactiveState<number>(
(prev) => (prev === undefined ? 0 : prev + 1),
@@ -170,8 +119,8 @@ export const makeGridLayout: CallLayout<GridLayoutModel> = ({
{
width,
"--gap": `${gap}px`,
"--width": `${Math.floor(slotWidth)}px`,
"--height": `${Math.floor(slotHeight)}px`,
"--width": `${Math.floor(tileWidth)}px`,
"--height": `${Math.floor(tileHeight)}px`,
} as GridCSSProperties
}
>

22
src/grid/LegacyGrid.module.css
View File

@@ -1,22 +0,0 @@
/*
Copyright 2022-2024 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
.grid {
position: relative;
overflow: hidden;
flex: 1;
touch-action: none;
}

1405
src/grid/LegacyGrid.tsx
View File

File diff suppressed because it is too large Load Diff

63
src/grid/OneOnOneLayout.module.css Normal file
View File

@@ -0,0 +1,63 @@
/*
Copyright 2024 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
.layer {
margin-inline: var(--inline-content-inset);
block-size: 100%;
display: grid;
place-items: center;
}
.container {
position: relative;
}
.local {
position: absolute;
inline-size: 135px;
block-size: 160px;
inset: var(--cpd-space-4x);
}
@media (min-width: 600px) {
.local {
inline-size: 170px;
block-size: 110px;
}
}
.spotlight {
position: absolute;
inline-size: 404px;
block-size: 233px;
inset: -12px;
}
.slot[data-block-alignment="start"] {
inset-block-end: unset;
}
.slot[data-block-alignment="end"] {
inset-block-start: unset;
}
.slot[data-inline-alignment="start"] {
inset-inline-end: unset;
}
.slot[data-inline-alignment="end"] {
inset-inline-start: unset;
}

132
src/grid/OneOnOneLayout.tsx Normal file
View File

@@ -0,0 +1,132 @@
/*
Copyright 2024 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { forwardRef, useCallback, useMemo } from "react";
import { useObservableEagerState } from "observable-hooks";
import classNames from "classnames";
import { OneOnOneLayout as OneOnOneLayoutModel } from "../state/CallViewModel";
import {
CallLayout,
GridTileModel,
SpotlightTileModel,
arrangeTiles,
} from "./CallLayout";
import { useReactiveState } from "../useReactiveState";
import styles from "./OneOnOneLayout.module.css";
import { DragCallback } from "./Grid";
export const makeOneOnOneLayout: CallLayout<OneOnOneLayoutModel> = ({
minBounds,
spotlightAlignment,
pipAlignment,
}) => ({
fixed: forwardRef(function OneOnOneLayoutFixed({ model, Slot }, ref) {
const { width, height } = useObservableEagerState(minBounds);
const spotlightAlignmentValue = useObservableEagerState(spotlightAlignment);
const [generation] = useReactiveState<number>(
(prev) => (prev === undefined ? 0 : prev + 1),
[width, height, model.spotlight === undefined, spotlightAlignmentValue],
);
const spotlightTileModel: SpotlightTileModel | undefined = useMemo(
() =>
model.spotlight && {
type: "spotlight",
vms: model.spotlight,
maximised: false,
},
[model.spotlight],
);
const onDragSpotlight: DragCallback = useCallback(
({ xRatio, yRatio }) =>
spotlightAlignment.next({
block: yRatio < 0.5 ? "start" : "end",
inline: xRatio < 0.5 ? "start" : "end",
}),
[],
);
return (
<div ref={ref} data-generation={generation} className={styles.layer}>
{spotlightTileModel && (
<Slot
className={classNames(styles.slot, styles.spotlight)}
id="spotlight"
model={spotlightTileModel}
onDrag={onDragSpotlight}
data-block-alignment={spotlightAlignmentValue.block}
data-inline-alignment={spotlightAlignmentValue.inline}
/>
)}
</div>
);
}),
scrolling: forwardRef(function OneOnOneLayoutScrolling({ model, Slot }, ref) {
const { width, height } = useObservableEagerState(minBounds);
const pipAlignmentValue = useObservableEagerState(pipAlignment);
const { tileWidth, tileHeight } = useMemo(
() => arrangeTiles(width, height, 1),
[width, height],
);
const [generation] = useReactiveState<number>(
(prev) => (prev === undefined ? 0 : prev + 1),
[width, height, pipAlignmentValue],
);
const remoteTileModel: GridTileModel = useMemo(
() => ({ type: "grid", vm: model.remote }),
[model.remote],
);
const localTileModel: GridTileModel = useMemo(
() => ({ type: "grid", vm: model.local }),
[model.local],
);
const onDragLocalTile: DragCallback = useCallback(
({ xRatio, yRatio }) =>
pipAlignment.next({
block: yRatio < 0.5 ? "start" : "end",
inline: xRatio < 0.5 ? "start" : "end",
}),
[],
);
return (
<div ref={ref} data-generation={generation} className={styles.layer}>
<Slot
id={remoteTileModel.vm.id}
model={remoteTileModel}
className={styles.container}
style={{ width: tileWidth, height: tileHeight }}
>
<Slot
className={classNames(styles.slot, styles.local)}
id={localTileModel.vm.id}
model={localTileModel}
onDrag={onDragLocalTile}
data-block-alignment={pipAlignmentValue.block}
data-inline-alignment={pipAlignmentValue.inline}
/>
</Slot>
</div>
);
}),
});

5
src/grid/SpotlightLayout.module.css
View File

@@ -16,6 +16,7 @@ limitations under the License.
.layer {
margin-inline: var(--inline-content-inset);
block-size: 100%;
display: grid;
--grid-gap: 20px;
gap: 30px;
@@ -30,10 +31,6 @@ limitations under the License.
grid-template-rows: minmax(1fr, auto);
}
.scrolling {
block-size: 100%;
}
.spotlight {
container: spotlight / size;
display: grid;

8
src/grid/SpotlightLayout.tsx
View File

@@ -69,10 +69,8 @@ export const makeSpotlightLayout: CallLayout<SpotlightLayoutModel> = ({
ref={ref}
data-generation={generation}
data-orientation={layout.orientation}
className={classNames(styles.layer, styles.fixed)}
style={
{ "--grid-columns": layout.gridColumns, height } as GridCSSProperties
}
className={styles.layer}
style={{ "--grid-columns": layout.gridColumns } as GridCSSProperties}
>
<div className={styles.spotlight}>
<Slot className={styles.slot} id="spotlight" model={tileModel} />
@@ -102,7 +100,7 @@ export const makeSpotlightLayout: CallLayout<SpotlightLayoutModel> = ({
ref={ref}
data-generation={generation}
data-orientation={layout.orientation}
className={classNames(styles.layer, styles.scrolling)}
className={styles.layer}
style={{ "--grid-columns": layout.gridColumns } as GridCSSProperties}
>
<div

255
src/room/InCallView.tsx
View File

@@ -18,10 +18,9 @@ import {
RoomAudioRenderer,
RoomContext,
useLocalParticipant,
useTracks,
} from "@livekit/components-react";
import { usePreventScroll } from "@react-aria/overlays";
import { ConnectionState, Room, Track } from "livekit-client";
import { ConnectionState, Room } from "livekit-client";
import { MatrixClient } from "matrix-js-sdk/src/client";
import {
FC,
@@ -38,7 +37,6 @@ import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import classNames from "classnames";
import { BehaviorSubject, map } from "rxjs";
import { useObservableEagerState } from "observable-hooks";
import { useTranslation } from "react-i18next";
import LogoMark from "../icons/LogoMark.svg?react";
import LogoType from "../icons/LogoType.svg?react";
@@ -51,10 +49,8 @@ import {
SettingsButton,
} from "../button";
import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
import { LegacyGrid, useLegacyGridLayout } from "../grid/LegacyGrid";
import { useUrlParams } from "../UrlParams";
import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
import { usePrefersReducedMotion } from "../usePrefersReducedMotion";
import { ElementWidgetActions, widget } from "../widget";
import styles from "./InCallView.module.css";
import { GridTile } from "../tile/GridTile";
@@ -72,14 +68,8 @@ import { InviteButton } from "../button/InviteButton";
import { LayoutToggle } from "./LayoutToggle";
import { ECConnectionState } from "../livekit/useECConnectionState";
import { useOpenIDSFU } from "../livekit/openIDSFU";
import {
GridMode,
Layout,
TileDescriptor,
useCallViewModel,
} from "../state/CallViewModel";
import { GridMode, Layout, useCallViewModel } from "../state/CallViewModel";
import { Grid, TileProps } from "../grid/Grid";
import { MediaViewModel } from "../state/MediaViewModel";
import { useObservable } from "../state/useObservable";
import { useInitial } from "../useInitial";
import { SpotlightTile } from "../tile/SpotlightTile";
@@ -87,21 +77,16 @@ import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
import { E2eeType } from "../e2ee/e2eeType";
import { makeGridLayout } from "../grid/GridLayout";
import { makeSpotlightLayout } from "../grid/SpotlightLayout";
import { CallLayout, GridTileModel, TileModel } from "../grid/CallLayout";
import {
CallLayout,
TileModel,
defaultPipAlignment,
defaultSpotlightAlignment,
} from "../grid/CallLayout";
import { makeOneOnOneLayout } from "../grid/OneOnOneLayout";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
export interface Alignment {
inline: "start" | "end";
block: "start" | "end";
}
const defaultAlignment: Alignment = { inline: "end", block: "end" };
const dummySpotlightItem = {
id: "spotlight",
} as TileDescriptor<MediaViewModel>;
export interface ActiveCallProps
extends Omit<InCallViewProps, "livekitRoom" | "connState"> {
e2eeSystem: EncryptionSystem;
@@ -155,11 +140,9 @@ export const InCallView: FC<InCallViewProps> = ({
participantCount,
onLeave,
hideHeader,
otelGroupCallMembership,
connState,
onShareClick,
}) => {
const { t } = useTranslation();
usePreventScroll();
useWakeLock();
@@ -177,15 +160,6 @@ export const InCallView: FC<InCallViewProps> = ({
// Merge the refs so they can attach to the same element
const containerRef = useMergedRefs(containerRef1, containerRef2);
const screenSharingTracks = useTracks(
[{ source: Track.Source.ScreenShare, withPlaceholder: false }],
{
room: livekitRoom,
},
);
const { layout: legacyLayout, setLayout: setLegacyLayout } =
useLegacyGridLayout(screenSharingTracks.length > 0);
const { hideScreensharing, showControls } = useUrlParams();
const { isScreenShareEnabled, localParticipant } = useLocalParticipant({
@@ -210,42 +184,6 @@ export const InCallView: FC<InCallViewProps> = ({
(muted) => muteStates.audio.setEnabled?.(!muted),
);
useEffect(() => {
widget?.api.transport.send(
legacyLayout === "grid"
? ElementWidgetActions.TileLayout
: ElementWidgetActions.SpotlightLayout,
{},
);
}, [legacyLayout]);
useEffect(() => {
if (widget) {
const onTileLayout = (ev: CustomEvent<IWidgetApiRequest>): void => {
setLegacyLayout("grid");
widget!.api.transport.reply(ev.detail, {});
};
const onSpotlightLayout = (ev: CustomEvent<IWidgetApiRequest>): void => {
setLegacyLayout("spotlight");
widget!.api.transport.reply(ev.detail, {});
};
widget.lazyActions.on(ElementWidgetActions.TileLayout, onTileLayout);
widget.lazyActions.on(
ElementWidgetActions.SpotlightLayout,
onSpotlightLayout,
);
return (): void => {
widget!.lazyActions.off(ElementWidgetActions.TileLayout, onTileLayout);
widget!.lazyActions.off(
ElementWidgetActions.SpotlightLayout,
onSpotlightLayout,
);
};
}
}, [setLegacyLayout]);
const mobile = boundsValid && bounds.width <= 660;
const reducedControls = boundsValid && bounds.width <= 340;
const noControls = reducedControls && bounds.height <= 400;
@@ -256,15 +194,12 @@ export const InCallView: FC<InCallViewProps> = ({
matrixInfo.e2eeSystem.kind !== E2eeType.NONE,
connState,
);
const items = useObservableEagerState(vm.tiles);
const layout = useObservableEagerState(vm.layout);
const gridMode = useObservableEagerState(vm.gridMode);
const hasSpotlight = layout.spotlight !== undefined;
// Hack: We insert a dummy "spotlight" tile into the tiles we pass to
// useFullscreen so that we can control the fullscreen state of the
// spotlight tile in the new layouts with this same hook.
const fullscreenItems = useMemo(
() => (hasSpotlight ? [...items, dummySpotlightItem] : items),
[items, hasSpotlight],
() => (hasSpotlight ? ["spotlight"] : []),
[hasSpotlight],
);
const { fullscreenItem, toggleFullscreen, exitFullscreen } =
useFullscreen(fullscreenItems);
@@ -274,18 +209,9 @@ export const InCallView: FC<InCallViewProps> = ({
);
// The maximised participant: either the participant that the user has
// manually put in fullscreen, or the focused (active) participant if the
// window is too small to show everyone
const maximisedParticipant = useMemo(
() =>
fullscreenItem ??
(noControls
? items.find((item) => item.isSpeaker) ?? items.at(0) ?? null
: null),
[fullscreenItem, noControls, items],
);
const prefersReducedMotion = usePrefersReducedMotion();
// manually put in fullscreen, or (TODO) the spotlight if the window is too
// small to show everyone
const maximisedParticipant = fullscreenItem;
const [settingsModalOpen, setSettingsModalOpen] = useState(false);
const [settingsTab, setSettingsTab] = useState(defaultSettingsTab);
@@ -321,8 +247,11 @@ export const InCallView: FC<InCallViewProps> = ({
);
const gridBoundsObservable = useObservable(gridBounds);
const floatingAlignment = useInitial(
() => new BehaviorSubject(defaultAlignment),
const spotlightAlignment = useInitial(
() => new BehaviorSubject(defaultSpotlightAlignment),
);
const pipAlignment = useInitial(
() => new BehaviorSubject(defaultPipAlignment),
);
const layoutSystem = useObservableEagerState(
@@ -330,18 +259,18 @@ export const InCallView: FC<InCallViewProps> = ({
vm.layout.pipe(
map((l) => {
let makeLayout: CallLayout<Layout>;
if (
l.type === "grid" &&
!(l.grid.length === 2 && l.spotlight === undefined)
)
if (l.type === "grid")
makeLayout = makeGridLayout as CallLayout<Layout>;
else if (l.type === "spotlight")
makeLayout = makeSpotlightLayout as CallLayout<Layout>;
else return null; // Not yet implemented
else if (l.type === "one-on-one")
makeLayout = makeOneOnOneLayout as CallLayout<Layout>;
else throw new Error(`Unimplemented layout: ${l.type}`);
return makeLayout({
minBounds: gridBoundsObservable,
floatingAlignment,
spotlightAlignment,
pipAlignment,
});
}),
),
@@ -349,13 +278,46 @@ export const InCallView: FC<InCallViewProps> = ({
);
const setGridMode = useCallback(
(mode: GridMode) => {
setLegacyLayout(mode);
vm.setGridMode(mode);
},
[setLegacyLayout, vm],
(mode: GridMode) => vm.setGridMode(mode),
[vm],
);
useEffect(() => {
widget?.api.transport.send(
gridMode === "grid"
? ElementWidgetActions.TileLayout
: ElementWidgetActions.SpotlightLayout,
{},
);
}, [gridMode]);
useEffect(() => {
if (widget) {
const onTileLayout = (ev: CustomEvent<IWidgetApiRequest>): void => {
setGridMode("grid");
widget!.api.transport.reply(ev.detail, {});
};
const onSpotlightLayout = (ev: CustomEvent<IWidgetApiRequest>): void => {
setGridMode("spotlight");
widget!.api.transport.reply(ev.detail, {});
};
widget.lazyActions.on(ElementWidgetActions.TileLayout, onTileLayout);
widget.lazyActions.on(
ElementWidgetActions.SpotlightLayout,
onSpotlightLayout,
);
return (): void => {
widget!.lazyActions.off(ElementWidgetActions.TileLayout, onTileLayout);
widget!.lazyActions.off(
ElementWidgetActions.SpotlightLayout,
onSpotlightLayout,
);
};
}
}, [setGridMode]);
const showSpotlightIndicators = useObservable(layout.type === "spotlight");
const showSpeakingIndicators = useObservable(
layout.type === "spotlight" ||
@@ -416,33 +378,10 @@ export const InCallView: FC<InCallViewProps> = ({
],
);
const LegacyTile = useMemo(
() =>
forwardRef<
HTMLDivElement,
PropsWithoutRef<TileProps<MediaViewModel, HTMLDivElement>>
>(function LegacyTile({ model: legacyModel, ...props }, ref) {
const model: GridTileModel = useMemo(
() => ({ type: "grid", vm: legacyModel }),
[legacyModel],
);
return <Tile ref={ref} model={model} {...props} />;
}),
[Tile],
);
const renderContent = (): JSX.Element => {
if (items.length === 0) {
return (
<div className={styles.centerMessage}>
<p>{t("waiting_for_participants")}</p>
</div>
);
}
if (maximisedParticipant !== null) {
const fullscreen = maximisedParticipant === fullscreenItem;
if (maximisedParticipant.id === "spotlight") {
if (maximisedParticipant === "spotlight") {
return (
<SpotlightTile
className={classNames(styles.tile, styles.maximised)}
@@ -456,52 +395,28 @@ export const InCallView: FC<InCallViewProps> = ({
/>
);
}
return (
<GridTile
className={classNames(styles.tile, styles.maximised)}
vm={maximisedParticipant.data}
maximised={true}
fullscreen={fullscreen}
onToggleFullscreen={toggleFullscreen}
targetHeight={gridBounds.height}
targetWidth={gridBounds.width}
key={maximisedParticipant.id}
showSpeakingIndicators={false}
onOpenProfile={openProfile}
/>
);
}
if (layoutSystem === null) {
// This new layout doesn't yet have an implemented layout system, so fall
// back to the legacy grid system
return (
<LegacyGrid
items={items}
layout={legacyLayout}
disableAnimations={prefersReducedMotion}
Tile={LegacyTile}
return (
<>
<Grid
className={styles.scrollingGrid}
model={layout}
Layout={layoutSystem.scrolling}
Tile={Tile}
/>
);
} else {
return (
<>
<Grid
className={styles.scrollingGrid}
model={layout}
Layout={layoutSystem.scrolling}
Tile={Tile}
/>
<Grid
className={styles.fixedGrid}
style={{ insetBlockStart: headerBounds.bottom }}
model={layout}
Layout={layoutSystem.fixed}
Tile={Tile}
/>
</>
);
}
<Grid
className={styles.fixedGrid}
style={{
insetBlockStart: headerBounds.bottom,
height: gridBounds.height,
}}
model={layout}
Layout={layoutSystem.fixed}
Tile={Tile}
/>
</>
);
};
const rageshakeRequestModalProps = useRageshakeRequestModal(
@@ -590,7 +505,7 @@ export const InCallView: FC<InCallViewProps> = ({
{!mobile && !hideHeader && showControls && (
<LayoutToggle
className={styles.layout}
layout={legacyLayout}
layout={gridMode}
setLayout={setGridMode}
/>
)}

26
src/room/useFullscreen.ts
View File

@@ -20,7 +20,6 @@ import { useCallback, useLayoutEffect, useRef } from "react";
import { useReactiveState } from "../useReactiveState";
import { useEventTarget } from "../useEvents";
import { TileDescriptor } from "../state/CallViewModel";
const isFullscreen = (): boolean =>
Boolean(document.fullscreenElement) ||
@@ -55,31 +54,30 @@ function useFullscreenChange(onFullscreenChange: () => void): void {
* Provides callbacks for controlling the full-screen view, which can hold one
* item at a time.
*/
export function useFullscreen<T>(items: TileDescriptor<T>[]): {
fullscreenItem: TileDescriptor<T> | null;
// TODO: Simplify this. Nowadays we only allow the spotlight to be fullscreen,
// so we don't need to bother with multiple items.
export function useFullscreen(items: string[]): {
fullscreenItem: string | null;
toggleFullscreen: (itemId: string) => void;
exitFullscreen: () => void;
} {
const [fullscreenItem, setFullscreenItem] =
useReactiveState<TileDescriptor<T> | null>(
(prevItem) =>
prevItem == null
? null
: items.find((i) => i.id === prevItem.id) ?? null,
[items],
);
const [fullscreenItem, setFullscreenItem] = useReactiveState<string | null>(
(prevItem) =>
prevItem == null ? null : items.find((i) => i === prevItem) ?? null,
[items],
);
const latestItems = useRef<TileDescriptor<T>[]>(items);
const latestItems = useRef<string[]>(items);
latestItems.current = items;
const latestFullscreenItem = useRef<TileDescriptor<T> | null>(fullscreenItem);
const latestFullscreenItem = useRef<string | null>(fullscreenItem);
latestFullscreenItem.current = fullscreenItem;
const toggleFullscreen = useCallback(
(itemId: string) => {
setFullscreenItem(
latestFullscreenItem.current === null
? latestItems.current.find((i) => i.id === itemId) ?? null
? latestItems.current.find((i) => i === itemId) ?? null
: null,
);
},

177
src/state/CallViewModel.ts
View File

@@ -82,22 +82,6 @@ const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
// entirely with the spotlight tile, if we workshop this further.
const largeGridThreshold = 20;
// Represents something that should get a tile on the layout,
// ie. a user's video feed or a screen share feed.
// TODO: This exposes too much information to the view layer, let's keep this
// information internal to the view model and switch to using Tile<T> instead
export interface TileDescriptor<T> {
id: string;
focused: boolean;
isPresenter: boolean;
isSpeaker: boolean;
hasVideo: boolean;
local: boolean;
largeBaseSize: boolean;
placeNear?: string;
data: T;
}
export interface GridLayout {
type: "grid";
spotlight?: MediaViewModel[];
@@ -110,6 +94,13 @@ export interface SpotlightLayout {
grid: UserMediaViewModel[];
}
export interface OneOnOneLayout {
type: "one-on-one";
spotlight?: ScreenShareViewModel[];
local: LocalUserMediaViewModel;
remote: RemoteUserMediaViewModel;
}
export interface FullScreenLayout {
type: "full screen";
spotlight: MediaViewModel[];
@@ -128,6 +119,7 @@ export interface PipLayout {
export type Layout =
| GridLayout
| SpotlightLayout
| OneOnOneLayout
| FullScreenLayout
| PipLayout;
@@ -247,7 +239,8 @@ function findMatrixMember(
room: MatrixRoom,
id: string,
): RoomMember | undefined {
if (!id) return undefined;
if (id === "local")
return room.getMember(room.client.getUserId()!) ?? undefined;
const parts = id.split(":");
// must be at least 3 parts because we know the first part is a userId which must necessarily contain a colon
@@ -351,21 +344,15 @@ export class CallViewModel extends ViewModel {
prevItems,
[remoteParticipants, { participant: localParticipant }, duplicateTiles],
) => {
let allGhosts = true;
const newItems = new Map(
function* (this: CallViewModel): Iterable<[string, MediaItem]> {
for (const p of [localParticipant, ...remoteParticipants]) {
const member = findMatrixMember(this.matrixRoom, p.identity);
allGhosts &&= member === undefined;
// We always start with a local participant with the empty string as
// their ID before we're connected, this is fine and we'll be in
// "all ghosts" mode.
if (p.identity !== "" && member === undefined) {
const userMediaId = p === localParticipant ? "local" : p.identity;
const member = findMatrixMember(this.matrixRoom, userMediaId);
if (member === undefined)
logger.warn(
`Ruh, roh! No matrix member found for SFU participant '${p.identity}': creating g-g-g-ghost!`,
);
}
// Create as many tiles for this participant as called for by
// the duplicateTiles option
@@ -390,9 +377,8 @@ export class CallViewModel extends ViewModel {
}.bind(this)(),
);
// If every item is a ghost, that probably means we're still connecting
// and shouldn't bother showing anything yet
return allGhosts ? new Map() : newItems;
for (const [id, t] of prevItems) if (!newItems.has(id)) t.destroy();
return newItems;
},
new Map<string, MediaItem>(),
),
@@ -545,15 +531,28 @@ export class CallViewModel extends ViewModel {
case "grid":
return combineLatest(
[this.grid, this.spotlight, this.screenShares],
(grid, spotlight, screenShares): Layout => ({
type: "grid",
spotlight:
screenShares.length > 0 ||
grid.length > largeGridThreshold
? spotlight
: undefined,
grid,
}),
(grid, spotlight, screenShares): Layout =>
grid.length == 2
? {
type: "one-on-one",
spotlight:
screenShares.length > 0 ? spotlight : undefined,
local: grid.find(
(vm) => vm.local,
) as LocalUserMediaViewModel,
remote: grid.find(
(vm) => !vm.local,
) as RemoteUserMediaViewModel,
}
: {
type: "grid",
spotlight:
screenShares.length > 0 ||
grid.length > largeGridThreshold
? spotlight
: undefined,
grid,
},
);
case "spotlight":
return combineLatest(
@@ -572,108 +571,6 @@ export class CallViewModel extends ViewModel {
shareReplay(1),
);
/**
* The media tiles to be displayed in the call view.
*/
// TODO: Get rid of this field, replacing it with the 'layout' field above
// which keeps more details of the layout order internal to the view model
public readonly tiles: Observable<TileDescriptor<MediaViewModel>[]> =
combineLatest([
this.remoteParticipants,
observeParticipantMedia(this.livekitRoom.localParticipant),
]).pipe(
scan((ts, [remoteParticipants, { participant: localParticipant }]) => {
const ps = [localParticipant, ...remoteParticipants];
const tilesById = new Map(ts.map((t) => [t.id, t]));
const now = Date.now();
let allGhosts = true;
const newTiles = ps.flatMap((p) => {
const userMediaId = p.identity;
const member = findMatrixMember(this.matrixRoom, userMediaId);
allGhosts &&= member === undefined;
const spokeRecently =
p.lastSpokeAt !== undefined && now - +p.lastSpokeAt <= 10000;
// We always start with a local participant with the empty string as
// their ID before we're connected, this is fine and we'll be in
// "all ghosts" mode.
if (userMediaId !== "" && member === undefined) {
logger.warn(
`Ruh, roh! No matrix member found for SFU participant '${userMediaId}': creating g-g-g-ghost!`,
);
}
const userMediaVm =
tilesById.get(userMediaId)?.data ??
(p instanceof LocalParticipant
? new LocalUserMediaViewModel(
userMediaId,
member,
p,
this.encrypted,
)
: new RemoteUserMediaViewModel(
userMediaId,
member,
p,
this.encrypted,
));
tilesById.delete(userMediaId);
const userMediaTile: TileDescriptor<MediaViewModel> = {
id: userMediaId,
focused: false,
isPresenter: p.isScreenShareEnabled,
isSpeaker: (p.isSpeaking || spokeRecently) && !p.isLocal,
hasVideo: p.isCameraEnabled,
local: p.isLocal,
largeBaseSize: false,
data: userMediaVm,
};
if (p.isScreenShareEnabled) {
const screenShareId = `${userMediaId}:screen-share`;
const screenShareVm =
tilesById.get(screenShareId)?.data ??
new ScreenShareViewModel(
screenShareId,
member,
p,
this.encrypted,
);
tilesById.delete(screenShareId);
const screenShareTile: TileDescriptor<MediaViewModel> = {
id: screenShareId,
focused: true,
isPresenter: false,
isSpeaker: false,
hasVideo: true,
local: p.isLocal,
largeBaseSize: true,
placeNear: userMediaId,
data: screenShareVm,
};
return [userMediaTile, screenShareTile];
} else {
return [userMediaTile];
}
});
// Any tiles left in the map are unused and should be destroyed
for (const t of tilesById.values()) t.data.destroy();
// If every item is a ghost, that probably means we're still connecting
// and shouldn't bother showing anything yet
return allGhosts ? [] : newTiles;
}, [] as TileDescriptor<MediaViewModel>[]),
finalizeValue((ts) => {
for (const t of ts) t.data.destroy();
}),
shareReplay(1),
);
public constructor(
// A call is permanently tied to a single Matrix room and LiveKit room
private readonly matrixRoom: MatrixRoom,

112
src/tile/GridTile.tsx
View File

@@ -33,7 +33,6 @@ import VolumeOffIcon from "@vector-im/compound-design-tokens/icons/volume-off.sv
import VisibilityOnIcon from "@vector-im/compound-design-tokens/icons/visibility-on.svg?react";
import UserProfileIcon from "@vector-im/compound-design-tokens/icons/user-profile.svg?react";
import ExpandIcon from "@vector-im/compound-design-tokens/icons/expand.svg?react";
import CollapseIcon from "@vector-im/compound-design-tokens/icons/collapse.svg?react";
import {
ContextMenu,
MenuItem,
@@ -44,8 +43,6 @@ import { useObservableEagerState } from "observable-hooks";
import styles from "./GridTile.module.css";
import {
ScreenShareViewModel,
MediaViewModel,
UserMediaViewModel,
useNameData,
LocalUserMediaViewModel,
@@ -63,45 +60,12 @@ interface TileProps {
maximised: boolean;
displayName: string;
nameTag: string;
showSpeakingIndicators: boolean;
}
interface MediaTileProps
extends TileProps,
Omit<ComponentProps<typeof animated.div>, "className"> {
vm: MediaViewModel;
videoEnabled: boolean;
videoFit: "contain" | "cover";
mirror: boolean;
nameTagLeadingIcon?: ReactNode;
primaryButton: ReactNode;
secondaryButton?: ReactNode;
}
const MediaTile = forwardRef<HTMLDivElement, MediaTileProps>(
({ vm, className, maximised, ...props }, ref) => {
const video = useObservableEagerState(vm.video);
const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning);
return (
<MediaView
ref={ref}
className={classNames(className, styles.tile)}
data-maximised={maximised}
video={video}
member={vm.member}
unencryptedWarning={unencryptedWarning}
{...props}
/>
);
},
);
MediaTile.displayName = "MediaTile";
interface UserMediaTileProps extends TileProps {
vm: UserMediaViewModel;
mirror: boolean;
showSpeakingIndicators: boolean;
menuStart?: ReactNode;
menuEnd?: ReactNode;
}
@@ -115,11 +79,14 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
menuEnd,
className,
nameTag,
maximised,
...props
},
ref,
) => {
const { t } = useTranslation();
const video = useObservableEagerState(vm.video);
const unencryptedWarning = useObservableEagerState(vm.unencryptedWarning);
const audioEnabled = useObservableEagerState(vm.audioEnabled);
const videoEnabled = useObservableEagerState(vm.videoEnabled);
const speaking = useObservableEagerState(vm.speaking);
@@ -148,12 +115,14 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
);
const tile = (
<MediaTile
<MediaView
ref={ref}
vm={vm}
video={video}
member={vm.member}
unencryptedWarning={unencryptedWarning}
videoEnabled={videoEnabled}
videoFit={cropVideo ? "cover" : "contain"}
className={classNames(className, {
className={classNames(className, styles.tile, {
[styles.speaking]: showSpeakingIndicators && speaking,
})}
nameTagLeadingIcon={
@@ -182,6 +151,7 @@ const UserMediaTile = forwardRef<HTMLDivElement, UserMediaTileProps>(
{menu}
</Menu>
}
data-maximised={maximised}
{...props}
/>
);
@@ -199,7 +169,6 @@ UserMediaTile.displayName = "UserMediaTile";
interface LocalUserMediaTileProps extends TileProps {
vm: LocalUserMediaViewModel;
onOpenProfile: () => void;
showSpeakingIndicators: boolean;
}
const LocalUserMediaTile = forwardRef<HTMLDivElement, LocalUserMediaTileProps>(
@@ -248,7 +217,6 @@ LocalUserMediaTile.displayName = "LocalUserMediaTile";
interface RemoteUserMediaTileProps extends TileProps {
vm: RemoteUserMediaViewModel;
showSpeakingIndicators: boolean;
}
const RemoteUserMediaTile = forwardRef<
@@ -303,53 +271,8 @@ const RemoteUserMediaTile = forwardRef<
RemoteUserMediaTile.displayName = "RemoteUserMediaTile";
interface ScreenShareTileProps extends TileProps {
vm: ScreenShareViewModel;
fullscreen: boolean;
onToggleFullscreen: (itemId: string) => void;
}
const ScreenShareTile = forwardRef<HTMLDivElement, ScreenShareTileProps>(
({ vm, fullscreen, onToggleFullscreen, ...props }, ref) => {
const { t } = useTranslation();
const onClickFullScreen = useCallback(
() => onToggleFullscreen(vm.id),
[onToggleFullscreen, vm],
);
const FullScreenIcon = fullscreen ? CollapseIcon : ExpandIcon;
return (
<MediaTile
ref={ref}
vm={vm}
videoEnabled
videoFit="contain"
mirror={false}
primaryButton={
!vm.local && (
<button
aria-label={
fullscreen
? t("video_tile.full_screen")
: t("video_tile.exit_full_screen")
}
onClick={onClickFullScreen}
>
<FullScreenIcon aria-hidden width={20} height={20} />
</button>
)
}
{...props}
/>
);
},
);
ScreenShareTile.displayName = "ScreenShareTile";
interface GridTileProps {
vm: MediaViewModel;
vm: UserMediaViewModel;
maximised: boolean;
fullscreen: boolean;
onToggleFullscreen: (itemId: string) => void;
@@ -375,19 +298,8 @@ export const GridTile = forwardRef<HTMLDivElement, GridTileProps>(
{...nameData}
/>
);
} else if (vm instanceof RemoteUserMediaViewModel) {
return <RemoteUserMediaTile ref={ref} vm={vm} {...props} {...nameData} />;
} else {
return (
<ScreenShareTile
ref={ref}
vm={vm}
fullscreen={fullscreen}
onToggleFullscreen={onToggleFullscreen}
{...props}
{...nameData}
/>
);
return <RemoteUserMediaTile ref={ref} vm={vm} {...props} {...nameData} />;
}
},
);

69
test/grid/LegacyGrid-test.ts
View File

@@ -1,69 +0,0 @@
/*
Copyright 2023-2024 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
import { TileDescriptor } from "../../src/state/CallViewModel";
import { Tile, reorderTiles } from "../../src/grid/LegacyGrid";
const alice: Tile<unknown> = {
key: "alice",
order: 0,
item: { local: false } as unknown as TileDescriptor<unknown>,
remove: false,
focused: false,
isPresenter: false,
isSpeaker: false,
hasVideo: true,
};
const bob: Tile<unknown> = {
key: "bob",
order: 1,
item: { local: false } as unknown as TileDescriptor<unknown>,
remove: false,
focused: false,
isPresenter: false,
isSpeaker: false,
hasVideo: false,
};
test("reorderTiles does not promote a non-speaker", () => {
const tiles = [{ ...alice }, { ...bob }];
reorderTiles(tiles, "spotlight", 1);
expect(tiles).toEqual([
expect.objectContaining({ key: "alice", order: 0 }),
expect.objectContaining({ key: "bob", order: 1 }),
]);
});
test("reorderTiles promotes a speaker into the visible area", () => {
const tiles = [{ ...alice }, { ...bob, isSpeaker: true }];
reorderTiles(tiles, "spotlight", 1);
expect(tiles).toEqual([
expect.objectContaining({ key: "alice", order: 1 }),
expect.objectContaining({ key: "bob", order: 0 }),
]);
});
test("reorderTiles keeps a promoted speaker in the visible area", () => {
const tiles = [
{ ...alice, order: 1 },
{ ...bob, isSpeaker: true, order: 0 },
];
reorderTiles(tiles, "spotlight", 1);
expect(tiles).toEqual([
expect.objectContaining({ key: "alice", order: 1 }),
expect.objectContaining({ key: "bob", order: 0 }),
]);
});