rzerres/element-call
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Implement the new unified grid layout

Browse Source 
Here I've implemented an MVP for the new unified grid layout, which scales smoothly up to arbitrarily many participants. It doesn't yet have a special 1:1 layout, so in spotlight mode and 1:1s, we will still fall back to the legacy grid systems.

Things that happened along the way:
- The part of VideoTile that is common to both spotlight and grid tiles, I refactored into MediaView
- VideoTile renamed to GridTile
- Added SpotlightTile for the new, glassy spotlight designs
- NewVideoGrid renamed to Grid, and refactored to be even more generic
- I extracted the media name logic into a custom React hook
- Deleted the BigGrid experiment
This commit is contained in:
Robin
2024-05-02 18:44:36 -04:00
parent 5ad2a27a92
commit 20602c122b
32 changed files with 1863 additions and 2586 deletions
Download Patch File Download Diff File Expand all files Collapse all files

37
src/room/InCallView.module.css
View File

@@ -1,5 +1,5 @@
/*
Copyright 2021 New Vector Ltd
Copyright 2021-2024 New Vector Ltd
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@@ -19,6 +19,7 @@ limitations under the License.
flex-direction: column;
height: 100%;
width: 100%;
overflow-y: auto;
}
.controlsOverlay {
@@ -46,16 +47,28 @@ limitations under the License.
margin-bottom: 0;
}
.header {
position: sticky;
inset-block-start: 0;
z-index: 1;
background: linear-gradient(
0deg,
rgba(0, 0, 0, 0) 0%,
var(--cpd-color-bg-canvas-default) 100%
);
}
.footer {
position: sticky;
inset-block-end: 0;
z-index: 1;
display: grid;
grid-template-columns: 1fr auto 1fr;
grid-template-areas: "logo buttons layout";
align-items: center;
gap: var(--cpd-space-3x);
padding-block: var(--cpd-space-4x);
padding-inline: var(--inline-content-inset);
margin-inline: var(--inline-content-inset);
background: linear-gradient(
180deg,
rgba(0, 0, 0, 0) 0%,
@@ -109,3 +122,23 @@ limitations under the License.
.footerHidden {
display: none;
}
.fixedGrid {
position: absolute;
inline-size: calc(100% - 2 * var(--inline-content-inset));
align-self: center;
/* Disable pointer events so the overlay doesn't block interaction with
elements behind it */
pointer-events: none;
}
.fixedGrid > :not(:first-child) {
pointer-events: initial;
}
.scrollingGrid {
position: relative;
flex-grow: 1;
inline-size: calc(100% - 2 * var(--inline-content-inset));
align-self: center;
}

307
src/room/InCallView.tsx
View File

@@ -14,7 +14,6 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
import { ResizeObserver } from "@juggle/resize-observer";
import {
RoomAudioRenderer,
RoomContext,
@@ -26,19 +25,19 @@ import { ConnectionState, Room, Track } from "livekit-client";
import { MatrixClient } from "matrix-js-sdk/src/client";
import {
FC,
ReactNode,
Ref,
forwardRef,
useCallback,
useEffect,
useMemo,
useRef,
useState,
} from "react";
import { useTranslation } from "react-i18next";
import useMeasure from "react-use-measure";
import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
import classNames from "classnames";
import { useStateObservable } from "@react-rxjs/core";
import { state, useStateObservable } from "@react-rxjs/core";
import { BehaviorSubject } from "rxjs";
import { useTranslation } from "react-i18next";
import LogoMark from "../icons/LogoMark.svg?react";
import LogoType from "../icons/LogoType.svg?react";
@@ -51,21 +50,19 @@ import {
SettingsButton,
} from "../button";
import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header";
import { useVideoGridLayout, VideoGrid } from "../video-grid/VideoGrid";
import { LegacyGrid, useLegacyGridLayout } from "../grid/LegacyGrid";
import { useUrlParams } from "../UrlParams";
import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts";
import { usePrefersReducedMotion } from "../usePrefersReducedMotion";
import { ElementWidgetActions, widget } from "../widget";
import styles from "./InCallView.module.css";
import { VideoTile } from "../video-grid/VideoTile";
import { NewVideoGrid } from "../video-grid/NewVideoGrid";
import { GridTile } from "../tile/GridTile";
import { OTelGroupCallMembership } from "../otel/OTelGroupCallMembership";
import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal";
import { useRageshakeRequestModal } from "../settings/submit-rageshake";
import { RageshakeRequestModal } from "./RageshakeRequestModal";
import { useLiveKit } from "../livekit/useLiveKit";
import { useFullscreen } from "./useFullscreen";
import { useLayoutStates } from "../video-grid/Layout";
import { useWakeLock } from "../useWakeLock";
import { useMergedRefs } from "../useMergedRefs";
import { MuteStates } from "./MuteStates";
@@ -74,13 +71,33 @@ import { InviteButton } from "../button/InviteButton";
import { LayoutToggle } from "./LayoutToggle";
import { ECConnectionState } from "../livekit/useECConnectionState";
import { useOpenIDSFU } from "../livekit/openIDSFU";
import { useCallViewModel } from "../state/CallViewModel";
import {
GridMode,
TileDescriptor,
useCallViewModel,
} from "../state/CallViewModel";
import { subscribe } from "../state/subscribe";
import { Grid, TileProps } from "../grid/Grid";
import { MediaViewModel } from "../state/MediaViewModel";
import { gridLayoutSystems } from "../grid/GridLayout";
import { useObservable } from "../state/useObservable";
import { useInitial } from "../useInitial";
import { SpotlightTile } from "../tile/SpotlightTile";
import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
import { E2eeType } from "../e2ee/e2eeType";
const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
export interface Alignment {
inline: "start" | "end";
block: "start" | "end";
}
const defaultAlignment: Alignment = { inline: "end", block: "end" };
const dummySpotlightItem = {
id: "spotlight",
} as TileDescriptor<MediaViewModel>;
export interface ActiveCallProps
extends Omit<InCallViewProps, "livekitRoom" | "connState"> {
@@ -153,7 +170,7 @@ export const InCallView: FC<InCallViewProps> = subscribe(
}, [connState, onLeave]);
const containerRef1 = useRef<HTMLDivElement | null>(null);
const [containerRef2, bounds] = useMeasure({ polyfill: ResizeObserver });
const [containerRef2, bounds] = useMeasure();
const boundsValid = bounds.height > 0;
// Merge the refs so they can attach to the same element
const containerRef = useMergedRefs(containerRef1, containerRef2);
@@ -164,9 +181,8 @@ export const InCallView: FC<InCallViewProps> = subscribe(
room: livekitRoom,
},
);
const { layout, setLayout } = useVideoGridLayout(
screenSharingTracks.length > 0,
);
const { layout: legacyLayout, setLayout: setLegacyLayout } =
useLegacyGridLayout(screenSharingTracks.length > 0);
const { hideScreensharing, showControls } = useUrlParams();
@@ -194,23 +210,23 @@ export const InCallView: FC<InCallViewProps> = subscribe(
useEffect(() => {
widget?.api.transport.send(
layout === "grid"
legacyLayout === "grid"
? ElementWidgetActions.TileLayout
: ElementWidgetActions.SpotlightLayout,
{},
);
}, [layout]);
}, [legacyLayout]);
useEffect(() => {
if (widget) {
const onTileLayout = (ev: CustomEvent<IWidgetApiRequest>): void => {
setLayout("grid");
setLegacyLayout("grid");
widget!.api.transport.reply(ev.detail, {});
};
const onSpotlightLayout = (
ev: CustomEvent<IWidgetApiRequest>,
): void => {
setLayout("spotlight");
setLegacyLayout("spotlight");
widget!.api.transport.reply(ev.detail, {});
};
@@ -231,7 +247,7 @@ export const InCallView: FC<InCallViewProps> = subscribe(
);
};
}
}, [setLayout]);
}, [setLegacyLayout]);
const mobile = boundsValid && bounds.width <= 660;
const reducedControls = boundsValid && bounds.width <= 340;
@@ -244,8 +260,21 @@ export const InCallView: FC<InCallViewProps> = subscribe(
connState,
);
const items = useStateObservable(vm.tiles);
const layout = useStateObservable(vm.layout);
const hasSpotlight = layout.spotlight !== undefined;
// Hack: We insert a dummy "spotlight" tile into the tiles we pass to
// useFullscreen so that we can control the fullscreen state of the
// spotlight tile in the new layouts with this same hook.
const fullscreenItems = useMemo(
() => [...items, ...(hasSpotlight ? [dummySpotlightItem] : [])],
[items, hasSpotlight],
);
const { fullscreenItem, toggleFullscreen, exitFullscreen } =
useFullscreen(items);
useFullscreen(fullscreenItems);
const toggleSpotlightFullscreen = useCallback(
() => toggleFullscreen("spotlight"),
[toggleFullscreen],
);
// The maximised participant: either the participant that the user has
// manually put in fullscreen, or the focused (active) participant if the
@@ -259,66 +288,8 @@ export const InCallView: FC<InCallViewProps> = subscribe(
[fullscreenItem, noControls, items],
);
const Grid =
items.length > 12 && layout === "grid" ? NewVideoGrid : VideoGrid;
const prefersReducedMotion = usePrefersReducedMotion();
// This state is lifted out of NewVideoGrid so that layout states can be
// restored after a layout switch or upon exiting fullscreen
const layoutStates = useLayoutStates();
const renderContent = (): JSX.Element => {
if (items.length === 0) {
return (
<div className={styles.centerMessage}>
<p>{t("waiting_for_participants")}</p>
</div>
);
}
if (maximisedParticipant) {
return (
<VideoTile
vm={maximisedParticipant.data}
maximised={true}
fullscreen={maximisedParticipant === fullscreenItem}
onToggleFullscreen={toggleFullscreen}
targetHeight={bounds.height}
targetWidth={bounds.width}
key={maximisedParticipant.id}
showSpeakingIndicator={false}
onOpenProfile={openProfile}
/>
);
}
return (
<Grid
items={items}
layout={layout}
disableAnimations={prefersReducedMotion || isSafari}
layoutStates={layoutStates}
>
{({ data: vm, ...props }): ReactNode => (
<VideoTile
vm={vm}
maximised={false}
fullscreen={false}
onToggleFullscreen={toggleFullscreen}
showSpeakingIndicator={items.length > 2}
onOpenProfile={openProfile}
{...props}
ref={props.ref as Ref<HTMLDivElement>}
/>
)}
</Grid>
);
};
const rageshakeRequestModalProps = useRageshakeRequestModal(
rtcSession.room.roomId,
);
const [settingsModalOpen, setSettingsModalOpen] = useState(false);
const [settingsTab, setSettingsTab] = useState(defaultSettingsTab);
@@ -336,6 +307,169 @@ export const InCallView: FC<InCallViewProps> = subscribe(
setSettingsModalOpen(true);
}, [setSettingsTab, setSettingsModalOpen]);
const [headerRef, headerBounds] = useMeasure();
const [footerRef, footerBounds] = useMeasure();
const gridBounds = useMemo(
() => ({
width: footerBounds.width,
height: bounds.height - headerBounds.height - footerBounds.height,
}),
[
footerBounds.width,
bounds.height,
headerBounds.height,
footerBounds.height,
],
);
const gridBoundsObservable = useObservable(gridBounds);
const floatingAlignment = useInitial(
() => new BehaviorSubject(defaultAlignment),
);
const { fixed, scrolling } = useInitial(() =>
gridLayoutSystems(state(gridBoundsObservable), floatingAlignment),
);
const setGridMode = useCallback(
(mode: GridMode) => {
setLegacyLayout(mode);
vm.setGridMode(mode);
},
[setLegacyLayout, vm],
);
const showSpeakingIndicators =
layout.type === "spotlight" ||
(layout.type === "grid" && layout.grid.length > 2);
const SpotlightTileView = useMemo(
() =>
forwardRef<HTMLDivElement, TileProps<MediaViewModel[], HTMLDivElement>>(
function SpotlightTileView(
{ className, style, targetWidth, targetHeight, model },
ref,
) {
return (
<SpotlightTile
ref={ref}
vms={model}
maximised={false}
fullscreen={false}
onToggleFullscreen={toggleSpotlightFullscreen}
targetWidth={targetWidth}
targetHeight={targetHeight}
className={className}
style={style}
/>
);
},
),
[toggleSpotlightFullscreen],
);
const GridTileView = useMemo(
() =>
forwardRef<HTMLDivElement, TileProps<MediaViewModel, HTMLDivElement>>(
function GridTileView(
{ className, style, targetWidth, targetHeight, model },
ref,
) {
return (
<GridTile
ref={ref}
vm={model}
maximised={false}
fullscreen={false}
onToggleFullscreen={toggleFullscreen}
onOpenProfile={openProfile}
targetWidth={targetWidth}
targetHeight={targetHeight}
className={className}
style={style}
showSpeakingIndicator={showSpeakingIndicators}
/>
);
},
),
[toggleFullscreen, openProfile, showSpeakingIndicators],
);
const renderContent = (): JSX.Element => {
if (items.length === 0) {
return (
<div className={styles.centerMessage}>
<p>{t("waiting_for_participants")}</p>
</div>
);
}
if (maximisedParticipant !== null) {
const fullscreen = maximisedParticipant === fullscreenItem;
if (maximisedParticipant.id === "spotlight") {
return (
<SpotlightTile
vms={layout.spotlight!}
maximised={true}
fullscreen={fullscreen}
onToggleFullscreen={toggleSpotlightFullscreen}
targetWidth={gridBounds.height}
targetHeight={gridBounds.width}
/>
);
}
return (
<GridTile
vm={maximisedParticipant.data}
maximised={true}
fullscreen={fullscreen}
onToggleFullscreen={toggleFullscreen}
targetHeight={gridBounds.height}
targetWidth={gridBounds.width}
key={maximisedParticipant.id}
showSpeakingIndicator={false}
onOpenProfile={openProfile}
/>
);
}
// The only new layout we've implemented so far is grid layout for non-1:1
// calls. All other layouts use the legacy grid system for now.
if (
legacyLayout === "grid" &&
layout.type === "grid" &&
!(layout.grid.length === 2 && layout.spotlight === undefined)
) {
return (
<>
<Grid
className={styles.scrollingGrid}
model={layout}
system={scrolling}
Tile={GridTileView}
/>
<Grid
className={styles.fixedGrid}
style={{ insetBlockStart: headerBounds.bottom }}
model={layout}
system={fixed}
Tile={SpotlightTileView}
/>
</>
);
} else {
return (
<LegacyGrid
items={items}
layout={legacyLayout}
disableAnimations={prefersReducedMotion}
Tile={GridTileView}
/>
);
}
};
const rageshakeRequestModalProps = useRageshakeRequestModal(
rtcSession.room.roomId,
);
const toggleScreensharing = useCallback(async () => {
exitFullscreen();
await localParticipant.setScreenShareEnabled(!isScreenShareEnabled, {
@@ -395,6 +529,7 @@ export const InCallView: FC<InCallViewProps> = subscribe(
);
footer = (
<div
ref={footerRef}
className={classNames(
showControls
? styles.footer
@@ -417,8 +552,8 @@ export const InCallView: FC<InCallViewProps> = subscribe(
{!mobile && !hideHeader && showControls && (
<LayoutToggle
className={styles.layout}
layout={layout}
setLayout={setLayout}
layout={legacyLayout}
setLayout={setGridMode}
/>
)}
</div>
@@ -428,7 +563,7 @@ export const InCallView: FC<InCallViewProps> = subscribe(
return (
<div className={styles.inRoom} ref={containerRef}>
{!hideHeader && maximisedParticipant === null && (
<Header>
<Header className={styles.header} ref={headerRef}>
<LeftNav>
<RoomHeaderInfo
id={matrixInfo.roomId}
@@ -445,11 +580,9 @@ export const InCallView: FC<InCallViewProps> = subscribe(
</RightNav>
</Header>
)}
<div className={styles.controlsOverlay}>
<RoomAudioRenderer />
{renderContent()}
{footer}
</div>
<RoomAudioRenderer />
{renderContent()}
{footer}
{!noControls && (
<RageshakeRequestModal {...rageshakeRequestModalProps} />
)}

23
src/room/VideoPreview.tsx
View File

@@ -18,11 +18,7 @@ import { useEffect, useMemo, useRef, FC, ReactNode, useCallback } from "react";
import useMeasure from "react-use-measure";
import { ResizeObserver } from "@juggle/resize-observer";
import { usePreviewTracks } from "@livekit/components-react";
import {
CreateLocalTracksOptions,
LocalVideoTrack,
Track,
} from "livekit-client";
import { LocalVideoTrack, Track } from "livekit-client";
import classNames from "classnames";
import { logger } from "matrix-js-sdk/src/logger";
import { Glass } from "@vector-im/compound-web";
@@ -32,6 +28,7 @@ import styles from "./VideoPreview.module.css";
import { useMediaDevices } from "../livekit/MediaDevicesContext";
import { MuteStates } from "./MuteStates";
import { useMediaQuery } from "../useMediaQuery";
import { useInitial } from "../useInitial";
import { EncryptionSystem } from "../e2ee/sharedKeyManagement";
export type MatrixInfo = {
@@ -63,10 +60,10 @@ export const VideoPreview: FC<Props> = ({
// Capture the audio options as they were when we first mounted, because
// we're not doing anything with the audio anyway so we don't need to
// re-open the devices when they change (see below).
const initialAudioOptions = useRef<CreateLocalTracksOptions["audio"]>();
initialAudioOptions.current ??= muteStates.audio.enabled && {
deviceId: devices.audioInput.selectedId,
};
const initialAudioOptions = useInitial(
() =>
muteStates.audio.enabled && { deviceId: devices.audioInput.selectedId },
);
const localTrackOptions = useMemo(
() => ({
@@ -76,12 +73,16 @@ export const VideoPreview: FC<Props> = ({
// reference the initial values here.
// We also pass in a clone because livekit mutates the object passed in,
// which would cause the devices to be re-opened on the next render.
audio: Object.assign({}, initialAudioOptions.current),
audio: Object.assign({}, initialAudioOptions),
video: muteStates.video.enabled && {
deviceId: devices.videoInput.selectedId,
},
}),
[devices.videoInput.selectedId, muteStates.video.enabled],
[
initialAudioOptions,
devices.videoInput.selectedId,
muteStates.video.enabled,
],
);
const onError = useCallback(