Merge branch 'livekit' into SimonBrandner/feat/gradient

src/App.tsx

@@ -30,7 +30,6 @@ import { LoginPage } from "./auth/LoginPage";
 import { RegisterPage } from "./auth/RegisterPage";
 import { RoomPage } from "./room/RoomPage";
 import { ClientProvider } from "./ClientContext";
-import { usePageFocusStyle } from "./usePageFocusStyle";
 import { SequenceDiagramViewerPage } from "./SequenceDiagramViewerPage";
 import { InspectorContextProvider } from "./room/GroupCallInspector";
 import { CrashView, LoadingView } from "./FullScreenView";
@@ -73,8 +72,6 @@ export default function App({ history }: AppProps) {
     });
   });
 
-  usePageFocusStyle();
-
   const errorPage = <CrashView />;
 
   return (

src/button/Button.module.css

@@ -50,14 +50,14 @@ limitations under the License.
   background-color: var(--cpd-color-text-action-accent);
 }
 
-.button:focus,
-.toolbarButton:focus,
-.toolbarButtonSecondary:focus,
-.iconButton:focus,
-.iconCopyButton:focus,
-.secondary:focus,
-.secondaryHangup:focus,
-.copyButton:focus {
+.button:focus-visible,
+.toolbarButton:focus-visible,
+.toolbarButtonSecondary:focus-visible,
+.iconButton:focus-visible,
+.iconCopyButton:focus-visible,
+.secondary:focus-visible,
+.secondaryHangup:focus-visible,
+.copyButton:focus-visible {
   outline: auto;
 }
 
@@ -74,7 +74,7 @@ limitations under the License.
   background-color: var(--cpd-color-bg-canvas-default);
   color: var(--cpd-color-icon-primary);
   border: 1px solid var(--cpd-color-gray-400);
-  box-shadow: 0px 1px 2px 0px rgba(16, 24, 40, 0.05);
+  box-shadow: var(--subtle-drop-shadow);
 }
 
 .toolbarButton.on,

src/index.css

@@ -39,21 +39,23 @@ limitations under the License.
   --font-size-title: calc(24px * var(--font-scale));
   --font-size-headline: calc(32px * var(--font-scale));
 
-  --cpd-color-border-accent: var(--cpd-color-green-1100);
   /* These colors are needed during the transitionary period between the old and
   new Compound design systems, but should be removed ASAP */
   --stopgap-color-on-solid-accent: var(--cpd-color-bg-canvas-default);
   --stopgap-background-85: rgba(255, 255, 255, 0.85);
   --stopgap-bgColor3: #444;
 
+  --cpd-color-border-accent: var(--cpd-color-green-800);
   /* The distance to inset non-full-width content from the edge of the window
   along the inline axis */
   --inline-content-inset: max(var(--cpd-space-4x), calc((100vw - 1180px) / 2));
-
+  --small-drop-shadow: 0px 1.2px 2.4px 0px rgba(0, 0, 0, 0.15);
+  --subtle-drop-shadow: 0px 1px 2px 0px rgba(16, 24, 40, 0.05);
   --background-gradient: url("graphics/backgroundGradient.svg");
 }
 
 .cpd-theme-dark {
+  --cpd-color-border-accent: var(--cpd-color-green-1100);
   --stopgap-color-on-solid-accent: var(--cpd-color-text-primary);
   --stopgap-background-85: rgba(16, 19, 23, 0.85);
 

src/livekit/useECConnectionState.ts

@@ -14,7 +14,12 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-import { ConnectionState, Room, RoomEvent } from "livekit-client";
+import {
+  AudioCaptureOptions,
+  ConnectionState,
+  Room,
+  RoomEvent,
+} from "livekit-client";
 import { useCallback, useEffect, useRef, useState } from "react";
 import { logger } from "matrix-js-sdk/src/logger";
 
@@ -42,7 +47,28 @@ function sfuConfigValid(sfuConfig?: SFUConfig): boolean {
   return Boolean(sfuConfig?.url) && Boolean(sfuConfig?.jwt);
 }
 
+async function doConnect(
+  livekitRoom: Room,
+  sfuConfig: SFUConfig,
+  audioEnabled: boolean,
+  audioOptions: AudioCaptureOptions
+): Promise<void> {
+  await livekitRoom!.connect(sfuConfig!.url, sfuConfig!.jwt);
+  const audioTracks = await livekitRoom!.localParticipant.createTracks({
+    audio: audioOptions,
+  });
+  if (audioTracks.length < 1) {
+    logger.info("Tried to pre-create local audio track but got no tracks");
+    return;
+  }
+  if (!audioEnabled) await audioTracks[0].mute();
+
+  await livekitRoom?.localParticipant.publishTrack(audioTracks[0]);
+}
+
 export function useECConnectionState(
+  initialAudioOptions: AudioCaptureOptions,
+  initialAudioEnabled: boolean,
   livekitRoom?: Room,
   sfuConfig?: SFUConfig
 ): ECConnectionState {
@@ -89,12 +115,33 @@ export function useECConnectionState(
       (async () => {
         setSwitchingFocus(true);
         await livekitRoom?.disconnect();
-        await livekitRoom?.connect(sfuConfig!.url, sfuConfig!.jwt);
+        await doConnect(
+          livekitRoom!,
+          sfuConfig!,
+          initialAudioEnabled,
+          initialAudioOptions
+        );
       })();
+    } else if (
+      !sfuConfigValid(currentSFUConfig.current) &&
+      sfuConfigValid(sfuConfig)
+    ) {
+      // if we're transitioning from an invalid config to a valid one (ie. connecting)
+      // then do an initial connection, including publishing the microphone track:
+      // livekit (by default) keeps the mic track open when you mute, but if you start muted,
+      // doesn't publish it until you unmute. We want to publish it from the start so we're
+      // always capturing audio: it helps keep bluetooth headsets in the right mode and
+      // mobile browsers to know we're doing a call.
+      doConnect(
+        livekitRoom!,
+        sfuConfig!,
+        initialAudioEnabled,
+        initialAudioOptions
+      );
     }
 
     currentSFUConfig.current = Object.assign({}, sfuConfig);
-  }, [sfuConfig, livekitRoom]);
+  }, [sfuConfig, livekitRoom, initialAudioOptions, initialAudioEnabled]);
 
   return isSwitchingFocus ? ECAddonConnectionState.ECSwitchingFocus : connState;
 }

src/livekit/useLiveKit.ts

@@ -108,9 +108,17 @@ export function useLiveKit(
     audio: initialMuteStates.current.audio.enabled,
     video: initialMuteStates.current.video.enabled,
     room: roomWithoutProps,
+    connect: false,
   });
 
-  const connectionState = useECConnectionState(room, sfuConfig);
+  const connectionState = useECConnectionState(
+    {
+      deviceId: initialDevices.current.audioOutput.selectedId,
+    },
+    initialMuteStates.current.audio.enabled,
+    room,
+    sfuConfig
+  );
 
   useEffect(() => {
     // Sync the requested mute states with LiveKit's mute states. We do it this

src/room/InCallView.tsx

@@ -27,7 +27,7 @@ import { ConnectionState, Room, Track } from "livekit-client";
 import { MatrixClient } from "matrix-js-sdk/src/client";
 import { RoomMember } from "matrix-js-sdk/src/models/room-member";
 import { Room as MatrixRoom } from "matrix-js-sdk/src/models/room";
-import { Ref, useCallback, useEffect, useMemo, useRef } from "react";
+import { Ref, useCallback, useEffect, useMemo, useRef, useState } from "react";
 import { useTranslation } from "react-i18next";
 import useMeasure from "react-use-measure";
 import { OverlayTriggerState } from "@react-stately/overlays";
@@ -73,7 +73,10 @@ import { MuteStates } from "./MuteStates";
 import { MatrixInfo } from "./VideoPreview";
 import { ShareButton } from "../button/ShareButton";
 import { LayoutToggle } from "./LayoutToggle";
-import { ECConnectionState } from "../livekit/useECConnectionState";
+import {
+  ECAddonConnectionState,
+  ECConnectionState,
+} from "../livekit/useECConnectionState";
 import { useOpenIDSFU } from "../livekit/openIDSFU";
 
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
@@ -82,6 +85,9 @@ const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 // For now we can disable screensharing in Safari.
 const isSafari = /^((?!chrome|android).)*safari/i.test(navigator.userAgent);
 
+// How long we wait after a focus switch before showing the real participant list again
+const POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS = 3000;
+
 export interface ActiveCallProps
   extends Omit<InCallViewProps, "livekitRoom" | "connState"> {
   e2eeConfig?: E2EEConfig;
@@ -236,7 +242,7 @@ export function InCallView({
   const reducedControls = boundsValid && bounds.width <= 340;
   const noControls = reducedControls && bounds.height <= 400;
 
-  const items = useParticipantTiles(livekitRoom, rtcSession.room);
+  const items = useParticipantTiles(livekitRoom, rtcSession.room, connState);
   const { fullscreenItem, toggleFullscreen, exitFullscreen } =
     useFullscreen(items);
 
@@ -382,8 +388,12 @@ export function InCallView({
       <div className={styles.footer}>
         {!mobile && !hideHeader && (
           <div className={styles.logo}>
-            <LogoMark width={24} height={24} />
-            <LogoType width={80} height={11} />
+            <LogoMark width={24} height={24} aria-hidden />
+            <LogoType
+              width={80}
+              height={11}
+              aria-label={import.meta.env.VITE_PRODUCT_NAME || "Element Call"}
+            />
           </div>
         )}
         <div className={styles.buttons}>{buttons}</div>
@@ -472,8 +482,11 @@ function findMatrixMember(
 
 function useParticipantTiles(
   livekitRoom: Room,
-  matrixRoom: MatrixRoom
+  matrixRoom: MatrixRoom,
+  connState: ECConnectionState
 ): TileDescriptor<ItemData>[] {
+  const previousTiles = useRef<TileDescriptor<ItemData>[]>([]);
+
   const sfuParticipants = useParticipants({
     room: livekitRoom,
   });
@@ -555,5 +568,44 @@ function useParticipantTiles(
     return allGhosts ? [] : tiles;
   }, [matrixRoom, sfuParticipants]);
 
-  return items;
+  // We carry over old tiles from the previous focus for some time after a focus switch
+  // so that the video tiles don't all disappear and reappear.
+  // This is set to true when the state transitions to Switching Focus and remains
+  // true for a short time after it changes (ie. connState is only switching focus for
+  // the time it takes us to reconnect to the conference).
+  // If there are still members that haven't reconnected after that time, they'll just
+  // appear to disconnect and will reappear once they reconnect.
+  const [isSwitchingFocus, setIsSwitchingFocus] = useState(false);
+
+  useEffect(() => {
+    if (connState === ECAddonConnectionState.ECSwitchingFocus) {
+      setIsSwitchingFocus(true);
+    } else if (isSwitchingFocus) {
+      setTimeout(() => {
+        setIsSwitchingFocus(false);
+      }, POST_FOCUS_PARTICIPANT_UPDATE_DELAY_MS);
+    }
+  }, [connState, setIsSwitchingFocus, isSwitchingFocus]);
+
+  if (
+    connState === ECAddonConnectionState.ECSwitchingFocus ||
+    isSwitchingFocus
+  ) {
+    logger.debug("Switching focus: injecting previous tiles");
+
+    // inject the previous tile for members that haven't rejoined yet
+    const newItems = items.slice(0);
+    const rejoined = new Set(newItems.map((p) => p.id));
+
+    for (const prevTile of previousTiles.current) {
+      if (!rejoined.has(prevTile.id)) {
+        newItems.push(prevTile);
+      }
+    }
+
+    return newItems;
+  } else {
+    previousTiles.current = items;
+    return items;
+  }
 }

src/room/LayoutToggle.module.css

@@ -34,7 +34,7 @@ limitations under the License.
   border-radius: var(--cpd-radius-pill-effect);
   color: var(--cpd-color-icon-primary);
   background: var(--cpd-color-bg-action-secondary-rest);
-  box-shadow: 0px 1.2px 2.4px 0px rgba(0, 0, 0, 0.15);
+  box-shadow: var(--small-drop-shadow);
 }
 
 @media (hover: hover) {

src/usePageFocusStyle.module.css

@@ -1,19 +0,0 @@
-/*
-Copyright 2022 New Vector Ltd
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-.hideFocus * {
-  outline: none !important;
-}

src/usePageFocusStyle.ts

@@ -1,39 +0,0 @@
-/*
-Copyright 2022 New Vector Ltd
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-import { useEffect } from "react";
-import { useFocusVisible } from "@react-aria/interactions";
-
-import styles from "./usePageFocusStyle.module.css";
-
-export function usePageFocusStyle(): void {
-  const { isFocusVisible } = useFocusVisible();
-
-  useEffect(() => {
-    const classList = document.body.classList;
-    const hasClass = classList.contains(styles.hideFocus);
-
-    if (isFocusVisible && hasClass) {
-      classList.remove(styles.hideFocus);
-    } else if (!isFocusVisible && !hasClass) {
-      classList.add(styles.hideFocus);
-    }
-
-    return () => {
-      classList.remove(styles.hideFocus);
-    };
-  }, [isFocusVisible]);
-}

src/video-grid/BigGrid.module.css

@@ -16,14 +16,13 @@ limitations under the License.
 
 .bigGrid {
   display: grid;
-  grid-auto-rows: 163px;
-  gap: 8px;
+  grid-auto-rows: 130px;
+  gap: var(--cpd-space-2x);
 }
 
 @media (min-width: 800px) {
   .bigGrid {
-    grid-auto-rows: 183px;
-    column-gap: 18px;
-    row-gap: 21px;
+    grid-auto-rows: 135px;
+    gap: var(--cpd-space-5x);
   }
 }

src/video-grid/BigGrid.tsx

@@ -957,7 +957,7 @@ function updateTiles(g: Grid, tiles: TileDescriptor<unknown>[]): Grid {
 }
 
 function updateBounds(g: Grid, bounds: RectReadOnly): Grid {
-  const columns = Math.max(2, Math.floor(bounds.width * 0.0045));
+  const columns = Math.max(2, Math.floor(bounds.width * 0.0055));
   return columns === g.columns ? g : resize(g, columns);
 }
 

src/video-grid/NewVideoGrid.tsx

@@ -178,7 +178,7 @@ export function NewVideoGrid<T>({
       from: ({ x, y, width, height }: Tile<T>) => ({
         opacity: 0,
         scale: 0,
-        shadow: 1,
+        shadow: 0,
         shadowSpread: 0,
         zIndex: 1,
         x,
@@ -221,7 +221,7 @@ export function NewVideoGrid<T>({
           ? {
               scale: 1,
               zIndex: 1,
-              shadow: 1,
+              shadow: 0,
               x: tile.x,
               y: tile.y,
               width: tile.width,

src/video-grid/VideoGrid.tsx

@@ -1082,7 +1082,7 @@ export function VideoGrid<T>({
             y?: number;
             width?: number;
             height?: number;
-          } = { shadow: 1, scale: 0, opacity: 0 };
+          } = { shadow: 0, scale: 0, opacity: 0 };
           let reset = false;
 
           if (!tilePositionsWereValid) {
@@ -1105,7 +1105,7 @@ export function VideoGrid<T>({
             scale: remove ? 0 : 1,
             opacity: remove ? 0 : 1,
             zIndex: tilePosition.zIndex,
-            shadow: 1,
+            shadow: oneOnOneLayout && tile.item.local ? 1 : 0,
             shadowSpread: oneOnOneLayout && tile.item.local ? 1 : 0,
             from,
             reset,

src/video-grid/VideoTile.module.css

@@ -20,14 +20,11 @@ limitations under the License.
   top: 0;
   container-name: videoTile;
   container-type: size;
-  --tileRadius: 8px;
-  border-radius: var(--tileRadius);
+  border-radius: var(--cpd-space-4x);
   overflow: hidden;
   cursor: pointer;
-
-  /* HACK: This has no visual effect due to the short duration, but allows the
-  JS to detect movement via the transform property for audio spatialization */
-  transition: transform 0.000000001s;
+  outline: 2px solid rgba(0, 0, 0, 0);
+  transition: outline-radius ease 0.15s, outline-color ease 0.15s;
 }
 
 .videoTile * {
@@ -45,21 +42,14 @@ limitations under the License.
   transform: scaleX(-1);
 }
 
-.videoTile::after {
-  position: absolute;
-  top: -1px;
-  left: -1px;
-  right: -1px;
-  bottom: -1px;
-  content: "";
-  border-radius: var(--tileRadius);
-  box-shadow: inset 0 0 0 4px var(--cpd-color-border-accent) !important;
-  opacity: 0;
-  transition: opacity ease 0.15s;
+.videoTile.speaking {
+  outline: 4px solid var(--cpd-color-border-accent);
 }
 
-.videoTile.speaking::after {
-  opacity: 1;
+@media (hover: hover) {
+  .videoTile:hover {
+    outline: 2px solid var(--cpd-color-gray-1400);
+  }
 }
 
 .videoTile.maximised {
@@ -73,30 +63,47 @@ limitations under the License.
   object-fit: contain;
 }
 
-.infoBubble {
+.nameTag {
   position: absolute;
-  height: 24px;
-  padding: 0 8px;
+  inset-inline-start: var(--cpd-space-1x);
+  inset-block-end: var(--cpd-space-1x);
+  padding: var(--cpd-space-1x);
+  padding-block: var(--cpd-space-1x);
   color: var(--cpd-color-text-primary);
-  background-color: var(--stopgap-background-85);
+  /* TODO: un-hardcode this color. It comes from the dark theme. */
+  background-color: rgba(237, 244, 252, 0.79);
   display: flex;
   align-items: center;
-  justify-content: center;
-  border-radius: 4px;
+  border-radius: var(--cpd-radius-pill-effect);
   user-select: none;
   max-width: calc(100% - 48px);
   overflow: hidden;
   z-index: 1;
+  box-shadow: var(--small-drop-shadow);
 }
 
-.infoBubble > svg {
-  height: 16px;
-  width: 16px;
-  margin-right: 4px;
+:global(.cpd-theme-dark) .nameTag {
+  /* TODO: un-hardcode this color. It comes from the light theme. */
+  background-color: rgba(2, 7, 13, 0.77);
 }
 
-.infoBubble > svg * {
-  fill: var(--cpd-color-icon-primary);
+.nameTag > svg {
+  flex-shrink: 0;
+}
+
+.nameTag > svg[data-muted="true"] {
+  color: var(--cpd-color-icon-secondary);
+}
+
+.nameTag > svg[data-muted="false"] {
+  color: var(--cpd-color-icon-primary);
+}
+
+.nameTag span {
+  text-overflow: ellipsis;
+  overflow: hidden;
+  white-space: nowrap;
+  padding-inline: var(--cpd-space-2x);
 }
 
 .toolbar {
@@ -137,24 +144,6 @@ limitations under the License.
   height: 16px;
 }
 
-.memberName {
-  left: 16px;
-  bottom: 16px;
-}
-
-.memberName > :last-child {
-  margin-right: 0px;
-}
-
-.memberName span {
-  font-size: var(--font-size-caption);
-  font-weight: 400;
-  line-height: var(--font-size-body);
-  text-overflow: ellipsis;
-  overflow: hidden;
-  white-space: nowrap;
-}
-
 .videoMutedOverlay {
   width: 100%;
   height: 100%;
@@ -192,12 +181,6 @@ limitations under the License.
   background-color: rgba(0, 0, 0, 0.5);
 }
 
-@media (min-width: 800px) {
-  .videoTile {
-    --tileRadius: 20px;
-  }
-}
-
 /* CSS makes us put a condition here, even though all we want to do is
 unconditionally select the container so we can use cqmin units */
 @container videoTile (width > 0) {

src/video-grid/VideoTile.tsx

@@ -28,11 +28,12 @@ import {
   RoomMember,
   RoomMemberEvent,
 } from "matrix-js-sdk/src/models/room-member";
+import { ReactComponent as MicOnSolidIcon } from "@vector-im/compound-design-tokens/icons/mic-on-solid.svg";
+import { ReactComponent as MicOffSolidIcon } from "@vector-im/compound-design-tokens/icons/mic-off-solid.svg";
+import { Text } from "@vector-im/compound-web";
 
 import { Avatar } from "../Avatar";
 import styles from "./VideoTile.module.css";
-import { ReactComponent as MicIcon } from "../icons/Mic.svg";
-import { ReactComponent as MicMutedIcon } from "../icons/MicMuted.svg";
 import { useReactiveState } from "../useReactiveState";
 import { AudioButton, FullscreenButton } from "../button/Button";
 import { useModalTriggerState } from "../Modal";
@@ -102,12 +103,15 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
       }
     }, [member, setDisplayName]);
 
-    const { isMuted: microphoneMuted } = useMediaTrack(
-      content === TileContent.UserMedia
-        ? Track.Source.Microphone
-        : Track.Source.ScreenShareAudio,
-      sfuParticipant
-    );
+    const muted =
+      useMediaTrack(
+        content === TileContent.UserMedia
+          ? Track.Source.Microphone
+          : Track.Source.ScreenShareAudio,
+        sfuParticipant
+      ).isMuted !== false;
+
+    const MicIcon = muted ? MicOffSolidIcon : MicOnSolidIcon;
 
     const onFullscreen = useCallback(() => {
       onToggleFullscreen(data.id);
@@ -153,7 +157,6 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
             sfuParticipant.isSpeaking &&
             content === TileContent.UserMedia &&
             showSpeakingIndicator,
-          [styles.muted]: microphoneMuted,
           [styles.screenshare]: content === TileContent.ScreenShare,
           [styles.maximised]: maximised,
         })}
@@ -182,9 +185,16 @@ export const VideoTile = forwardRef<HTMLDivElement, Props>(
             <span>{t("{{displayName}} is presenting", { displayName })}</span>
           </div>
         ) : (
-          <div className={classNames(styles.infoBubble, styles.memberName)}>
-            {microphoneMuted === false ? <MicIcon /> : <MicMutedIcon />}
-            <span title={displayName}>{displayName}</span>
+          <div className={styles.nameTag}>
+            <MicIcon
+              width={20}
+              height={20}
+              aria-label={muted ? t("Microphone off") : t("Microphone on")}
+              data-muted={muted}
+            />
+            <Text as="span" size="sm" weight="medium">
+              {sfuParticipant.isLocal ? t("You") : displayName}
+            </Text>
             {showConnectionStats && (
               <ConnectionQualityIndicator participant={sfuParticipant} />
             )}