Merge remote-tracking branch 'origin/livekit' into dbkr/react_to_livekit_disconnect

src/App.tsx

@@ -29,6 +29,7 @@ import { usePageFocusStyle } from "./usePageFocusStyle";
 import { SequenceDiagramViewerPage } from "./SequenceDiagramViewerPage";
 import { InspectorContextProvider } from "./room/GroupCallInspector";
 import { CrashView, LoadingView } from "./FullScreenView";
+import { DisconnectedBanner } from "./DisconnectedBanner";
 import { Initializer } from "./initializer";
 
 const SentryRoute = Sentry.withSentryRouting(Route);
@@ -60,6 +61,7 @@ export default function App({ history }: AppProps) {
             <InspectorContextProvider>
               <Sentry.ErrorBoundary fallback={errorPage}>
                 <OverlayProvider>
+                  <DisconnectedBanner />
                   <Switch>
                     <SentryRoute exact path="/">
                       <HomePage />

src/ClientContext.tsx

@@ -25,9 +25,10 @@ import {
   useMemo,
 } from "react";
 import { useHistory } from "react-router-dom";
-import { MatrixClient } from "matrix-js-sdk/src/client";
+import { ClientEvent, MatrixClient } from "matrix-js-sdk/src/client";
 import { logger } from "matrix-js-sdk/src/logger";
 import { useTranslation } from "react-i18next";
+import { ISyncStateData, SyncState } from "matrix-js-sdk/src/sync";
 
 import { ErrorView } from "./FullScreenView";
 import {
@@ -56,6 +57,9 @@ export type ClientState = ValidClientState | ErrorState;
 export type ValidClientState = {
   state: "valid";
   authenticated?: AuthenticatedClient;
+  // 'Disconnected' rather than 'connected' because it tracks specifically
+  // whether the client is supposed to be connected but is not
+  disconnected: boolean;
   setClient: (params?: SetClientParams) => void;
 };
 
@@ -149,8 +153,9 @@ interface Props {
 export const ClientProvider: FC<Props> = ({ children }) => {
   const history = useHistory();
 
+  // null = signed out, undefined = loading
   const [initClientState, setInitClientState] = useState<
-    InitResult | undefined
+    InitResult | null | undefined
   >(undefined);
 
   const initializing = useRef(false);
@@ -162,14 +167,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
     initializing.current = true;
 
     loadClient()
-      .then((maybeClient) => {
-        if (!maybeClient) {
-          logger.error("Failed to initialize client");
-          return;
-        }
-
-        setInitClientState(maybeClient);
-      })
+      .then(setInitClientState)
       .catch((err) => logger.error(err))
       .finally(() => (initializing.current = false));
   }, []);
@@ -264,23 +262,46 @@ export const ClientProvider: FC<Props> = ({ children }) => {
     }, [initClientState?.client, setAlreadyOpenedErr, t])
   );
 
-  const state: ClientState = useMemo(() => {
+  const [isDisconnected, setIsDisconnected] = useState(false);
+
+  const state: ClientState | undefined = useMemo(() => {
     if (alreadyOpenedErr) {
       return { state: "error", error: alreadyOpenedErr };
     }
 
-    let authenticated = undefined;
-    if (initClientState) {
-      authenticated = {
-        client: initClientState.client,
-        isPasswordlessUser: initClientState.passwordlessUser,
-        changePassword,
-        logout,
-      };
-    }
+    if (initClientState === undefined) return undefined;
 
-    return { state: "valid", authenticated, setClient };
-  }, [alreadyOpenedErr, changePassword, initClientState, logout, setClient]);
+    const authenticated =
+      initClientState === null
+        ? undefined
+        : {
+            client: initClientState.client,
+            isPasswordlessUser: initClientState.passwordlessUser,
+            changePassword,
+            logout,
+          };
+
+    return {
+      state: "valid",
+      authenticated,
+      setClient,
+      disconnected: isDisconnected,
+    };
+  }, [
+    alreadyOpenedErr,
+    changePassword,
+    initClientState,
+    logout,
+    setClient,
+    isDisconnected,
+  ]);
+
+  const onSync = useCallback(
+    (state: SyncState, _old: SyncState | null, data?: ISyncStateData) => {
+      setIsDisconnected(clientIsDisconnected(state, data));
+    },
+    []
+  );
 
   useEffect(() => {
     if (!initClientState) {
@@ -292,7 +313,17 @@ export const ClientProvider: FC<Props> = ({ children }) => {
 
     if (PosthogAnalytics.hasInstance())
       PosthogAnalytics.instance.onLoginStatusChanged();
-  }, [initClientState]);
+
+    if (initClientState.client) {
+      initClientState.client.on(ClientEvent.Sync, onSync);
+    }
+
+    return () => {
+      if (initClientState.client) {
+        initClientState.client.removeListener(ClientEvent.Sync, onSync);
+      }
+    };
+  }, [initClientState, onSync]);
 
   if (alreadyOpenedErr) {
     return <ErrorView error={alreadyOpenedErr} />;
@@ -308,7 +339,7 @@ type InitResult = {
   passwordlessUser: boolean;
 };
 
-async function loadClient(): Promise<InitResult> {
+async function loadClient(): Promise<InitResult | null> {
   if (widget) {
     // We're inside a widget, so let's engage *matryoshka mode*
     logger.log("Using a matryoshka client");
@@ -322,7 +353,8 @@ async function loadClient(): Promise<InitResult> {
     try {
       const session = loadSession();
       if (!session) {
-        throw new Error("No session stored");
+        logger.log("No session stored; continuing without a client");
+        return null;
       }
 
       logger.log("Using a standalone client");
@@ -387,3 +419,8 @@ const loadSession = (): Session | undefined => {
 
   return JSON.parse(data);
 };
+
+const clientIsDisconnected = (
+  syncState: SyncState,
+  syncData?: ISyncStateData
+) => syncState === "ERROR" && syncData?.error?.name === "ConnectionError";

src/DisconnectedBanner.module.css

@@ -0,0 +1,27 @@
+/*
+Copyright 2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+.banner {
+  position: absolute;
+  padding: 29px;
+  background-color: var(--quaternary-content);
+  vertical-align: middle;
+  font-size: var(--font-size-body);
+  text-align: center;
+  z-index: 1;
+  top: 76px;
+  width: calc(100% - 58px);
+}

src/DisconnectedBanner.tsx

@@ -0,0 +1,53 @@
+/*
+Copyright 2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import classNames from "classnames";
+import { HTMLAttributes, ReactNode } from "react";
+import { useTranslation } from "react-i18next";
+
+import styles from "./DisconnectedBanner.module.css";
+import { ValidClientState, useClientState } from "./ClientContext";
+
+interface DisconnectedBannerProps extends HTMLAttributes<HTMLElement> {
+  children?: ReactNode;
+  className?: string;
+}
+
+export function DisconnectedBanner({
+  children,
+  className,
+  ...rest
+}: DisconnectedBannerProps) {
+  const { t } = useTranslation();
+  const clientState = useClientState();
+  let shouldShowBanner = false;
+
+  if (clientState?.state === "valid") {
+    const validClientState = clientState as ValidClientState;
+    shouldShowBanner = validClientState.disconnected;
+  }
+
+  return (
+    <>
+      {shouldShowBanner && (
+        <div className={classNames(styles.banner, className)} {...rest}>
+          {children}
+          {t("Connectivity to the server has been lost.")}
+        </div>
+      )}
+    </>
+  );
+}

src/FullScreenView.tsx

@@ -18,6 +18,7 @@ import { ReactNode, useCallback, useEffect } from "react";
 import { useLocation } from "react-router-dom";
 import classNames from "classnames";
 import { Trans, useTranslation } from "react-i18next";
+import * as Sentry from "@sentry/react";
 
 import { Header, HeaderLogo, LeftNav, RightNav } from "./Header";
 import { LinkButton, Button } from "./button";
@@ -57,6 +58,7 @@ export function ErrorView({ error }: ErrorViewProps) {
 
   useEffect(() => {
     console.error(error);
+    Sentry.captureException(error);
   }, [error]);
 
   const onReload = useCallback(() => {

src/initializer.tsx

@@ -45,6 +45,12 @@ class DependencyLoadStates {
 
 export class Initializer {
   private static internalInstance: Initializer;
+  private isInitialized = false;
+
+  public static isInitialized(): boolean {
+    return Initializer.internalInstance?.isInitialized;
+  }
+
   public static initBeforeReact() {
     // this maybe also needs to return a promise in the future,
     // if we have to do async inits before showing the loading screen
@@ -227,6 +233,7 @@ export class Initializer {
     if (this.loadStates.allDepsAreLoaded()) {
       // resolve if there is no dependency that is not loaded
       resolve();
+      this.isInitialized = true;
     }
   }
 

src/livekit/useLiveKit.ts

@@ -11,7 +11,7 @@ export type UserChoices = {
 };
 
 export type DeviceChoices = {
-  selectedId: string;
+  selectedId?: string;
   enabled: boolean;
 };
 

src/matrix-utils.ts

@@ -60,11 +60,11 @@ function waitForSync(client: MatrixClient) {
       data?: ISyncStateData
     ) => {
       if (state === "PREPARED") {
+        client.removeListener(ClientEvent.Sync, onSync);
         resolve();
-        client.removeListener(ClientEvent.Sync, onSync);
       } else if (state === "ERROR") {
-        reject(data?.error);
         client.removeListener(ClientEvent.Sync, onSync);
+        reject(data?.error);
       }
     };
     client.on(ClientEvent.Sync, onSync);

src/room/GroupCallView.tsx

@@ -293,6 +293,7 @@ export function GroupCallView({
           setUserChoices(choices);
           enter();
         }}
+        muteAudio={participants.size > 8}
         isEmbedded={isEmbedded}
         hideHeader={hideHeader}
       />

src/room/InCallView.tsx

@@ -85,6 +85,7 @@ import { useLayoutStates } from "../video-grid/Layout";
 import { useSFUConfig } from "../livekit/OpenIDLoader";
 import { E2EELock } from "../E2EELock";
 import { useEventEmitterThree } from "../useEvents";
+import { useWakeLock } from "../useWakeLock";
 
 const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {});
 // There is currently a bug in Safari our our code with cloning and sending MediaStreams
@@ -134,6 +135,7 @@ export function InCallView({
 }: InCallViewProps) {
   const { t } = useTranslation();
   usePreventScroll();
+  useWakeLock();
 
   const containerRef1 = useRef<HTMLDivElement | null>(null);
   const [containerRef2, bounds] = useMeasure({ polyfill: ResizeObserver });

src/room/LobbyView.tsx

@@ -33,6 +33,7 @@ interface Props {
   onEnter: (userChoices: UserChoices) => void;
   isEmbedded: boolean;
   hideHeader: boolean;
+  muteAudio: boolean;
 }
 
 export function LobbyView(props: Props) {
@@ -66,6 +67,7 @@ export function LobbyView(props: Props) {
         <div className={styles.joinRoomContent}>
           <VideoPreview
             matrixInfo={props.matrixInfo}
+            muteAudio={props.muteAudio}
             onUserChoicesChanged={setUserChoices}
           />
           <Trans>

src/room/VideoPreview.tsx

@@ -40,10 +40,15 @@ export type MatrixInfo = {
 
 interface Props {
   matrixInfo: MatrixInfo;
+  muteAudio: boolean;
   onUserChoicesChanged: (choices: UserChoices) => void;
 }
 
-export function VideoPreview({ matrixInfo, onUserChoicesChanged }: Props) {
+export function VideoPreview({
+  matrixInfo,
+  muteAudio,
+  onUserChoicesChanged,
+}: Props) {
   const { client } = useClient();
   const [previewRef, previewBounds] = useMeasure({ polyfill: ResizeObserver });
 
@@ -64,7 +69,13 @@ export function VideoPreview({ matrixInfo, onUserChoicesChanged }: Props) {
 
   // Create local media tracks.
   const [videoEnabled, setVideoEnabled] = useState<boolean>(true);
-  const [audioEnabled, setAudioEnabled] = useState<boolean>(true);
+  const [audioEnabled, setAudioEnabled] = useState<boolean>(!muteAudio);
+
+  useEffect(() => {
+    if (muteAudio) {
+      setAudioEnabled(false);
+    }
+  }, [muteAudio]);
 
   // The settings are updated as soon as the device changes. We wrap the settings value in a ref to store their initial value.
   // Not changing the device options prohibits the usePreviewTracks hook to recreate the tracks.

src/room/useGroupCall.ts

@@ -15,6 +15,7 @@ limitations under the License.
 */
 
 import { useCallback, useEffect, useReducer, useState } from "react";
+import * as Sentry from "@sentry/react";
 import {
   GroupCallEvent,
   GroupCallState,
@@ -331,6 +332,7 @@ export function useGroupCall(
     }
 
     function onError(e: GroupCallError): void {
+      Sentry.captureException(e);
       if (e.code === GroupCallErrorCode.UnknownDevice) {
         const unknownDeviceError = e as GroupCallUnknownDeviceError;
         addUnencryptedEventUser(unknownDeviceError.userId);

src/settings/SettingsModal.tsx

@@ -118,7 +118,7 @@ export const SettingsModal = (props: Props) => {
 
   const devices = props.mediaDevicesSwitcher;
 
-  const tabs = [
+  const audioTab = (
     <TabItem
       key="audio"
       title={
@@ -130,7 +130,10 @@ export const SettingsModal = (props: Props) => {
     >
       {devices && generateDeviceSelection(devices.audioIn, t("Microphone"))}
       {devices && generateDeviceSelection(devices.audioOut, t("Speaker"))}
-    </TabItem>,
+    </TabItem>
+  );
+
+  const videoTab = (
     <TabItem
       key="video"
       title={
@@ -141,7 +144,24 @@ export const SettingsModal = (props: Props) => {
       }
     >
       {devices && generateDeviceSelection(devices.videoIn, t("Camera"))}
-    </TabItem>,
+    </TabItem>
+  );
+
+  const profileTab = (
+    <TabItem
+      key="profile"
+      title={
+        <>
+          <UserIcon width={15} height={15} />
+          <span>{t("Profile")}</span>
+        </>
+      }
+    >
+      <ProfileSettingsTab client={props.client} />
+    </TabItem>
+  );
+
+  const feedbackTab = (
     <TabItem
       key="feedback"
       title={
@@ -152,7 +172,10 @@ export const SettingsModal = (props: Props) => {
       }
     >
       <FeedbackSettingsTab roomId={props.roomId} />
-    </TabItem>,
+    </TabItem>
+  );
+
+  const moreTab = (
     <TabItem
       key="more"
       title={
@@ -188,73 +211,61 @@ export const SettingsModal = (props: Props) => {
           }}
         />
       </FieldRow>
-    </TabItem>,
-  ];
+    </TabItem>
+  );
 
-  if (!isEmbedded) {
-    tabs.push(
-      <TabItem
-        key="profile"
-        title={
-          <>
-            <UserIcon width={15} height={15} />
-            <span>{t("Profile")}</span>
-          </>
-        }
-      >
-        <ProfileSettingsTab client={props.client} />
-      </TabItem>
-    );
-  }
+  const developerTab = (
+    <TabItem
+      key="developer"
+      title={
+        <>
+          <DeveloperIcon width={16} height={16} />
+          <span>{t("Developer")}</span>
+        </>
+      }
+    >
+      <FieldRow>
+        <Body className={styles.fieldRowText}>
+          {t("Version: {{version}}", {
+            version: import.meta.env.VITE_APP_VERSION || "dev",
+          })}
+        </Body>
+      </FieldRow>
+      <FieldRow>
+        <InputField
+          id="showInspector"
+          name="inspector"
+          label={t("Show call inspector")}
+          type="checkbox"
+          checked={showInspector}
+          onChange={(e: ChangeEvent<HTMLInputElement>) =>
+            setShowInspector(e.target.checked)
+          }
+        />
+      </FieldRow>
+      <FieldRow>
+        <InputField
+          id="showConnectionStats"
+          name="connection-stats"
+          label={t("Show connection stats")}
+          type="checkbox"
+          checked={showConnectionStats}
+          onChange={(e: ChangeEvent<HTMLInputElement>) =>
+            setShowConnectionStats(e.target.checked)
+          }
+        />
+      </FieldRow>
+      <FieldRow>
+        <Button onPress={downloadDebugLog}>{t("Download debug logs")}</Button>
+      </FieldRow>
+    </TabItem>
+  );
 
-  if (developerSettingsTab) {
-    tabs.push(
-      <TabItem
-        key="developer"
-        title={
-          <>
-            <DeveloperIcon width={16} height={16} />
-            <span>{t("Developer")}</span>
-          </>
-        }
-      >
-        <FieldRow>
-          <Body className={styles.fieldRowText}>
-            {t("Version: {{version}}", {
-              version: import.meta.env.VITE_APP_VERSION || "dev",
-            })}
-          </Body>
-        </FieldRow>
-        <FieldRow>
-          <InputField
-            id="showInspector"
-            name="inspector"
-            label={t("Show call inspector")}
-            type="checkbox"
-            checked={showInspector}
-            onChange={(e: ChangeEvent<HTMLInputElement>) =>
-              setShowInspector(e.target.checked)
-            }
-          />
-        </FieldRow>
-        <FieldRow>
-          <InputField
-            id="showConnectionStats"
-            name="connection-stats"
-            label={t("Show connection stats")}
-            type="checkbox"
-            checked={showConnectionStats}
-            onChange={(e: ChangeEvent<HTMLInputElement>) =>
-              setShowConnectionStats(e.target.checked)
-            }
-          />
-        </FieldRow>
-        <FieldRow>
-          <Button onPress={downloadDebugLog}>{t("Download debug logs")}</Button>
-        </FieldRow>
-      </TabItem>
-    );
-  }
+  const tabs: JSX.Element[] = [];
+  if (devices) tabs.push(audioTab, videoTab);
+  if (!isEmbedded) tabs.push(profileTab);
+  tabs.push(feedbackTab, moreTab);
+  if (developerSettingsTab) tabs.push(developerTab);
 
   return (
     <Modal

src/useWakeLock.ts

@@ -0,0 +1,60 @@
+/*
+Copyright 2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import { logger } from "matrix-js-sdk/src/logger";
+import { useEffect } from "react";
+
+/**
+ * React hook that inhibits the device from automatically going to sleep.
+ */
+export const useWakeLock = () => {
+  useEffect(() => {
+    if ("wakeLock" in navigator) {
+      let mounted = true;
+      let lock: WakeLockSentinel | null = null;
+
+      // The lock is automatically released whenever the window goes invisible,
+      // so we need to reacquire it on visiblity changes
+      const onVisiblityChange = async () => {
+        if (document.visibilityState === "visible") {
+          try {
+            lock = await navigator.wakeLock.request("screen");
+            // Handle the edge case where this component unmounts before the
+            // promise resolves
+            if (!mounted)
+              lock
+                .release()
+                .catch((e) => logger.warn("Can't release wake lock", e));
+          } catch (e) {
+            logger.warn("Can't acquire wake lock", e);
+          }
+        }
+      };
+
+      onVisiblityChange();
+      document.addEventListener("visiblitychange", onVisiblityChange);
+
+      return () => {
+        mounted = false;
+        if (lock !== null)
+          lock
+            .release()
+            .catch((e) => logger.warn("Can't release wake lock", e));
+        document.removeEventListener("visiblitychange", onVisiblityChange);
+      };
+    }
+  }, []);
+};