Force to allow calls without video and audio in embedded mode (#1134)

* force to allow calls without video and audio in embedded mode

.github/workflows/netlify-livekit.yaml

@@ -0,0 +1,88 @@
+name: Netlify LiveKit Experiment
+on:
+  workflow_run:
+    workflows: ["Build"]
+    types:
+      - completed
+    branches:
+      - "livekit-experiment"
+
+jobs:
+  deploy:
+    runs-on: ubuntu-latest
+    permissions:
+      deployments: write
+    # Important: the 'branches' filter above will match the 'livekit-experiment' branch on forks,
+    # so we need to check the head repo too in order to not run on PRs from forks
+    # We check the branch name again too just for completeness
+    # (Is there a nicer way to see if a PR is from a fork?)
+    if: github.event.workflow_run.conclusion == 'success' && github.event.workflow_run.head_repository.full_name == 'vector-im/element-call' && github.event.workflow_run.head_branch == 'livekit-experiment'
+    steps:
+      - name: Create Deployment
+        uses: bobheadxi/deployments@v1
+        id: deployment
+        with:
+          step: start
+          token: ${{ secrets.GITHUB_TOKEN }}
+          env: livekit-experiment-branch-cd
+          ref: ${{ github.event.workflow_run.head_sha }}
+
+      - name: "Download artifact"
+        uses: actions/github-script@v3.1.0
+        with:
+          script: |
+            const artifacts = await github.actions.listWorkflowRunArtifacts({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: ${{ github.event.workflow_run.id }},
+            });
+            const matchArtifact = artifacts.data.artifacts.filter((artifact) => {
+                return artifact.name == "build"
+            })[0];
+            const download = await github.actions.downloadArtifact({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                artifact_id: matchArtifact.id,
+                archive_format: 'zip',
+            });
+            const fs = require('fs');
+            fs.writeFileSync('${{github.workspace}}/build.zip', Buffer.from(download.data));
+
+      - name: Extract Artifacts
+        run: unzip -d dist build.zip && rm build.zip
+
+      - name: Add redirects file
+        # We fetch from github directly as we don't bother checking out the repo
+        run: curl -s https://raw.githubusercontent.com/vector-im/element-call/livekit-experiment/config/netlify_redirects > dist/_redirects
+
+      - name: Add config file
+        run: curl -s https://raw.githubusercontent.com/vector-im/element-call/livekit-experiment/config/element_io_preview.json > dist/config.json
+
+      - name: Deploy to Netlify
+        id: netlify
+        uses: nwtgck/actions-netlify@v1.2.3
+        with:
+          publish-dir: dist
+          deploy-message: "Deploy from GitHub Actions"
+          production-branch: livekit-experiment
+          production-deploy: true
+          # These don't work because we're in workflow_run
+          enable-pull-request-comment: false
+          enable-commit-comment: false
+          github-deployment-environment: livekit-experiment
+        env:
+          NETLIFY_AUTH_TOKEN: ${{ secrets.NETLIFY_AUTH_TOKEN }}
+          NETLIFY_SITE_ID: e3b9fa82-c040-4db6-b4bf-42b524d57423
+        timeout-minutes: 1
+
+      - name: Update deployment status
+        uses: bobheadxi/deployments@v1
+        if: always()
+        with:
+          step: finish
+          override: false
+          token: ${{ secrets.GITHUB_TOKEN }}
+          status: ${{ job.status }}
+          env: ${{ steps.deployment.outputs.env }}
+          deployment_id: ${{ steps.deployment.outputs.deployment_id }}
+          env_url: ${{ steps.netlify.outputs.deploy-url }}

.github/workflows/test.yaml

@@ -1,6 +1,8 @@
 name: Run jest tests
 on:
   pull_request: {}
+  push:
+    branches: [livekit]
 jobs:
   jest:
     name: Run jest tests
@@ -16,3 +18,7 @@ jobs:
         run: "yarn install"
       - name: Jest
         run: "yarn run test"
+      - name: Upload to codecov
+        uses: codecov/codecov-action@v3
+        with:
+          flags: unittests

package.json

@@ -53,7 +53,7 @@
     "i18next-browser-languagedetector": "^6.1.8",
     "i18next-http-backend": "^1.4.4",
     "lodash": "^4.17.21",
-    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#3cfad3cdeb7b19b8e0e7015784efd803cb9542f1",
+    "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#f884c78579c336a03bc20ff8f4e92c46582822b6",
     "matrix-widget-api": "^1.3.1",
     "mermaid": "^8.13.8",
     "normalize.css": "^8.0.1",
@@ -118,6 +118,11 @@
       "\\.(css|less|svg)+$": "identity-obj-proxy",
       "^\\./IndexedDBWorker\\?worker$": "<rootDir>/test/mocks/workerMock.ts",
       "^\\./olm$": "<rootDir>/test/mocks/olmMock.ts"
-    }
+    },
+    "collectCoverage": true,
+    "coverageReporters": [
+      "text",
+      "cobertura"
+    ]
   }
 }

public/locales/en-GB/app.json

@@ -36,6 +36,7 @@
   "Close": "Close",
   "Confirm password": "Confirm password",
   "Connection lost": "Connection lost",
+  "Connectivity to the server has been lost.": "Connectivity to the server has been lost.",
   "Copied!": "Copied!",
   "Copy": "Copy",
   "Copy and share this call link": "Copy and share this call link",

src/App.tsx

@@ -29,6 +29,7 @@ import { usePageFocusStyle } from "./usePageFocusStyle";
 import { SequenceDiagramViewerPage } from "./SequenceDiagramViewerPage";
 import { InspectorContextProvider } from "./room/GroupCallInspector";
 import { CrashView, LoadingView } from "./FullScreenView";
+import { DisconnectedBanner } from "./DisconnectedBanner";
 import { Initializer } from "./initializer";
 import { MediaHandlerProvider } from "./settings/useMediaHandler";
 
@@ -60,6 +61,7 @@ export default function App({ history }: AppProps) {
               <InspectorContextProvider>
                 <Sentry.ErrorBoundary fallback={errorPage}>
                   <OverlayProvider>
+                    <DisconnectedBanner />
                     <Switch>
                       <SentryRoute exact path="/">
                         <HomePage />

src/ClientContext.tsx

@@ -25,9 +25,10 @@ import React, {
   useRef,
 } from "react";
 import { useHistory } from "react-router-dom";
-import { MatrixClient } from "matrix-js-sdk/src/client";
+import { ClientEvent, MatrixClient } from "matrix-js-sdk/src/client";
 import { logger } from "matrix-js-sdk/src/logger";
 import { useTranslation } from "react-i18next";
+import { ISyncStateData, SyncState } from "matrix-js-sdk/src/sync";
 
 import { ErrorView } from "./FullScreenView";
 import {
@@ -70,6 +71,8 @@ const loadSession = (): Session => {
 const saveSession = (session: Session) =>
   localStorage.setItem("matrix-auth-store", JSON.stringify(session));
 const clearSession = () => localStorage.removeItem("matrix-auth-store");
+const isDisconnected = (syncState, syncData) =>
+  syncState === "ERROR" && syncData?.error?.name === "ConnectionError";
 
 interface ClientState {
   loading: boolean;
@@ -81,6 +84,7 @@ interface ClientState {
   logout: () => void;
   setClient: (client: MatrixClient, session: Session) => void;
   error?: Error;
+  disconnected: boolean;
 }
 
 const ClientContext = createContext<ClientState>(null);
@@ -98,7 +102,15 @@ export const ClientProvider: FC<Props> = ({ children }) => {
   const history = useHistory();
   const initializing = useRef(false);
   const [
-    { loading, isAuthenticated, isPasswordlessUser, client, userName, error },
+    {
+      loading,
+      isAuthenticated,
+      isPasswordlessUser,
+      client,
+      userName,
+      error,
+      disconnected,
+    },
     setState,
   ] = useState<ClientProviderState>({
     loading: true,
@@ -107,8 +119,18 @@ export const ClientProvider: FC<Props> = ({ children }) => {
     client: undefined,
     userName: null,
     error: undefined,
+    disconnected: false,
   });
 
+  const onSync = (state: SyncState, _old: SyncState, data: ISyncStateData) => {
+    setState((currentState) => {
+      const disconnected = isDisconnected(state, data);
+      return disconnected === currentState.disconnected
+        ? currentState
+        : { ...currentState, disconnected };
+    });
+  };
+
   useEffect(() => {
     // In case the component is mounted, unmounted, and remounted quickly (as
     // React does in strict mode), we need to make sure not to doubly initialize
@@ -183,9 +205,10 @@ export const ClientProvider: FC<Props> = ({ children }) => {
         }
       }
     };
-
+    let clientWithListener: MatrixClient;
     init()
       .then(({ client, isPasswordlessUser }) => {
+        clientWithListener = client;
         setState({
           client,
           loading: false,
@@ -193,7 +216,12 @@ export const ClientProvider: FC<Props> = ({ children }) => {
           isPasswordlessUser,
           userName: client?.getUserIdLocalpart(),
           error: undefined,
+          disconnected: isDisconnected(
+            client?.getSyncState,
+            client?.getSyncStateData
+          ),
         });
+        clientWithListener?.on(ClientEvent.Sync, onSync);
       })
       .catch((err) => {
         logger.error(err);
@@ -204,9 +232,13 @@ export const ClientProvider: FC<Props> = ({ children }) => {
           isPasswordlessUser: false,
           userName: null,
           error: undefined,
+          disconnected: false,
         });
       })
       .finally(() => (initializing.current = false));
+    return () => {
+      clientWithListener?.removeListener(ClientEvent.Sync, onSync);
+    };
   }, []);
 
   const changePassword = useCallback(
@@ -235,6 +267,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
         isPasswordlessUser: false,
         userName: client.getUserIdLocalpart(),
         error: undefined,
+        disconnected: false,
       });
     },
     [client]
@@ -256,6 +289,10 @@ export const ClientProvider: FC<Props> = ({ children }) => {
           isPasswordlessUser: session.passwordlessUser,
           userName: newClient.getUserIdLocalpart(),
           error: undefined,
+          disconnected: isDisconnected(
+            newClient.getSyncState(),
+            newClient.getSyncStateData()
+          ),
         });
       } else {
         clearSession();
@@ -267,6 +304,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
           isPasswordlessUser: false,
           userName: null,
           error: undefined,
+          disconnected: false,
         });
       }
     },
@@ -284,6 +322,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
       isPasswordlessUser: true,
       userName: "",
       error: undefined,
+      disconnected: false,
     });
     history.push("/");
     PosthogAnalytics.instance.setRegistrationType(RegistrationType.Guest);
@@ -326,6 +365,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
       userName,
       setClient,
       error: undefined,
+      disconnected,
     }),
     [
       loading,
@@ -336,6 +376,7 @@ export const ClientProvider: FC<Props> = ({ children }) => {
       logout,
       userName,
       setClient,
+      disconnected,
     ]
   );
 

src/DisconnectedBanner.module.css

@@ -0,0 +1,27 @@
+/*
+Copyright 2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+.banner {
+  position: absolute;
+  padding: 29px;
+  background-color: var(--quaternary-content);
+  vertical-align: middle;
+  font-size: var(--font-size-body);
+  text-align: center;
+  z-index: 1;
+  top: 76px;
+  width: calc(100% - 58px);
+}

src/DisconnectedBanner.tsx

@@ -0,0 +1,47 @@
+/*
+Copyright 2023 New Vector Ltd
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+import classNames from "classnames";
+import React, { HTMLAttributes, ReactNode } from "react";
+import { useTranslation } from "react-i18next";
+
+import styles from "./DisconnectedBanner.module.css";
+import { useClient } from "./ClientContext";
+
+interface DisconnectedBannerProps extends HTMLAttributes<HTMLElement> {
+  children?: ReactNode;
+  className?: string;
+}
+
+export function DisconnectedBanner({
+  children,
+  className,
+  ...rest
+}: DisconnectedBannerProps) {
+  const { t } = useTranslation();
+  const { disconnected } = useClient();
+
+  return (
+    <>
+      {disconnected && (
+        <div className={classNames(styles.banner, className)} {...rest}>
+          {children}
+          {t("Connectivity to the server has been lost.")}
+        </div>
+      )}
+    </>
+  );
+}

src/UrlParams.ts

@@ -84,6 +84,10 @@ export interface UrlParams {
    * user's homeserver doesn't provide any.
    */
   allowIceFallback: boolean;
+  /**
+   * Whether the app is allowed screen share only mode
+   */
+  allowVoipWithNoMedia: boolean;
 }
 
 /**
@@ -141,6 +145,7 @@ export const getUrlParams = (
     fontScale: Number.isNaN(fontScale) ? null : fontScale,
     analyticsID: getParam("analyticsID"),
     allowIceFallback: hasParam("allowIceFallback"),
+    allowVoipWithNoMedia: hasParam("allowVoipWithNoMedia"),
   };
 };
 

src/analytics/PosthogAnalytics.ts

@@ -70,6 +70,7 @@ export enum RegistrationType {
 interface PlatformProperties {
   appVersion: string;
   matrixBackend: "embedded" | "jssdk";
+  callBackend: "livekit" | "full-mesh";
 }
 
 interface PosthogSettings {
@@ -191,6 +192,7 @@ export class PosthogAnalytics {
     return {
       appVersion,
       matrixBackend: widget ? "embedded" : "jssdk",
+      callBackend: "full-mesh",
     };
   }
 

src/initializer.tsx

@@ -45,6 +45,12 @@ class DependencyLoadStates {
 
 export class Initializer {
   private static internalInstance: Initializer;
+  private isInitialized = false;
+
+  public static isInitialized(): boolean {
+    return Initializer.internalInstance?.isInitialized;
+  }
+
   public static initBeforeReact() {
     // this maybe also needs to return a promise in the future,
     // if we have to do async inits before showing the loading screen
@@ -223,6 +229,7 @@ export class Initializer {
     if (this.loadStates.allDepsAreLoaded()) {
       // resolve if there is no dependency that is not loaded
       resolve();
+      this.isInitialized = true;
     }
   }
   private initPromise: Promise<void> | null;

src/matrix-utils.ts

@@ -61,11 +61,11 @@ function waitForSync(client: MatrixClient) {
       data: ISyncStateData
     ) => {
       if (state === "PREPARED") {
+        client.removeListener(ClientEvent.Sync, onSync);
         resolve();
-        client.removeListener(ClientEvent.Sync, onSync);
       } else if (state === "ERROR") {
-        reject(data?.error);
         client.removeListener(ClientEvent.Sync, onSync);
+        reject(data?.error);
       }
     };
     client.on(ClientEvent.Sync, onSync);

src/room/GroupCallView.tsx

@@ -102,12 +102,17 @@ export function GroupCallView({
         // Get the available devices so we can match the selected device
         // to its ID. This involves getting a media stream (see docs on
         // the function) so we only do it once and re-use the result.
-        const devices = await getNamedDevices();
+        //
+        // But we only want to preload the devices if we get audio or video devices included in the widget request!
+        // By default, the device list is null!
+        let devices: MediaDeviceInfo[] | null = null;
 
         const { audioInput, videoInput } = ev.detail
           .data as unknown as JoinCallData;
 
         if (audioInput !== null) {
+          // we load the devices because w have an audio device in the widget request
+          devices = await getNamedDevices();
           const deviceId = await findDeviceByName(
             audioInput,
             "audioinput",
@@ -124,6 +129,11 @@ export function GroupCallView({
         }
 
         if (videoInput !== null) {
+          // we only need to load the devices once time
+          if (devices === null) {
+            // we load the devices because w have a video device in the widget request
+            devices = await getNamedDevices();
+          }
           const deviceId = await findDeviceByName(
             videoInput,
             "videoinput",

src/settings/submit-rageshake.ts

@@ -101,6 +101,7 @@ export function useSubmitRageshake(): {
         body.append("user_agent", userAgent);
         body.append("installed_pwa", "false");
         body.append("touch_input", touchInput);
+        body.append("call_backend", "full-mesh");
 
         if (client) {
           const userId = client.getUserId();

src/widget.ts

@@ -108,6 +108,7 @@ export const widget: WidgetHelpers | null = (() => {
         baseUrl,
         e2eEnabled,
         allowIceFallback,
+        allowVoipWithNoMedia,
       } = getUrlParams();
       if (!roomId) throw new Error("Room ID must be supplied");
       if (!userId) throw new Error("User ID must be supplied");
@@ -156,6 +157,7 @@ export const widget: WidgetHelpers | null = (() => {
           timelineSupport: true,
           useE2eForGroupCall: e2eEnabled,
           fallbackICEServerAllowed: allowIceFallback,
+          isVoipWithNoMediaAllowed: allowVoipWithNoMedia,
         }
       );
       const clientPromise = client.startClient().then(() => client);

yarn.lock

@@ -1828,10 +1828,10 @@
   resolved "https://registry.yarnpkg.com/@juggle/resize-observer/-/resize-observer-3.3.1.tgz#b50a781709c81e10701004214340f25475a171a0"
   integrity sha512-zMM9Ds+SawiUkakS7y94Ymqx+S0ORzpG3frZirN3l+UlXUmSUR7hF4wxCVqW+ei94JzV5kt0uXBcoOEAuiydrw==
 
-"@matrix-org/matrix-sdk-crypto-js@^0.1.0-alpha.9":
+"@matrix-org/matrix-sdk-crypto-js@^0.1.0-alpha.10":
-  version "0.1.0-alpha.9"
+  version "0.1.0-alpha.11"
-  resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.0-alpha.9.tgz#00bc266781502641a661858a5a521dd4d95275fc"
+  resolved "https://registry.yarnpkg.com/@matrix-org/matrix-sdk-crypto-js/-/matrix-sdk-crypto-js-0.1.0-alpha.11.tgz#24d705318c3159ef7dbe43bca464ac2bdd11e45d"
-  integrity sha512-g5cjpFwA9h0CbEGoAqNVI2QcyDsbI8FHoLo9+OXWHIezEKITsSv78mc5ilIwN+2YpmVlH0KNeQWTHw4vi0BMnw==
+  integrity sha512-HD3rskPkqrUUSaKzGLg97k/bN+OZrkcX7ODB/pNBs/jqq+/A0wDKqsszJotzFwsQcDPpWn78BmMyvBo4tLxKjw==
 
 "@matrix-org/olm@https://gitlab.matrix.org/api/v4/projects/27/packages/npm/@matrix-org/olm/-/@matrix-org/olm-3.2.14.tgz":
   version "3.2.14"
@@ -10557,12 +10557,12 @@ matrix-events-sdk@0.0.1:
   resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1.tgz#c8c38911e2cb29023b0bbac8d6f32e0de2c957dd"
   integrity sha512-1QEOsXO+bhyCroIe2/A5OwaxHvBm7EsSQ46DEDn8RBIfQwN5HWBpFvyWWR4QY0KHPPnnJdI99wgRiAl7Ad5qaA==
 
-"matrix-js-sdk@github:matrix-org/matrix-js-sdk#3cfad3cdeb7b19b8e0e7015784efd803cb9542f1":
+"matrix-js-sdk@github:matrix-org/matrix-js-sdk#f884c78579c336a03bc20ff8f4e92c46582822b6":
-  version "26.0.0"
+  version "26.1.0"
-  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/3cfad3cdeb7b19b8e0e7015784efd803cb9542f1"
+  resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/f884c78579c336a03bc20ff8f4e92c46582822b6"
   dependencies:
     "@babel/runtime" "^7.12.5"
-    "@matrix-org/matrix-sdk-crypto-js" "^0.1.0-alpha.9"
+    "@matrix-org/matrix-sdk-crypto-js" "^0.1.0-alpha.10"
     another-json "^0.2.0"
     bs58 "^5.0.0"
     content-type "^1.0.4"