Load focus information from well known and use client config only as a fallback. (#2358)

* Load focus information from well known and use client config only as a fallback.

Signed-off-by: Timo K <toger5@hotmail.de>
Co-authored-by: Andrew Ferrazzutti <andrewf@element.io>

src/ClientContext.tsx

@@ -25,7 +25,11 @@ import {
   useMemo,
 } from "react";
 import { useHistory } from "react-router-dom";
-import { ClientEvent, MatrixClient } from "matrix-js-sdk/src/client";
+import {
+  ClientEvent,
+  ICreateClientOpts,
+  MatrixClient,
+} from "matrix-js-sdk/src/client";
 import { logger } from "matrix-js-sdk/src/logger";
 import { useTranslation } from "react-i18next";
 import { ISyncStateData, SyncState } from "matrix-js-sdk/src/sync";
@@ -360,13 +364,13 @@ async function loadClient(): Promise<InitResult | null> {
 
       /* eslint-disable camelcase */
       const { user_id, device_id, access_token, passwordlessUser } = session;
-      const initClientParams = {
+      const initClientParams: ICreateClientOpts = {
         baseUrl: Config.defaultHomeserverUrl()!,
         accessToken: access_token,
         userId: user_id,
         deviceId: device_id,
         fallbackICEServerAllowed: fallbackICEServerAllowed,
-        livekitServiceURL: Config.get().livekit!.livekit_service_url,
+        livekitServiceURL: Config.get().livekit?.livekit_service_url,
       };
 
       try {

src/config/ConfigOptions.ts

@@ -55,7 +55,12 @@ export interface ConfigOptions {
 
   // Describes the LiveKit configuration to be used.
   livekit?: {
-    // The link to the service that returns a livekit url and token to use it
+    // The link to the service that returns a livekit url and token to use it.
+    // This is a fallback link in case the homeserver in use does not advertise
+    // a livekit service url in the client well-known.
+    // The well known needs to be formatted like so:
+    // {"type":"livekit", "livekit_service_url":"https://livekit.example.com"}
+    // and stored under the key: "livekit_focus"
     livekit_service_url: string;
   };
 

src/livekit/LivekitFocus.ts

@@ -1,23 +0,0 @@
-/*
-Copyright 2023 New Vector Ltd
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-import { Focus } from "matrix-js-sdk/src/matrixrtc/focus";
-
-export interface LivekitFocus extends Focus {
-  type: "livekit";
-  livekit_service_url: string;
-  livekit_alias: string;
-}

src/livekit/openIDSFU.ts

@@ -18,9 +18,9 @@ import { IOpenIDToken, MatrixClient } from "matrix-js-sdk";
 import { logger } from "matrix-js-sdk/src/logger";
 import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
 import { useEffect, useState } from "react";
+import { LivekitFocus } from "matrix-js-sdk/src/matrixrtc/LivekitFocus";
 
-import { LivekitFocus } from "./LivekitFocus";
-import { useActiveFocus } from "../room/useActiveFocus";
+import { useActiveLivekitFocus } from "../room/useActiveFocus";
 
 export interface SFUConfig {
   url: string;
@@ -46,7 +46,7 @@ export function useOpenIDSFU(
 ): SFUConfig | undefined {
   const [sfuConfig, setSFUConfig] = useState<SFUConfig | undefined>(undefined);
 
-  const activeFocus = useActiveFocus(rtcSession);
+  const activeFocus = useActiveLivekitFocus(rtcSession);
 
   useEffect(() => {
     (async (): Promise<void> => {

src/matrix-utils.ts

@@ -206,7 +206,7 @@ export async function initClient(
   // Otherwise, a sync may complete before the listener gets applied,
   // and we will miss it.
   const syncPromise = waitForSync(client);
-  await client.startClient();
+  await client.startClient({ clientWellKnownPollPeriod: 60 * 10 });
   await syncPromise;
 
   return client;

src/room/GroupCallView.tsx

@@ -196,7 +196,7 @@ export const GroupCallView: FC<Props> = ({
         ev: CustomEvent<IWidgetApiRequest>,
       ): Promise<void> => {
         defaultDeviceSetup(ev.detail.data as unknown as JoinCallData);
-        enterRTCSession(rtcSession, perParticipantE2EE);
+        await enterRTCSession(rtcSession, perParticipantE2EE);
         await widget!.api.transport.reply(ev.detail, {});
       };
       widget.lazyActions.on(ElementWidgetActions.JoinCall, onJoin);
@@ -318,7 +318,7 @@ export const GroupCallView: FC<Props> = ({
         client={client}
         matrixInfo={matrixInfo}
         muteStates={muteStates}
-        onEnter={(): void => enterRTCSession(rtcSession, perParticipantE2EE)}
+        onEnter={() => void enterRTCSession(rtcSession, perParticipantE2EE)}
         confineToRoom={confineToRoom}
         hideHeader={hideHeader}
         participantCount={participantCount}

src/room/useActiveFocus.ts

@@ -21,33 +21,28 @@ import {
 import { useCallback, useEffect, useState } from "react";
 import { deepCompare } from "matrix-js-sdk/src/utils";
 import { logger } from "matrix-js-sdk/src/logger";
-
-import { LivekitFocus } from "../livekit/LivekitFocus";
-
-function getActiveFocus(
-  rtcSession: MatrixRTCSession,
-): LivekitFocus | undefined {
-  const oldestMembership = rtcSession.getOldestMembership();
-  const focus = oldestMembership?.getActiveFoci()[0] as LivekitFocus;
-
-  return focus;
-}
+import {
+  LivekitFocus,
+  isLivekitFocus,
+} from "matrix-js-sdk/src/matrixrtc/LivekitFocus";
 
 /**
  * Gets the currently active (livekit) focus for a MatrixRTC session
  * This logic is specific to livekit foci where the whole call must use one
  * and the same focus.
  */
-export function useActiveFocus(
+export function useActiveLivekitFocus(
   rtcSession: MatrixRTCSession,
 ): LivekitFocus | undefined {
-  const [activeFocus, setActiveFocus] = useState(() =>
-    getActiveFocus(rtcSession),
-  );
+  const [activeFocus, setActiveFocus] = useState(() => {
+    const f = rtcSession.getActiveFocus();
+    // Only handle foci with type="livekit" for now.
+    return !!f && isLivekitFocus(f) ? f : undefined;
+  });
 
   const onMembershipsChanged = useCallback(() => {
-    const newActiveFocus = getActiveFocus(rtcSession);
-
+    const newActiveFocus = rtcSession.getActiveFocus();
+    if (!!newActiveFocus && !isLivekitFocus(newActiveFocus)) return;
     if (!deepCompare(activeFocus, newActiveFocus)) {
       const oldestMembership = rtcSession.getOldestMembership();
       logger.warn(

src/room/useLoadGroupCall.ts

@@ -217,8 +217,6 @@ export const useLoadGroupCall = (
             "Room not found. The widget-api did not pass over the relevant room events/information.",
           );
 
-        // If the room does not exist we first search for it with viaServers
-        const roomSummary = await client.getRoomSummary(roomId, viaServers);
         if (membership === KnownMembership.Ban) {
           throw bannedError();
         } else if (membership === KnownMembership.Invite) {
@@ -226,6 +224,8 @@ export const useLoadGroupCall = (
             viaServers,
           });
         } else {
+          // If the room does not exist we first search for it with viaServers
+          const roomSummary = await client.getRoomSummary(roomId, viaServers);
           if (roomSummary.join_rule === JoinRule.Public) {
             room = await client.joinRoom(roomSummary.room_id, {
               viaServers,

src/rtcSessionHelpers.ts

@@ -15,29 +15,90 @@ limitations under the License.
 */
 
 import { MatrixRTCSession } from "matrix-js-sdk/src/matrixrtc/MatrixRTCSession";
+import { logger } from "matrix-js-sdk/src/logger";
+import {
+  LivekitFocus,
+  LivekitFocusActive,
+  isLivekitFocus,
+  isLivekitFocusConfig,
+} from "matrix-js-sdk/src/matrixrtc/LivekitFocus";
 
 import { PosthogAnalytics } from "./analytics/PosthogAnalytics";
-import { LivekitFocus } from "./livekit/LivekitFocus";
 import { Config } from "./config/Config";
 import { ElementWidgetActions, WidgetHelpers, widget } from "./widget";
 
-function makeFocus(livekitAlias: string): LivekitFocus {
-  const urlFromConf = Config.get().livekit!.livekit_service_url;
-  if (!urlFromConf) {
-    throw new Error("No livekit_service_url is configured!");
-  }
+const FOCI_WK_KEY = "org.matrix.msc4143.rtc_foci";
 
+export function makeActiveFocus(): LivekitFocusActive {
   return {
     type: "livekit",
-    livekit_service_url: urlFromConf,
-    livekit_alias: livekitAlias,
+    focus_selection: "oldest_membership",
   };
 }
 
-export function enterRTCSession(
+async function makePreferredLivekitFoci(
+  rtcSession: MatrixRTCSession,
+  livekitAlias: string,
+): Promise<LivekitFocus[]> {
+  logger.log("Start building foci_preferred list: ", rtcSession.room.roomId);
+
+  const preferredFoci: LivekitFocus[] = [];
+
+  // Make the Focus from the running rtc session the highest priority one
+  // This minimizes how often we need to switch foci during a call.
+  const focusInUse = rtcSession.getFocusInUse();
+  if (focusInUse && isLivekitFocus(focusInUse)) {
+    logger.log("Adding livekit focus from oldest member: ", focusInUse);
+    preferredFoci.push(focusInUse);
+  }
+
+  // Prioritize the client well known over the configured sfu.
+  const wellKnownFoci =
+    rtcSession.room.client.getClientWellKnown()?.[FOCI_WK_KEY];
+  if (Array.isArray(wellKnownFoci)) {
+    preferredFoci.push(
+      ...wellKnownFoci
+        .filter((f) => !!f)
+        .filter(isLivekitFocusConfig)
+        .map((wellKnownFocus) => {
+          logger.log("Adding livekit focus from well known: ", wellKnownFocus);
+          return { ...wellKnownFocus, livekit_alias: livekitAlias };
+        }),
+    );
+  }
+
+  const urlFromConf = Config.get().livekit?.livekit_service_url;
+  if (urlFromConf) {
+    const focusFormConf: LivekitFocus = {
+      type: "livekit",
+      livekit_service_url: urlFromConf,
+      livekit_alias: livekitAlias,
+    };
+    logger.log("Adding livekit focus from config: ", focusFormConf);
+    preferredFoci.push(focusFormConf);
+  }
+
+  if (preferredFoci.length === 0)
+    throw new Error(
+      `No livekit_service_url is configured so we could not create a focus.
+    Currently we skip computing a focus based on other users in the room.`,
+    );
+
+  return preferredFoci;
+
+  // TODO: we want to do something like this:
+  //
+  // const focusOtherMembers = await focusFromOtherMembers(
+  //   rtcSession,
+  //   livekitAlias,
+  // );
+  // if (focusOtherMembers) preferredFoci.push(focusOtherMembers);
+}
+
+export async function enterRTCSession(
   rtcSession: MatrixRTCSession,
   encryptMedia: boolean,
-): void {
+): Promise<void> {
   PosthogAnalytics.instance.eventCallEnded.cacheStartCall(new Date());
   PosthogAnalytics.instance.eventCallStarted.track(rtcSession.room.roomId);
 
@@ -47,8 +108,11 @@ export function enterRTCSession(
 
   // right now we assume everything is a room-scoped call
   const livekitAlias = rtcSession.room.roomId;
-
-  rtcSession.joinRoomSession([makeFocus(livekitAlias)], encryptMedia);
+  rtcSession.joinRoomSession(
+    await makePreferredLivekitFoci(rtcSession, livekitAlias),
+    makeActiveFocus(),
+    { manageMediaKeys: encryptMedia },
+  );
 }
 
 const widgetPostHangupProcedure = async (

src/useMatrixRTCSessionJoinState.ts

@@ -26,14 +26,17 @@ export function useMatrixRTCSessionJoinState(
 ): boolean {
   const [isJoined, setJoined] = useState(rtcSession.isJoined());
 
-  const onJoinStateChanged = useCallback(() => {
-    logger.info(
-      `Session in room ${rtcSession.room.roomId} changed to ${
-        rtcSession.isJoined() ? "joined" : "left"
-      }`,
-    );
-    setJoined(rtcSession.isJoined());
-  }, [rtcSession]);
+  const onJoinStateChanged = useCallback(
+    (isJoined: boolean) => {
+      logger.info(
+        `Session in room ${rtcSession.room.roomId} changed to ${
+          isJoined ? "joined" : "left"
+        }`,
+      );
+      setJoined(isJoined);
+    },
+    [rtcSession],
+  );
 
   useEffect(() => {
     rtcSession.on(MatrixRTCSessionEvent.JoinStateChanged, onJoinStateChanged);

src/widget.ts

@@ -77,8 +77,6 @@ export const widget = ((): WidgetHelpers | null => {
       logger.info("Widget API is available");
       const api = new WidgetApi(widgetId, parentOrigin);
       api.requestCapability(MatrixCapabilities.AlwaysOnScreen);
-      api.requestCapabilityToSendEvent(EventType.CallEncryptionKeysPrefix);
-      api.requestCapabilityToReceiveEvent(EventType.CallEncryptionKeysPrefix);
 
       // Set up the lazy action emitter, but only for select actions that we
       // intend for the app to handle
@@ -116,9 +114,15 @@ export const widget = ((): WidgetHelpers | null => {
       if (!baseUrl) throw new Error("Base URL must be supplied");
 
       // These are all the event types the app uses
-      const sendRecvEvent = ["org.matrix.rageshake_request"];
+      const sendRecvEvent = [
+        "org.matrix.rageshake_request",
+        EventType.CallEncryptionKeysPrefix,
+      ];
       const sendState = [
-        { eventType: EventType.GroupCallMemberPrefix, stateKey: userId },
+        {
+          eventType: EventType.GroupCallMemberPrefix,
+          stateKey: userId, // TODO: based on if we use the new format we want the key to be: `_${userId}_${deviceId}`
+        },
       ];
       const receiveState = [
         { eventType: EventType.RoomMember },
@@ -167,7 +171,7 @@ export const widget = ((): WidgetHelpers | null => {
           // wait for the config file to be ready (we load very early on so it might not
           // be otherwise)
           await Config.init();
-          await client.startClient();
+          await client.startClient({ clientWellKnownPollPeriod: 60 * 10 });
           resolve(client);
         })();
       });