From 71dcc94166b51324e8e1d075af4b6d2f9a0480f2 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 21 Oct 2022 20:19:52 +0100 Subject: [PATCH 1/3] Fix screen sharing * Make the embedded mode screen sharing a request-each-way rather than request-and-reply, since replies time out and so can't wait for the user. * Try normal screen sharing first, then fall back to using the widget API if it fails (for lack of a good way of detecting when we should be using the widget API). Fixes https://github.com/vector-im/element-call/issues/649 --- src/room/useGroupCall.ts | 97 +++++++++++++++++++++++++++++----------- src/widget.ts | 22 ++++++++- 2 files changed, 93 insertions(+), 26 deletions(-) diff --git a/src/room/useGroupCall.ts b/src/room/useGroupCall.ts index 87bbdb2e..895ef3ee 100644 --- a/src/room/useGroupCall.ts +++ b/src/room/useGroupCall.ts @@ -27,10 +27,11 @@ import { MatrixCall } from "matrix-js-sdk/src/webrtc/call"; import { CallFeed } from "matrix-js-sdk/src/webrtc/callFeed"; import { RoomMember } from "matrix-js-sdk/src/models/room-member"; import { useTranslation } from "react-i18next"; +import { IWidgetApiRequest } from "matrix-widget-api"; import { usePageUnload } from "./usePageUnload"; import { TranslatedError, translatedError } from "../TranslatedError"; -import { ElementWidgetActions, widget } from "../widget"; +import { ElementWidgetActions, ScreenshareStartData, widget } from "../widget"; export interface UseGroupCallReturnType { state: GroupCallState; @@ -302,36 +303,82 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType { groupCall.setMicrophoneMuted(!groupCall.isMicrophoneMuted()); }, [groupCall]); - const toggleScreensharing = useCallback(() => { - updateState({ requestingScreenshare: true }); + const toggleScreensharing = useCallback(async () => { + if (!groupCall.isScreensharing()) { + // toggling on + updateState({ requestingScreenshare: true }); - if (groupCall.isScreensharing()) { - groupCall.setScreensharingEnabled(false).then(() => { + try { + await groupCall.setScreensharingEnabled(true, { + audio: true, + throwOnFail: true, + }); updateState({ requestingScreenshare: false }); - }); - } else { - widget.api.transport - .send(ElementWidgetActions.Screenshare, {}) - .then( - (reply: { desktopCapturerSourceId: string; failed?: boolean }) => { - if (reply.failed) { - updateState({ requestingScreenshare: false }); - return; - } - - groupCall - .setScreensharingEnabled(true, { - audio: !reply.desktopCapturerSourceId, - desktopCapturerSourceId: reply.desktopCapturerSourceId, - }) - .then(() => { - updateState({ requestingScreenshare: false }); - }); + } catch (e) { + // this will fail in Electron because getDisplayMedia just throws a permission + // error, so if we have a widget API, try requesting via that. + if (widget) { + const reply = await widget.api.transport.send( + ElementWidgetActions.ScreenshareRequest, + {} + ); + if (!reply.pending) { + updateState({ requestingScreenshare: false }); } - ); + } + } + } else { + // toggling off + groupCall.setScreensharingEnabled(false); } }, [groupCall]); + const onScreenshareStart = useCallback( + async (ev: CustomEvent) => { + updateState({ requestingScreenshare: false }); + await groupCall.setScreensharingEnabled(true, { + desktopCapturerSourceId: ev.detail.data + .desktopCapturerSourceId as string, + audio: !ev.detail.data.desktopCapturerSourceId, + }); + await widget.api.transport.reply(ev.detail, {}); + }, + [groupCall] + ); + + const onScreenshareStop = useCallback( + async (ev: CustomEvent) => { + updateState({ requestingScreenshare: false }); + await groupCall.setScreensharingEnabled(false); + await widget.api.transport.reply(ev.detail, {}); + }, + [groupCall] + ); + + useEffect(() => { + if (widget) { + widget.lazyActions.on( + ElementWidgetActions.ScreenshareStart, + onScreenshareStart + ); + widget.lazyActions.on( + ElementWidgetActions.ScreenshareStop, + onScreenshareStop + ); + + return () => { + widget.lazyActions.off( + ElementWidgetActions.ScreenshareStart, + onScreenshareStart + ); + widget.lazyActions.off( + ElementWidgetActions.ScreenshareStop, + onScreenshareStop + ); + }; + } + }, [onScreenshareStart, onScreenshareStop]); + const { t } = useTranslation(); useEffect(() => { diff --git a/src/widget.ts b/src/widget.ts index aa0be832..66ac5460 100644 --- a/src/widget.ts +++ b/src/widget.ts @@ -30,7 +30,21 @@ export enum ElementWidgetActions { HangupCall = "im.vector.hangup", TileLayout = "io.element.tile_layout", SpotlightLayout = "io.element.spotlight_layout", - Screenshare = "io.element.screenshare", + + // Element Call -> host requesting to start a screenshare + // (ie. expects a ScreenshareStart once the user has picked a source) + // Element Call -> host requesting to start a screenshare + // (ie. expects a ScreenshareStart once the user has picked a source) + // replies with { pending } where pending is true if the host has asked + // the user to choose a window and false if not (ie. if the host isn't + // running within Electron) + ScreenshareRequest = "io.element.screenshare_request", + // host -> Element Call telling EC to start screen sharing with + // the given source + ScreenshareStart = "io.element.screenshare_start", + // host -> Element Call telling EC to stop screen sharing, or that + // the user cancelled when selecting a source after a ScreenshareRequest + ScreenshareStop = "io.element.screenshare_stop", } export interface JoinCallData { @@ -38,6 +52,10 @@ export interface JoinCallData { videoInput: string | null; } +export interface ScreenshareStartData { + desktopCapturerSourceId: string; +} + interface WidgetHelpers { api: WidgetApi; lazyActions: LazyEventEmitter; @@ -69,6 +87,8 @@ export const widget: WidgetHelpers | null = (() => { ElementWidgetActions.HangupCall, ElementWidgetActions.TileLayout, ElementWidgetActions.SpotlightLayout, + ElementWidgetActions.ScreenshareStart, + ElementWidgetActions.ScreenshareStop, ].forEach((action) => { api.on(`action:${action}`, (ev: CustomEvent) => { ev.preventDefault(); From 821622f71ce1b5cccf21a58902b284637167eed8 Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 21 Oct 2022 20:28:33 +0100 Subject: [PATCH 2/3] Types --- src/room/useGroupCall.ts | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/room/useGroupCall.ts b/src/room/useGroupCall.ts index 895ef3ee..622ea2ee 100644 --- a/src/room/useGroupCall.ts +++ b/src/room/useGroupCall.ts @@ -336,10 +336,12 @@ export function useGroupCall(groupCall: GroupCall): UseGroupCallReturnType { const onScreenshareStart = useCallback( async (ev: CustomEvent) => { updateState({ requestingScreenshare: false }); + + const data = ev.detail.data as unknown as ScreenshareStartData; + await groupCall.setScreensharingEnabled(true, { - desktopCapturerSourceId: ev.detail.data - .desktopCapturerSourceId as string, - audio: !ev.detail.data.desktopCapturerSourceId, + desktopCapturerSourceId: data.desktopCapturerSourceId as string, + audio: !data.desktopCapturerSourceId, }); await widget.api.transport.reply(ev.detail, {}); }, From 3891b042e3a8556df73e0b9a3d1761cf1fcf50de Mon Sep 17 00:00:00 2001 From: David Baker Date: Fri, 21 Oct 2022 21:10:04 +0100 Subject: [PATCH 3/3] Use commit from js-sdk branch --- package.json | 2 +- yarn.lock | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/package.json b/package.json index 00c79bba..ecf3b027 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "i18next": "^21.10.0", "i18next-browser-languagedetector": "^6.1.8", "i18next-http-backend": "^1.4.4", - "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#5a0787349d4951012eabe72f3363c17bdcda0d56", + "matrix-js-sdk": "github:matrix-org/matrix-js-sdk#1666419beaf08cafefdc34cced417cb0ba120b2e", "matrix-widget-api": "^1.0.0", "mermaid": "^8.13.8", "normalize.css": "^8.0.1", diff --git a/yarn.lock b/yarn.lock index 4e453523..b5667ae0 100644 --- a/yarn.lock +++ b/yarn.lock @@ -8728,9 +8728,9 @@ matrix-events-sdk@^0.0.1-beta.7: resolved "https://registry.yarnpkg.com/matrix-events-sdk/-/matrix-events-sdk-0.0.1-beta.7.tgz#5ffe45eba1f67cc8d7c2377736c728b322524934" integrity sha512-9jl4wtWanUFSy2sr2lCjErN/oC8KTAtaeaozJtrgot1JiQcEI4Rda9OLgQ7nLKaqb4Z/QUx/fR3XpDzm5Jy1JA== -"matrix-js-sdk@github:matrix-org/matrix-js-sdk#5a0787349d4951012eabe72f3363c17bdcda0d56": +"matrix-js-sdk@github:matrix-org/matrix-js-sdk#1666419beaf08cafefdc34cced417cb0ba120b2e": version "20.1.0" - resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/5a0787349d4951012eabe72f3363c17bdcda0d56" + resolved "https://codeload.github.com/matrix-org/matrix-js-sdk/tar.gz/1666419beaf08cafefdc34cced417cb0ba120b2e" dependencies: "@babel/runtime" "^7.12.5" "@types/sdp-transform" "^2.4.5"