From 0e34f9a4648552100d541dd77903eb1f851c88e8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 12 Aug 2022 09:36:46 +0200 Subject: [PATCH 1/8] Add `useAudioOutputDevice()` MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/room/InCallView.tsx | 3 ++ src/video-grid/useAudioOutputDevice.ts | 40 ++++++++++++++++++++++++++ src/video-grid/useMediaStream.ts | 25 ++++------------ 3 files changed, 48 insertions(+), 20 deletions(-) create mode 100644 src/video-grid/useAudioOutputDevice.ts diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 5ff3924e..0e9cba17 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -48,6 +48,7 @@ import { useShowInspector } from "../settings/useSetting"; import { useModalTriggerState } from "../Modal"; import { useAudioContext } from "../video-grid/useMediaStream"; import { useFullscreen } from "../video-grid/useFullscreen"; +import { useAudioOutputDevice } from "../video-grid/useAudioOutputDevice"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); // There is currently a bug in Safari our our code with cloning and sending MediaStreams @@ -114,6 +115,8 @@ export function InCallView({ const { modalState: feedbackModalState, modalProps: feedbackModalProps } = useModalTriggerState(); + useAudioOutputDevice(audioRef, audioOutput); + const items = useMemo(() => { const participants: Participant[] = []; diff --git a/src/video-grid/useAudioOutputDevice.ts b/src/video-grid/useAudioOutputDevice.ts new file mode 100644 index 00000000..71b6b7e9 --- /dev/null +++ b/src/video-grid/useAudioOutputDevice.ts @@ -0,0 +1,40 @@ +/* +Copyright 2022 New Vector Ltd + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +import { RefObject, useEffect } from "react"; + +export function useAudioOutputDevice( + mediaRef: RefObject, + audioOutputDevice: string +): void { + useEffect(() => { + if ( + mediaRef.current && + mediaRef.current !== undefined && + audioOutputDevice + ) { + if (mediaRef.current.setSinkId) { + console.log( + `useMediaStream setting output setSinkId ${audioOutputDevice}` + ); + // Chrome for Android doesn't support this + mediaRef.current.setSinkId(audioOutputDevice); + } else { + console.log("Can't set output - no setsinkid"); + } + } + }, [mediaRef, audioOutputDevice]); +} diff --git a/src/video-grid/useMediaStream.ts b/src/video-grid/useMediaStream.ts index f9c5e474..dc915f19 100644 --- a/src/video-grid/useMediaStream.ts +++ b/src/video-grid/useMediaStream.ts @@ -22,6 +22,7 @@ import { } from "matrix-js-sdk/src/webrtc/audioContext"; import { useSpatialAudio } from "../settings/useSetting"; +import { useAudioOutputDevice } from "./useAudioOutputDevice"; declare global { interface Window { @@ -38,6 +39,8 @@ export const useMediaStream = ( ): RefObject => { const mediaRef = useRef(); + useAudioOutputDevice(mediaRef, audioOutputDevice); + useEffect(() => { console.log( `useMediaStream update stream mediaRef.current ${!!mediaRef.current} stream ${ @@ -67,24 +70,6 @@ export const useMediaStream = ( } }, [stream, mute]); - useEffect(() => { - if ( - mediaRef.current && - audioOutputDevice && - mediaRef.current !== undefined - ) { - if (mediaRef.current.setSinkId) { - console.log( - `useMediaStream setting output setSinkId ${audioOutputDevice}` - ); - // Chrome for Android doesn't support this - mediaRef.current.setSinkId(audioOutputDevice); - } else { - console.log("Can't set output - no setsinkid"); - } - } - }, [audioOutputDevice]); - useEffect(() => { if (!mediaRef.current) return; if (localVolume === null || localVolume === undefined) return; @@ -156,11 +141,11 @@ const createLoopback = async (stream: MediaStream): Promise => { export const useAudioContext = (): [ AudioContext, AudioNode, - RefObject + RefObject ] => { const context = useRef(); const destination = useRef(); - const audioRef = useRef(); + const audioRef = useRef(); useEffect(() => { if (audioRef.current && !context.current) { From 8b97904144c401ab7f269e4db61c2b9a17851871 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=C5=A0imon=20Brandner?= Date: Fri, 12 Aug 2022 09:53:44 +0200 Subject: [PATCH 2/8] Fix full-screen audio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Šimon Brandner --- src/room/InCallView.tsx | 15 +++-- src/video-grid/AudioContainer.tsx | 85 +++++++++++++++++++++++++++ src/video-grid/VideoTileContainer.jsx | 2 - src/video-grid/useMediaStream.ts | 11 +--- 4 files changed, 99 insertions(+), 14 deletions(-) create mode 100644 src/video-grid/AudioContainer.tsx diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 0e9cba17..70377658 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -44,10 +44,11 @@ import { UserMenuContainer } from "../UserMenuContainer"; import { useRageshakeRequestModal } from "../settings/submit-rageshake"; import { RageshakeRequestModal } from "./RageshakeRequestModal"; import { useMediaHandler } from "../settings/useMediaHandler"; -import { useShowInspector } from "../settings/useSetting"; +import { useShowInspector, useSpatialAudio } from "../settings/useSetting"; import { useModalTriggerState } from "../Modal"; import { useAudioContext } from "../video-grid/useMediaStream"; import { useFullscreen } from "../video-grid/useFullscreen"; +import { AudioContainer } from "../video-grid/AudioContainer"; import { useAudioOutputDevice } from "../video-grid/useAudioOutputDevice"; const canScreenshare = "getDisplayMedia" in (navigator.mediaDevices ?? {}); @@ -108,6 +109,8 @@ export function InCallView({ const [layout, setLayout] = useVideoGridLayout(screenshareFeeds.length > 0); const { toggleFullscreen, fullscreenParticipant } = useFullscreen(elementRef); + const [spatialAudio] = useSpatialAudio(); + const [audioContext, audioDestination, audioRef] = useAudioContext(); const { audioOutput } = useMediaHandler(); const [showInspector] = useShowInspector(); @@ -183,7 +186,6 @@ export function InCallView({ key={fullscreenParticipant.id} item={fullscreenParticipant} getAvatar={renderAvatar} - audioOutputDevice={audioOutput} audioContext={audioContext} audioDestination={audioDestination} disableSpeakingIndicator={true} @@ -201,7 +203,6 @@ export function InCallView({ item={item} getAvatar={renderAvatar} showName={items.length > 2 || item.focused} - audioOutputDevice={audioOutput} audioContext={audioContext} audioDestination={audioDestination} disableSpeakingIndicator={items.length < 3} @@ -217,7 +218,6 @@ export function InCallView({ items, audioContext, audioDestination, - audioOutput, layout, renderAvatar, toggleFullscreen, @@ -235,6 +235,13 @@ export function InCallView({ return (