From 0a5701b9fae35022f54658c0045be8854f529ab0 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 7 Nov 2022 11:41:49 -0500 Subject: [PATCH 1/2] Add logs to debug missing spatial audio And potentially fix it, by recreating the source node when the stream changes. --- src/video-grid/VideoTileContainer.tsx | 4 +- src/video-grid/useMediaStream.ts | 109 ++++++++++++++------------ 2 files changed, 62 insertions(+), 51 deletions(-) diff --git a/src/video-grid/VideoTileContainer.tsx b/src/video-grid/VideoTileContainer.tsx index cb93dbdd..fe3cb232 100644 --- a/src/video-grid/VideoTileContainer.tsx +++ b/src/video-grid/VideoTileContainer.tsx @@ -71,8 +71,8 @@ export function VideoTileContainer({ stream ?? null, audioContext, audioDestination, - isLocal, - localVolume + localVolume, + isLocal ); const { modalState: videoTileSettingsModalState, diff --git a/src/video-grid/useMediaStream.ts b/src/video-grid/useMediaStream.ts index 61c65240..d1879bd2 100644 --- a/src/video-grid/useMediaStream.ts +++ b/src/video-grid/useMediaStream.ts @@ -20,6 +20,7 @@ import { acquireContext, releaseContext, } from "matrix-js-sdk/src/webrtc/audioContext"; +import { logger } from "matrix-js-sdk/src/logger"; import { useSpatialAudio } from "../settings/useSetting"; import { useEventTarget } from "../useEvents"; @@ -213,10 +214,10 @@ export const useSpatialMediaStream = ( stream: MediaStream | null, audioContext: AudioContext, audioDestination: AudioNode, - mute = false, - localVolume?: number + localVolume: number, + mute = false ): [RefObject, RefObject] => { - const tileRef = useRef(); + const tileRef = useRef(null); const [spatialAudio] = useSpatialAudio(); // We always handle audio separately form the video element const mediaRef = useMediaStream(stream, null, true); @@ -227,53 +228,63 @@ export const useSpatialMediaStream = ( const sourceRef = useRef(); useEffect(() => { - if (spatialAudio && tileRef.current && !mute && audioTrackCount > 0) { - if (!pannerNodeRef.current) { - pannerNodeRef.current = new PannerNode(audioContext, { - panningModel: "HRTF", - refDistance: 3, - }); + if (spatialAudio) { + if (tileRef.current && !mute && audioTrackCount > 0) { + logger.debug(`Rendering spatial audio for ${stream!.id}`); + + if (!pannerNodeRef.current) { + pannerNodeRef.current = new PannerNode(audioContext, { + panningModel: "HRTF", + refDistance: 3, + }); + } + if (!gainNodeRef.current) { + gainNodeRef.current = new GainNode(audioContext, { + gain: localVolume, + }); + } + if (!sourceRef.current || sourceRef.current.mediaStream !== stream!) { + sourceRef.current = audioContext.createMediaStreamSource(stream!); + } + + const tile = tileRef.current; + const source = sourceRef.current; + const gainNode = gainNodeRef.current; + const pannerNode = pannerNodeRef.current; + + const updatePosition = () => { + const bounds = tile.getBoundingClientRect(); + const windowSize = Math.max(window.innerWidth, window.innerHeight); + // Position the source relative to its placement in the window + pannerNodeRef.current!.positionX.value = + (bounds.x + bounds.width / 2) / windowSize - 0.5; + pannerNodeRef.current!.positionY.value = + (bounds.y + bounds.height / 2) / windowSize - 0.5; + // Put the source in front of the listener + pannerNodeRef.current!.positionZ.value = -2; + }; + + gainNode.gain.value = localVolume; + updatePosition(); + source.connect(gainNode).connect(pannerNode).connect(audioDestination); + // HACK: We abuse the CSS transitionrun event to detect when the tile + // moves, because useMeasure, IntersectionObserver, etc. all have no + // ability to track changes in the CSS transform property + tile.addEventListener("transitionrun", updatePosition); + + return () => { + tile.removeEventListener("transitionrun", updatePosition); + source.disconnect(); + gainNode.disconnect(); + pannerNode.disconnect(); + }; + } else if (stream) { + logger.debug( + `Not rendering spatial audio for ${stream.id} (tile ref ${Boolean( + tileRef.current + )}, mute ${mute}, track count ${audioTrackCount})` + ); } - if (!gainNodeRef.current) { - gainNodeRef.current = new GainNode(audioContext, { - gain: localVolume, - }); - } - if (!sourceRef.current) { - sourceRef.current = audioContext.createMediaStreamSource(stream!); - } - - const tile = tileRef.current; - const source = sourceRef.current; - const gainNode = gainNodeRef.current; - const pannerNode = pannerNodeRef.current; - - const updatePosition = () => { - const bounds = tile.getBoundingClientRect(); - const windowSize = Math.max(window.innerWidth, window.innerHeight); - // Position the source relative to its placement in the window - pannerNodeRef.current!.positionX.value = - (bounds.x + bounds.width / 2) / windowSize - 0.5; - pannerNodeRef.current!.positionY.value = - (bounds.y + bounds.height / 2) / windowSize - 0.5; - // Put the source in front of the listener - pannerNodeRef.current!.positionZ.value = -2; - }; - - gainNode.gain.value = localVolume; - updatePosition(); - source.connect(gainNode).connect(pannerNode).connect(audioDestination); - // HACK: We abuse the CSS transitionrun event to detect when the tile - // moves, because useMeasure, IntersectionObserver, etc. all have no - // ability to track changes in the CSS transform property - tile.addEventListener("transitionrun", updatePosition); - - return () => { - tile.removeEventListener("transitionrun", updatePosition); - source.disconnect(); - gainNode.disconnect(); - pannerNode.disconnect(); - }; } }, [ stream, From 5f84cb5790861930687686a3134a1c22459564d1 Mon Sep 17 00:00:00 2001 From: Robin Townsend Date: Mon, 7 Nov 2022 14:03:28 -0500 Subject: [PATCH 2/2] Fix lint --- src/video-grid/VideoTileContainer.tsx | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/video-grid/VideoTileContainer.tsx b/src/video-grid/VideoTileContainer.tsx index afe3d643..0478bb4f 100644 --- a/src/video-grid/VideoTileContainer.tsx +++ b/src/video-grid/VideoTileContainer.tsx @@ -72,7 +72,7 @@ export function VideoTileContainer({ audioContext, audioDestination, localVolume, - isLocal || maximised, + isLocal || maximised ); const { modalState: videoTileSettingsModalState,