diff --git a/webui/src/avdummy.ts b/webui/src/avdummy.ts index 48ce56dd..761753fc 100644 --- a/webui/src/avdummy.ts +++ b/webui/src/avdummy.ts @@ -11,6 +11,10 @@ const DRAW_MAX = 150 let drawLoopIsRunning = false let drawCount = 0 +/* One shared AudioContext: Chrome hard-caps a page at ~6 of them, so a new + * one per listen()/hangup() cycle eventually throws. */ +let actx: AudioContext | null = null + function draw() { if (!ctx) return @@ -21,7 +25,8 @@ function draw() { ctx.textAlign = "center" ctx.fillStyle = "gray"; ctx.fillText(State.user.value.name, width / 2, height / 2 + image.height / 2); - ctx.drawImage(image, width / 2 - (image.width / 2), (height / 2 - image.height / 2) - 48) + if (image.complete && image.naturalWidth) + ctx.drawImage(image, width / 2 - (image.width / 2), (height / 2 - image.height / 2) - 48) } function drawLoop() { @@ -47,17 +52,32 @@ const black = () => { ctx.fillRect(0, 0, width, height) - image.src = '/avatars/' + api.user_id() + '.png' - image.onload = () => { + const start = () => { drawCount = 0 //Chrome workaround: needs canvas frame change to start webrtc rtp drawLoop() } + + image.src = '/avatars/' + api.user_id() + '.png' + image.onload = start + /* without this a missing avatar means the canvas never changes, and on + * Chrome the dummy video track then never starts sending RTP */ + image.onerror = () => { + console.warn('avdummy: avatar image could not be loaded') + start() + } + return stream.getVideoTracks()[0] } const silence = () => { - const actx = new AudioContext() + if (!actx || actx.state === 'closed') + actx = new AudioContext() + + /* created outside a user gesture the context starts suspended */ + if (actx.state === 'suspended') + actx.resume().catch((e) => console.log('avdummy: audiocontext resume', e)) + const oscillator = actx.createOscillator() const dst = actx.createMediaStreamDestination() oscillator.connect(dst) @@ -69,6 +89,10 @@ export const Avdummy = { stream: null, async init() { + /* reuse while the tracks are still live */ + if (this.stream?.getTracks().every((t) => t.readyState === 'live')) + return + this.stream = new MediaStream([black(), silence()]) }, @@ -90,5 +114,13 @@ export const Avdummy = { async stopDrawLoop() { drawCount = DRAW_MAX - } + }, + + close() { + this.stopDrawLoop() + this.stream?.getTracks().forEach((track) => track.stop()) + this.stream = null + actx?.close().catch((e) => console.log('avdummy: audiocontext close', e)) + actx = null + }, } diff --git a/webui/src/components/SettingsModal.vue b/webui/src/components/SettingsModal.vue index 03989314..bc8d49bf 100644 --- a/webui/src/components/SettingsModal.vue +++ b/webui/src/components/SettingsModal.vue @@ -70,6 +70,7 @@ v-slot="{ active, checked }" >
track.stop()) + + return null +} + +function drop_device_id(constraints: MediaStreamConstraints): boolean { + const track = (typeof constraints.video === 'object' ? constraints.video : null) + ?? (typeof constraints.audio === 'object' ? constraints.audio : null) + + if (!track || track.deviceId === undefined) + return false + + delete track.deviceId + return true +} + +async function get_media(constraints: MediaStreamConstraints, kind: MediaKind): Promise { + if (!media_supported()) + return null + + for (let i = 0; i < MEDIA_RETRIES; i++) { + try { + return await navigator.mediaDevices.getUserMedia(constraints) + } catch (e) { + const name = (e as DOMException)?.name + + /* Sometimes the device is not ready yet, since the previous stream is + * not stopped synchronously, so retry after timeout */ + if (name === 'NotReadableError' && i < MEDIA_RETRIES - 1) { + await sleep(50 * (i || 1)) + continue + } + + /* Stored device ids go stale (Safari rotates them every session, + * Chrome after site data is cleared) - retry on the default device */ + if (name === 'OverconstrainedError' && drop_device_id(constraints)) { + console.warn(`get_media/${kind}: stored device gone, using default`) + continue + } + + console.error(`get_media/${kind}:`, e) + throw e + } + } + + return null +} + +function audio_constraints(deviceId: string | undefined, echoCancellation: boolean): MediaStreamConstraints { + const audio: MediaTrackConstraints = { + echoCancellation, // disabling audio processing + autoGainControl: false, + noiseSuppression: false, + sampleRate: 48000, + /* not in lib.dom's MediaTrackConstraints, honoured by Chrome */ + ...({ latency: 0.02 } as any), //20ms + } + + /* 'ideal' rather than 'exact': a stale id degrades to the default device + * instead of rejecting with OverconstrainedError */ + if (deviceId) + audio.deviceId = { ideal: deviceId } + + return { audio, video: false } +} + +function video_constraints(deviceId: string | undefined, width: number, height: number): MediaStreamConstraints { + const video: MediaTrackConstraints = { + width: { ideal: width }, + height: { ideal: height }, + } + + if (deviceId) + video.deviceId = { ideal: deviceId } + + return { audio: false, video } +} + + /* Limits bandwidth in in [kbps] */ async function updateBandwidthRestriction(bandwidth: number) { @@ -93,7 +225,7 @@ function handle_answer(descr: any) { console.log('set remote description -- success') Webrtc.state.value = WebrtcState.Listening }, - function(error) { + function (error) { console.warn('setRemoteDescription: %s', error.toString()) } ) @@ -104,17 +236,17 @@ function pc_offer() { iceRestart: false, } pc?.createOffer(offerOptions) - .then(function(desc) { + .then(function (desc) { console.log('got local description: %s', desc.type) pc?.setLocalDescription(desc).then( () => { }, - function(error) { + function (error) { console.log('setLocalDescription: %s', error.toString()) } ) }) - .catch(function(error) { + .catch(function (error) { console.log('Failed to create session description: %s', error.toString()) }) } @@ -190,7 +322,23 @@ async function pc_stats(pc: RTCPeerConnection | null) { }); await api.rtc_stats(data) - setTimeout(pc_stats, 5000, pc) + + /* keep the handle so hangup() can cancel the chain */ + stats_timer = setTimeout(pc_stats, 5000, pc) +} + +function pc_stats_start(pc: RTCPeerConnection | null) { + /* ontrack fires per track - only ever run one stats chain */ + if (stats_timer !== null) + return + stats_timer = setTimeout(pc_stats, 0, pc) +} + +function pc_stats_stop() { + if (stats_timer === null) + return + clearTimeout(stats_timer) + stats_timer = null } @@ -221,7 +369,7 @@ async function pc_setup() { } } - pc.ontrack = function(event) { + pc.ontrack = function (event) { const track = event.track console.log('got remote track: kind=%s', track.kind) const stream = event.streams[0] @@ -234,7 +382,7 @@ async function pc_setup() { return } - pc_stats(pc) + pc_stats_start(pc) console.log('received remote audio stream') @@ -267,11 +415,11 @@ async function pc_setup() { console.log("Error attaching video stream to element", e) } - try { - video.play() - } catch (e) { + /* play() rejects, it does not throw - a try/catch never sees the + * autoplay error and it escapes as an unhandled rejection */ + video.play().catch(e => { console.log("Error video play", e) - } + }) } } @@ -304,54 +452,72 @@ async function pc_setup() { } async function pc_media_audio() { - audiostream?.getAudioTracks()[0].stop() + audiostream = stop_stream(audiostream) try { - audiostream = await navigator.mediaDevices.getUserMedia(constraintsAudio) + audiostream = await get_media( + audio_constraints(Webrtc.audio_input_id.value, Webrtc.echo.value), + 'Microphone' + ) } catch (e) { - console.error('pc_media_audio: permission denied...', e) - Error.error('Microphone permission denied!') - return + Error.error(media_error_message('Microphone', e)) } } -async function pc_media_video() { - videostream?.getVideoTracks()[0].stop() +async function pc_media_video(width: number, height: number) { + videostream = stop_stream(videostream) + /* null it out: leaving the stopped stream behind means join() later hands + * an ended track to replaceTrack(), which throws InvalidStateError */ - for (let i = 0; i < 5; i++) { - try { - Error.errorVideo('') - videostream = await navigator.mediaDevices.getUserMedia(constraintsVideo) - return - } catch (e) { - console.error('pc_media_video/error: ', e) - Error.errorVideo('Camera: ' + e) - if (e instanceof DOMException) { - /* Sometimes camera is not ready yet, since stream is not stopped - * synchronously, so retry after timeout */ - if (e.name === 'NotReadableError') { - await sleep(200) - continue - } - } - return - } + try { + Error.errorVideo('') + videostream = await get_media( + video_constraints(Webrtc.video_input_id.value, width, height), + 'Camera' + ) + } catch (e) { + Error.errorVideo(media_error_message('Camera', e)) } } -async function pc_screen() { - screenstream?.getVideoTracks()[0].stop() - const gdmOptions = { - video: true, - audio: false, - } - try { - screenstream = await navigator.mediaDevices.getDisplayMedia(gdmOptions) - } catch (e) { - console.error('pc_screen: ', e) - Error.error('Screen: ' + e) +/* Must be called synchronously from a click handler. Safari ties transient + * user activation to the current task, so calling getDisplayMedia() from a + * timer or after an await rejects with InvalidAccessError. */ +export function pc_screen_request(): Promise { + if (!has_display_media()) { + Error.error('Screen sharing is not supported by this browser.') + screen_request = Promise.resolve(null) + return screen_request } + + screen_request = navigator.mediaDevices + .getDisplayMedia({ video: true, audio: false }) + .catch((e) => { + /* the user closing the picker is not an error worth reporting */ + if ((e as DOMException)?.name !== 'NotAllowedError') + Error.error(media_error_message('Screen', e)) + else + console.log('pc_screen: cancelled by user') + return null + }) + + return screen_request +} + +async function pc_screen() { + screenstream = stop_stream(screenstream) + + const pending = screen_request ?? pc_screen_request() + screen_request = null + + screenstream = await pending + + /* browsers show their own "Stop sharing" control - follow it */ + screenstream?.getVideoTracks()[0]?.addEventListener('ended', () => { + if (Webrtc.video_select.value === 'Screen') + Webrtc.video_select.value = 'Disabled' + }) } async function pc_replace_tracks(audio_track: MediaStreamTrack | null, video_track: MediaStreamTrack | null) { @@ -364,40 +530,122 @@ async function pc_replace_tracks(audio_track: MediaStreamTrack | null, video_tra return } - if (audio_track && video_track) { - await Promise.all([audio.replaceTrack(audio_track), video.replaceTrack(video_track)]) - console.log('pc_replace_tracks: audio and video') - return + /* replaceTrack() throws InvalidStateError on an already ended track */ + if (audio_track?.readyState === 'ended') { + console.warn('pc_replace_tracks: audio track already ended') + audio_track = null } - if (audio_track) { - await Promise.all([audio.replaceTrack(audio_track)]) - console.log('pc_replace_tracks: audio') - return + if (video_track?.readyState === 'ended') { + console.warn('pc_replace_tracks: video track already ended') + video_track = null } - if (video_track) { - await Promise.all([video.replaceTrack(video_track)]) - console.log('pc_replace_tracks: video') - return + try { + const pending: Promise[] = [] + + if (audio_track) pending.push(audio.replaceTrack(audio_track)) + if (video_track) pending.push(video.replaceTrack(video_track)) + + if (!pending.length) + return + + await Promise.all(pending) + console.log('pc_replace_tracks:', audio_track ? 'audio' : '', video_track ? 'video' : '') + } catch (e) { + console.error('pc_replace_tracks:', e) } } async function audio_output(device: string | undefined) { - let audio_out: any = document.querySelector("audio#live"); - - if ( - typeof audio_out === "undefined" || - typeof audio_out?.setSinkId === "undefined" - ) { - console.log( - "webrtc: audio element not found or setSinkId not supported" - ); - return; + if (!device) + return + + /* querySelector() returns null, never undefined */ + const audio_out = document.querySelector('audio#live') + + /* setSinkId is unsupported on Safari and was disabled by default on + * Firefox until 116 */ + if (!audio_out || typeof (audio_out as any).setSinkId !== 'function') { + console.log('webrtc: audio element not found or setSinkId not supported') + return } - await audio_out?.setSinkId(device); - console.log("webrtc: changed output"); + try { + await (audio_out as any).setSinkId(device) + console.log('webrtc: changed output') + } catch (e) { + /* rejects with NotAllowedError without output device permission */ + console.warn('webrtc: setSinkId failed', e) + Error.error('Could not switch the speaker: ' + ((e as Error)?.message ?? String(e))) + } +} + +/* Device ids are not stable identifiers, so fall back to matching on the + * label before giving up on a remembered device. */ +function resolve_device( + devices: MediaDeviceInfo[], + kind: MediaDeviceKind, + id: string | undefined, + label: string | undefined +): string | undefined { + const list = devices.filter((d) => d.kind === kind && d.deviceId) + + if (id && list.some((d) => d.deviceId === id)) + return id + + if (label) + return list.find((d) => d.label === label)?.deviceId + + return undefined +} + +async function change_video_now(): Promise { + const select = Webrtc.video_select.value + + if (select !== 'Screen') { + screenstream = stop_stream(screenstream) + } + + if (select === 'Disabled') { + Webrtc.video_mute(true) + videostream = stop_stream(videostream) + Webrtc.videostream.value = null + return null + } + + if (select === 'Screen') { + videostream = stop_stream(videostream) + + await pc_screen() + Webrtc.videostream.value = screenstream + + /* picker cancelled or unsupported - do not leave the UI on "Screen" */ + if (!screenstream) + Webrtc.video_select.value = 'Disabled' + + return screenstream + } + + const res = resolutions[Webrtc.video_resolution.value] ?? resolutions['720p'] + await updateBandwidthRestriction(res.bandwidth) + await pc_media_video(res.width, res.height) + + const track = videostream?.getVideoTracks()[0] + if (track) { + /* remember what we actually got, not what we asked for */ + Webrtc.video_input_id.value = track.getSettings().deviceId ?? Webrtc.video_input_id.value + Webrtc.video_input_label.value = track.label + } + + Webrtc.videostream.value = videostream + if (!videostream) + return null + + Avdummy.stopDrawLoop() + + console.log('video changed', res) + return videostream } @@ -416,8 +664,10 @@ export const Webrtc = { state: ref(WebrtcState.Offline), deviceInfos: ref([]), audio_input_id: useStorage('audio_input_id', undefined as string | undefined), + audio_input_label: useStorage('audio_input_label', undefined as string | undefined), audio_output_id: ref(undefined), video_input_id: useStorage('video_input_id', undefined as string | undefined), + video_input_label: useStorage('video_input_label', undefined as string | undefined), video_select: ref('Disabled'), video_resolution: ref('720p'), audio_muted: ref(true), @@ -431,8 +681,12 @@ export const Webrtc = { async listen() { if (this.state.value != WebrtcState.Offline) return - pc_setup() + + /* set before pc_setup() so the first relay candidate cannot race the + * state transition and lose the offer */ this.state.value = WebrtcState.ICEGatheringRelay + await pc_setup() + const audio: HTMLAudioElement | null = document.querySelector('audio#live') const audio_return = audio?.play() @@ -443,53 +697,81 @@ export const Webrtc = { }, async update_avdevices() { + if (!media_supported()) + return Webrtc.deviceInfos.value = await navigator.mediaDevices.enumerateDevices() }, async init_avdevices() { - let audio_available = false - let video_available = false + if (!media_supported()) + return + await pc_media_audio() - await pc_media_video() + + /* do not prompt for the camera when the user is joining audio only */ + if (this.video_select.value !== 'Disabled') + await this.change_video() + + /* labels and ids are only exposed after permission was granted */ this.deviceInfos.value = await navigator.mediaDevices.enumerateDevices() - this.deviceInfos.value.forEach((device) => { - console.log(`${device.kind}: ${device.label} id = ${device.deviceId}`) - if (device.kind === 'audiooutput' && !this.audio_output_id.value) { - this.audio_output_id.value = device.deviceId - this.change_audio_out() - return - } + const audio_track = audiostream?.getAudioTracks()[0] + const audio_id = audio_track?.getSettings().deviceId + + /* the remembered id may be stale while the label still matches */ + const wanted_audio = resolve_device( + this.deviceInfos.value, 'audioinput', + this.audio_input_id.value, this.audio_input_label.value + ) + + if (wanted_audio && wanted_audio !== audio_id) { + this.audio_input_id.value = wanted_audio + await this.change_audio() + } else if (audio_track) { + this.audio_input_id.value = audio_id ?? undefined + this.audio_input_label.value = audio_track.label + } - if (this.audio_input_id.value) { - if (this.audio_input_id.value == device.deviceId) { - audio_available = true - } - } - if (this.video_input_id.value) { - if (this.video_input_id.value == device.deviceId) { - video_available = true - } + if (!this.audio_output_id.value) { + /* Safari does not enumerate audiooutput devices at all */ + const out = this.deviceInfos.value.find((d) => d.kind === 'audiooutput' && d.deviceId) + if (out) { + this.audio_output_id.value = out.deviceId + await this.change_audio_out() } - }) + } - if (audio_available) - this.change_audio() - else - this.audio_input_id.value = audiostream?.getAudioTracks()[0].getSettings().deviceId + const video_id = videostream?.getVideoTracks()[0]?.getSettings().deviceId + const wanted_video = resolve_device( + this.deviceInfos.value, 'videoinput', + this.video_input_id.value, this.video_input_label.value + ) - if (!video_available) + if (this.video_select.value !== 'Disabled' && wanted_video && wanted_video !== video_id) { + this.video_input_id.value = wanted_video + await this.change_video() + } else if (!wanted_video && this.video_select.value === 'Disabled') { this.video_input_id.value = undefined + this.video_input_label.value = undefined + } - useEventListener(navigator!.mediaDevices, 'devicechange', Webrtc.update_avdevices) + useEventListener(navigator.mediaDevices, 'devicechange', Webrtc.update_avdevices) }, async change_audio() { - constraintsAudio.audio.deviceId = { exact: this.audio_input_id.value } await pc_media_audio() + + const track = audiostream?.getAudioTracks()[0] + if (track) { + this.audio_input_id.value = track.getSettings().deviceId ?? this.audio_input_id.value + this.audio_input_label.value = track.label + } + this.audio_mute(this.audio_muted.value) - if (audiostream && this.state.value >= WebrtcState.ReadySpeaking) - await pc_replace_tracks(audiostream.getAudioTracks()[0], null) + + if (track && this.state.value >= WebrtcState.ReadySpeaking) + await pc_replace_tracks(track, null) + console.log('audio changed') }, @@ -497,61 +779,16 @@ export const Webrtc = { await audio_output(this.audio_output_id.value) }, + /* Serialized: concurrent getUserMedia() on one device fails with + * NotReadableError, and the shared constraint object used to be mutated + * by whichever caller ran last. */ async change_video(): Promise { - screenstream?.getVideoTracks()[0].stop() - - if (this.video_select.value === 'Disabled') { - this.video_mute(true) - videostream?.getVideoTracks()[0].stop() - return null - } - - if (this.video_select.value === 'Screen') { - videostream?.getVideoTracks()[0].stop() - await pc_screen() - this.videostream.value = screenstream - return screenstream - } - - if (this.video_resolution.value === '720p') { - constraintsVideo.video.width = 1280 - constraintsVideo.video.height = 720 - updateBandwidthRestriction(2000) - } - - if (this.video_resolution.value === '360p') { - constraintsVideo.video.width = 640 - constraintsVideo.video.height = 360 - updateBandwidthRestriction(1000) - } - - if (this.video_resolution.value === '360pl') { - constraintsVideo.video.width = 640 - constraintsVideo.video.height = 360 - updateBandwidthRestriction(400) - } - - if (this.video_input_id.value !== 'undefined') - constraintsVideo.video.deviceId = { exact: this.video_input_id.value } - - await pc_media_video() - if (!this.video_input_id.value) { - this.video_input_id.value = videostream?.getVideoTracks()[0].getSettings().deviceId - } - - this.videostream.value = videostream - if (videostream === null) - return null - - Avdummy.stopDrawLoop() - - console.log('video changed', constraintsVideo) - return videostream + video_queue = video_queue.catch(() => null).then(change_video_now) + return video_queue }, async change_echo() { - constraintsAudio.audio.echoCancellation = this.echo.value - console.log('echo changed', constraintsAudio) + console.log('echo changed', this.echo.value) await this.change_audio() }, @@ -563,20 +800,17 @@ export const Webrtc = { } if (this.state.value < WebrtcState.ReadySpeaking) { - if (this.video_select.value === 'Screen' && audiostream && screenstream) { - await pc_replace_tracks(audiostream.getAudioTracks()[0], screenstream.getVideoTracks()[0]) - this.audio_mute(false) - this.video_mute(false) - } - else if (videostream && audiostream) { - await pc_replace_tracks(audiostream.getAudioTracks()[0], videostream.getVideoTracks()[0]) - this.audio_mute(false) - this.video_mute(false) - } - else if (audiostream) { - await pc_replace_tracks(audiostream.getAudioTracks()[0], null) - this.audio_mute(false) + const audio_track = audiostream?.getAudioTracks()[0] ?? null + const video_track = this.video_select.value === 'Screen' + ? screenstream?.getVideoTracks()[0] ?? null + : videostream?.getVideoTracks()[0] ?? null + + if (audio_track || video_track) { + await pc_replace_tracks(audio_track, video_track) + if (audio_track) this.audio_mute(false) + if (video_track) this.video_mute(false) } + this.state.value = WebrtcState.ReadySpeaking } }, @@ -586,9 +820,12 @@ export const Webrtc = { audiostream?.getAudioTracks().forEach((track) => { track.enabled = !mute - this.audio_muted.value = mute - api.audio(!mute) }) + + /* was inside the forEach: without a track the mute state and the + * server never learned about the change */ + this.audio_muted.value = mute + api.audio(!mute) }, async video_mute(mute: boolean, local?: boolean) { @@ -618,11 +855,15 @@ export const Webrtc = { }, hangup() { - videostream?.getVideoTracks()[0].stop() - screenstream?.getVideoTracks()[0].stop() - audiostream?.getAudioTracks()[0].stop() + videostream = stop_stream(videostream) + screenstream = stop_stream(screenstream) + audiostream = stop_stream(audiostream) + this.videostream.value = null this.video_select.value = 'Disabled' + pc_stats_stop() + Avdummy.close() + pc?.close() pc = null this.state.value = WebrtcState.Offline diff --git a/webui/src/webrtc_source.ts b/webui/src/webrtc_source.ts index 04600ba3..bd6ab23c 100644 --- a/webui/src/webrtc_source.ts +++ b/webui/src/webrtc_source.ts @@ -42,7 +42,7 @@ export class WebRTCSource { } video.srcObject = this.video - video.play() + video.play().catch((e) => console.log('WebRTCSource: play', e)) console.log("WebRTCSource: video track added") }