diff --git a/.gitignore b/.gitignore index e9225072dd..35c3bf3b55 100644 --- a/.gitignore +++ b/.gitignore @@ -7,6 +7,8 @@ dist-ssr *.bkp .idea/ public/config.json +# DeepFilterNet3 WASM/model assets are downloaded by `pnpm setup:assets` +public/assets/deepfilternet3 backend/synapse_tmp/* backend/synapse_tmp_othersite/* /coverage diff --git a/locales/de/app.json b/locales/de/app.json index 32ee930cdf..86ba77d474 100644 --- a/locales/de/app.json +++ b/locales/de/app.json @@ -252,6 +252,8 @@ "muted_for_me": "Für mich stumm geschaltet", "screen_share_volume": "Lautstärke der Bildschirmfreigabe", "volume": "Lautstärke", - "waiting_for_media": "Warten auf Medien..." + "waiting_for_media": "Warten auf Medien...", + "stop_watching": "Nicht mehr zuschauen", + "watch_stream": "Stream zuschauen" } } diff --git a/locales/en/app.json b/locales/en/app.json index c5f8b34320..bb0a2d4e8b 100644 --- a/locales/en/app.json +++ b/locales/en/app.json @@ -204,6 +204,12 @@ "audio_processing_description": "Changes apply on next call join.", "audio_processing_header": "Audio processing", "audio_tab": { + "deepfilternet_description": "Uses a deep-learning model to remove background noise such as keyboard, traffic, and other non-stationary sounds.", + "deepfilternet_header": "AI noise suppression", + "deepfilternet_label": "Enable AI noise suppression (DeepFilterNet)", + "deepfilternet_level_description": "Higher levels remove more noise but can make your voice sound more processed.", + "deepfilternet_level_label": "Noise reduction level", + "deepfilternet_not_supported": "(AI noise suppression is not supported by this browser.)", "effect_volume_description": "Adjust the volume at which reactions and hand raised effects play.", "effect_volume_label": "Sound effect volume", "mic_cutoff_description": "Sound quieter than the cutoff volume will not be sent to other participants.", @@ -255,6 +261,8 @@ "preferences_tab": { "developer_mode_label": "Developer mode", "developer_mode_label_description": "Enable developer mode and show developer settings tab.", + "hide_avatars_when_camera_off_description": "Hide avatar tiles of participants whose camera is off.", + "hide_avatars_when_camera_off_label": "Hide avatar tiles when camera is off", "introduction": "Here you can configure extra options for an improved experience.", "reactions_play_sound_description": "Play a sound effect when anyone sends a reaction into a call.", "reactions_play_sound_label": "Play reaction sounds", @@ -290,7 +298,9 @@ "mute_for_me": "Mute for me", "muted_for_me": "Muted for me", "screen_share_volume": "Screen share volume", + "stop_watching": "Stop watching", "volume": "Volume", - "waiting_for_media": "Waiting for media..." + "waiting_for_media": "Waiting for media...", + "watch_stream": "Watch stream" } } diff --git a/package.json b/package.json index ec6e3d19ca..e4bb3105a7 100644 --- a/package.json +++ b/package.json @@ -7,7 +7,7 @@ "dev:full": "vite", "dev:embedded": "vite --config vite-embedded.config.js", "build": "pnpm build:full", - "build:full": "vite build", + "build:full": "pnpm setup:assets && vite build", "build:full:production": "pnpm build:full", "build:full:development": "pnpm build:full --mode development", "build:embedded": "pnpm build:full --config vite-embedded.config.ts", @@ -16,6 +16,7 @@ "build:sdk:development": "pnpm build:sdk --mode development", "build:sdk": "pnpm build:full --config vite-sdk.config.js", "build:sdk:production": "pnpm build:sdk", + "setup:assets": "node scripts/setup-noise-suppression-assets.js", "serve": "vite preview", "format": "oxfmt", "format:check": "oxfmt --check; rc=$?; [[ $rc -ne 0 ]] && printf '\\033[46;30m INFO \\033[0m To fix, run: pnpm format\\n' >&2; exit $rc", @@ -124,6 +125,7 @@ "vite-plugin-html": "^3.2.2", "vite-plugin-node-polyfills": "^0.28.0", "vite-plugin-node-stdlib-browser": "^0.2.1", + "vite-plugin-static-copy": "^4.1.1", "vite-plugin-svgr": "^4.0.0", "vite-plugin-wasm": "^3.6.0", "vitest": "^4.1.5", @@ -132,6 +134,7 @@ "packageManager": "pnpm@11.6.0+sha512.9a36518224080c6fe5165afdcfe79bfa118c29be703f3f462b1e32efe1e98e47e8750b148e08286250aad4113cc7993ca413c4e2cd447752708c2ee5751bc95f", "dependencies": { "@jitsi/rnnoise-wasm": "0.2.1", - "@phosphor-icons/react": "^2.1.10" + "@phosphor-icons/react": "^2.1.10", + "deepfilternet3-noise-filter": "^1.3.0" } } diff --git a/pnpm-lock.yaml b/pnpm-lock.yaml index 6dba68ba99..0317afd210 100644 --- a/pnpm-lock.yaml +++ b/pnpm-lock.yaml @@ -26,6 +26,9 @@ importers: '@phosphor-icons/react': specifier: ^2.1.10 version: 2.1.10(react-dom@19.2.6(react@19.2.6))(react@19.2.6) + deepfilternet3-noise-filter: + specifier: ^1.3.0 + version: 1.3.0(livekit-client@2.19.2(@types/dom-mediacapture-record@1.0.22)) devDependencies: '@codecov/vite-plugin': specifier: ^1.3.0 @@ -279,6 +282,9 @@ importers: vite-plugin-node-stdlib-browser: specifier: ^0.2.1 version: 0.2.1(node-stdlib-browser@1.3.1)(rollup@4.60.1)(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(sass@1.100.0)(terser@5.46.1)(yaml@2.9.0)) + vite-plugin-static-copy: + specifier: ^4.1.1 + version: 4.1.1(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(sass@1.100.0)(terser@5.46.1)(yaml@2.9.0)) vite-plugin-svgr: specifier: ^4.0.0 version: 4.5.0(rollup@4.60.1)(typescript@5.9.3)(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(sass@1.100.0)(terser@5.46.1)(yaml@2.9.0)) @@ -4073,6 +4079,12 @@ packages: deep-is@0.1.4: resolution: {integrity: sha512-oIPzksmTg4/MriiaYGO+okXDT7ztn/w3Eptv/+gSIdMdKsJo0u4CfYNFJPy+4SKMuCqGw2wxnA+URMg3t8a/bQ==} + deepfilternet3-noise-filter@1.3.0: + resolution: {integrity: sha512-yYFUlPuvPguqcd/R6/OSsr0noGqlqOE50JkCWYHogk+PjLj9qrNgwTt5zKraVkKnw0l4+eXJagkz2SUWtUl4sQ==} + engines: {node: '>=18.0.0'} + peerDependencies: + livekit-client: ^2.0.0 + default-browser-id@5.0.1: resolution: {integrity: sha512-x1VCxdX4t+8wVfd1so/9w+vQ4vx7lKd2Qp5tDRutErwmR85OgmfX7RlLRMWafRMY7hbEiXIbudNrjOAPa/hL8Q==} engines: {node: '>=18'} @@ -5203,6 +5215,10 @@ packages: resolution: {integrity: sha512-LaNjtRWUBY++zB5nE/NwcaoMylSPk+S+ZHNB1TzdbMJMny6dynpAGt7X/tl/QYq3TIeE6nxHppbo2LGymrG5Pw==} engines: {node: '>=10'} + p-map@7.0.6: + resolution: {integrity: sha512-I4Prw6ivkd6p8PiYR1tXASOAOBzIJwu0TB7fqaX0c/8c3QAehNYmX57EijyGGGBt3c/BIowGwV03RVBtXvHEVg==} + engines: {node: '>=18'} + p-retry@8.0.0: resolution: {integrity: sha512-kFVqH1HxOHp8LupNsOys7bSV09VYTRLxarH/mokO4Rqhk6wGi70E0jh4VzvVGXfEVNggHoHLAMWsQqHyU1Ey9A==} engines: {node: '>=22'} @@ -6222,6 +6238,12 @@ packages: node-stdlib-browser: ^1.2.0 vite: ^2.0.0 || ^3.0.0 || ^4.0.0 + vite-plugin-static-copy@4.1.1: + resolution: {integrity: sha512-GrlA8YklrAfSyxJ4M3fdQLOo9oNkp56IM9FYgX/WtEgeIFkPwhu4wzpufBCIuNKCa6Fn77FkRdYxkHqV0FwjAw==} + engines: {node: ^22.0.0 || >=24.0.0} + peerDependencies: + vite: ^6.0.0 || ^7.0.0 || ^8.0.0 + vite-plugin-svgr@4.5.0: resolution: {integrity: sha512-W+uoSpmVkSmNOGPSsDCWVW/DDAyv+9fap9AZXBvWiQqrboJ08j2vh0tFxTD/LjwqwAd3yYSVJgm54S/1GhbdnA==} peerDependencies: @@ -9836,6 +9858,10 @@ snapshots: deep-is@0.1.4: {} + deepfilternet3-noise-filter@1.3.0(livekit-client@2.19.2(@types/dom-mediacapture-record@1.0.22)): + dependencies: + livekit-client: 2.19.2(@types/dom-mediacapture-record@1.0.22) + default-browser-id@5.0.1: {} default-browser@5.5.0: @@ -11141,6 +11167,8 @@ snapshots: dependencies: p-limit: 3.1.0 + p-map@7.0.6: {} + p-retry@8.0.0: dependencies: is-network-error: 1.3.1 @@ -12260,6 +12288,14 @@ snapshots: transitivePeerDependencies: - rollup + vite-plugin-static-copy@4.1.1(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(sass@1.100.0)(terser@5.46.1)(yaml@2.9.0)): + dependencies: + chokidar: 3.6.0 + p-map: 7.0.6 + picocolors: 1.1.1 + tinyglobby: 0.2.17 + vite: 8.0.16(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(sass@1.100.0)(terser@5.46.1)(yaml@2.9.0) + vite-plugin-svgr@4.5.0(rollup@4.60.1)(typescript@5.9.3)(vite@8.0.16(@types/node@24.12.4)(esbuild@0.28.0)(jiti@2.7.0)(sass@1.100.0)(terser@5.46.1)(yaml@2.9.0)): dependencies: '@rollup/pluginutils': 5.3.0(rollup@4.60.1) diff --git a/scripts/setup-noise-suppression-assets.js b/scripts/setup-noise-suppression-assets.js new file mode 100644 index 0000000000..70ff0b88fd --- /dev/null +++ b/scripts/setup-noise-suppression-assets.js @@ -0,0 +1,172 @@ +#!/usr/bin/env node + +/** + * Setup script to download DeepFilterNet3 assets for local bundling. + * This downloads the WASM binary and AI model from Mezon's CDN + * and places them in public/assets/deepfilternet3/ for bundling. + * + * Usage: + * node scripts/setup-noise-suppression-assets.js + * + * Environment variables: + * DEEPFILTERNET3_CDN_URL: Override the default CDN URL (optional) + */ + +import fs from "fs"; +import path from "path"; +import https from "https"; +import { fileURLToPath } from "url"; + +const __filename = fileURLToPath(import.meta.url); +const __dirname = path.dirname(__filename); +const projectRoot = path.join(__dirname, ".."); + +const CDN_URL = + process.env.DEEPFILTERNET3_CDN_URL || + "https://cdn.mezon.ai/AI/models/datas/noise_suppression/deepfilternet3"; + +const ASSETS_DIR = path.join(projectRoot, "public", "assets", "deepfilternet3"); +// The deepfilternet3-noise-filter package (>= 1.3.0) requests assets from the +// `v3/` subdirectory. Keep these paths in sync with the installed package's +// AssetLoader (see node_modules/deepfilternet3-noise-filter/dist/index.esm.js). +const VERSION_DIR = path.join(ASSETS_DIR, "v3"); +const PKG_DIR = path.join(VERSION_DIR, "pkg"); +const MODELS_DIR = path.join(VERSION_DIR, "models"); + +const FILES_TO_DOWNLOAD = [ + { + url: `${CDN_URL}/v3/pkg/df_bg.wasm`, + path: path.join(PKG_DIR, "df_bg.wasm"), + description: "WASM binary", + }, + { + url: `${CDN_URL}/v3/pkg/df_bg.wasm.d.ts`, + path: path.join(PKG_DIR, "df_bg.wasm.d.ts"), + description: "WASM TypeScript definitions", + optional: true, + }, + { + url: `${CDN_URL}/v3/models/DeepFilterNet3_onnx.tar.gz`, + path: path.join(MODELS_DIR, "DeepFilterNet3_onnx.tar.gz"), + description: "AI Model (ONNX format)", + }, +]; + +function ensureDir(dir) { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }); + console.log(`✓ Created directory: ${dir}`); + } +} + +function downloadFile(fileUrl, filePath, isOptional = false) { + return new Promise((resolve, reject) => { + const fileName = path.basename(filePath); + + // Skip if already exists + if (fs.existsSync(filePath)) { + console.log(`✓ Already exists: ${fileName}`); + resolve(); + return; + } + + console.log(`⏳ Downloading ${fileName}...`); + + https + .get(fileUrl, (response) => { + // Handle redirects + if ( + response.statusCode === 301 || + response.statusCode === 302 || + response.statusCode === 307 + ) { + const redirectUrl = response.headers.location; + console.log(` Redirected to: ${redirectUrl}`); + downloadFile(redirectUrl, filePath, isOptional) + .then(resolve) + .catch(reject); + return; + } + + if (response.statusCode !== 200) { + const error = new Error( + `Download failed: HTTP ${response.statusCode} for ${fileName}`, + ); + if (isOptional) { + console.warn(`⚠ Optional file skipped: ${fileName}`); + resolve(); + } else { + reject(error); + } + return; + } + + const expectedSize = Number(response.headers["content-length"]); + const fileStream = fs.createWriteStream(filePath); + + response.pipe(fileStream); + + fileStream.on("finish", () => { + fileStream.close(); + const actualSize = fs.statSync(filePath).size; + // Guard against truncated downloads: if the server told us the + // expected size and we received fewer bytes, the file is corrupt. + if (Number.isFinite(expectedSize) && expectedSize > 0 && actualSize !== expectedSize) { + fs.unlinkSync(filePath); + reject( + new Error( + `Download incomplete for ${fileName}: expected ${expectedSize} bytes, got ${actualSize}`, + ), + ); + return; + } + const sizeMB = (actualSize / 1024 / 1024).toFixed(2); + console.log(`✓ Downloaded: ${fileName} (${sizeMB} MB)`); + resolve(); + }); + + fileStream.on("error", (err) => { + fs.unlink(filePath, () => {}); // Clean up partial file + reject(err); + }); + }) + .on("error", (err) => { + if (isOptional) { + console.warn(`⚠ Optional file skipped: ${fileName} (${err.message})`); + resolve(); + } else { + reject(err); + } + }); + }); +} + +async function main() { + try { + console.log("\n🚀 Setting up DeepFilterNet3 assets for bundling...\n"); + console.log(`📦 CDN URL: ${CDN_URL}`); + console.log(`📁 Asset directory: ${ASSETS_DIR}\n`); + + // Ensure directories exist + ensureDir(ASSETS_DIR); + ensureDir(VERSION_DIR); + ensureDir(PKG_DIR); + ensureDir(MODELS_DIR); + + // Download files + for (const file of FILES_TO_DOWNLOAD) { + await downloadFile(file.url, file.path, file.optional); + } + + console.log("\n✅ Asset setup complete!"); + console.log( + "\nAssets are ready for bundling. Next build will include them.\n", + ); + process.exit(0); + } catch (error) { + console.error("\n❌ Asset setup failed:", error.message); + process.exit(1); + } +} + +main(); diff --git a/sdk/main.ts b/sdk/main.ts index a001af65c0..dd79498da9 100644 --- a/sdk/main.ts +++ b/sdk/main.ts @@ -84,6 +84,8 @@ interface MatrixRTCSdk { connection: Connection | null; membership: CallMembership; participant: LocalParticipant | RemoteParticipant | null; + speaking: boolean; + audioLevel: number; }[] >; /** @@ -93,7 +95,17 @@ interface MatrixRTCSdk { connection: Connection | null; membership: CallMembership; participant: LocalParticipant | null; + speaking: boolean; + audioLevel: number; } | null>; + activeSpeakers$: Behavior< + { + connection: Connection | null; + membership: CallMembership; + participant: LocalParticipant | RemoteParticipant | null; + audioLevel: number; + }[] + >; /** Use the LocalMemberConnectionState returned from `join` for a more detailed connection state */ connected$: Behavior; sendData?: (data: unknown) => Promise; @@ -302,6 +314,87 @@ export async function createMatrixRTCSdk( logger.info("createMatrixRTCSdk done"); + const voiceActivityForMember$ = (member: { + userId: string; + membership$: Behavior; + }): Observable<{ speaking: boolean; audioLevel: number }> => + combineLatest([member.membership$, callViewModel.userMedia$]).pipe( + switchMap(([membership, mediaItems]) => { + const media = mediaItems.find( + (m) => + m.userId === member.userId && + m.id.startsWith(`${member.userId}:${membership.deviceId}:`), + ); + return media + ? combineLatest([media.voiceActivity$, media.audioLevel$]).pipe( + map(([speaking, audioLevel]) => ({ speaking, audioLevel })), + ) + : of({ speaking: false, audioLevel: 0 }); + }), + ); + + const localMember$ = scope.behavior( + callViewModel.localMatrixLivekitMember$.pipe( + tap((member) => logger.info("localMatrixLivekitMember$ next: ", member)), + switchMap((member) => { + if (member === null) return of(null); + return combineLatest([ + member.connection$, + member.membership$, + member.participant.value$, + voiceActivityForMember$(member), + ]).pipe( + map(([connection, membership, participant, voice]) => ({ + connection, + membership, + participant, + speaking: voice.speaking, + audioLevel: voice.audioLevel, + })), + ); + }), + tap((member) => logger.info("localMember$ next: ", member)), + ), + ); + + const remoteMembers$ = scope.behavior( + callViewModel.remoteMatrixLivekitMembers$.pipe( + switchMap((members) => { + const listOfMemberObservables = members.map((member) => + combineLatest([ + member.connection$, + member.membership$, + member.participant.value$, + voiceActivityForMember$(member), + ]).pipe( + map(([connection, membership, participant, voice]) => ({ + connection, + membership, + participant, + speaking: voice.speaking, + audioLevel: voice.audioLevel, + })), + // using shareReplay instead of a Behavior here because the behavior would need + // a tricky scope.end() setup. + shareReplay({ bufferSize: 1, refCount: true }), + ), + ); + return combineLatest(listOfMemberObservables); + }), + ), + [], + ); + const activeSpeakers$ = scope.behavior( + combineLatest([localMember$, remoteMembers$]).pipe( + map(([local, remote]) => + [...(local && local.speaking ? [local] : []), ...remote].filter( + (m) => m.speaking, + ), + ), + ), + [], + ); + return { join: (): void => { // first lets try making the widget sticky @@ -317,53 +410,10 @@ export async function createMatrixRTCSdk( scope.end(); }, data$, - localMember$: scope.behavior( - callViewModel.localMatrixLivekitMember$.pipe( - tap((member) => - logger.info("localMatrixLivekitMember$ next: ", member), - ), - switchMap((member) => { - if (member === null) return of(null); - return combineLatest([ - member.connection$, - member.membership$, - member.participant.value$, - ]).pipe( - map(([connection, membership, participant]) => ({ - connection, - membership, - participant, - })), - ); - }), - tap((member) => logger.info("localMember$ next: ", member)), - ), - ), + localMember$, connected$: callViewModel.connected$, - remoteMembers$: scope.behavior( - callViewModel.remoteMatrixLivekitMembers$.pipe( - switchMap((members) => { - const listOfMemberObservables = members.map((member) => - combineLatest([ - member.connection$, - member.membership$, - member.participant.value$, - ]).pipe( - map(([connection, membership, participant]) => ({ - connection, - membership, - participant, - })), - // using shareReplay instead of a Behavior here because the behavior would need - // a tricky scope.end() setup. - shareReplay({ bufferSize: 1, refCount: true }), - ), - ); - return combineLatest(listOfMemberObservables); - }), - ), - [], - ), + remoteMembers$, + activeSpeakers$, sendData, sendRoomMessage, }; diff --git a/src/UrlParams.test.ts b/src/UrlParams.test.ts index 75bf9bfb89..1639660317 100644 --- a/src/UrlParams.test.ts +++ b/src/UrlParams.test.ts @@ -376,6 +376,46 @@ describe("UrlParams", () => { }); }); + describe("deepFilterNetNoiseSuppression", () => { + it("is undefined by default", () => { + expect(computeUrlParams().deepFilterNetNoiseSuppression).toBeUndefined(); + }); + + it("is parsed as a flag", () => { + expect( + computeUrlParams("?deepFilterNetNoiseSuppression=true") + .deepFilterNetNoiseSuppression, + ).toBe(true); + expect( + computeUrlParams("?deepFilterNetNoiseSuppression=false") + .deepFilterNetNoiseSuppression, + ).toBe(false); + }); + }); + + describe("deepFilterNetNoiseSuppressionLevel", () => { + it("is undefined by default", () => { + expect( + computeUrlParams().deepFilterNetNoiseSuppressionLevel, + ).toBeUndefined(); + }); + + it("is parsed and clamped to 0-1", () => { + expect( + computeUrlParams("?deepFilterNetNoiseSuppressionLevel=0.5") + .deepFilterNetNoiseSuppressionLevel, + ).toBe(0.5); + expect( + computeUrlParams("?deepFilterNetNoiseSuppressionLevel=2") + .deepFilterNetNoiseSuppressionLevel, + ).toBe(1); + expect( + computeUrlParams("?deepFilterNetNoiseSuppressionLevel=-1") + .deepFilterNetNoiseSuppressionLevel, + ).toBe(0); + }); + }); + describe("echoCancellation", () => { it("defaults to true", () => { expect(computeUrlParams().echoCancellation).toBe(true); diff --git a/src/UrlParams.ts b/src/UrlParams.ts index 22f6b45873..26b3089bd1 100644 --- a/src/UrlParams.ts +++ b/src/UrlParams.ts @@ -255,6 +255,16 @@ export interface UrlConfiguration { * Defaults to true. */ noiseSuppression?: boolean; + /** + * Whether to enable DeepFilterNet-based noise suppression. + * Overrides the user setting when provided. + */ + deepFilterNetNoiseSuppression?: boolean; + /** + * The DeepFilterNet noise reduction level (0-1). + * Overrides the user setting when provided. + */ + deepFilterNetNoiseSuppressionLevel?: number; callIntent?: RTCCallIntent; } @@ -505,6 +515,15 @@ export const computeUrlParams = (search = "", hash = ""): UrlParams => { autoLeaveWhenOthersLeft: parser.getFlag("autoLeave"), noiseSuppression: parser.getFlagParam("noiseSuppression", true), echoCancellation: parser.getFlagParam("echoCancellation", true), + deepFilterNetNoiseSuppression: parser.getFlag( + "deepFilterNetNoiseSuppression", + ), + deepFilterNetNoiseSuppressionLevel: ((): number | undefined => { + const val = parseFloat( + parser.getParam("deepFilterNetNoiseSuppressionLevel") ?? "", + ); + return Number.isFinite(val) ? Math.max(0, Math.min(1, val)) : undefined; + })(), }; // Log the final configuration for debugging purposes. diff --git a/src/audio/DeepFilterNetProcessor.test.ts b/src/audio/DeepFilterNetProcessor.test.ts new file mode 100644 index 0000000000..ee6a39fdd4 --- /dev/null +++ b/src/audio/DeepFilterNetProcessor.test.ts @@ -0,0 +1,169 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { beforeEach, describe, expect, it, vi } from "vitest"; +import { DeepFilterNoiseFilterProcessor } from "deepfilternet3-noise-filter"; + +import { + DeepFilterNetProcessor, + DEEPFILTERNET_PROCESSOR_NAME, + supportsDeepFilterNetProcessor, +} from "./DeepFilterNetProcessor"; + +type DeepFilterNoiseFilterProcessorOptions = Record; + +type DeepFilterNoiseFilterProcessorContext = { + setEnabled?: unknown; + setSuppressionLevel?: unknown; + destroy?: unknown; + init?: unknown; + restart?: unknown; + processedTrack?: MediaStreamTrack; +}; + +type NoiseFilterProcessorMock = ReturnType & { + mockSetEnabled: ReturnType; + mockSetSuppressionLevel: ReturnType; + mockDestroy: ReturnType; + mockInit: ReturnType; + mockRestart: ReturnType; +}; + +vi.mock("deepfilternet3-noise-filter", () => { + const mockSetEnabled = vi.fn().mockResolvedValue(true); + const mockSetSuppressionLevel = vi.fn(); + const mockDestroy = vi.fn().mockResolvedValue(undefined); + const mockInit = vi.fn().mockResolvedValue(undefined); + const mockRestart = vi.fn().mockResolvedValue(undefined); + + const mockDeepFilterNoiseFilterProcessor = vi + .fn() + .mockImplementation(function DeepFilterNoiseFilterProcessor( + this: DeepFilterNoiseFilterProcessorContext, + options: DeepFilterNoiseFilterProcessorOptions, + ): void { + Object.assign(this, options); + this.setEnabled = mockSetEnabled; + this.setSuppressionLevel = mockSetSuppressionLevel; + this.destroy = mockDestroy; + this.init = mockInit; + this.restart = mockRestart; + this.processedTrack = {} as MediaStreamTrack; + }); + + Object.assign(mockDeepFilterNoiseFilterProcessor, { + mockSetEnabled, + mockSetSuppressionLevel, + mockDestroy, + mockInit, + mockRestart, + }); + + return { + __esModule: true, + DeepFilterNoiseFilterProcessor: mockDeepFilterNoiseFilterProcessor, + }; +}); + +const mockDeepFilterNoiseFilterProcessor = + DeepFilterNoiseFilterProcessor as unknown as NoiseFilterProcessorMock; + +const mockTrack = { kind: "audio" } as MediaStreamTrack; + +describe("DeepFilterNetProcessor", () => { + beforeEach((): void => { + mockDeepFilterNoiseFilterProcessor.mockSetEnabled.mockClear(); + mockDeepFilterNoiseFilterProcessor.mockSetSuppressionLevel.mockClear(); + mockDeepFilterNoiseFilterProcessor.mockDestroy.mockClear(); + mockDeepFilterNoiseFilterProcessor.mockInit.mockClear(); + mockDeepFilterNoiseFilterProcessor.mockRestart.mockClear(); + mockDeepFilterNoiseFilterProcessor.mockClear(); + }); + + it("has the expected processor name", (): void => { + const processor = new DeepFilterNetProcessor(); + expect(processor.name).toBe(DEEPFILTERNET_PROCESSOR_NAME); + }); + + it("initializes the underlying processor with the expected configuration", async (): Promise => { + const processor = new DeepFilterNetProcessor(0.5, false); + + await processor.init({ track: mockTrack } as never); + + expect(mockDeepFilterNoiseFilterProcessor).toHaveBeenCalledTimes(1); + expect(mockDeepFilterNoiseFilterProcessor).toHaveBeenCalledWith( + expect.objectContaining({ + sampleRate: 48000, + noiseReductionLevel: 50, + enabled: false, + assetConfig: expect.objectContaining({ + cdnUrl: expect.any(String), + }), + }), + ); + expect(mockDeepFilterNoiseFilterProcessor.mockInit).toHaveBeenCalledWith({ + track: mockTrack, + }); + expect(processor.processedTrack).toBeDefined(); + }); + + it("clamps the noise reduction level to the 0-1 range", async (): Promise => { + const processor = new DeepFilterNetProcessor(1.5, true); + await processor.init({ track: mockTrack } as never); + + expect(mockDeepFilterNoiseFilterProcessor).toHaveBeenCalledWith( + expect.objectContaining({ noiseReductionLevel: 100 }), + ); + }); + + it("forwards suppression level changes and clamps out-of-range values", async (): Promise => { + const processor = new DeepFilterNetProcessor(0.2, true); + await processor.init({ track: mockTrack } as never); + + processor.setSuppressionLevel(1.5); + processor.setSuppressionLevel(-0.2); + + expect( + mockDeepFilterNoiseFilterProcessor.mockSetSuppressionLevel, + ).toHaveBeenNthCalledWith(1, 100); + expect( + mockDeepFilterNoiseFilterProcessor.mockSetSuppressionLevel, + ).toHaveBeenNthCalledWith(2, 0); + }); + + it("forwards enabled state changes to the underlying processor", async (): Promise => { + const processor = new DeepFilterNetProcessor(0.4, true); + await processor.init({ track: mockTrack } as never); + + await processor.setEnabled(false); + await processor.setEnabled(true); + + expect( + mockDeepFilterNoiseFilterProcessor.mockSetEnabled, + ).toHaveBeenNthCalledWith(1, false); + expect( + mockDeepFilterNoiseFilterProcessor.mockSetEnabled, + ).toHaveBeenNthCalledWith(2, true); + }); + + it("destroys the processor and resets internal state", async (): Promise => { + const processor = new DeepFilterNetProcessor(0.6, true); + await processor.init({ track: mockTrack } as never); + + await processor.destroy(); + + expect( + mockDeepFilterNoiseFilterProcessor.mockDestroy, + ).toHaveBeenCalledTimes(1); + expect(processor.processedTrack).toBeUndefined(); + }); + + it("reports support based on the runtime APIs", (): void => { + // In the jsdom test environment AudioContext/WebAssembly may be present. + expect(typeof supportsDeepFilterNetProcessor()).toBe("boolean"); + }); +}); diff --git a/src/audio/DeepFilterNetProcessor.ts b/src/audio/DeepFilterNetProcessor.ts new file mode 100644 index 0000000000..68a2915b55 --- /dev/null +++ b/src/audio/DeepFilterNetProcessor.ts @@ -0,0 +1,168 @@ +/* +Copyright 2026 Element Creations Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { DeepFilterNoiseFilterProcessor } from "deepfilternet3-noise-filter"; +import { logger } from "matrix-js-sdk/lib/logger"; + +import type { + AudioProcessorOptions, + Track, + TrackProcessor, +} from "livekit-client"; + +/** + * The sample rate DeepFilterNet is trained for. + */ +const DEEPFILTERNET_SAMPLE_RATE = 48000; + +/** + * The default noise reduction level (0-1), mapped to the package's 0-100 scale. + */ +const DEFAULT_NOISE_REDUCTION_LEVEL = 0.75; + +/** + * The name used to identify this processor on a LiveKit track. + */ +export const DEEPFILTERNET_PROCESSOR_NAME = "deepfilternet-noise-suppression"; + +/** + * The base path where the DeepFilterNet WASM binary and ONNX model are served + * from. Resolves `import.meta.env.BASE_URL` against the current page location + * so the assets resolve correctly even when the app is hosted under a subpath + * (e.g. `/element-call/`). Overridable via the `VITE_NOISE_SUPPRESSION_CDN_URL` + * env var for custom deployments. + */ +function resolveAssetUrl(): string { + if (import.meta.env.VITE_NOISE_SUPPRESSION_CDN_URL) { + return import.meta.env.VITE_NOISE_SUPPRESSION_CDN_URL; + } + + // BASE_URL is `./` in the embedded build, so resolve it against the current + // page to get the absolute app root (e.g. https://host/element-call/). + const baseUrl = new URL(import.meta.env.BASE_URL, window.location.href); + return `${baseUrl.href}assets/deepfilternet3`; +} + +/** + * Whether the current runtime supports the APIs required by DeepFilterNet. + */ +export function supportsDeepFilterNetProcessor(): boolean { + return ( + typeof AudioContext !== "undefined" && + typeof WebAssembly !== "undefined" && + typeof MediaStreamAudioDestinationNode !== "undefined" && + typeof MediaStreamAudioSourceNode !== "undefined" + ); +} + +/** + * A LiveKit TrackProcessor that applies DeepFilterNet3-based noise + * suppression to a local audio track. + * + * DeepFilterNet is a deep-learning speech enhancement model that provides + * significantly better noise suppression than RNNoise, especially for + * non-stationary noise (keyboard, traffic, etc.). It runs in an AudioWorklet + * and loads its WASM binary and ONNX model from locally-bundled assets. + * + * The underlying `DeepFilterNoiseFilterProcessor` from the + * `deepfilternet3-noise-filter` package implements the LiveKit + * `TrackProcessor` interface directly, so this wrapper primarily manages the + * lifecycle and exposes a stable API for the rest of the app. + */ +export class DeepFilterNetProcessor implements TrackProcessor< + Track.Kind.Audio, + AudioProcessorOptions +> { + public name = DEEPFILTERNET_PROCESSOR_NAME; + public processedTrack?: MediaStreamTrack; + + // oxlint-disable-next-line typescript/no-redundant-type-constituents -- The + // DeepFilterNoiseFilterProcessor type is not resolvable by oxlint's type-aware + // analysis (it imports from livekit-client), so it is treated as `any`. + private processor: DeepFilterNoiseFilterProcessor | null = null; + private level: number; + private enabled: boolean; + + public constructor( + level: number = DEFAULT_NOISE_REDUCTION_LEVEL, + enabled = true, + ) { + this.level = level; + this.enabled = enabled; + } + + /** + * Creates (or reuses) the underlying DeepFilterNet processor. + */ + private ensureProcessor(): DeepFilterNoiseFilterProcessor { + if (!this.processor) { + this.processor = new DeepFilterNoiseFilterProcessor({ + sampleRate: DEEPFILTERNET_SAMPLE_RATE, + noiseReductionLevel: this.clampLevel(this.level) * 100, + enabled: this.enabled, + assetConfig: { + cdnUrl: resolveAssetUrl(), + }, + }); + } + return this.processor; + } + + private clampLevel(level: number): number { + return Math.max(0, Math.min(1, level)); + } + + public async init(opts: AudioProcessorOptions): Promise { + const processor = this.ensureProcessor(); + try { + await processor.init({ track: opts.track }); + this.processedTrack = processor.processedTrack; + } catch (e) { + logger.error("[DeepFilterNetProcessor] init failed", e); + throw e; + } + } + + public async restart(opts: AudioProcessorOptions): Promise { + const processor = this.ensureProcessor(); + try { + await processor.restart({ track: opts.track }); + this.processedTrack = processor.processedTrack; + } catch (e) { + logger.error("[DeepFilterNetProcessor] restart failed", e); + throw e; + } + } + + public async destroy(): Promise { + if (this.processor) { + try { + await this.processor.destroy(); + } catch (e) { + logger.warn("[DeepFilterNetProcessor] destroy failed", e); + } + this.processor = null; + } + this.processedTrack = undefined; + } + + /** + * Sets the noise reduction level (0-1). + */ + public setSuppressionLevel(level: number): void { + this.level = this.clampLevel(level); + this.processor?.setSuppressionLevel(this.level * 100); + } + + /** + * Enables or disables noise suppression without tearing down the processor. + */ + public async setEnabled(enabled: boolean): Promise { + this.enabled = enabled; + await this.processor?.setEnabled(enabled); + } +} diff --git a/src/grid/CallLayout.ts b/src/grid/CallLayout.ts index 3128087bc4..e53c48b0d0 100644 --- a/src/grid/CallLayout.ts +++ b/src/grid/CallLayout.ts @@ -107,3 +107,18 @@ export function arrangeTiles( return { tileWidth, tileHeight, gap, columns }; } + +/** + * @param cameraCount - Number of regular participant tiles + * @param streamCount - Number of screen shares + */ +export function arrangeTilesWithStreams( + width: number, + minHeight: number, + cameraCount: number, + streamCount: number, +): GridArrangement { + // Each stream occupies a 2x2 block, so it consumes four regular cells worth of space. + const effectiveTileCount = cameraCount + streamCount * 4; + return arrangeTiles(width, minHeight, effectiveTileCount); +} diff --git a/src/grid/GridLayout.module.css b/src/grid/GridLayout.module.css index 984755d4cf..febe694551 100644 --- a/src/grid/GridLayout.module.css +++ b/src/grid/GridLayout.module.css @@ -11,16 +11,34 @@ Please see LICENSE in the repository root for full details. } .scrolling { - display: flex; - flex-wrap: wrap; + display: grid; + grid-template-columns: repeat(var(--columns), var(--width)); + grid-auto-rows: var(--height); + grid-auto-flow: row; justify-content: center; align-content: center; gap: var(--gap); } .scrolling > .slot { - width: var(--width); - height: var(--height); + min-width: 0; + min-height: 0; +} + +/* Larger tiles for Screen shares in Grids */ +.scrolling > .slot[data-stream="true"] { + grid-column: span 2; + grid-row: span 2; +} + +/* Focused tile takes up the entire grid area */ +.scrolling.focused { + display: block; +} + +.scrolling.focused > .slot { + width: 100%; + height: 100%; } .fixed { diff --git a/src/grid/GridLayout.tsx b/src/grid/GridLayout.tsx index 79c2b3a4a1..5d48c2731b 100644 --- a/src/grid/GridLayout.tsx +++ b/src/grid/GridLayout.tsx @@ -13,17 +13,19 @@ import { } from "react"; import { distinctUntilChanged } from "rxjs"; import { useObservableEagerState } from "observable-hooks"; +import classNames from "classnames"; import { type GridLayout as GridLayoutModel } from "../state/layout-types.ts"; import styles from "./GridLayout.module.css"; import { useInitial } from "../useInitial"; -import { type CallLayout, arrangeTiles } from "./CallLayout"; +import { type CallLayout, arrangeTilesWithStreams } from "./CallLayout"; import { type DragCallback, useUpdateLayout, useVisibleTiles } from "./Grid"; interface GridCSSProperties extends CSSProperties { "--gap": string; "--width": string; "--height": string; + "--columns": string; } /** @@ -79,26 +81,54 @@ export const makeGridLayout: CallLayout = ({ useUpdateLayout(); useVisibleTiles(model.setVisibleTiles); const { width, height: minHeight } = useObservableEagerState(minBounds$); - const { gap, tileWidth, tileHeight } = useMemo( - () => arrangeTiles(width, minHeight, model.grid.length), - [width, minHeight, model.grid.length], + // Screen shares are shown as larger 2x2 tiles + const { gap, tileWidth, tileHeight, columns } = useMemo(() => { + const streamCount = model.grid.filter( + (m) => m.media$.value.type === "screen share", + ).length; + return arrangeTilesWithStreams( + width, + minHeight, + model.grid.length - streamCount, + streamCount, + ); + }, [width, minHeight, model.grid]); + + // Render camera tiles before screen shares + const orderedTiles = useMemo( + () => [ + ...model.grid.filter((m) => m.media$.value.type !== "screen share"), + ...model.grid.filter((m) => m.media$.value.type === "screen share"), + ], + [model.grid], ); return (
- {model.grid.map((m) => ( - + {orderedTiles.map((m) => ( + ))}
); diff --git a/src/livekit/MatrixAudioRenderer.test.tsx b/src/livekit/MatrixAudioRenderer.test.tsx index bc6ef66873..e2dc2c5601 100644 --- a/src/livekit/MatrixAudioRenderer.test.tsx +++ b/src/livekit/MatrixAudioRenderer.test.tsx @@ -19,10 +19,12 @@ import { } from "livekit-client"; import { type ReactNode } from "react"; import { useTracks } from "@livekit/components-react"; +import { MemoryRouter } from "react-router-dom"; import { testAudioContext } from "../useAudioContext.test"; import * as MediaDevicesContext from "../MediaDevicesContext"; import { LivekitRoomAudioRenderer } from "./MatrixAudioRenderer"; +import { setParticipantBoosted } from "../state/participantVolume"; import { mockMediaDevices, mockRemoteParticipant, @@ -42,11 +44,13 @@ const MediaDevicesProvider = MediaDevicesContext.MediaDevicesContext.Provider; beforeEach(() => { vi.stubGlobal("AudioContext", TestAudioContextConstructor); + setParticipantBoosted("@bob:DEV0", false); }); afterEach(() => { vi.unstubAllGlobals(); vi.clearAllMocks(); + setParticipantBoosted("@bob:DEV0", false); }); vi.mock("@livekit/components-react", async (importOriginal) => { @@ -112,13 +116,15 @@ function renderTestComponent( vi.mocked(useTracks).mockReturnValue(tracks); return render( - - p.identity)} - livekitRoom={livekitRoom} - url={""} - /> - , + + + p.identity)} + livekitRoom={livekitRoom} + url={""} + /> + + , ); } @@ -258,7 +264,7 @@ it.each(TEST_CASES)( }, ); -it("should not setup audioContext gain and pan if there is no need to.", () => { +it("should not setup audioContext gain and pan if there is no need to", () => { renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]); const audioTrack = tracks[0].publication.track! as RemoteAudioTrack; @@ -286,3 +292,23 @@ it("should setup audioContext gain and pan", () => { expect(testAudioContext.gain.gain.value).toEqual(0.1); expect(testAudioContext.pan.pan.value).toEqual(1); }); + +it("should render a boosted volume through the WebAudio gain node", () => { + vi.spyOn(MediaDevicesContext, "useEarpieceAudioConfig").mockReturnValue({ + pan: 0, + volume: 1, + }); + + // Bob's volume is boosted above 100%, so the audio context must be + // attached so that the boosted volume is applied to the WebAudio gain node + // rather than being clamped to 1 on the HTMLMediaElement. + setParticipantBoosted("@bob:DEV0", true); + renderTestComponent([{ userId: "@bob", deviceId: "DEV0" }], ["@bob:DEV0"]); + const audioTrack = tracks[0].publication.track! as RemoteAudioTrack; + + expect(audioTrack.setAudioContext).toHaveBeenLastCalledWith(testAudioContext); + expect(audioTrack.setWebAudioPlugins).toHaveBeenLastCalledWith([ + testAudioContext.gain, + testAudioContext.pan, + ]); +}); diff --git a/src/livekit/MatrixAudioRenderer.tsx b/src/livekit/MatrixAudioRenderer.tsx index e3970e9f36..9be2e4607e 100644 --- a/src/livekit/MatrixAudioRenderer.tsx +++ b/src/livekit/MatrixAudioRenderer.tsx @@ -16,8 +16,15 @@ import { } from "@livekit/components-react"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; -import { useEarpieceAudioConfig } from "../MediaDevicesContext"; +import { + useEarpieceAudioConfig, + useMediaDevices, +} from "../MediaDevicesContext"; import { useReactiveState } from "../useReactiveState"; +import { useBehavior } from "../useBehavior"; +import { useObservableEagerState } from "observable-hooks"; +import { useUrlParams } from "../UrlParams"; +import { boostedParticipants$ } from "../state/participantVolume"; import * as controls from "../controls"; export interface MatrixAudioRendererProps { @@ -107,7 +114,22 @@ export function LivekitRoomAudioRenderer({ // shouldUseAudioContext is set to false if stereoPan === 0 to allow standby bluetooth playback. const { pan: stereoPan, volume: volumeFactor } = useEarpieceAudioConfig(); - const shouldUseAudioContext = stereoPan !== 0; + // A participant whose volume is above 100% needs WebAudio routing: the gain + // node supports volumes above 1, whereas the volume of a plain + // HTMLMediaElement is clamped to 1. When nobody is boosted we keep the + // previous behavior and only use the audio context for the earpiece. + const boosted = useBehavior(boostedParticipants$); + const anyBoosted = validIdentities.some((id) => boosted.has(id)); + const shouldUseAudioContext = anyBoosted || stereoPan !== 0; + + // The selected output device (e.g. NVIDIA Broadcast). When audio is routed + // through the WebAudio context for a boosted participant it would otherwise + // play out of the context's default device, bypassing the user's chosen + // output device and any processing applied there (e.g. noise suppression). + const audioOutputId = useObservableEagerState( + useMediaDevices().audioOutput.selected$, + )?.id; + const { controlledAudioDevices } = useUrlParams(); // initialize the potentially used audio context. const [audioContext, setAudioContext] = useState( @@ -120,6 +142,25 @@ export function LivekitRoomAudioRenderer({ void ctx.close(); }; }, []); + // The AudioContext starts suspended until a user gesture; it must be running + // for volumes above 100% (applied via the WebAudio gain node) to amplify. + useEffect(() => { + if (audioContext === undefined) return; + const resume = (): void => { + if (audioContext.state === "suspended") void audioContext.resume(); + }; + resume(); + // Browsers require a user gesture to resume an AudioContext, so retry on + // any interaction. + document.addEventListener("pointerdown", resume); + document.addEventListener("keydown", resume); + document.addEventListener("touchstart", resume); + return (): void => { + document.removeEventListener("pointerdown", resume); + document.removeEventListener("keydown", resume); + document.removeEventListener("touchstart", resume); + }; + }, [audioContext]); const audioNodes = useMemo( () => ({ gain: audioContext?.createGain(), @@ -128,6 +169,23 @@ export function LivekitRoomAudioRenderer({ [audioContext], ); + // Route the audio context to the selected output device so boosted audio + // doesn't bypass it (e.g. NVIDIA Broadcast noise suppression). Mirrors the + // sink handling in useAudioContext.tsx. + useEffect(() => { + if ( + audioContext && + "setSinkId" in audioContext && + !controlledAudioDevices + ) { + // https://developer.mozilla.org/en-US/docs/Web/API/AudioContext/setSinkId + // @ts-expect-error - setSinkId doesn't exist yet in types, maybe because it's not supported everywhere. + audioContext.setSinkId(audioOutputId).catch((ex) => { + logger.warn("Unable to change sink for audio context", ex); + }); + } + }, [audioContext, audioOutputId, controlledAudioDevices, logger]); + // Simple effects to update the gain and pan node based on the props useEffect(() => { if (audioNodes.pan) audioNodes.pan.pan.value = stereoPan; @@ -185,11 +243,14 @@ function AudioTrackWithAudioNodes({ // This is used to unmount/remount the AudioTrack component. // Mounting needs to happen after the audioContext is set. // (adding the audio context when already mounted did not work outside strict mode) + const mediaStream = trackRef?.publication.track?.mediaStream; const [trackReady, setTrackReady] = useReactiveState( () => false, - // We only want the track to reset once both (audioNodes and audioContext) are set. - // for unsetting the audioContext its enough if one of the two is undefined. - [audioContext && audioNodes], + // We want the track to reset when the audio context becomes available, + // and when the underlying media stream changes (e.g. on encryption + // renegotiation, where the WebAudio source node would otherwise stay + // bound to the old stream). + [audioContext && audioNodes, mediaStream], ); useEffect(() => { diff --git a/src/reactions/index.ts b/src/reactions/index.ts index acf7e18161..a6fcdf0754 100644 --- a/src/reactions/index.ts +++ b/src/reactions/index.ts @@ -29,6 +29,8 @@ import waveSoundOgg from "../sound/reactions/wave.ogg?url"; import waveSoundMp3 from "../sound/reactions/wave.mp3?url"; import baduntssSoundOgg from "../sound/reactions/baduntss.ogg?url"; import baduntssSoundMp3 from "../sound/reactions/baduntss.mp3?url"; +import vineBoomSoundMp3 from "../sound/reactions/vine-boom.mp3?url"; +import vineBoomSoundOgg from "../sound/reactions/vine-boom.ogg?url"; export const ElementCallReactionEventType = "io.element.call.reaction"; @@ -202,6 +204,15 @@ export const ReactionSet: ReactionOption[] = [ mp3: baduntssSoundMp3, }, }, + { + emoji: "🗿", + name: "Moai", + alias: ["vine-boom"], + sound: { + ogg: vineBoomSoundOgg, + mp3: vineBoomSoundMp3, + }, + }, ]; export interface RaisedHandInfo { diff --git a/src/room/InCallView.tsx b/src/room/InCallView.tsx index 58b378aec2..32c0c16169 100644 --- a/src/room/InCallView.tsx +++ b/src/room/InCallView.tsx @@ -20,7 +20,7 @@ import { import useMeasure from "react-use-measure"; import { type MatrixRTCSession } from "matrix-js-sdk/lib/matrixrtc"; import classNames from "classnames"; -import { map } from "rxjs"; +import { combineLatest, map, switchMap } from "rxjs"; import { useObservable } from "observable-hooks"; import { logger as rootLogger } from "matrix-js-sdk/lib/logger"; import { useTranslation } from "react-i18next"; @@ -28,7 +28,7 @@ import { useTranslation } from "react-i18next"; import { Header, LeftNav, RightNav, RoomHeaderInfo } from "../Header"; import { HeaderStyle, useUrlParams } from "../UrlParams"; import { useCallViewKeyboardShortcuts } from "../useCallViewKeyboardShortcuts"; -import { widget } from "../widget"; +import { widget, ElementWidgetActions } from "../widget"; import styles from "./InCallView.module.css"; import { GridTile } from "../tile/GridTile"; import { SettingsModal, defaultSettingsTab } from "../settings/SettingsModal"; @@ -142,6 +142,58 @@ export const ActiveCall: FC = (props) => { vm.leave$.pipe(scope.bind()).subscribe(props.onLeft); + // Forward currently speaking user IDs to the host client + if (widget) { + const widgetApi = widget.api; + vm.activeSpeakers$.pipe(scope.bind()).subscribe((speakers) => { + const userIds = speakers + .map((m) => m.userId) + .filter((id): id is string => typeof id === "string" && id !== ""); + widgetApi.transport + .send(ElementWidgetActions.ActiveSpeakers, { userIds }) + .catch((e) => + rootLogger.error("Failed to send active speakers action", e), + ); + }); + + // Forward each participant's mute state to the host client so it can + // show per-user mute indicators for the whole roster. Rebuild the + // payload whenever the roster or any participant's audio/video flips. + const mediaState$ = vm.userMedia$.pipe( + switchMap((mediaItems) => { + const sources = mediaItems.flatMap((media) => [ + media.audioEnabled$, + media.videoEnabled$, + ]); + return combineLatest(sources).pipe( + map(() => + mediaItems.flatMap((media) => + media.userId + ? [ + { + userId: media.userId, + audioEnabled: media.audioEnabled$.getValue(), + videoEnabled: media.videoEnabled$.getValue(), + }, + ] + : [], + ), + ), + ); + }), + ); + mediaState$.pipe(scope.bind()).subscribe((participants) => { + widgetApi.transport + .send(ElementWidgetActions.ParticipantMediaState, { participants }) + .catch((e) => + rootLogger.error( + "Failed to send participant media state action", + e, + ), + ); + }); + } + return (): void => { scope.end(); }; @@ -454,6 +506,8 @@ export const InCallView: FC = ({ showRingingStatus={showRingingStatus} showOutline={showOutline} focusable={!contentObscured} + focusedStream$={vm.focusedStream$} + onToggleFocusedStream={vm.setFocusedStream} /> ) : ( { playReactionsSoundSetting, ); + const [hideAvatarTilesWhenCameraOff, setHideAvatarTilesWhenCameraOff] = + useSetting(hideAvatarTilesWhenCameraOffSetting); + const onChangeSetting = ( e: ChangeEvent, fn: (value: boolean) => void, @@ -76,6 +80,20 @@ export const PreferencesSettingsTab: FC = () => { onChange={(e) => onChangeSetting(e, setPlayReactionSound)} /> + + onChangeSetting(e, setHideAvatarTilesWhenCameraOff)} + /> + = ({ ); }; + const DeepFilterNetCheckbox: React.FC = (): ReactNode => { + const supported = supportsDeepFilterNetProcessor(); + const [dfEnabled, setDfEnabled] = useSetting( + deepFilterNetNoiseSuppressionSetting, + ); + const [dfLevel, setDfLevel] = useSetting( + deepFilterNetNoiseSuppressionLevelSetting, + ); + const [dfLevelRaw, setDfLevelRaw] = useState(dfLevel); + const effectiveDfEnabled = supported && !!dfEnabled; + + useEffect(() => { + setDfLevelRaw(dfLevel); + }, [dfLevel]); + + return ( + <> +

{t("settings.audio_tab.deepfilternet_header")}

+ + setDfEnabled(e.target.checked)} + disabled={!supported} + /> + + {effectiveDfEnabled && ( +
+ +

{t("settings.audio_tab.deepfilternet_level_description")}

+ `${Math.round(v * 100)}%`} + /> +
+ )} + + ); + }; + const MicrophoneCutoffSettings: React.FC = (): ReactNode => { const supported = supportsRNNoiseProcessor(); const [cutoffEnabled, setCutoffEnabled] = useSetting( @@ -556,6 +618,8 @@ export const SettingsModal: FC = ({ + + diff --git a/src/settings/settings.test.ts b/src/settings/settings.test.ts index 9fd445e9ae..27f6942781 100644 --- a/src/settings/settings.test.ts +++ b/src/settings/settings.test.ts @@ -124,6 +124,13 @@ describe("saveTileVolume", () => { }); }); + it("stores boosted volumes above 1", () => { + saveTileVolume("@alice:example.org:DEVICE", 2); + expect(tileVolumes.getValue()).toEqual({ + "@alice:example.org:DEVICE": 2, + }); + }); + it("removes the entry when set back to the default volume", () => { saveTileVolume("@alice:example.org:DEVICE", 0.5); saveTileVolume("@alice:example.org:DEVICE", 1); diff --git a/src/settings/settings.ts b/src/settings/settings.ts index 71661412ce..b1d18bd463 100644 --- a/src/settings/settings.ts +++ b/src/settings/settings.ts @@ -128,6 +128,23 @@ export const rnnoiseNoiseSuppressionPreset = "conservative", ); +/** + * Whether DeepFilterNet-based noise suppression is enabled. + * Defaults to off; when enabled it takes precedence over RNNoise. + */ +export const deepFilterNetNoiseSuppression = new Setting( + "deepfilternet-noise-suppression", + false, +); + +/** + * The DeepFilterNet noise reduction level (0-1). + */ +export const deepFilterNetNoiseSuppressionLevel = new Setting( + "deepfilternet-noise-suppression-level", + 0.75, +); + export const micCutoffEnabled = new Setting( "mic-cutoff-enabled", false, @@ -158,6 +175,11 @@ export const muteAllAudio = new Setting("mute-all-audio", false); export const alwaysShowSelf = new Setting("always-show-self", true); +export const hideAvatarTilesWhenCameraOff = new Setting( + "hide-avatars-when-camera-off", + false, +); + export const alwaysShowIphoneEarpiece = new Setting( "always-show-iphone-earpiece", false, diff --git a/src/sound/reactions/vine-boom.mp3 b/src/sound/reactions/vine-boom.mp3 new file mode 100644 index 0000000000..ee830e24fc Binary files /dev/null and b/src/sound/reactions/vine-boom.mp3 differ diff --git a/src/sound/reactions/vine-boom.ogg b/src/sound/reactions/vine-boom.ogg new file mode 100644 index 0000000000..61c072019a Binary files /dev/null and b/src/sound/reactions/vine-boom.ogg differ diff --git a/src/state/CallViewModel/CallViewModel.test.ts b/src/state/CallViewModel/CallViewModel.test.ts index 440ae35e9f..9f4acaae44 100644 --- a/src/state/CallViewModel/CallViewModel.test.ts +++ b/src/state/CallViewModel/CallViewModel.test.ts @@ -123,6 +123,7 @@ export interface GridLayoutSummary { type: "grid"; spotlight?: string[]; grid: string[]; + focused?: boolean; } export interface SpotlightLandscapeLayoutSummary { @@ -184,6 +185,7 @@ function summarizeLayout$(l$: Observable): Observable { type: l.type, spotlight: spotlight?.map((vm) => vm.id), grid: grid.map((vm) => vm.id), + ...(l.focused ? { focused: true as const } : {}), }), ); case "spotlight-landscape": @@ -302,82 +304,129 @@ describe.each([ }); }); - test("remote screen sharing activates spotlight layout", () => { - withTestScheduler(({ behavior, schedule, expectObservable }) => { - // Start with no screen shares, then have Alice and Bob share their screens, - // then return to no screen shares, then have just Alice share for a bit - const aliceSharingInputMarbles = " ny-n--yn"; - const bobSharingInputMarbles = " n-y-n---"; - // While there are no screen shares, switch to spotlight manually, and then - // switch back to grid at the end - const modeInputMarbles = " -----s--g"; - // We should automatically enter spotlight for the first round of screen - // sharing, then return to grid, then manually go into spotlight, and - // remain in spotlight until we manually go back to grid - const expectedLayoutMarbles = " abcdaefeg"; - const expectedShowSpeakingMarbles = "y----nyny"; + test("remote screen sharing shows streams in grid", () => { + withTestScheduler(({ expectObservable }) => { + // Both Alice and Bob share their screens at the same time. withCallViewModel( { remoteParticipants$: constant([aliceParticipant, bobParticipant]), rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), sharingScreen: new Map([ - [aliceParticipant, behavior(aliceSharingInputMarbles, yesNo)], - [bobParticipant, behavior(bobSharingInputMarbles, yesNo)], + [aliceParticipant, constant(true)], + [bobParticipant, constant(true)], ]), }, (vm) => { - schedule(modeInputMarbles, { + expectObservable(summarizeLayout$(vm.layout$)).toBe("a", { + a: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + `${aliceId}:0`, + `${bobId}:0`, + `${aliceId}:0:screen-share`, + `${bobId}:0:screen-share`, + ], + }, + }); + expectObservable(vm.showSpeakingIndicators$).toBe("y", yesNo); + }, + ); + }); + }); + + test("manually switching to spotlight still spotlights screen shares", () => { + withTestScheduler(({ schedule, expectObservable }) => { + // Alice shares her screen; the user manually switches to spotlight and + // back to grid. + withCallViewModel( + { + remoteParticipants$: constant([aliceParticipant, bobParticipant]), + rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), + sharingScreen: new Map([[aliceParticipant, constant(true)]]), + }, + (vm) => { + schedule(" s g", { s: () => vm.setGridMode("spotlight"), g: () => vm.setGridMode("grid"), }); - expectObservable(summarizeLayout$(vm.layout$)).toBe( - expectedLayoutMarbles, - { - a: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - b: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0:screen-share`], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - c: { - type: "spotlight-landscape", - spotlight: [ - `${aliceId}:0:screen-share`, - `${bobId}:0:screen-share`, - ], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - d: { - type: "spotlight-landscape", - spotlight: [`${bobId}:0:screen-share`], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], - }, - e: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0`], - grid: [`${localId}:0`, `${bobId}:0`], - }, - f: { - type: "spotlight-landscape", - spotlight: [`${aliceId}:0:screen-share`], - grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], - }, - g: { - type: "grid", - spotlight: undefined, - grid: [`${localId}:0`, `${bobId}:0`, `${aliceId}:0`], - }, + expectObservable(summarizeLayout$(vm.layout$)).toBe("ba", { + a: { + type: "grid", + spotlight: undefined, + grid: [ + `${localId}:0`, + `${aliceId}:0`, + `${bobId}:0`, + `${aliceId}:0:screen-share`, + ], }, - ); - expectObservable(vm.showSpeakingIndicators$).toBe( - expectedShowSpeakingMarbles, - yesNo, - ); + b: { + type: "spotlight-landscape", + spotlight: [`${aliceId}:0:screen-share`], + grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + }, + }); + }, + ); + }); + }); + + test("focused stream fills the grid and hides other tiles", () => { + withTestScheduler(({ schedule, expectObservable }) => { + withCallViewModel( + { + remoteParticipants$: constant([aliceParticipant, bobParticipant]), + rtcMembers$: constant([localRtcMember, aliceRtcMember, bobRtcMember]), + sharingScreen: new Map([ + [aliceParticipant, constant(true)], + [bobParticipant, constant(true)], + ]), + }, + (vm) => { + // Focus Alice's screen share using the live view model from the + // current layout, then unfocus it again. + const focusAlice = (): void => { + const layout = vm.layout$.value; + if (layout.type !== "grid") return; + const share = layout.grid + .map((tile) => tile.media$.value) + .find( + (m) => + m.type === "screen share" && + m.id === `${aliceId}:0:screen-share`, + ); + if (share !== undefined && share.type === "screen share") + vm.setFocusedStream(share); + }; + schedule(" f u", { + f: focusAlice, + u: (): void => vm.setFocusedStream(null), + }); + + expectObservable(summarizeLayout$(vm.layout$)).toBe("ba", { + a: { + type: "grid", + spotlight: undefined, + grid: [ + // After unfocusing, the TileStore keeps the previously focused + // stream tile in its spot (index 0) and appends the rest. + `${aliceId}:0:screen-share`, + `${localId}:0`, + `${aliceId}:0`, + `${bobId}:0`, + `${bobId}:0:screen-share`, + ], + }, + b: { + type: "grid", + focused: true, + spotlight: undefined, + grid: [`${aliceId}:0:screen-share`], + }, + }); }, ); }); @@ -387,7 +436,7 @@ describe.each([ withTestScheduler(({ behavior, expectObservable }) => { // Local participant shares their screen, then stops sharing const sharingInputMarbles = " nyn"; - // Layout should show the screen share but stay in type: "grid" + // Layout should show the screen share as a grid tile but stay in grid const expectedLayoutMarbles = "aba"; withCallViewModel( { @@ -408,8 +457,13 @@ describe.each([ }, b: { type: "grid", - spotlight: [`${localId}:0:screen-share`], - grid: [`${localId}:0`, `${aliceId}:0`, `${bobId}:0`], + spotlight: undefined, + grid: [ + `${localId}:0`, + `${aliceId}:0`, + `${bobId}:0`, + `${localId}:0:screen-share`, + ], }, }, ); @@ -443,8 +497,12 @@ describe.each([ }, b: { type: "grid", - spotlight: [`${localId}:0:screen-share`], - grid: [`${localId}:0`, `${aliceId}:0`], + spotlight: undefined, + grid: [ + `${localId}:0`, + `${aliceId}:0`, + `${localId}:0:screen-share`, + ], }, }, ); diff --git a/src/state/CallViewModel/CallViewModel.ts b/src/state/CallViewModel/CallViewModel.ts index d34e9160f8..be0b7450c7 100644 --- a/src/state/CallViewModel/CallViewModel.ts +++ b/src/state/CallViewModel/CallViewModel.ts @@ -61,6 +61,7 @@ import { import { duplicateTiles, echoCancellationSetting, + hideAvatarTilesWhenCameraOff, noiseSuppressionSetting, playReactionsSound, rnnoiseNoiseSuppression, @@ -312,6 +313,8 @@ export interface CallViewModel { /** use the layout instead, this is just for the sdk export. */ remoteMatrixLivekitMembers$: Behavior; localMatrixLivekitMember$: Behavior; + /** All user media (local + remote) with their live speaking status */ + userMedia$: Behavior; /** List of participants raising their hand */ handsRaised$: Behavior>; /** List of reactions. Keys are: membership.membershipId (currently predefined as: `${membershipEvent.userId}:${membershipEvent.deviceId}`)*/ @@ -353,10 +356,13 @@ export interface CallViewModel { showSpotlightIndicators$: Behavior; showSpeakingIndicators$: Behavior; showNameTags$: Behavior; + activeSpeakers$: Behavior; spotlightExpanded$: Behavior; toggleSpotlightExpanded$: Behavior<(() => void) | null>; gridMode$: Behavior; setGridMode: (value: GridMode) => void; + focusedStream$: Behavior; + setFocusedStream: (vm: ScreenShareViewModel | null) => void; // header/footer visibility showHeader$: Behavior; @@ -939,9 +945,47 @@ export function createCallViewModel$( }, undefined), ), ); + // All active speakers in a call + const activeSpeakers$ = scope.behavior( + userMedia$.pipe( + switchMap((mediaItems) => + mediaItems.length === 0 + ? of([]) + : combineLatest( + mediaItems.map((m) => + m.voiceActivity$.pipe(map((v) => [m, v] as const)), + ), + ), + ), + map((mediaItems) => mediaItems.filter(([, v]) => v).map(([m]) => m)), + distinctUntilChanged(shallowEquals), + ), + ); const grid$ = scope.behavior( - userMedia$.pipe( + combineLatest([userMedia$, hideAvatarTilesWhenCameraOff.value$]).pipe( + switchMap(([mediaItems, hideAvatars]) => + hideAvatars + ? // When enabled, only generate tiles for participants whose camera + // is on. Participants with their camera off remain audible but + // have no tile, keeping voice calls tidy. + mediaItems.length === 0 + ? of([]) + : combineLatest( + mediaItems.map((m) => + m.videoEnabled$.pipe( + map((videoEnabled) => [m, videoEnabled] as const), + ), + ), + ).pipe( + map((pairs) => + pairs + .filter(([, videoEnabled]) => videoEnabled) + .map(([m]) => m), + ), + ) + : of(mediaItems), + ), switchMap((mediaItems) => { const bins = mediaItems.map((m) => m.bin$.pipe(map((bin) => [m, bin] as const)), @@ -1016,14 +1060,6 @@ export function createCallViewModel$( ), ); - const hasRemoteScreenShares$ = scope.behavior( - spotlight$.pipe( - map((spotlight) => - spotlight.some((vm) => vm.type === "screen share" && !vm.local), - ), - ), - ); - const pipEnabled$ = scope.behavior(setPipEnabled$, false); const windowSize$ = @@ -1066,22 +1102,51 @@ export function createCallViewModel$( spotlightExpandedToggle$, ); - const { setGridMode, gridMode$ } = createLayoutModeSwitch( - scope, - windowMode$, - hasRemoteScreenShares$, + const { setGridMode, gridMode$ } = createLayoutModeSwitch(scope, windowMode$); + + // A single screen share can be focused (maximised) to fill the grid + const focusedStreamRequest$ = new Subject(); + const focusedStream$ = scope.behavior( + focusedStreamRequest$.pipe( + startWith(null), + switchMap((requested) => + requested === null + ? of(null) + : screenShares$.pipe( + map( + (shares) => shares.find((s) => s.id === requested.id) ?? null, + ), + distinctUntilChanged(), + ), + ), + ), ); + const setFocusedStream = (requested: ScreenShareViewModel | null): void => + focusedStreamRequest$.next(requested); const gridLayoutMedia$: Observable = combineLatest( - [grid$, spotlight$], - (grid, spotlight) => ({ - type: "grid", - edgeToEdge: false, - spotlight: spotlight.some((vm) => vm.type === "screen share") - ? spotlight - : undefined, - grid, - }), + [grid$, spotlight$, focusedStream$], + (grid, spotlight, focusedStream) => { + if (focusedStream !== null) + return { + type: "grid", + edgeToEdge: false, + focused: true, + grid: [focusedStream], + }; + // Screen shares are rendered as larger tiles inside the + // grid layout, so multiple screen shares can be seen at once. + // May be not elegant to get them from spotlight. + const screenShares = spotlight.filter( + (vm): vm is ScreenShareViewModel => vm.type === "screen share", + ); + return { + type: "grid", + edgeToEdge: false, + focused: false, + grid: [...grid, ...screenShares], + }; + }, ); const spotlightLandscapeLayoutMedia$ = ( @@ -1781,8 +1846,11 @@ export function createCallViewModel$( toggleSpotlightExpanded$: toggleSpotlightExpanded$, gridMode$: gridMode$, setGridMode: setGridMode, + focusedStream$, + setFocusedStream, layout$: layout$, localMatrixLivekitMember$, + userMedia$, remoteMatrixLivekitMembers$: scope.behavior( remoteMatrixLivekitMembers$.pipe( map((members) => members.value), @@ -1803,6 +1871,7 @@ export function createCallViewModel$( showSpotlightIndicators$: showSpotlightIndicators$, showSpeakingIndicators$: showSpeakingIndicators$, showNameTags$, + activeSpeakers$, showHeader$: showHeader$, showFooter$: showFooter$, settingsOpen$: settingsOpen$, diff --git a/src/state/CallViewModel/LayoutSwitch.test.ts b/src/state/CallViewModel/LayoutSwitch.test.ts index 0d184017b1..0cf61f276b 100644 --- a/src/state/CallViewModel/LayoutSwitch.test.ts +++ b/src/state/CallViewModel/LayoutSwitch.test.ts @@ -12,12 +12,10 @@ import { testScope, withTestScheduler } from "../../utils/test"; function testLayoutSwitch({ windowMode = "n", - hasScreenShares = "n", userSelection = "", expectedGridMode, }: { windowMode?: string; - hasScreenShares?: string; userSelection?: string; expectedGridMode: string; }): void { @@ -25,7 +23,6 @@ function testLayoutSwitch({ const { gridMode$, setGridMode } = createLayoutModeSwitch( testScope(), behavior(windowMode, { n: "normal", N: "narrow", f: "flat" }), - behavior(hasScreenShares, { y: true, n: false }), ); schedule(userSelection, { g: () => setGridMode("grid"), @@ -57,50 +54,6 @@ test("allows switching modes manually", () => expectedGridMode: "g-sgs", })); -test("switches to spotlight mode when there is a remote screen share", () => - testLayoutSwitch({ - hasScreenShares: " n--y", - expectedGridMode: "g--s", - })); - -test("can manually switch to grid when there is a screenshare", () => - testLayoutSwitch({ - hasScreenShares: " n-y", - userSelection: " ---g", - expectedGridMode: "g-sg", - })); - -test("auto-switches after manually selecting grid", () => - testLayoutSwitch({ - // Two screenshares will happen in sequence. There is a screen share that - // forces spotlight, then the user manually switches back to grid. - hasScreenShares: " n-y-ny", - userSelection: " ---g", - expectedGridMode: "g-sg-s", - // If we did want to respect manual selection, the expectation would be: g-sg - })); - -test("switches back to grid mode when the remote screen share ends", () => - testLayoutSwitch({ - hasScreenShares: " n--y--n", - expectedGridMode: "g--s--g", - })); - -test("auto-switches to spotlight again after first screen share ends", () => - testLayoutSwitch({ - hasScreenShares: " nyny", - expectedGridMode: "gsgs", - })); - -test("switches manually to grid after screen share while manually in spotlight", () => - testLayoutSwitch({ - // Initially, no one is sharing. Then the user manually switches to spotlight. - // After a screen share starts, the user manually switches to grid. - hasScreenShares: " n-y", - userSelection: " -s-g", - expectedGridMode: "gs-g", - })); - test("auto-switches to spotlight when in flat window mode", () => testLayoutSwitch({ // First normal, then narrow, then flat. @@ -117,16 +70,9 @@ test("allows switching modes manually when in flat window mode", () => expectedGridMode: "gsgsg", })); -test("stays in spotlight while there are screen shares even when window mode changes", () => - testLayoutSwitch({ - windowMode: " nfn", - hasScreenShares: " y", - expectedGridMode: "s", - })); - -test("ignores end of screen share until window mode returns to normal", () => +test("returns to grid mode when the window returns to a normal shape", () => testLayoutSwitch({ - windowMode: " nf-n", - hasScreenShares: " y-n", - expectedGridMode: "s--g", + // Window starts flat (spotlight), then returns to a normal shape. + windowMode: "f n", + expectedGridMode: "sg", })); diff --git a/src/state/CallViewModel/LayoutSwitch.ts b/src/state/CallViewModel/LayoutSwitch.ts index 97a4ee6fe4..34bd4e7214 100644 --- a/src/state/CallViewModel/LayoutSwitch.ts +++ b/src/state/CallViewModel/LayoutSwitch.ts @@ -5,14 +5,7 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { - combineLatest, - map, - Subject, - startWith, - skipWhile, - switchMap, -} from "rxjs"; +import { map, Subject, startWith, skipWhile, switchMap } from "rxjs"; import { type GridMode, type WindowMode } from "./CallViewModel.ts"; import { constant, type Behavior } from "../Behavior.ts"; @@ -25,12 +18,10 @@ import { type ObservableScope } from "../ObservableScope.ts"; * * @param scope - The observable scope to manage subscriptions. * @param windowMode$ - The current window mode. - * @param hasRemoteScreenShares$ - A behavior indicating if there are remote screen shares active. */ export function createLayoutModeSwitch( scope: ObservableScope, windowMode$: Behavior, - hasRemoteScreenShares$: Behavior, ): { gridMode$: Behavior; setGridMode: (value: GridMode) => void; @@ -38,7 +29,7 @@ export function createLayoutModeSwitch( const userSelection$ = new Subject(); // Callback to set the grid mode desired by the user. // Notice that this is only a preference, the actual grid mode can be overridden - // if there is a remote screen share active. + // if the window mode is flat. const setGridMode = (value: GridMode): void => userSelection$.next(value); /** @@ -46,15 +37,11 @@ export function createLayoutModeSwitch( * not accounting for the user's manual selections. */ const naturalGridMode$ = scope.behavior( - combineLatest( - [hasRemoteScreenShares$, windowMode$], - (hasRemoteScreenShares, windowMode) => - // When there are screen shares or the window is flat (as with a phone - // in landscape orientation), spotlight is a better experience. - // We want screen shares to be big and readable, and we want flipping - // your phone into landscape to be a quick way of maximising the - // spotlight tile. - hasRemoteScreenShares || windowMode === "flat" ? "spotlight" : "grid", + // When the window is flat (as with a phone in landscape orientation), + // spotlight is a better experience: flipping your phone into landscape is + // a quick way of maximising the spotlight tile. + windowMode$.pipe( + map((windowMode) => (windowMode === "flat" ? "spotlight" : "grid")), ), ); diff --git a/src/state/CallViewModel/localMember/Publisher.test.ts b/src/state/CallViewModel/localMember/Publisher.test.ts index f48612de35..c0b3beb904 100644 --- a/src/state/CallViewModel/localMember/Publisher.test.ts +++ b/src/state/CallViewModel/localMember/Publisher.test.ts @@ -29,12 +29,15 @@ import { type Connection } from "../remoteMembers/Connection"; import { type MuteStates } from "../../MuteStates"; import { autoGainControlSetting, + deepFilterNetNoiseSuppression, + deepFilterNetNoiseSuppressionLevel, micCutoffEnabled, micCutoffThresholdDb, rnnoiseNoiseSuppression, rnnoiseNoiseSuppressionPreset, } from "../../../settings/settings"; import type { RNNoiseProcessor } from "../../../audio/RNNoiseProcessor"; +import type { DeepFilterNetProcessor } from "../../../audio/DeepFilterNetProcessor"; import { MIC_CUTOFF_DEFAULT_DB } from "../../../audio/microphoneGate"; let scope: ObservableScope; @@ -817,6 +820,174 @@ describe("Publisher", () => { expect(micCutoffEnabled.getValue()).toBe(false); }); }); + + describe("DeepFilterNet", () => { + beforeEach(() => { + vi.stubGlobal("AudioWorkletNode", class AudioWorkletNode {}); + vi.stubGlobal( + "AudioWorklet", + class AudioWorklet { + public async addModule(): Promise { + await Promise.resolve(); + } + }, + ); + vi.stubGlobal( + "MediaStreamAudioDestinationNode", + class MediaStreamAudioDestinationNode {}, + ); + vi.stubGlobal( + "MediaStreamAudioSourceNode", + class MediaStreamAudioSourceNode {}, + ); + vi.stubGlobal("AudioContext", class AudioContext {}); + vi.stubGlobal("WebAssembly", {}); + deepFilterNetNoiseSuppression.setValue(false); + deepFilterNetNoiseSuppressionLevel.setValue(0.75); + }); + + afterEach(() => { + vi.unstubAllGlobals(); + deepFilterNetNoiseSuppression.setValue(false); + deepFilterNetNoiseSuppressionLevel.setValue(0.75); + rnnoiseNoiseSuppression.setValue(false); + }); + + it("enabling setting applies DeepFilterNet processor on microphone track", async () => { + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { setProcessor: (...args: unknown[]) => void }; + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + + deepFilterNetNoiseSuppression.setValue(true); + await flushPromises(); + + expect(micTrack.setProcessor).toHaveBeenCalledOnce(); + }); + + it("disabling setting removes DeepFilterNet processor on microphone track", async () => { + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { + setProcessor: (...args: unknown[]) => void; + stopProcessor: () => void; + }; + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + + deepFilterNetNoiseSuppression.setValue(true); + await flushPromises(); + deepFilterNetNoiseSuppression.setValue(false); + await flushPromises(); + + expect(micTrack.setProcessor).toHaveBeenCalledOnce(); + expect(micTrack.stopProcessor).toHaveBeenCalledOnce(); + }); + + it("restarts microphone track with native noise suppression disabled when DeepFilterNet is enabled", async () => { + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { restartTrack: (...args: unknown[]) => void }; + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + + deepFilterNetNoiseSuppression.setValue(true); + await flushPromises(); + + expect(micTrack.restartTrack).toHaveBeenCalledWith( + expect.objectContaining({ + noiseSuppression: false, + }), + ); + }); + + it("updates active DeepFilterNet processor level when level setting changes", async () => { + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { getProcessor: () => unknown }; + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + + deepFilterNetNoiseSuppression.setValue(true); + await flushPromises(); + + const processor = micTrack.getProcessor() as DeepFilterNetProcessor; + expect(processor).toBeDefined(); + const setSuppressionLevelSpy = vi.spyOn(processor, "setSuppressionLevel"); + + deepFilterNetNoiseSuppressionLevel.setValue(0.5); + await flushPromises(); + + expect(setSuppressionLevelSpy).toHaveBeenCalledWith(0.5); + }); + + it("stops any existing processor before attaching DeepFilterNet (mutual exclusion)", async () => { + const micTrack = createMockLocalTrack( + Track.Source.Microphone, + ) as LocalTrack & { + setProcessor: (...args: unknown[]) => void; + stopProcessor: () => void; + }; + trackPublications.push({ + source: Track.Source.Microphone, + track: micTrack, + audioTrack: micTrack, + } as unknown as LocalTrackPublication); + localParticipant.emit( + ParticipantEvent.LocalTrackPublished, + trackPublications[0], + ); + + // First attach an RNNoise processor. + rnnoiseNoiseSuppression.setValue(true); + await flushPromises(); + expect(micTrack.setProcessor).toHaveBeenCalledOnce(); + + // Then enable DeepFilterNet; it should stop the existing processor + // before attaching itself. + vi.mocked(micTrack.stopProcessor).mockClear(); + vi.mocked(micTrack.setProcessor).mockClear(); + deepFilterNetNoiseSuppression.setValue(true); + for (let i = 0; i < 5; i++) { + await flushPromises(); + } + + expect(micTrack.stopProcessor).toHaveBeenCalled(); + const processors = vi + .mocked(micTrack.setProcessor) + .mock.calls.map((call) => (call[0] as { name: string }).name); + expect(processors).toContain("deepfilternet-noise-suppression"); + }); + }); }); describe("Bug fix", () => { diff --git a/src/state/CallViewModel/localMember/Publisher.ts b/src/state/CallViewModel/localMember/Publisher.ts index 579c037cd3..4a876b14bc 100644 --- a/src/state/CallViewModel/localMember/Publisher.ts +++ b/src/state/CallViewModel/localMember/Publisher.ts @@ -41,9 +41,16 @@ import { RNNoiseProcessor, supportsRNNoiseProcessor, } from "../../../audio/RNNoiseProcessor.ts"; +import { + DeepFilterNetProcessor, + DEEPFILTERNET_PROCESSOR_NAME, + supportsDeepFilterNetProcessor, +} from "../../../audio/DeepFilterNetProcessor.ts"; import { shouldEnableNativeNoiseSuppression } from "../../../audio/noiseSuppressionPolicy.ts"; import { autoGainControlSetting, + deepFilterNetNoiseSuppression, + deepFilterNetNoiseSuppressionLevel, echoCancellationSetting, micCutoffEnabled, micCutoffThresholdDb, @@ -97,6 +104,8 @@ export class Publisher { this.observeTrackProcessors(this.scope, room, trackerProcessorState$); this.observeRNNoiseProcessor(this.scope, room, devices); this.observeRNNoiseSettingRestart(this.scope, room, devices); + this.observeDeepFilterNetProcessor(this.scope, room, devices); + this.observeDeepFilterNetSettingRestart(this.scope, room, devices); // Observe media device changes and update LiveKit active devices accordingly this.observeMediaDevices(this.scope, devices, controlledAudioDevices); @@ -633,7 +642,9 @@ export class Publisher { return; } - if (processorActive) { + // Stop any existing processor (DeepFilterNet or otherwise) before + // attaching RNNoise, since only one processor can be active at a time. + if (processor) { await microphoneTrack.stopProcessor(); } await microphoneTrack.setProcessor( @@ -657,4 +668,147 @@ export class Publisher { } } } + + private observeDeepFilterNetProcessor( + scope: ObservableScope, + room: LivekitRoom, + devices: MediaDevices, + ): void { + const microphoneTrack$ = scope.behavior( + observeTrackReference$( + room.localParticipant, + Track.Source.Microphone, + ).pipe( + map((trackRef) => { + const track = trackRef?.publication.track; + return track?.kind === Track.Kind.Audio + ? (track as LocalAudioTrack) + : null; + }), + ), + null, + ); + + combineLatest([ + microphoneTrack$, + deepFilterNetNoiseSuppression.value$, + deepFilterNetNoiseSuppressionLevel.value$, + ]) + .pipe( + scope.bind(), + // Changes to the DeepFilterNet enabled setting are deliberately + // ignored here; they need a track restart and are handled by + // observeDeepFilterNetSettingRestart. + distinctUntilChanged( + ([aTrack, _aEnabled, aLevel], [bTrack, _bEnabled, bLevel]) => { + return aTrack === bTrack && aLevel === bLevel; + }, + ), + ) + .subscribe(([microphoneTrack, dfEnabled, dfLevel]) => { + const dfSupported = supportsDeepFilterNetProcessor(); + if (!microphoneTrack || !dfSupported) { + return; + } + + this.enqueueRNNoiseOperation(async () => { + await this.syncDeepFilterNetProcessor( + microphoneTrack, + dfEnabled, + dfLevel, + ); + }); + }); + } + + private observeDeepFilterNetSettingRestart( + scope: ObservableScope, + room: LivekitRoom, + devices: MediaDevices, + ): void { + deepFilterNetNoiseSuppression.value$ + .pipe(scope.bind(), distinctUntilChanged(), skip(1)) + .subscribe((dfEnabled) => { + const audioTrack = room.localParticipant.getTrackPublication( + Track.Source.Microphone, + )?.audioTrack; + if (!audioTrack) return; + + const dfSupported = supportsDeepFilterNetProcessor(); + this.enqueueRNNoiseOperation(async () => { + await this.restartMicrophoneTrackForDeepFilterNetPolicy( + audioTrack, + devices, + dfEnabled, + ); + await this.syncDeepFilterNetProcessor( + audioTrack, + dfEnabled && dfSupported, + deepFilterNetNoiseSuppressionLevel.getValue(), + ); + }); + }); + } + + private async restartMicrophoneTrackForDeepFilterNetPolicy( + audioTrack: LocalAudioTrack, + devices: MediaDevices, + dfEnabled: boolean, + ): Promise { + const activeProcessor = audioTrack.getProcessor(); + if (activeProcessor?.name === DEEPFILTERNET_PROCESSOR_NAME) { + await audioTrack.stopProcessor(); + } + + await audioTrack.restartTrack({ + deviceId: devices.audioInput.selected$.value?.id, + autoGainControl: autoGainControlSetting.getValue(), + echoCancellation: echoCancellationSetting.getValue(), + noiseSuppression: shouldEnableNativeNoiseSuppression({ + urlNoiseSuppression: noiseSuppressionSetting.getValue(), + rnnoiseEnabled: dfEnabled, + rnnoiseSupported: supportsDeepFilterNetProcessor(), + }), + }); + } + + private async syncDeepFilterNetProcessor( + microphoneTrack: LocalAudioTrack, + dfEnabled: boolean, + dfLevel: number, + ): Promise { + try { + const processor = microphoneTrack.getProcessor(); + const processorActive = processor?.name === DEEPFILTERNET_PROCESSOR_NAME; + const dfProcessor = + processor instanceof DeepFilterNetProcessor ? processor : undefined; + + if (dfEnabled) { + if (dfProcessor) { + dfProcessor.setSuppressionLevel(dfLevel); + await dfProcessor.setEnabled(true); + return; + } + + // Stop any existing processor (RNNoise or otherwise) before attaching + // DeepFilterNet, since only one processor can be active at a time. + if (processor) { + await microphoneTrack.stopProcessor(); + } + await microphoneTrack.setProcessor( + new DeepFilterNetProcessor(dfLevel, true), + ); + } else if (processorActive) { + await microphoneTrack.stopProcessor(); + } + } catch (e) { + this.logger.error("Failed to apply DeepFilterNet audio processor", e); + if (dfEnabled && deepFilterNetNoiseSuppression.getValue()) { + this.logger.warn( + "Disabling DeepFilterNet setting after processor setup failure", + ); + deepFilterNetNoiseSuppression.setValue(false); + } + } + } } diff --git a/src/state/GridLikeLayout.ts b/src/state/GridLikeLayout.ts index f91f8e310b..30ec231ff5 100644 --- a/src/state/GridLikeLayout.ts +++ b/src/state/GridLikeLayout.ts @@ -31,7 +31,7 @@ export function gridLikeLayout( prevTiles: TileStore, ): [Layout & { type: GridLikeLayoutType }, TileStore] { const update = prevTiles.from(visibleTiles); - if (media.spotlight !== undefined) + if (media.type !== "grid") update.registerSpotlight( media.spotlight, media.type === "spotlight-portrait", @@ -44,6 +44,7 @@ export function gridLikeLayout( type: media.type, spotlight: tiles.spotlightTile, grid: tiles.gridTiles, + focused: media.type === "grid" ? (media.focused ?? false) : undefined, spotlightAlignment$, setVisibleTiles, } as Layout & { type: GridLikeLayoutType }, diff --git a/src/state/TileStore.ts b/src/state/TileStore.ts index 132d1b9461..e70a75be07 100644 --- a/src/state/TileStore.ts +++ b/src/state/TileStore.ts @@ -13,7 +13,6 @@ import { fillGaps } from "../utils/iter"; import { debugTileLayout } from "../settings/settings"; import { type MediaViewModel } from "./media/MediaViewModel"; import { type UserMediaViewModel } from "./media/UserMediaViewModel"; -import { type RingingMediaViewModel } from "./media/RingingMediaViewModel"; type SpotlightBackground = "solid" | "transparent"; @@ -68,19 +67,17 @@ class SpotlightTileData { } class GridTileData { - private readonly media$: BehaviorSubject< - UserMediaViewModel | RingingMediaViewModel - >; - public get media(): UserMediaViewModel | RingingMediaViewModel { + private readonly media$: BehaviorSubject; + public get media(): MediaViewModel { return this.media$.value; } - public set media(value: UserMediaViewModel) { + public set media(value: MediaViewModel) { this.media$.next(value); } public readonly vm: GridTileViewModel; - public constructor(media: UserMediaViewModel | RingingMediaViewModel) { + public constructor(media: MediaViewModel) { this.media$ = new BehaviorSubject(media); this.vm = new GridTileViewModel(this.media$); } @@ -205,9 +202,7 @@ export class TileStoreBuilder { * Sets up a grid tile for the given media. If this is never called for some * media, then that media will have no grid tile. */ - public registerGridTile( - media: UserMediaViewModel | RingingMediaViewModel, - ): void { + public registerGridTile(media: MediaViewModel): void { if (DEBUG_ENABLED) logger.debug( `[TileStore, ${this.generation}] register grid tile: ${media.displayName$.value}`, diff --git a/src/state/TileViewModel.ts b/src/state/TileViewModel.ts index 6a5d9175da..bdc06b7e2d 100644 --- a/src/state/TileViewModel.ts +++ b/src/state/TileViewModel.ts @@ -9,8 +9,6 @@ import { BehaviorSubject } from "rxjs"; import { type Behavior } from "./Behavior"; import { type MediaViewModel } from "./media/MediaViewModel"; -import { type RingingMediaViewModel } from "./media/RingingMediaViewModel"; -import { type UserMediaViewModel } from "./media/UserMediaViewModel"; let nextId = 0; function createId(): string { @@ -22,11 +20,7 @@ export class GridTileViewModel { private readonly _showOutline$ = new BehaviorSubject(false); public readonly showOutline$: Behavior = this._showOutline$; - public constructor( - public readonly media$: Behavior< - UserMediaViewModel | RingingMediaViewModel - >, - ) {} + public constructor(public readonly media$: Behavior) {} public setShowOutline(value: boolean): void { this._showOutline$.next(value); diff --git a/src/state/VolumeControls.test.ts b/src/state/VolumeControls.test.ts index cfe2bd459e..7ebe2c45e1 100644 --- a/src/state/VolumeControls.test.ts +++ b/src/state/VolumeControls.test.ts @@ -7,7 +7,7 @@ Please see LICENSE in the repository root for full details. import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; -import { createVolumeControls } from "./VolumeControls"; +import { createVolumeControls, MAX_PLAYBACK_VOLUME } from "./VolumeControls"; import { ObservableScope } from "./ObservableScope"; import { constant } from "./Behavior"; @@ -23,6 +23,7 @@ describe("createVolumeControls", () => { function create(options?: { initialVolume?: number; onVolumeCommitted?: (volume: number) => void; + onBoostedChange?: (boosted: boolean) => void; }): { controls: ReturnType; sink: ReturnType; @@ -82,4 +83,40 @@ describe("createVolumeControls", () => { controls.togglePlaybackMuted(); expect(controls.playbackVolume$.value).toBe(0.6); }); + + it("supports volumes above 100%", () => { + const { controls, sink } = create(); + + controls.adjustPlaybackVolume(1.5); + expect(controls.playbackVolume$.value).toBe(1.5); + expect(sink).toHaveBeenLastCalledWith(1.5); + }); + + it("clamps volumes above the maximum", () => { + const { controls, sink } = create(); + + controls.adjustPlaybackVolume(2.5); + expect(controls.playbackVolume$.value).toBe(MAX_PLAYBACK_VOLUME); + expect(sink).toHaveBeenLastCalledWith(MAX_PLAYBACK_VOLUME); + }); + + it("clamps an out-of-range initial volume", () => { + const { controls, sink } = create({ initialVolume: 5 }); + + expect(controls.playbackVolume$.value).toBe(MAX_PLAYBACK_VOLUME); + expect(sink).toHaveBeenCalledWith(MAX_PLAYBACK_VOLUME); + }); + + it("reports whether the volume is boosted above the base volume", () => { + const onBoostedChange = vi.fn(); + const { controls } = create({ onBoostedChange }); + + expect(onBoostedChange).toHaveBeenLastCalledWith(false); + + controls.adjustPlaybackVolume(1.01); + expect(onBoostedChange).toHaveBeenLastCalledWith(true); + + controls.adjustPlaybackVolume(0.5); + expect(onBoostedChange).toHaveBeenLastCalledWith(false); + }); }); diff --git a/src/state/VolumeControls.ts b/src/state/VolumeControls.ts index 67f5e2c5f6..4132008d34 100644 --- a/src/state/VolumeControls.ts +++ b/src/state/VolumeControls.ts @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. import { combineLatest, + distinctUntilChanged, map, merge, of, @@ -19,17 +20,17 @@ import { type Behavior } from "./Behavior"; import { type ObservableScope } from "./ObservableScope"; import { accumulate } from "../utils/observable"; +/** + * The maximum playback volume, as a scalar multiplier of the stream's base + * volume. Values above 1 boost the volume past 100%. + */ +export const MAX_PLAYBACK_VOLUME = 2; + /** * Controls for audio playback volume. */ export interface VolumeControls { - /** - * The volume to which the audio is set, as a scalar multiplier. - */ playbackVolume$: Behavior; - /** - * Whether playback of this audio is disabled. - */ playbackMuted$: Behavior; togglePlaybackMuted: () => void; adjustPlaybackVolume: (value: number) => void; @@ -38,23 +39,15 @@ export interface VolumeControls { interface VolumeControlsInputs { pretendToBeDisconnected$: Behavior; - /** - * The callback to run to notify the module performing audio playback of the - * requested volume. - */ sink$: Behavior<(volume: number) => void>; - /** - * The volume to start at, e.g. restored from a saved preference. Defaults - * to 1. - */ initialVolume?: number; + onVolumeCommitted?: (volume: number) => void; /** - * Called with the newly committed volume whenever the user finishes - * adjusting it (i.e. on commit, not while dragging). Not called for mute - * toggles or when the slider is released at zero, since those keep the - * previous committed volume. + * Called with whether the volume is above 100% whenever it changes, so the + * audio renderer can route the participant's audio through a WebAudio gain + * node (required to amplify past the HTMLMediaElement's volume cap of 1). */ - onVolumeCommitted?: (volume: number) => void; + onBoostedChange?: (boosted: boolean) => void; } /** @@ -68,8 +61,11 @@ export function createVolumeControls( sink$, initialVolume = 1, onVolumeCommitted, + onBoostedChange, }: VolumeControlsInputs, ): VolumeControls { + const clamp = (v: number): number => + Math.max(0, Math.min(MAX_PLAYBACK_VOLUME, v)); const toggleMuted$ = new Subject<"toggle mute">(); const adjustVolume$ = new Subject(); const commitVolume$ = new Subject<"commit">(); @@ -77,7 +73,10 @@ export function createVolumeControls( const playbackVolume$ = scope.behavior( merge(toggleMuted$, adjustVolume$, commitVolume$).pipe( accumulate( - { volume: initialVolume, committedVolume: initialVolume }, + { + volume: clamp(initialVolume), + committedVolume: clamp(initialVolume), + }, (state, event) => { switch (event) { case "toggle mute": @@ -95,8 +94,9 @@ export function createVolumeControls( state.volume === 0 ? state.committedVolume : state.volume, }; default: - // Volume adjustment - return { ...state, volume: event }; + // Clamp so nothing above the maximum can slip through (e.g. a + // stale saved preference). + return { ...state, volume: clamp(event) }; } }, ), @@ -128,6 +128,17 @@ export function createVolumeControls( .pipe(scope.bind()) .subscribe(([sink, volume]) => sink(volume)); + // Notify the audio renderer when this stream starts/stops needing a boost. + if (onBoostedChange !== undefined) { + playbackVolume$ + .pipe( + map((volume) => volume > 1), + distinctUntilChanged(), + scope.bind(), + ) + .subscribe(onBoostedChange); + } + return { playbackVolume$, playbackMuted$: scope.behavior( diff --git a/src/state/layout-types.ts b/src/state/layout-types.ts index 2b0d459daa..d813d99b8f 100644 --- a/src/state/layout-types.ts +++ b/src/state/layout-types.ts @@ -20,8 +20,8 @@ import { type Behavior } from "./Behavior.ts"; export interface GridLayoutMedia { type: "grid"; edgeToEdge: false; - spotlight?: MediaViewModel[]; - grid: UserMediaViewModel[]; + grid: MediaViewModel[]; + focused?: boolean; } export interface SpotlightLandscapeLayoutMedia { @@ -85,6 +85,7 @@ export interface GridLayout { grid: GridTileViewModel[]; spotlightAlignment$: BehaviorSubject; setVisibleTiles: (value: number) => void; + focused?: boolean; } export interface SpotlightLandscapeLayout { diff --git a/src/state/media/MediaViewModel.test.ts b/src/state/media/MediaViewModel.test.ts index 9d873ccba2..f79b55bba9 100644 --- a/src/state/media/MediaViewModel.test.ts +++ b/src/state/media/MediaViewModel.test.ts @@ -9,6 +9,8 @@ import { expect, onTestFinished, test, vi } from "vitest"; import { type LocalTrackPublication, LocalVideoTrack, + ParticipantEvent, + RemoteTrackPublication, Track, TrackEvent, } from "livekit-client"; @@ -24,6 +26,7 @@ import { mockRemoteParticipant, mockRemoteScreenShare, } from "../../utils/test"; +import { tileVolumes } from "../../settings/settings"; import { constant } from "../Behavior"; global.MediaStreamTrack = class {} as unknown as { @@ -93,6 +96,36 @@ test("control a participant's volume", () => { }); }); +test("a participant's volume can be boosted above 100%", () => { + // Don't let volumes persisted by earlier tests leak into this one. + tileVolumes.setValue({}); + const setVolumeSpy = vi.fn(); + const vm = mockRemoteMedia( + rtcMembership, + {}, + mockRemoteParticipant({ setVolume: setVolumeSpy }), + ); + withTestScheduler(({ expectObservable, schedule }) => { + schedule("-ab|", { + a() { + // Boost the volume above the base volume + vm.adjustPlaybackVolume(1.5); + expect(setVolumeSpy).toHaveBeenLastCalledWith(1.5); + }, + b() { + // Back below the base volume + vm.adjustPlaybackVolume(0.9); + expect(setVolumeSpy).toHaveBeenLastCalledWith(0.9); + }, + }); + expectObservable(vm.playbackVolume$).toBe("abc", { + a: 1, + b: 1.5, + c: 0.9, + }); + }); +}); + test("control a participant's screen share volume", () => { const setVolumeSpy = vi.fn(); const vm = mockRemoteScreenShare( @@ -160,6 +193,86 @@ test("control a participant's screen share volume", () => { }); }); +test("stop watching a remote screen share actually unsubscribes from the LiveKit track", () => { + const videoPublication = new RemoteTrackPublication( + Track.Kind.Video, + { + sid: "TR_screen", + name: "screen", + muted: false, + } as unknown as ConstructorParameters[1], + true, + {}, + ); + const audioPublication = new RemoteTrackPublication( + Track.Kind.Audio, + { + sid: "TR_screen_audio", + name: "screen_audio", + muted: false, + } as unknown as ConstructorParameters[1], + true, + {}, + ); + const setVideoSubscribedSpy = vi.spyOn(videoPublication, "setSubscribed"); + const setAudioSubscribedSpy = vi.spyOn(audioPublication, "setSubscribed"); + const vm = mockRemoteScreenShare( + rtcMembership, + {}, + mockRemoteParticipant({ + getTrackPublication: (source) => { + if (source === Track.Source.ScreenShare) return videoPublication; + if (source === Track.Source.ScreenShareAudio) return audioPublication; + return undefined; + }, + }), + ); + + // Watching starts out enabled, so we should be subscribed to both the video + // and the screen share audio track. + expect(setVideoSubscribedSpy).toHaveBeenCalledWith(true); + expect(setAudioSubscribedSpy).toHaveBeenCalledWith(true); + + // Stopping watching should unsubscribe both so that the data stops flowing. + vm.setWatching(false); + expect(setVideoSubscribedSpy).toHaveBeenLastCalledWith(false); + expect(setAudioSubscribedSpy).toHaveBeenLastCalledWith(false); + + // Watching again should resubscribe both. + vm.setWatching(true); + expect(setVideoSubscribedSpy).toHaveBeenLastCalledWith(true); + expect(setAudioSubscribedSpy).toHaveBeenLastCalledWith(true); +}); + +test("screen share mute is re-applied when the audio track is re-subscribed", () => { + const setVolumeSpy = vi.fn(); + const participant = mockRemoteParticipant({ setVolume: setVolumeSpy }); + const vm = mockRemoteScreenShare(rtcMembership, {}, participant); + + // Muting should set the screen share audio volume to 0. + vm.togglePlaybackMuted(); + expect(setVolumeSpy).toHaveBeenLastCalledWith( + 0, + Track.Source.ScreenShareAudio, + ); + + // Simulate the audio track being re-attached (e.g. after the user resumes + // watching): the current volume must be re-applied, otherwise the mute + // would be lost and the sound would come back. + const callsBefore = setVolumeSpy.mock.calls.length; + ( + participant.emit as unknown as ( + event: string, + ...args: unknown[] + ) => boolean + )(ParticipantEvent.TrackSubscribed, {}); + expect(setVolumeSpy.mock.calls.length).toBeGreaterThan(callsBefore); + expect(setVolumeSpy).toHaveBeenLastCalledWith( + 0, + Track.Source.ScreenShareAudio, + ); +}); + test("local media remembers whether it should always be shown", () => { const vm1 = mockLocalMedia( rtcMembership, diff --git a/src/state/media/RemoteScreenShareViewModel.ts b/src/state/media/RemoteScreenShareViewModel.ts index cc3221cfa3..8dd4e30674 100644 --- a/src/state/media/RemoteScreenShareViewModel.ts +++ b/src/state/media/RemoteScreenShareViewModel.ts @@ -6,8 +6,14 @@ SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial Please see LICENSE in the repository root for full details. */ -import { Track, type RemoteParticipant } from "livekit-client"; -import { map, of, switchMap } from "rxjs"; +import { + ParticipantEvent, + RemoteTrackPublication, + Track, + type RemoteParticipant, +} from "livekit-client"; +import { observeParticipantEvents } from "@livekit/components-core"; +import { combineLatest, distinctUntilChanged, map, of, switchMap } from "rxjs"; import { type Behavior } from "../Behavior"; import { @@ -19,6 +25,7 @@ import { type ObservableScope } from "../ObservableScope"; import { createVolumeControls, type VolumeControls } from "../VolumeControls"; import { observeTrackReference$ } from "../observeTrackReference"; import { saveTileVolume, tileVolumes } from "../../settings/settings"; +import { setParticipantBoosted } from "../participantVolume"; export interface RemoteScreenShareViewModel extends BaseScreenShareViewModel, VolumeControls { @@ -43,23 +50,68 @@ export function createRemoteScreenShare( scope: ObservableScope, { pretendToBeDisconnected$, ...inputs }: RemoteScreenShareInputs, ): RemoteScreenShareViewModel { + const base = createBaseScreenShare(scope, inputs); + + // The screen share's video and audio publications. + const videoPublication$ = base.video$.pipe(map((ref) => ref?.publication)); + const audioPublication$ = inputs.participant$.pipe( + switchMap((p) => + p + ? observeTrackReference$(p, Track.Source.ScreenShareAudio) + : of(undefined), + ), + map((ref) => ref?.publication), + ); + combineLatest([base.watching$, videoPublication$, audioPublication$]) + .pipe( + scope.bind(), + distinctUntilChanged( + ([watching, video, audio], [nextWatching, nextVideo, nextAudio]) => + watching === nextWatching && + video === nextVideo && + audio === nextAudio, + ), + ) + .subscribe(([watching, video, audio]) => { + for (const publication of [video, audio]) { + if (publication instanceof RemoteTrackPublication) + publication.setSubscribed(watching); + } + }); + + // Emits whenever any remote track subscribes or unsubscribes. + const audioTrackEvents$ = inputs.participant$.pipe( + switchMap((p) => + p === null + ? of(undefined) + : observeParticipantEvents( + p, + ParticipantEvent.TrackSubscribed, + ParticipantEvent.TrackUnsubscribed, + ), + ), + ); + // Screen share audio gets its own saved volume, separate from the // participant's voice volume. const savedVolumeKey = `${inputs.rtcBackendIdentity}:screen-share`; return { - ...createBaseScreenShare(scope, inputs), + ...base, ...createVolumeControls(scope, { pretendToBeDisconnected$, sink$: scope.behavior( - inputs.participant$.pipe( + combineLatest([inputs.participant$, audioTrackEvents$]).pipe( map( - (p) => (volume) => - p?.setVolume(volume, Track.Source.ScreenShareAudio), + ([p]) => + (volume) => + p?.setVolume(volume, Track.Source.ScreenShareAudio), ), ), ), initialVolume: tileVolumes.getValue()[savedVolumeKey], onVolumeCommitted: (volume) => saveTileVolume(savedVolumeKey, volume), + onBoostedChange: (boosted) => + setParticipantBoosted(inputs.rtcBackendIdentity, boosted), }), local: false, videoEnabled$: scope.behavior( diff --git a/src/state/media/RemoteUserMediaViewModel.ts b/src/state/media/RemoteUserMediaViewModel.ts index 7d0ed9111d..47e48cab17 100644 --- a/src/state/media/RemoteUserMediaViewModel.ts +++ b/src/state/media/RemoteUserMediaViewModel.ts @@ -12,6 +12,7 @@ import { combineLatest, map, of, switchMap } from "rxjs"; import { type Behavior } from "../Behavior"; import { createVolumeControls, type VolumeControls } from "../VolumeControls"; import { saveTileVolume, tileVolumes } from "../../settings/settings"; +import { setParticipantBoosted } from "../participantVolume"; import { type BaseUserMediaInputs, type BaseUserMediaViewModel, @@ -57,6 +58,8 @@ export function createRemoteUserMedia( initialVolume: tileVolumes.getValue()[inputs.rtcBackendIdentity], onVolumeCommitted: (volume) => saveTileVolume(inputs.rtcBackendIdentity, volume), + onBoostedChange: (boosted) => + setParticipantBoosted(inputs.rtcBackendIdentity, boosted), }), local: false, speaking$: scope.behavior( @@ -66,6 +69,20 @@ export function createRemoteUserMedia( ), ), ), + audioLevel$: scope.behavior( + pretendToBeDisconnected$.pipe( + switchMap((disconnected) => + disconnected ? of(0) : baseUserMedia.audioLevel$, + ), + ), + ), + voiceActivity$: scope.behavior( + pretendToBeDisconnected$.pipe( + switchMap((disconnected) => + disconnected ? of(false) : baseUserMedia.voiceActivity$, + ), + ), + ), videoEnabled$: scope.behavior( pretendToBeDisconnected$.pipe( switchMap((disconnected) => diff --git a/src/state/media/ScreenShareViewModel.ts b/src/state/media/ScreenShareViewModel.ts index 8336f0a6ba..adb2b16b92 100644 --- a/src/state/media/ScreenShareViewModel.ts +++ b/src/state/media/ScreenShareViewModel.ts @@ -7,7 +7,9 @@ Please see LICENSE in the repository root for full details. */ import { Track } from "livekit-client"; +import { Subject, startWith } from "rxjs"; +import { type Behavior } from "../Behavior"; import { type ObservableScope } from "../ObservableScope"; import { type LocalScreenShareViewModel } from "./LocalScreenShareViewModel"; import { @@ -29,6 +31,8 @@ export type ScreenShareViewModel = */ export interface BaseScreenShareViewModel extends BaseMemberMediaViewModel { type: "screen share"; + watching$: Behavior; + setWatching: (watching: boolean) => void; } export type BaseScreenShareInputs = Omit< @@ -40,6 +44,8 @@ export function createBaseScreenShare( scope: ObservableScope, inputs: BaseScreenShareInputs, ): BaseScreenShareViewModel { + const watchingRequest$ = new Subject(); + const watching$ = scope.behavior(watchingRequest$.pipe(startWith(true))); return { ...createMemberMedia(scope, { ...inputs, @@ -47,5 +53,7 @@ export function createBaseScreenShare( videoSource: Track.Source.ScreenShare, }), type: "screen share", + watching$, + setWatching: (watching: boolean): void => watchingRequest$.next(watching), }; } diff --git a/src/state/media/UserMediaViewModel.ts b/src/state/media/UserMediaViewModel.ts index ea03310302..2d5cb9453f 100644 --- a/src/state/media/UserMediaViewModel.ts +++ b/src/state/media/UserMediaViewModel.ts @@ -29,6 +29,10 @@ import { type MemberMediaInputs, type BaseMemberMediaViewModel, } from "./MemberMediaViewModel"; +import { + observeSpeakingFromLevel$, + observeTrackAudioLevel$, +} from "./observeAudioLevel"; import { type RemoteUserMediaViewModel } from "./RemoteUserMediaViewModel"; import { type ObservableScope } from "../ObservableScope"; import { showConnectionStats } from "../../settings/settings"; @@ -45,6 +49,8 @@ export type UserMediaViewModel = export interface BaseUserMediaViewModel extends BaseMemberMediaViewModel { type: "user"; speaking$: Behavior; + audioLevel$: Behavior; + voiceActivity$: Behavior; audioEnabled$: Behavior; videoEnabled$: Behavior; videoFit$: Behavior<"cover" | "contain">; @@ -106,6 +112,19 @@ export function createBaseUserMedia( >(undefined); const videoSize$ = videoSizeFromParticipant$(participant$); + + // Client-side voice activity detection using the audio track itself + const audioLevel$ = scope.behavior( + participant$.pipe( + switchMap((p) => { + if (!p) return of(0); + return observeTrackAudioLevel$( + observeParticipantMedia(p).pipe(map((m) => m.microphoneTrack?.track)), + ); + }), + ), + ); + return { ...createMemberMedia(scope, { ...inputs, @@ -125,6 +144,8 @@ export function createBaseUserMedia( ), ), ), + audioLevel$, + voiceActivity$: scope.behavior(observeSpeakingFromLevel$(audioLevel$)), audioEnabled$: scope.behavior( media$.pipe(map((m) => m?.microphoneTrack?.isMuted === false)), ), diff --git a/src/state/media/observeAudioLevel.test.ts b/src/state/media/observeAudioLevel.test.ts new file mode 100644 index 0000000000..f991c791dc --- /dev/null +++ b/src/state/media/observeAudioLevel.test.ts @@ -0,0 +1,187 @@ +/* +SableCall +Copyright (C) 2026 TomOdellSheetMusic + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ + +import { describe, expect, test, vi, beforeEach, afterEach } from "vitest"; +import { BehaviorSubject, of } from "rxjs"; +import type { LocalAudioTrack } from "livekit-client"; + +import { + observeSpeakingFromLevel$, + observeTrackAudioLevel$, + type AudioAnalyserFactory, +} from "./observeAudioLevel"; + +function mockAudioTrack(): LocalAudioTrack { + return { + kind: "audio", + mediaStreamTrack: {} as MediaStreamTrack, + isMuted: false, + } as unknown as LocalAudioTrack; +} + +describe("observeTrackAudioLevel$", () => { + let analyserFactory: ReturnType>; + let cleanup: ReturnType Promise>>; + + beforeEach(() => { + vi.useFakeTimers(); + cleanup = vi.fn<() => Promise>().mockResolvedValue(undefined); + analyserFactory = vi.fn(() => ({ + calculateVolume: () => 0, + cleanup, + })); + }); + + afterEach(() => { + vi.useRealTimers(); + }); + + test("emits 0 when there is no track", () => { + const levels: number[] = []; + observeTrackAudioLevel$(of(undefined), analyserFactory).subscribe((level) => + levels.push(level), + ); + expect(levels).toEqual([0]); + expect(analyserFactory).not.toHaveBeenCalled(); + }); + + test("emits the calculated volume for an audio track", async () => { + analyserFactory = vi.fn(() => ({ + calculateVolume: () => 0.7, + cleanup, + })); + + const levels: number[] = []; + observeTrackAudioLevel$(of(mockAudioTrack()), analyserFactory).subscribe( + (level) => levels.push(level), + ); + // Initial emission (startWith(0)) + expect(levels).toEqual([0]); + expect(analyserFactory).toHaveBeenCalled(); + + // Advance the interval timer + await vi.advanceTimersByTimeAsync(200); + expect(levels).toEqual([0, 0.7]); + }); + + test("cleans up the analyser when unsubscribed", () => { + const sub = observeTrackAudioLevel$( + of(mockAudioTrack()), + analyserFactory, + ).subscribe(); + sub.unsubscribe(); + + expect(cleanup).toHaveBeenCalled(); + }); +}); + +describe("observeSpeakingFromLevel$", () => { + let levels: BehaviorSubject; + let speaking: boolean[]; + let sub: ReturnType; + + function subscribeToSpeaking( + options?: Parameters[1], + ) { + speaking = []; + const s = observeSpeakingFromLevel$(levels, options).subscribe((v) => + speaking.push(v), + ); + return s; + } + + beforeEach(() => { + vi.useFakeTimers(); + levels = new BehaviorSubject(0.01); // below threshold + }); + + afterEach(() => { + sub?.unsubscribe(); + vi.useRealTimers(); + }); + + test("starts as not speaking and stays silent when level is low", () => { + sub = subscribeToSpeaking({ confirmMs: 300, dropOffMs: 1000 }); + expect(speaking).toEqual([false]); + levels.next(0.01); + expect(speaking).toEqual([false]); + }); + + test("brief blip above threshold does not trigger speaking", async () => { + sub = subscribeToSpeaking({ confirmMs: 300, dropOffMs: 1000 }); + levels.next(0.2); // blip above threshold + await vi.advanceTimersByTimeAsync(100); // blip lasts 100ms < confirmMs + levels.next(0.01); // back below threshold + await vi.advanceTimersByTimeAsync(1000); // more than confirmMs + expect(speaking).toEqual([false]); + }); + + test("sustained voice becomes speaking after confirm period", async () => { + sub = subscribeToSpeaking({ confirmMs: 300, dropOffMs: 1000 }); + levels.next(0.2); // above threshold + await vi.advanceTimersByTimeAsync(200); + expect(speaking).toEqual([false]); // not yet confirmed + await vi.advanceTimersByTimeAsync(100); // total 300ms + expect(speaking).toEqual([false, true]); // confirmed speaking + }); + + test("stops speaking after drop-off once level falls below hold threshold", async () => { + sub = subscribeToSpeaking({ confirmMs: 300, dropOffMs: 1000 }); + levels.next(0.2); + await vi.advanceTimersByTimeAsync(300); + expect(speaking).toEqual([false, true]); // confirmed speaking + levels.next(0.01); // below hold threshold + await vi.advanceTimersByTimeAsync(500); + expect(speaking).toEqual([false, true]); // still speaking during drop-off + await vi.advanceTimersByTimeAsync(500); // total 1000ms drop-off + expect(speaking).toEqual([false, true, false]); // stopped speaking + }); + + test("holds speaking through brief dips (hysteresis)", async () => { + sub = subscribeToSpeaking({ confirmMs: 300, dropOffMs: 1000 }); + levels.next(0.2); + await vi.advanceTimersByTimeAsync(300); + expect(speaking).toEqual([false, true]); // confirmed speaking + levels.next(0.01); // brief dip below hold threshold + await vi.advanceTimersByTimeAsync(100); // shorter than drop-off + levels.next(0.2); // resume speaking + await vi.advanceTimersByTimeAsync(1000); + expect(speaking).toEqual([false, true]); // never stopped speaking + }); + + test("hysteresis: requires higher level to start than to keep speaking", async () => { + sub = subscribeToSpeaking({ + threshold: 0.05, + holdThreshold: 0.02, + confirmMs: 300, + dropOffMs: 1000, + }); + // 0.03 is above hold but below threshold: should NOT start speaking + levels.next(0.03); + await vi.advanceTimersByTimeAsync(1000); + expect(speaking).toEqual([false]); + // 0.1 is above threshold: starts speaking after confirm + levels.next(0.1); + await vi.advanceTimersByTimeAsync(300); + expect(speaking).toEqual([false, true]); + // 0.03 is below threshold but above hold: keeps speaking + levels.next(0.03); + await vi.advanceTimersByTimeAsync(500); + expect(speaking).toEqual([false, true]); + }); +}); diff --git a/src/state/media/observeAudioLevel.ts b/src/state/media/observeAudioLevel.ts new file mode 100644 index 0000000000..175b8370c3 --- /dev/null +++ b/src/state/media/observeAudioLevel.ts @@ -0,0 +1,109 @@ +/* +SableCall +Copyright (C) 2026 TomOdellSheetMusic + +This program is free software: you can redistribute it and/or modify +it under the terms of the GNU Affero General Public License as published by +the Free Software Foundation, either version 3 of the License, or +(at your option) any later version. + +This program is distributed in the hope that it will be useful, +but WITHOUT ANY WARRANTY; without even the implied warranty of +MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +GNU Affero General Public License for more details. + +You should have received a copy of the GNU Affero General Public License +along with this program. If not, see . +*/ +import { + createAudioAnalyser, + type AudioAnalyserOptions, + type LocalAudioTrack, + type RemoteAudioTrack, + type Track, +} from "livekit-client"; +import { + distinctUntilChanged, + finalize, + interval, + map, + of, + scan, + startWith, + switchMap, + timer, + type Observable, +} from "rxjs"; + +// Constants for audio level detection and debounce +export const AUDIO_LEVEL_SAMPLE_INTERVAL_MS = 100; +export const VOICE_ACTIVITY_THRESHOLD = 0.1; +export const VOICE_ACTIVITY_HOLD_THRESHOLD = 0.05; +export const VOICE_ACTIVITY_CONFIRM_MS = 50; +export const VOICE_ACTIVITY_DROP_OFF_MS = 50; + +export type AudioAnalyserFactory = ( + track: LocalAudioTrack | RemoteAudioTrack, + options?: AudioAnalyserOptions, +) => { calculateVolume: () => number; cleanup: () => Promise }; + +function isAudioTrack( + track: Track, +): track is LocalAudioTrack | RemoteAudioTrack { + return track.kind === "audio" && typeof track.mediaStreamTrack === "object"; +} + +// Raw audio level (0-1) of a participant's microphone track, sampled continuously. +export function observeTrackAudioLevel$( + track$: Observable, + analyserFactory: AudioAnalyserFactory = createAudioAnalyser, +): Observable { + return track$.pipe( + switchMap((track) => { + if (!track || !isAudioTrack(track)) return of(0); + const { calculateVolume, cleanup } = analyserFactory(track, { + cloneTrack: true, + smoothingTimeConstant: 0.1, + }); + return interval(AUDIO_LEVEL_SAMPLE_INTERVAL_MS).pipe( + map(() => calculateVolume()), + startWith(0), + distinctUntilChanged(), + finalize(() => void cleanup()), + ); + }), + ); +} + +export interface SpeakingOptions { + threshold?: number; + holdThreshold?: number; + confirmMs?: number; + dropOffMs?: number; +} + +// Debounced speaking detection +export function observeSpeakingFromLevel$( + level$: Observable, + { + threshold = VOICE_ACTIVITY_THRESHOLD, + holdThreshold = VOICE_ACTIVITY_HOLD_THRESHOLD, + confirmMs = VOICE_ACTIVITY_CONFIRM_MS, + dropOffMs = VOICE_ACTIVITY_DROP_OFF_MS, + }: SpeakingOptions = {}, +): Observable { + return level$.pipe( + scan( + (speaking, level) => + speaking ? level > holdThreshold : level > threshold, + false, + ), + distinctUntilChanged(), + switchMap((speaking, index) => + index === 0 + ? of(speaking) + : timer(speaking ? confirmMs : dropOffMs).pipe(map(() => speaking)), + ), + distinctUntilChanged(), + ); +} diff --git a/src/state/participantVolume.test.ts b/src/state/participantVolume.test.ts new file mode 100644 index 0000000000..45321bc1c5 --- /dev/null +++ b/src/state/participantVolume.test.ts @@ -0,0 +1,44 @@ +/* +Copyright 2026 Element Software Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { describe, expect, it } from "vitest"; + +import { + boostedParticipants$, + setParticipantBoosted, +} from "./participantVolume"; + +describe("participantVolume", () => { + it("tracks which participants are boosted", () => { + setParticipantBoosted("@alice:example.org:AAAA", false); + setParticipantBoosted("@alice:example.org:AAAA", true); + expect(boostedParticipants$.value).toEqual( + new Set(["@alice:example.org:AAAA"]), + ); + + setParticipantBoosted("@bob:example.org:BBBB", true); + expect(boostedParticipants$.value).toEqual( + new Set(["@alice:example.org:AAAA", "@bob:example.org:BBBB"]), + ); + + setParticipantBoosted("@alice:example.org:AAAA", false); + expect(boostedParticipants$.value).toEqual( + new Set(["@bob:example.org:BBBB"]), + ); + + setParticipantBoosted("@bob:example.org:BBBB", false); + expect(boostedParticipants$.value).toEqual(new Set()); + }); + + it("does not emit when nothing changes", () => { + const values: Set[] = []; + const sub = boostedParticipants$.subscribe((v) => values.push(v)); + setParticipantBoosted("@alice:example.org:AAAA", false); + expect(values.length).toBe(1); + sub.unsubscribe(); + }); +}); diff --git a/src/state/participantVolume.ts b/src/state/participantVolume.ts new file mode 100644 index 0000000000..8b68b0dede --- /dev/null +++ b/src/state/participantVolume.ts @@ -0,0 +1,29 @@ +/* +Copyright 2026 Element Software Ltd. + +SPDX-License-Identifier: AGPL-3.0-only OR LicenseRef-Element-Commercial +Please see LICENSE in the repository root for full details. +*/ + +import { BehaviorSubject } from "rxjs"; + +/** + * Identities of remote participants whose playback volume is currently boosted + * above 100%. Module-level so the audio renderer can react to it without + * threading state through the view model tree. + */ +export const boostedParticipants$ = new BehaviorSubject>(new Set()); + +/** + * Mark a participant's volume as boosted (above 100%) or not. + */ +export function setParticipantBoosted( + identity: string, + boosted: boolean, +): void { + if (boostedParticipants$.value.has(identity) === boosted) return; + const next = new Set(boostedParticipants$.value); + if (boosted) next.add(identity); + else next.delete(identity); + boostedParticipants$.next(next); +} diff --git a/src/tile/GridTile.module.css b/src/tile/GridTile.module.css index 3ebb9bf757..2d67e33117 100644 --- a/src/tile/GridTile.module.css +++ b/src/tile/GridTile.module.css @@ -94,6 +94,79 @@ borders don't support gradients */ width: 100%; } +.maximise { + display: flex; + align-items: center; +} + +.maximise > svg { + display: block; + color: var(--cpd-color-icon-primary); +} + +.streamOverlayInner { + position: relative; + width: 100%; + height: 100%; + display: grid; + place-items: center; +} + +.streamOverlayInner > * { + pointer-events: auto; +} + +.streamOverlayScrim { + position: absolute; + inset: 0; + background: rgb(0 0 0 / 0.35); + backdrop-filter: blur(10px); +} + +.frozenFrame { + position: absolute; + inset: 0; + width: 100%; + height: 100%; + object-fit: contain; + background: var(--cpd-color-bg-canvas-default); + filter: blur(8px); + transform: scale(1.05); +} + +.watchStream { + appearance: none; + border: none; + border-radius: var(--cpd-radius-pill-effect); + padding: var(--cpd-space-3x) var(--cpd-space-5x); + background: var(--cpd-color-bg-action-primary-rest); + color: var(--cpd-color-text-on-primary); + box-shadow: var(--small-drop-shadow); + cursor: pointer; + display: flex; + align-items: center; + gap: var(--cpd-space-2x); + font: inherit; + font-weight: 600; + font-size: var(--cpd-font-size-body-lg); + z-index: 1; +} + +.watchStream > svg { + display: block; + color: var(--cpd-color-text-on-primary); +} + +@media (hover) { + .watchStream:hover { + background: var(--cpd-color-bg-action-primary-hovered); + } +} + +.watchStream:active { + background: var(--cpd-color-bg-action-primary-pressed); +} + .tile .switchCamera { opacity: 1; background: var(--cpd-color-bg-action-secondary-rest); diff --git a/src/tile/GridTile.tsx b/src/tile/GridTile.tsx index 1544e41da1..da19c82827 100644 --- a/src/tile/GridTile.tsx +++ b/src/tile/GridTile.tsx @@ -27,6 +27,9 @@ import { MicrophoneSlash, DotsThreeOutline, Eye, + EyeSlash, + Monitor, + Play, } from "@phosphor-icons/react"; import { ContextMenu, @@ -35,10 +38,15 @@ import { Menu, Text, } from "@vector-im/compound-web"; +import { + ExpandIcon, + CollapseIcon, +} from "@vector-im/compound-design-tokens/assets/web/icons"; import { useObservableEagerState } from "observable-hooks"; import styles from "./GridTile.module.css"; import { Slider } from "../Slider"; +import { MAX_PLAYBACK_VOLUME } from "../state/VolumeControls"; import { MediaView } from "./MediaView"; import { useLatest } from "../useLatest"; import { type GridTileViewModel } from "../state/TileViewModel"; @@ -48,7 +56,10 @@ import { useBehavior } from "../useBehavior"; import { type LocalUserMediaViewModel } from "../state/media/LocalUserMediaViewModel"; import { type RemoteUserMediaViewModel } from "../state/media/RemoteUserMediaViewModel"; import { type UserMediaViewModel } from "../state/media/UserMediaViewModel"; +import { type ScreenShareViewModel } from "../state/media/ScreenShareViewModel"; +import { type RemoteScreenShareViewModel } from "../state/media/RemoteScreenShareViewModel"; import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel"; +import { constant, type Behavior } from "../state/Behavior"; import { RingingStatus } from "./RingingStatus"; interface TileProps { @@ -388,7 +399,7 @@ const RemoteUserMediaTile: FC = ({ onValueChange={vm.adjustPlaybackVolume} onValueCommit={vm.commitPlaybackVolume} min={0} - max={1} + max={MAX_PLAYBACK_VOLUME} step={0.01} /> @@ -402,6 +413,289 @@ const RemoteUserMediaTile: FC = ({ RemoteUserMediaTile.displayName = "RemoteUserMediaTile"; +interface ScreenShareTileProps extends TileProps { + vm: ScreenShareViewModel; + /** + * The currently focused (maximised) stream, used to decide whether this tile + * shows a "maximise" or "restore" button. + */ + focusedStream$?: Behavior; + /** + * Focuses (maximises) the given stream so it fills the grid and hides every + * other tile, or unfocuses when passed null. + */ + onToggleFocusedStream?: (vm: ScreenShareViewModel | null) => void; +} + +/** + * New Tile for screen sharing participants. + */ +const ScreenShareTile: FC = (props) => { + const { vm, ...rest } = props; + return vm.local ? ( + + ) : ( + + ); +}; + +const RemoteScreenShareTileContent: FC< + Omit & { vm: RemoteScreenShareViewModel } +> = ({ vm, ...props }) => { + const { t } = useTranslation(); + const videoEnabled = useBehavior(vm.videoEnabled$); + const playbackMuted = useBehavior(vm.playbackMuted$); + const playbackVolume = useBehavior(vm.playbackVolume$); + const watching = useBehavior(vm.watching$); + + const onSelectMute = useCallback( + (e: Event) => { + e.preventDefault(); + vm.togglePlaybackMuted(); + }, + [vm], + ); + + const onSelectWatching = useCallback( + (e: Event) => { + e.preventDefault(); + vm.setWatching(!watching); + }, + [vm, watching], + ); + + const VolumeIcon = playbackMuted ? SpeakerSlash : SpeakerHigh; + + return ( + + + + {/* TODO: Figure out how to make this slider keyboard accessible */} + + + + + } + /> + ); +}; + +RemoteScreenShareTileContent.displayName = "RemoteScreenShareTileContent"; + +interface ScreenShareTileContentProps extends ScreenShareTileProps { + videoEnabled: boolean; + menu?: ReactNode; +} + +const ScreenShareTileContent: FC = ({ + ref, + vm, + videoEnabled, + menu, + focusedStream$, + onToggleFocusedStream, + className, + focusable, + targetWidth, + targetHeight, + displayName, + mxcAvatarUrl, + ...props +}) => { + const { t } = useTranslation(); + const video = useBehavior(vm.video$); + const unencryptedWarning = useBehavior(vm.unencryptedWarning$); + const focusUrl = useBehavior(vm.focusUrl$); + const watching = useBehavior(vm.watching$); + const [menuOpen, setMenuOpen] = useState(false); + const focusedStream = useBehavior(focusedStream$ ?? constant(null)); + const isFocused = focusedStream?.id === vm.id; + + // A ref to the tile root so we can freeze the video element when the user + // stops watching the stream. + const contentRef = useRef(null); + const mergedRef = useMergedRefs(contentRef, ref); + + const [frozenFrame, setFrozenFrame] = useState(null); + + useEffect(() => { + if (watching) { + setFrozenFrame(null); + return; + } + const video = contentRef.current?.querySelector("video"); + if (video && video.videoWidth > 0 && video.videoHeight > 0) { + const canvas = document.createElement("canvas"); + canvas.width = video.videoWidth; + canvas.height = video.videoHeight; + canvas.getContext("2d")?.drawImage(video, 0, 0); + setFrozenFrame(canvas.toDataURL()); + } else { + setFrozenFrame(null); + } + }, [watching]); + + // Freeze the video (pause it) while not watching, and resume when watching. + // While stopped we also watch for new video elements (e.g. LiveKit + // re-attaching) and pause those too. + useEffect(() => { + const root = contentRef.current; + if (root === null) return; + const apply = (): void => { + root.querySelectorAll("video").forEach((v) => { + if (watching) void v.play().catch(() => {}); + else v.pause(); + }); + }; + apply(); + if (watching) return; + const observer = new MutationObserver(apply); + observer.observe(root, { childList: true, subtree: true }); + return (): void => observer.disconnect(); + }, [watching]); + + const FocusIcon = isFocused ? CollapseIcon : ExpandIcon; + + const tile = ( + + {frozenFrame !== null ? ( + + ) : ( +
+ )} + +
+ ) + } + userId={vm.userId} + unencryptedWarning={unencryptedWarning} + videoEnabled={videoEnabled} + videoFit="contain" + mirror={false} + className={classNames(className, styles.tile)} + nameTagLeadingIcon={} + displayName={displayName} + mxcAvatarUrl={mxcAvatarUrl} + focusable={focusable} + primaryButton={ + onToggleFocusedStream === undefined && + menu === undefined ? undefined : ( + <> + {onToggleFocusedStream !== undefined && ( + + )} + {menu !== undefined && ( + + + + } + side="left" + align="start" + > + {menu} + + )} + + ) + } + focusUrl={focusUrl} + targetWidth={targetWidth} + targetHeight={targetHeight} + {...props} + /> + ); + + return menu === undefined ? ( + tile + ) : ( + + {menu} + + ); +}; + +ScreenShareTileContent.displayName = "ScreenShareTileContent"; + interface GridTileProps { ref?: Ref; vm: GridTileViewModel; @@ -415,6 +709,8 @@ interface GridTileProps { showRingingStatus: boolean; showOutline: boolean; focusable: boolean; + focusedStream$?: Behavior; + onToggleFocusedStream?: (vm: ScreenShareViewModel | null) => void; } export const GridTile: FC = ({ @@ -424,6 +720,8 @@ export const GridTile: FC = ({ showRingingStatus, showOutline, onOpenProfile, + focusedStream$, + onToggleFocusedStream, className, ...props }) => { @@ -445,6 +743,19 @@ export const GridTile: FC = ({ {...props} /> ); + } else if (media.type === "screen share") { + return ( + + ); } else if (media.local) { return ( { rtcBackendIdentity?: string; // The focus url, mainly for debugging purposes focusUrl?: string; + streamOverlay?: ReactNode; } export const MediaView: FC = ({ @@ -85,6 +86,7 @@ export const MediaView: FC = ({ videoStreamStats, rtcBackendIdentity, focusUrl, + streamOverlay, ...props }) => { const { t } = useTranslation(); @@ -211,6 +213,9 @@ export const MediaView: FC = ({ )} {primaryButton} + {streamOverlay !== undefined && ( +
{streamOverlay}
+ )} ); }; diff --git a/src/tile/SpotlightTile.tsx b/src/tile/SpotlightTile.tsx index dc0e0c5e07..a929ac3637 100644 --- a/src/tile/SpotlightTile.tsx +++ b/src/tile/SpotlightTile.tsx @@ -48,6 +48,7 @@ import { type ScreenShareViewModel } from "../state/media/ScreenShareViewModel"; import { type RemoteScreenShareViewModel } from "../state/media/RemoteScreenShareViewModel"; import { type MediaViewModel } from "../state/media/MediaViewModel"; import { Slider } from "../Slider"; +import { MAX_PLAYBACK_VOLUME } from "../state/VolumeControls"; import { platform } from "../Platform"; import { type RingingMediaViewModel } from "../state/media/RingingMediaViewModel"; import { RingingStatus } from "./RingingStatus"; @@ -377,7 +378,7 @@ const ScreenShareVolumeButton: FC = ({ vm }) => { label={t("video_tile.volume")} value={playbackVolume} min={0} - max={1} + max={MAX_PLAYBACK_VOLUME} step={0.01} onValueChange={onVolumeChange} onValueCommit={onVolumeCommit} diff --git a/src/utils/test.ts b/src/utils/test.ts index 06eb2548e9..639bac3e23 100644 --- a/src/utils/test.ts +++ b/src/utils/test.ts @@ -548,6 +548,9 @@ export const mockTrack = ( setAudioContext: vi.fn(), setWebAudioPlugins: vi.fn(), setVolume: vi.fn(), + // The audio renderer remounts the track element when the media stream + // changes, so the mock needs a stable mediaStream identity. + mediaStream: {}, }, }, track: {}, diff --git a/src/widget.ts b/src/widget.ts index 6bb326e7d7..5d7ba70769 100644 --- a/src/widget.ts +++ b/src/widget.ts @@ -28,6 +28,21 @@ export enum ElementWidgetActions { JoinCall = "io.element.join", HangupCall = "im.vector.hangup", Close = "io.element.close", + ActiveSpeakers = "io.element.active_speakers", + // fromWidget: updates the client with the mute state of every call + // participant (including the local user), so the host can show per-user + // mute indicators for the whole roster. Sent whenever any + // participant's media state changes. + // + // The data of the widget action request is: + // { + // participants: Array<{ + // userId: string, + // audioEnabled?: boolean, // microphone enabled + // videoEnabled?: boolean, // camera enabled + // }> + // } + ParticipantMediaState = "io.element.participant_media_state", // This can be sent as from or to widget // fromWidget: updates the client about the current device mute state // toWidget: the client requests a specific device mute configuration diff --git a/vite-embedded.config.ts b/vite-embedded.config.ts index 27a42fbbf3..22268c50f8 100644 --- a/vite-embedded.config.ts +++ b/vite-embedded.config.ts @@ -7,6 +7,7 @@ Please see LICENSE in the repository root for full details. import { defineConfig, mergeConfig } from "vite"; import generateFile from "vite-plugin-generate-file"; +import { viteStaticCopy } from "vite-plugin-static-copy"; import fullConfig from "./vite.config"; @@ -33,6 +34,21 @@ export default defineConfig((env) => }, }, ]), + // The embedded build disables publicDir, so the DeepFilterNet WASM + // binary and ONNX model (downloaded by `pnpm setup:assets`) would + // otherwise be omitted from the build output. Copy them explicitly so + // they are served from /assets/deepfilternet3/ at runtime. + viteStaticCopy({ + targets: [ + { + src: "public/assets/deepfilternet3/**/*", + dest: "assets/deepfilternet3", + // Strip the `public/assets/deepfilternet3` prefix (3 segments) + // so files land at dist/assets/deepfilternet3/v3/... + rename: { stripBase: 3 }, + }, + ], + }), ], }), ), diff --git a/vite.config.ts b/vite.config.ts index 6d224b7e87..3fcc19f71f 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -117,6 +117,16 @@ export default ({ key: fs.readFileSync("./backend/dev_tls_m.localhost.key"), cert: fs.readFileSync("./backend/dev_tls_m.localhost.crt"), }, + proxy: { + // Proxy for DeepFilterNet3 assets to avoid CORS issues during + // development when the assets have not been downloaded locally. + "/assets/deepfilternet3": { + target: + "https://cdn.mezon.ai/AI/models/datas/noise_suppression/deepfilternet3", + changeOrigin: true, + rewrite: (path) => path.replace(/^\/assets\/deepfilternet3/, ""), + }, + }, }, worker: { format: "es",