From a97886cfae51545dccc553eb8f13851f249274a0 Mon Sep 17 00:00:00 2001 From: locoholy Date: Mon, 26 Oct 2026 00:53:14 +0500 Subject: [PATCH] Fix macOS microphone stereo output for multichannel devices ScreenCaptureKit delivers microphone audio in the device's native layout. Multichannel interfaces (for example a 4-channel 24-bit USB input) and mono devices were written straight to stereo AAC writers, so only one channel carried the voice while the other stayed silent. Convert microphone buffers to stereo float before enqueueing: keep the channel that carries signal and duplicate it, or preserve the first two channels when both carry signal. Handles float and signed integer PCM, interleaved and planar. AVAudioFormat cannot represent 4-channel 24-bit PCM, so the conversion reads the retained audio buffer list directly. Add a Swift-compiled regression test covering mono, right-only stereo, true stereo and 24-bit multichannel buffers. --- ...creenCaptureKitRecorder.microphone.test.ts | 142 ++++++++++++++++++ .../native/ScreenCaptureKitRecorder.swift | 106 ++++++++++++- 2 files changed, 246 insertions(+), 2 deletions(-) diff --git a/electron/native/ScreenCaptureKitRecorder.microphone.test.ts b/electron/native/ScreenCaptureKitRecorder.microphone.test.ts index b1d35ed44..29912659d 100644 --- a/electron/native/ScreenCaptureKitRecorder.microphone.test.ts +++ b/electron/native/ScreenCaptureKitRecorder.microphone.test.ts @@ -98,3 +98,145 @@ print(Matcher.run(Array(CommandLine.arguments.dropFirst())) ?? "") expect(run("match", label, deviceId)).toBe(expected); }); }); + +// Compile the production converter and feed it microphone buffers shaped like real +// devices: mono, right-only stereo and multi-channel 24-bit integer interfaces. +describe.skipIf(process.platform !== "darwin")("recorder microphone channel layout", () => { + let directory: string; + let executable: string; + beforeAll(() => { + directory = mkdtempSync(join(tmpdir(), "recordly-upmix-")); + executable = join(directory, "upmix"); + const source = readFileSync( + new URL("./ScreenCaptureKitRecorder.swift", import.meta.url), + "utf8", + ); + const converter = source.slice( + source.indexOf("private func microphoneSampleWithStereoChannels"), + source.indexOf("private static func audioOutputSettings"), + ); + writeFileSync( + join(directory, "main.swift"), + ` +import AVFoundation +import CoreMedia + +final class Upmix { +${converter} + + static func floatBuffer(channels: AVAudioChannelCount, left: Float, right: Float, frames: Int = 64) -> CMSampleBuffer? { + guard let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 48_000, + channels: channels, interleaved: false), + let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frames)), + let data = pcm.floatChannelData else { return nil } + pcm.frameLength = AVAudioFrameCount(frames) + for frame in 0.. 1 { data[1][frame] = right } + } + return Upmix().makeMicrophoneSample(pcm, format: format.formatDescription, at: .zero) + } + + static func int24Buffer(channels: Int, values: [Float], frames: Int = 64) -> CMSampleBuffer? { + var asbd = AudioStreamBasicDescription(mSampleRate: 48_000, mFormatID: kAudioFormatLinearPCM, + mFormatFlags: kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked, + mBytesPerPacket: UInt32(3 * channels), mFramesPerPacket: 1, mBytesPerFrame: UInt32(3 * channels), + mChannelsPerFrame: UInt32(channels), mBitsPerChannel: 24, mReserved: 0) + var layout = AudioChannelLayout() + layout.mChannelLayoutTag = kAudioChannelLayoutTag_DiscreteInOrder | UInt32(channels) + layout.mChannelBitmap = [] + layout.mNumberChannelDescriptions = 0 + var format: CMAudioFormatDescription? + guard CMAudioFormatDescriptionCreate(allocator: kCFAllocatorDefault, asbd: &asbd, + layoutSize: MemoryLayout.size, layout: &layout, + magicCookieSize: 0, magicCookie: nil, extensions: nil, + formatDescriptionOut: &format) == noErr, let format else { return nil } + let frameBytes = 3 * channels + let length = frameBytes * frames + var block: CMBlockBuffer? + guard CMBlockBufferCreateWithMemoryBlock(allocator: kCFAllocatorDefault, memoryBlock: nil, + blockLength: length, blockAllocator: nil, customBlockSource: nil, offsetToData: 0, + dataLength: length, flags: 0, blockBufferOut: &block) == noErr, let block else { return nil } + guard CMBlockBufferAssureBlockMemory(block) == noErr else { return nil } + var pointer: UnsafeMutablePointer? + guard CMBlockBufferGetDataPointer(block, atOffset: 0, lengthAtOffsetOut: nil, totalLengthOut: nil, + dataPointerOut: &pointer) == noErr, let pointer else { return nil } + let bytes = UnsafeMutableRawPointer(pointer) + for frame in 0..> 8), toByteOffset: offset + 1, as: UInt8.self) + bytes.storeBytes(of: UInt8(truncatingIfNeeded: value >> 16), toByteOffset: offset + 2, as: UInt8.self) + } + } + var sample: CMSampleBuffer? + guard CMAudioSampleBufferCreateReadyWithPacketDescriptions(allocator: kCFAllocatorDefault, + dataBuffer: block, formatDescription: format, sampleCount: frames, + presentationTimeStamp: .zero, packetDescriptions: nil, sampleBufferOut: &sample) == noErr else { return nil } + return sample + } + + static func peaks(_ sample: CMSampleBuffer) -> (Float, Float) { + guard let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 48_000, + channels: 2, interleaved: false), + let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(sample.numSamples)) else { return (-1, -1) } + pcm.frameLength = AVAudioFrameCount(sample.numSamples) + guard CMSampleBufferCopyPCMDataIntoAudioBufferList(sample, at: 0, frameCount: Int32(sample.numSamples), + into: pcm.mutableAudioBufferList) == noErr, let data = pcm.floatChannelData else { return (-1, -1) } + var left: Float = 0 + var right: Float = 0 + for frame in 0.. String { + guard let sample, + let converted = Upmix().microphoneSampleWithStereoChannels(sample, at: .zero) else { return "nil" } + let (left, right) = peaks(converted) + return String(format: "%.3f %.3f", left, right) + } +} + +print(Upmix.render(Upmix.floatBuffer(channels: 1, left: 0.25, right: 0))) +print(Upmix.render(Upmix.floatBuffer(channels: 2, left: 0, right: 0.5))) +print(Upmix.render(Upmix.floatBuffer(channels: 2, left: 0.5, right: 0.25))) +print(Upmix.render(Upmix.int24Buffer(channels: 4, values: [0.5, 0.0005, 0, 0]))) +print(Upmix.render(Upmix.int24Buffer(channels: 4, values: [0, 0, 0.25, 0]))) +print(Upmix.render(Upmix.int24Buffer(channels: 4, values: [0.5, 0.25, 0, 0]))) +`, + ); + const build = spawnSync( + "swiftc", + [ + "-module-cache-path", + join(directory, "cache"), + join(directory, "main.swift"), + "-o", + executable, + ], + { encoding: "utf8", timeout: 120_000 }, + ); + expect(build.status, build.stderr).toBe(0); + }, 120_000); + afterAll(() => { + if (directory) rmSync(directory, { recursive: true, force: true }); + }); + + it("duplicates the only active channel for every device layout", () => { + const result = spawnSync(executable, [], { encoding: "utf8", timeout: 5_000 }); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "0.250 0.250", + "0.500 0.500", + "0.500 0.250", + "0.500 0.500", + "0.250 0.250", + "0.500 0.250", + ]); + }); +}); diff --git a/electron/native/ScreenCaptureKitRecorder.swift b/electron/native/ScreenCaptureKitRecorder.swift index 52f9b20ac..7dbe8e36b 100644 --- a/electron/native/ScreenCaptureKitRecorder.swift +++ b/electron/native/ScreenCaptureKitRecorder.swift @@ -684,12 +684,15 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } if outputType.rawValue == microphoneOutputTypeRawValue { + // The microphone arrives in the device's native layout (often wider than stereo, + // with the voice on one channel) while both writers expect stereo. + let microphoneBuffer = microphoneSampleWithStereoChannels(sampleBuffer, at: presentationTime) ?? sampleBuffer if let microphoneOnlyInput { - appendAudioSampleBuffer(sampleBuffer, to: microphoneOnlyInput, of: microphoneOnlyWriter, firstSampleTime: &firstMicrophoneSampleTime, lastPresentationTime: &lastMicrophonePresentationTime, presentationTime: presentationTime) + appendAudioSampleBuffer(microphoneBuffer, to: microphoneOnlyInput, of: microphoneOnlyWriter, firstSampleTime: &firstMicrophoneSampleTime, lastPresentationTime: &lastMicrophonePresentationTime, presentationTime: presentationTime) } // Write mic to inline video track only if there's no system audio (avoids double-writing) if !capturesSystemAudio, let inlineAudioInput { - appendAudioSampleBuffer(sampleBuffer, to: inlineAudioInput, of: assetWriter, firstSampleTime: &firstInlineAudioSampleTime, lastPresentationTime: &lastInlineAudioPresentationTime, presentationTime: presentationTime) + appendAudioSampleBuffer(microphoneBuffer, to: inlineAudioInput, of: assetWriter, firstSampleTime: &firstInlineAudioSampleTime, lastPresentationTime: &lastInlineAudioPresentationTime, presentationTime: presentationTime) } return } @@ -1055,6 +1058,105 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { if input === inlineAudioInput { lastInlineAudioDuration = sampleBuffer.duration } } + /// ScreenCaptureKit forwards the microphone at the device's native format, which can be + /// multi-channel integer PCM (e.g. a 4-channel 24-bit audio interface). The AAC writers + /// expect stereo float, so keep the channel that carries the voice and duplicate it. + private func microphoneSampleWithStereoChannels(_ sampleBuffer: CMSampleBuffer, at presentationTime: CMTime) -> CMSampleBuffer? { + guard let format = sampleBuffer.formatDescription, + let asbd = format.audioStreamBasicDescription, + asbd.mFormatID == kAudioFormatLinearPCM else { return nil } + let frameCount = Int(sampleBuffer.numSamples) + let channels = Int(asbd.mChannelsPerFrame) + let bits = Int(asbd.mBitsPerChannel) + let bytesPerSample = bits / 8 + guard frameCount > 0, channels > 0, [16, 24, 32, 64].contains(bits) else { return nil } + let isNonInterleaved = asbd.mFormatFlags & kAudioFormatFlagIsNonInterleaved != 0 + let frameStride = isNonInterleaved ? bytesPerSample : max(Int(asbd.mBytesPerFrame), bytesPerSample * channels) + + var sizeNeeded = 0 + var block: CMBlockBuffer? + guard CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, bufferListSizeNeededOut: &sizeNeeded, + bufferListOut: nil, bufferListSize: 0, blockBufferAllocator: nil, + blockBufferMemoryAllocator: nil, flags: 0, blockBufferOut: nil) == noErr, + sizeNeeded > 0 else { return nil } + let storage = UnsafeMutableRawPointer.allocate(byteCount: sizeNeeded, alignment: 16) + defer { storage.deallocate() } + let list = storage.assumingMemoryBound(to: AudioBufferList.self) + guard CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, bufferListSizeNeededOut: nil, + bufferListOut: list, bufferListSize: sizeNeeded, blockBufferAllocator: kCFAllocatorDefault, + blockBufferMemoryAllocator: kCFAllocatorDefault, flags: 0, blockBufferOut: &block) == noErr else { return nil } + let buffers = UnsafeMutableAudioBufferListPointer(list) + guard !buffers.isEmpty else { return nil } + + let isFloat = asbd.mFormatFlags & kAudioFormatFlagIsFloat != 0 + guard isFloat || asbd.mFormatFlags & kAudioFormatFlagIsSignedInteger != 0 else { return nil } + + func sampleValue(channel: Int, frame: Int) -> Float { + let bufferIndex = isNonInterleaved ? channel : 0 + guard bufferIndex < buffers.count else { return 0 } + let offset = frame * frameStride + (isNonInterleaved ? 0 : channel * bytesPerSample) + guard let data = buffers[bufferIndex].mData, + offset >= 0, offset + bytesPerSample <= Int(buffers[bufferIndex].mDataByteSize) else { return 0 } + if isFloat { + if bits == 32 { return data.loadUnaligned(fromByteOffset: offset, as: Float32.self) } + if bits == 64 { return Float(data.loadUnaligned(fromByteOffset: offset, as: Float64.self)) } + return 0 + } + if bits == 16 { return Float(data.loadUnaligned(fromByteOffset: offset, as: Int16.self)) / 32_768 } + if bits == 24 { + let b0 = data.loadUnaligned(fromByteOffset: offset, as: UInt8.self) + let b1 = data.loadUnaligned(fromByteOffset: offset + 1, as: UInt8.self) + let b2 = data.loadUnaligned(fromByteOffset: offset + 2, as: UInt8.self) + var value = Int32(b0) | Int32(b1) << 8 | Int32(b2) << 16 + if value & 0x80_0000 != 0 { value -= 0x100_0000 } + return Float(value) / 8_388_608 + } + if bits == 32 { return Float(data.loadUnaligned(fromByteOffset: offset, as: Int32.self)) / 2_147_483_648 } + return 0 + } + + var peaks = [Float](repeating: 0, count: channels) + for channel in 0.. 0.001, peaks[1] > 0.001 { return sampleBuffer } + let keepStereoPair = channels > 2 && peaks[0] > 0.001 && peaks[1] > 0.001 + let primary = peaks.indices.max { peaks[$0] < peaks[$1] } ?? 0 + + guard let stereoFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: asbd.mSampleRate, + channels: 2, interleaved: false), + let stereo = AVAudioPCMBuffer(pcmFormat: stereoFormat, frameCapacity: AVAudioFrameCount(frameCount)), + let stereoData = stereo.floatChannelData else { return nil } + stereo.frameLength = AVAudioFrameCount(frameCount) + for frame in 0.. CMSampleBuffer? { + var timing = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: CMTimeScale(pcm.format.sampleRate)), presentationTimeStamp: time, decodeTimeStamp: .invalid) + var sample: CMSampleBuffer? + guard CMSampleBufferCreate(allocator: kCFAllocatorDefault, dataBuffer: nil, + dataReady: false, makeDataReadyCallback: nil, refcon: nil, formatDescription: format, + sampleCount: Int(pcm.frameLength), sampleTimingEntryCount: 1, sampleTimingArray: &timing, + sampleSizeEntryCount: 0, sampleSizeArray: nil, sampleBufferOut: &sample) == noErr, + let sample, + CMSampleBufferSetDataBufferFromAudioBufferList(sample, blockBufferAllocator: nil, + blockBufferMemoryAllocator: nil, flags: 0, bufferList: pcm.audioBufferList) == noErr else { return nil } + return sample + } + private static func audioOutputSettings(bitRate: Int) -> [String: Any] { [ AVFormatIDKey: kAudioFormatMPEG4AAC,