diff --git a/electron/native/ScreenCaptureKitRecorder.microphone.test.ts b/electron/native/ScreenCaptureKitRecorder.microphone.test.ts index b1d35ed44..29912659d 100644 --- a/electron/native/ScreenCaptureKitRecorder.microphone.test.ts +++ b/electron/native/ScreenCaptureKitRecorder.microphone.test.ts @@ -98,3 +98,145 @@ print(Matcher.run(Array(CommandLine.arguments.dropFirst())) ?? "") expect(run("match", label, deviceId)).toBe(expected); }); }); + +// Compile the production converter and feed it microphone buffers shaped like real +// devices: mono, right-only stereo and multi-channel 24-bit integer interfaces. +describe.skipIf(process.platform !== "darwin")("recorder microphone channel layout", () => { + let directory: string; + let executable: string; + beforeAll(() => { + directory = mkdtempSync(join(tmpdir(), "recordly-upmix-")); + executable = join(directory, "upmix"); + const source = readFileSync( + new URL("./ScreenCaptureKitRecorder.swift", import.meta.url), + "utf8", + ); + const converter = source.slice( + source.indexOf("private func microphoneSampleWithStereoChannels"), + source.indexOf("private static func audioOutputSettings"), + ); + writeFileSync( + join(directory, "main.swift"), + ` +import AVFoundation +import CoreMedia + +final class Upmix { +${converter} + + static func floatBuffer(channels: AVAudioChannelCount, left: Float, right: Float, frames: Int = 64) -> CMSampleBuffer? { + guard let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 48_000, + channels: channels, interleaved: false), + let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(frames)), + let data = pcm.floatChannelData else { return nil } + pcm.frameLength = AVAudioFrameCount(frames) + for frame in 0.. 1 { data[1][frame] = right } + } + return Upmix().makeMicrophoneSample(pcm, format: format.formatDescription, at: .zero) + } + + static func int24Buffer(channels: Int, values: [Float], frames: Int = 64) -> CMSampleBuffer? { + var asbd = AudioStreamBasicDescription(mSampleRate: 48_000, mFormatID: kAudioFormatLinearPCM, + mFormatFlags: kAudioFormatFlagIsSignedInteger | kAudioFormatFlagIsPacked, + mBytesPerPacket: UInt32(3 * channels), mFramesPerPacket: 1, mBytesPerFrame: UInt32(3 * channels), + mChannelsPerFrame: UInt32(channels), mBitsPerChannel: 24, mReserved: 0) + var layout = AudioChannelLayout() + layout.mChannelLayoutTag = kAudioChannelLayoutTag_DiscreteInOrder | UInt32(channels) + layout.mChannelBitmap = [] + layout.mNumberChannelDescriptions = 0 + var format: CMAudioFormatDescription? + guard CMAudioFormatDescriptionCreate(allocator: kCFAllocatorDefault, asbd: &asbd, + layoutSize: MemoryLayout.size, layout: &layout, + magicCookieSize: 0, magicCookie: nil, extensions: nil, + formatDescriptionOut: &format) == noErr, let format else { return nil } + let frameBytes = 3 * channels + let length = frameBytes * frames + var block: CMBlockBuffer? + guard CMBlockBufferCreateWithMemoryBlock(allocator: kCFAllocatorDefault, memoryBlock: nil, + blockLength: length, blockAllocator: nil, customBlockSource: nil, offsetToData: 0, + dataLength: length, flags: 0, blockBufferOut: &block) == noErr, let block else { return nil } + guard CMBlockBufferAssureBlockMemory(block) == noErr else { return nil } + var pointer: UnsafeMutablePointer? + guard CMBlockBufferGetDataPointer(block, atOffset: 0, lengthAtOffsetOut: nil, totalLengthOut: nil, + dataPointerOut: &pointer) == noErr, let pointer else { return nil } + let bytes = UnsafeMutableRawPointer(pointer) + for frame in 0..> 8), toByteOffset: offset + 1, as: UInt8.self) + bytes.storeBytes(of: UInt8(truncatingIfNeeded: value >> 16), toByteOffset: offset + 2, as: UInt8.self) + } + } + var sample: CMSampleBuffer? + guard CMAudioSampleBufferCreateReadyWithPacketDescriptions(allocator: kCFAllocatorDefault, + dataBuffer: block, formatDescription: format, sampleCount: frames, + presentationTimeStamp: .zero, packetDescriptions: nil, sampleBufferOut: &sample) == noErr else { return nil } + return sample + } + + static func peaks(_ sample: CMSampleBuffer) -> (Float, Float) { + guard let format = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: 48_000, + channels: 2, interleaved: false), + let pcm = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: AVAudioFrameCount(sample.numSamples)) else { return (-1, -1) } + pcm.frameLength = AVAudioFrameCount(sample.numSamples) + guard CMSampleBufferCopyPCMDataIntoAudioBufferList(sample, at: 0, frameCount: Int32(sample.numSamples), + into: pcm.mutableAudioBufferList) == noErr, let data = pcm.floatChannelData else { return (-1, -1) } + var left: Float = 0 + var right: Float = 0 + for frame in 0.. String { + guard let sample, + let converted = Upmix().microphoneSampleWithStereoChannels(sample, at: .zero) else { return "nil" } + let (left, right) = peaks(converted) + return String(format: "%.3f %.3f", left, right) + } +} + +print(Upmix.render(Upmix.floatBuffer(channels: 1, left: 0.25, right: 0))) +print(Upmix.render(Upmix.floatBuffer(channels: 2, left: 0, right: 0.5))) +print(Upmix.render(Upmix.floatBuffer(channels: 2, left: 0.5, right: 0.25))) +print(Upmix.render(Upmix.int24Buffer(channels: 4, values: [0.5, 0.0005, 0, 0]))) +print(Upmix.render(Upmix.int24Buffer(channels: 4, values: [0, 0, 0.25, 0]))) +print(Upmix.render(Upmix.int24Buffer(channels: 4, values: [0.5, 0.25, 0, 0]))) +`, + ); + const build = spawnSync( + "swiftc", + [ + "-module-cache-path", + join(directory, "cache"), + join(directory, "main.swift"), + "-o", + executable, + ], + { encoding: "utf8", timeout: 120_000 }, + ); + expect(build.status, build.stderr).toBe(0); + }, 120_000); + afterAll(() => { + if (directory) rmSync(directory, { recursive: true, force: true }); + }); + + it("duplicates the only active channel for every device layout", () => { + const result = spawnSync(executable, [], { encoding: "utf8", timeout: 5_000 }); + expect(result.status, result.stderr).toBe(0); + expect(result.stdout.trim().split("\n")).toEqual([ + "0.250 0.250", + "0.500 0.500", + "0.500 0.250", + "0.500 0.500", + "0.250 0.250", + "0.500 0.250", + ]); + }); +}); diff --git a/electron/native/ScreenCaptureKitRecorder.swift b/electron/native/ScreenCaptureKitRecorder.swift index 52f9b20ac..7dbe8e36b 100644 --- a/electron/native/ScreenCaptureKitRecorder.swift +++ b/electron/native/ScreenCaptureKitRecorder.swift @@ -684,12 +684,15 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { } if outputType.rawValue == microphoneOutputTypeRawValue { + // The microphone arrives in the device's native layout (often wider than stereo, + // with the voice on one channel) while both writers expect stereo. + let microphoneBuffer = microphoneSampleWithStereoChannels(sampleBuffer, at: presentationTime) ?? sampleBuffer if let microphoneOnlyInput { - appendAudioSampleBuffer(sampleBuffer, to: microphoneOnlyInput, of: microphoneOnlyWriter, firstSampleTime: &firstMicrophoneSampleTime, lastPresentationTime: &lastMicrophonePresentationTime, presentationTime: presentationTime) + appendAudioSampleBuffer(microphoneBuffer, to: microphoneOnlyInput, of: microphoneOnlyWriter, firstSampleTime: &firstMicrophoneSampleTime, lastPresentationTime: &lastMicrophonePresentationTime, presentationTime: presentationTime) } // Write mic to inline video track only if there's no system audio (avoids double-writing) if !capturesSystemAudio, let inlineAudioInput { - appendAudioSampleBuffer(sampleBuffer, to: inlineAudioInput, of: assetWriter, firstSampleTime: &firstInlineAudioSampleTime, lastPresentationTime: &lastInlineAudioPresentationTime, presentationTime: presentationTime) + appendAudioSampleBuffer(microphoneBuffer, to: inlineAudioInput, of: assetWriter, firstSampleTime: &firstInlineAudioSampleTime, lastPresentationTime: &lastInlineAudioPresentationTime, presentationTime: presentationTime) } return } @@ -1055,6 +1058,105 @@ final class ScreenCaptureRecorder: NSObject, SCStreamOutput, SCStreamDelegate { if input === inlineAudioInput { lastInlineAudioDuration = sampleBuffer.duration } } + /// ScreenCaptureKit forwards the microphone at the device's native format, which can be + /// multi-channel integer PCM (e.g. a 4-channel 24-bit audio interface). The AAC writers + /// expect stereo float, so keep the channel that carries the voice and duplicate it. + private func microphoneSampleWithStereoChannels(_ sampleBuffer: CMSampleBuffer, at presentationTime: CMTime) -> CMSampleBuffer? { + guard let format = sampleBuffer.formatDescription, + let asbd = format.audioStreamBasicDescription, + asbd.mFormatID == kAudioFormatLinearPCM else { return nil } + let frameCount = Int(sampleBuffer.numSamples) + let channels = Int(asbd.mChannelsPerFrame) + let bits = Int(asbd.mBitsPerChannel) + let bytesPerSample = bits / 8 + guard frameCount > 0, channels > 0, [16, 24, 32, 64].contains(bits) else { return nil } + let isNonInterleaved = asbd.mFormatFlags & kAudioFormatFlagIsNonInterleaved != 0 + let frameStride = isNonInterleaved ? bytesPerSample : max(Int(asbd.mBytesPerFrame), bytesPerSample * channels) + + var sizeNeeded = 0 + var block: CMBlockBuffer? + guard CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, bufferListSizeNeededOut: &sizeNeeded, + bufferListOut: nil, bufferListSize: 0, blockBufferAllocator: nil, + blockBufferMemoryAllocator: nil, flags: 0, blockBufferOut: nil) == noErr, + sizeNeeded > 0 else { return nil } + let storage = UnsafeMutableRawPointer.allocate(byteCount: sizeNeeded, alignment: 16) + defer { storage.deallocate() } + let list = storage.assumingMemoryBound(to: AudioBufferList.self) + guard CMSampleBufferGetAudioBufferListWithRetainedBlockBuffer(sampleBuffer, bufferListSizeNeededOut: nil, + bufferListOut: list, bufferListSize: sizeNeeded, blockBufferAllocator: kCFAllocatorDefault, + blockBufferMemoryAllocator: kCFAllocatorDefault, flags: 0, blockBufferOut: &block) == noErr else { return nil } + let buffers = UnsafeMutableAudioBufferListPointer(list) + guard !buffers.isEmpty else { return nil } + + let isFloat = asbd.mFormatFlags & kAudioFormatFlagIsFloat != 0 + guard isFloat || asbd.mFormatFlags & kAudioFormatFlagIsSignedInteger != 0 else { return nil } + + func sampleValue(channel: Int, frame: Int) -> Float { + let bufferIndex = isNonInterleaved ? channel : 0 + guard bufferIndex < buffers.count else { return 0 } + let offset = frame * frameStride + (isNonInterleaved ? 0 : channel * bytesPerSample) + guard let data = buffers[bufferIndex].mData, + offset >= 0, offset + bytesPerSample <= Int(buffers[bufferIndex].mDataByteSize) else { return 0 } + if isFloat { + if bits == 32 { return data.loadUnaligned(fromByteOffset: offset, as: Float32.self) } + if bits == 64 { return Float(data.loadUnaligned(fromByteOffset: offset, as: Float64.self)) } + return 0 + } + if bits == 16 { return Float(data.loadUnaligned(fromByteOffset: offset, as: Int16.self)) / 32_768 } + if bits == 24 { + let b0 = data.loadUnaligned(fromByteOffset: offset, as: UInt8.self) + let b1 = data.loadUnaligned(fromByteOffset: offset + 1, as: UInt8.self) + let b2 = data.loadUnaligned(fromByteOffset: offset + 2, as: UInt8.self) + var value = Int32(b0) | Int32(b1) << 8 | Int32(b2) << 16 + if value & 0x80_0000 != 0 { value -= 0x100_0000 } + return Float(value) / 8_388_608 + } + if bits == 32 { return Float(data.loadUnaligned(fromByteOffset: offset, as: Int32.self)) / 2_147_483_648 } + return 0 + } + + var peaks = [Float](repeating: 0, count: channels) + for channel in 0.. 0.001, peaks[1] > 0.001 { return sampleBuffer } + let keepStereoPair = channels > 2 && peaks[0] > 0.001 && peaks[1] > 0.001 + let primary = peaks.indices.max { peaks[$0] < peaks[$1] } ?? 0 + + guard let stereoFormat = AVAudioFormat(commonFormat: .pcmFormatFloat32, sampleRate: asbd.mSampleRate, + channels: 2, interleaved: false), + let stereo = AVAudioPCMBuffer(pcmFormat: stereoFormat, frameCapacity: AVAudioFrameCount(frameCount)), + let stereoData = stereo.floatChannelData else { return nil } + stereo.frameLength = AVAudioFrameCount(frameCount) + for frame in 0.. CMSampleBuffer? { + var timing = CMSampleTimingInfo(duration: CMTime(value: 1, timescale: CMTimeScale(pcm.format.sampleRate)), presentationTimeStamp: time, decodeTimeStamp: .invalid) + var sample: CMSampleBuffer? + guard CMSampleBufferCreate(allocator: kCFAllocatorDefault, dataBuffer: nil, + dataReady: false, makeDataReadyCallback: nil, refcon: nil, formatDescription: format, + sampleCount: Int(pcm.frameLength), sampleTimingEntryCount: 1, sampleTimingArray: &timing, + sampleSizeEntryCount: 0, sampleSizeArray: nil, sampleBufferOut: &sample) == noErr, + let sample, + CMSampleBufferSetDataBufferFromAudioBufferList(sample, blockBufferAllocator: nil, + blockBufferMemoryAllocator: nil, flags: 0, bufferList: pcm.audioBufferList) == noErr else { return nil } + return sample + } + private static func audioOutputSettings(bitRate: Int) -> [String: Any] { [ AVFormatIDKey: kAudioFormatMPEG4AAC,