From 9389469f609ad3e7c6d7aeb7b8bbb74dd1aff26d Mon Sep 17 00:00:00 2001 From: silentbil Date: Fri, 6 Nov 2026 10:32:46 +0200 Subject: [PATCH 1/8] Live TV AI subtitles: Phase 0 look-ahead audio probe Copies the channel's compressed audio at the extractor (progressive via AudioSyncExtractorsFactory, HLS via a chunk-extractor wrapper), decodes it on a side thread and cuts it into speech segments with Silero, logging segment media times, look-ahead and player period position under tag LiveAiSubs. Adds SubtitleLookaheadLoadControl, which holds playback until D seconds are buffered, only while required. Everything is inert unless switched on over adb in a debug build (debug.arvio.liveaisubs*). No UI and no AI calls yet. Also makes the tap's output wrappers internal and the segmenter's maximum segment length configurable. --- .../audiosync/AudioSubtitleSyncController.kt | 4 +- .../player/audiosync/AudioSyncExtractors.kt | 4 +- .../player/audiosync/asr/SpeechSegmenter.kt | 11 +- .../tv/ui/screens/tv/live/LiveTvScreen.kt | 66 ++++- .../screens/tv/live/aisubs/LiveAiSubsDebug.kt | 74 +++++ .../screens/tv/live/aisubs/LiveAiSubsLog.kt | 22 ++ .../screens/tv/live/aisubs/LiveAudioTapHls.kt | 99 +++++++ .../screens/tv/live/aisubs/LiveSpeechProbe.kt | 266 ++++++++++++++++++ .../aisubs/SubtitleLookaheadLoadControl.kt | 79 ++++++ 9 files changed, 614 insertions(+), 11 deletions(-) create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsDebug.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsLog.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechProbe.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/SubtitleLookaheadLoadControl.kt diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSubtitleSyncController.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSubtitleSyncController.kt index 9e5f92070f..80216f237d 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSubtitleSyncController.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSubtitleSyncController.kt @@ -1052,8 +1052,8 @@ internal class AudioSubtitleSyncController( ) recognizer?.let(engine::setRecognizer) asr = engine - analyzer.chunkListener = SpeechSegmenter(engine::offerSegment) - liveAnalyzer.chunkListener = SpeechSegmenter(engine::offerSegment) + analyzer.chunkListener = SpeechSegmenter(onSegment = engine::offerSegment) + liveAnalyzer.chunkListener = SpeechSegmenter(onSegment = engine::offerSegment) return AudioSyncDecoder(analyzer, liveAnalyzer) { mime -> SyncLog.i("look-ahead capture unavailable for $mime; syncing from playback audio instead") liveOnly = true diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncExtractors.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncExtractors.kt index 808d4ca3d8..cb816fca80 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncExtractors.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncExtractors.kt @@ -69,7 +69,7 @@ private class AudioTapExtractor( override fun getUnderlyingImplementation(): Extractor = delegate.underlyingImplementation } -private class AudioTapExtractorOutput( +internal class AudioTapExtractorOutput( private val delegate: ExtractorOutput, private val sink: AudioSampleSink, ) : ExtractorOutput { @@ -91,7 +91,7 @@ private class AudioTapExtractorOutput( * report its metadata later (with [offset] bytes of following samples already written), so the * pending bytes are kept until [sampleMetadata] identifies the sample. */ -private class AudioTapTrackOutput( +internal class AudioTapTrackOutput( private val delegate: TrackOutput, private val sink: AudioSampleSink, ) : TrackOutput { diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/asr/SpeechSegmenter.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/asr/SpeechSegmenter.kt index 03c5d7ce5e..611dfa4052 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/asr/SpeechSegmenter.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/asr/SpeechSegmenter.kt @@ -6,9 +6,14 @@ import com.arflix.tv.ui.screens.player.audiosync.SpeechAnalyzer /** * Cuts the analysed 16 kHz audio into speech segments using the Silero probabilities that are * already being computed, so the recogniser only ever sees dialogue: no silence, music-only - * stretches or effects. Segments are at most [MAX_SEGMENT_FRAMES] long so words come back quickly. + * stretches or effects. Segments are at most [maxSegmentFrames] long so words come back quickly. */ internal class SpeechSegmenter( + /** + * Longest segment in 32 ms frames. Live subtitles keep this short (about 5 s): a line can only + * be translated once its segment has ended, so it bounds how far ahead playback must be held. + */ + private val maxSegmentFrames: Int = MAX_SEGMENT_FRAMES, private val onSegment: (startFrame: Int, samples: FloatArray) -> Unit, ) : SpeechAnalyzer.ChunkListener { private val chunk = SileroVad.CHUNK_SAMPLES @@ -42,7 +47,7 @@ internal class SpeechSegmenter( } append(chunk) if (speech) lastSpeechFrame = frame - if (frame - lastSpeechFrame > HANGOVER_FRAMES || bufferFrames >= MAX_SEGMENT_FRAMES) finish() + if (frame - lastSpeechFrame > HANGOVER_FRAMES || bufferFrames >= maxSegmentFrames) finish() } @Synchronized @@ -80,6 +85,6 @@ internal class SpeechSegmenter( private const val HANGOVER_FRAMES = 8 private const val MIN_SPEECH_FRAMES = 6 /** 12 s. */ - private const val MAX_SEGMENT_FRAMES = 375 + const val MAX_SEGMENT_FRAMES = 375 } } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt index 8224314b3f..20d8ae6524 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt @@ -119,6 +119,11 @@ import androidx.media3.common.Player import androidx.media3.datasource.HttpDataSource import androidx.media3.datasource.okhttp.OkHttpDataSource import androidx.media3.exoplayer.DefaultLoadControl +import com.arflix.tv.ui.screens.player.audiosync.AudioSyncExtractorsFactory +import com.arflix.tv.ui.screens.tv.live.aisubs.AudioTapHlsExtractorFactory +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsDebug +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveSpeechProbe +import com.arflix.tv.ui.screens.tv.live.aisubs.SubtitleLookaheadLoadControl import androidx.media3.exoplayer.ExoPlayer import androidx.media3.exoplayer.hls.HlsMediaSource import androidx.media3.exoplayer.source.DefaultMediaSourceFactory @@ -3086,7 +3091,11 @@ fun LiveTvScreen( isLowRamDevice = activityManager?.isLowRamDevice == true ) } - val exoPlayer = remember(livePlaybackBufferProfile) { + // Live TV AI subtitles, Phase 0 probe: taps the channel audio ahead of playback and logs + // segment timing under "LiveAiSubs". Inert unless switched on over adb in a debug build. + val liveSpeechProbe = remember(context) { LiveSpeechProbe(context) } + DisposableEffect(liveSpeechProbe) { onDispose { liveSpeechProbe.release() } } + val liveLoadControl = remember(livePlaybackBufferProfile) { val loadControl = DefaultLoadControl.Builder() .setBufferDurationsMs( livePlaybackBufferProfile.minBufferMs, @@ -3104,9 +3113,31 @@ fun LiveTvScreen( .setPrioritizeTimeOverSizeThresholds(false) .setBackBuffer(livePlaybackBufferProfile.backBufferMs, true) .build() + // Holds playback back so AI subtitles get audio ahead of time; a no-op until asked to. + SubtitleLookaheadLoadControl(loadControl) + } + val exoPlayer = remember(liveLoadControl) { ExoPlayer.Builder(context) .setMediaSourceFactory(mediaSourceFactory) - .setLoadControl(loadControl) + .setLoadControl(liveLoadControl) + .apply { + // Debug builds only: hear the played audio too, to check the tap's time base. + if (com.arflix.tv.BuildConfig.DEBUG) { + setRenderersFactory(object : androidx.media3.exoplayer.DefaultRenderersFactory(context) { + override fun buildAudioSink( + context: Context, + enableFloatOutput: Boolean, + enableAudioTrackPlaybackParams: Boolean, + ): androidx.media3.exoplayer.audio.AudioSink? { + val sink = super.buildAudioSink(context, enableFloatOutput, enableAudioTrackPlaybackParams) + ?: return null + return com.arflix.tv.ui.screens.player.audiosync.PlaybackAudioTap( + sink, liveSpeechProbe.playbackListener, + ) + } + }) + } + } // A live stream whose buffer stops moving while buffering raises an error after // this long, which the retry below recovers, instead of a spinner for ten minutes. .setStuckBufferingDetectionTimeoutMs(LIVE_STUCK_BUFFERING_TIMEOUT_MS) @@ -3139,6 +3170,27 @@ fun LiveTvScreen( } } + val probeFullscreen by rememberUpdatedState(isFullScreen && !fullscreenGuideOpen && !quickZapOpen) + LaunchedEffect(exoPlayer, liveSpeechProbe) { + val window = androidx.media3.common.Timeline.Window() + while (true) { + liveSpeechProbe.fullscreen = probeFullscreen + if (LiveAiSubsDebug.enabled) { + // currentPosition is relative to the live window; the tapped samples are timed on + // the period, so add the window's offset into the period. + val timeline = exoPlayer.currentTimeline + val windowOffsetUs = if (timeline.isEmpty) 0L else + timeline.getWindow(exoPlayer.currentMediaItemIndex, window).positionInFirstPeriodUs + liveSpeechProbe.onPosition( + periodPositionUs = exoPlayer.currentPosition * 1_000L + windowOffsetUs, + bufferedAheadMs = exoPlayer.bufferedPosition - exoPlayer.currentPosition, + playing = exoPlayer.isPlaying, + ) + } + delay(250L) + } + } + var playbackQuality by remember(exoPlayer) { mutableStateOf(null) } // Subtitles: start every channel on the profile's default subtitle language; a choice made in @@ -3382,14 +3434,20 @@ fun LiveTvScreen( } // HLS gets its own factory: DefaultMediaSourceFactory cannot set the HLS extractors, // and channels without IDR frames need iptvHlsExtractorFactory() to show video. + // Live TV AI subtitles: every channel source gets a fresh audio tap (Phase 0 probe), so a + // load still running for the previous channel can never feed this one. + val liveAudioSink = liveSpeechProbe.newSink(if (isHls) "hls" else "progressive") + liveLoadControl.requiredLookaheadUs = + if (playingCatchupProgram == null) LiveAiSubsDebug.delayMs * 1_000L else 0L val source = if (isHls) { HlsMediaSource.Factory(sourceDataFactory) .setAllowChunklessPreparation(IPTV_HLS_CHUNKLESS_PREPARATION) - .setExtractorFactory(iptvHlsExtractorFactory()) + .setExtractorFactory(AudioTapHlsExtractorFactory(iptvHlsExtractorFactory(), liveAudioSink)) .setLoadErrorHandlingPolicy(IptvLoadErrorHandlingPolicy()) .createMediaSource(mediaItem) } else { - DefaultMediaSourceFactory(context, iptvExtractorsFactory()).setDataSourceFactory(sourceDataFactory) + DefaultMediaSourceFactory(context, AudioSyncExtractorsFactory(iptvExtractorsFactory(), liveAudioSink)) + .setDataSourceFactory(sourceDataFactory) .setLoadErrorHandlingPolicy( IptvLoadErrorHandlingPolicy( liveMidStreamDrop( diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsDebug.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsDebug.kt new file mode 100644 index 0000000000..c1ef08e87a --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsDebug.kt @@ -0,0 +1,74 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import com.arflix.tv.BuildConfig +import java.util.concurrent.TimeUnit + +/** + * Phase 0 switches for the Live TV AI subtitles proof of concept, set over adb so no UI is + * needed yet (debug builds only; release builds always read "off"): + * + * - `adb shell setprop debug.arvio.liveaisubs 1` taps the channel's audio and logs segments. + * - `adb shell setprop debug.arvio.liveaisubs.delay 10` holds playback until 10 s are buffered + * (0 = no hold). Read when a channel is prepared. + * - `adb shell setprop debug.arvio.liveaisubs.verify 1` also runs the VAD on the audio as it is + * played, to compare its segment times with the look-ahead tap's (doubles the VAD cost). + * + * `debug.*` properties are writable from `adb shell` without root. They are polled on a daemon + * thread, because reading them means running `getprop`, which must not happen on the UI thread. + */ +internal object LiveAiSubsDebug { + @Volatile + var enabled: Boolean = false + private set + + @Volatile + var delayMs: Long = 0L + private set + + @Volatile + var verify: Boolean = false + private set + + @Volatile + private var started = false + + fun start() { + if (!BuildConfig.DEBUG || started) return + synchronized(this) { + if (started) return + started = true + Thread({ pollLoop() }, "LiveAiSubsProps").apply { + isDaemon = true + start() + } + } + } + + private fun pollLoop() { + while (true) { + val nowEnabled = read("debug.arvio.liveaisubs") == "1" + val nowDelay = read("debug.arvio.liveaisubs.delay")?.toLongOrNull()?.coerceIn(0L, 30L) ?: 0L + val nowVerify = read("debug.arvio.liveaisubs.verify") == "1" + if (nowEnabled != enabled || nowDelay * 1_000L != delayMs || nowVerify != verify) { + LiveAiSubsLog.i("debug switches: enabled=$nowEnabled delay=${nowDelay}s verify=$nowVerify") + } + enabled = nowEnabled + delayMs = if (nowEnabled) nowDelay * 1_000L else 0L + verify = nowEnabled && nowVerify + try { + Thread.sleep(POLL_MS) + } catch (_: InterruptedException) { + return + } + } + } + + private fun read(name: String): String? = runCatching { + val process = ProcessBuilder("getprop", name).redirectErrorStream(true).start() + val value = process.inputStream.bufferedReader().use { it.readText().trim() } + process.waitFor(1, TimeUnit.SECONDS) + value.ifEmpty { null } + }.getOrNull() + + private const val POLL_MS = 2_000L +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsLog.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsLog.kt new file mode 100644 index 0000000000..3d17f5864a --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsLog.kt @@ -0,0 +1,22 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.util.Log + +/** + * Live TV AI subtitles' log (tag `LiveAiSubs`). Links are reduced to their host, since IPTV stream + * URLs carry account credentials; the log holds timings and decisions, never audio or text. + */ +internal object LiveAiSubsLog { + private const val TAG = "LiveAiSubs" + private val url = Regex("""https?://([^/\s?#]+)[^\s]*""") + + fun i(message: String) { + Log.i(TAG, redact(message)) + } + + fun w(message: String) { + Log.w(TAG, redact(message)) + } + + private fun redact(message: String): String = url.replace(message) { "<${it.groupValues[1]}>" } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt new file mode 100644 index 0000000000..3c042f6c34 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt @@ -0,0 +1,99 @@ +@file:androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) + +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.net.Uri +import androidx.media3.common.Format +import androidx.media3.common.util.TimestampAdjuster +import androidx.media3.exoplayer.analytics.PlayerId +import androidx.media3.exoplayer.hls.HlsExtractorFactory +import androidx.media3.exoplayer.hls.HlsMediaChunkExtractor +import androidx.media3.extractor.ExtractorInput +import androidx.media3.extractor.ExtractorOutput +import androidx.media3.extractor.text.SubtitleParser +import com.arflix.tv.ui.screens.player.audiosync.AudioSampleSink +import com.arflix.tv.ui.screens.player.audiosync.AudioTapExtractorOutput + +/** + * HLS counterpart of `AudioSyncExtractorsFactory`: HLS segments are not read through an + * `ExtractorsFactory`, so the audio copy is made here, around whatever [delegate] returns. Hooking + * the chunk extractor's output covers TS segments (ARVIO's reused TS extractor) and every other + * segment type (packed audio, fMP4) in one place. + * + * Samples reach the sink with the timestamps the extractor wrote, after the HLS timestamp + * adjuster, i.e. on the period timeline the player's position is measured in. + */ +internal class AudioTapHlsExtractorFactory( + private val delegate: HlsExtractorFactory, + private val sink: AudioSampleSink, +) : HlsExtractorFactory { + override fun createExtractor( + uri: Uri, + format: Format, + muxedCaptionFormats: List?, + timestampAdjuster: TimestampAdjuster, + responseHeaders: Map>, + sniffingExtractorInput: ExtractorInput, + playerId: PlayerId, + ): HlsMediaChunkExtractor { + val extractor = delegate.createExtractor( + uri, format, muxedCaptionFormats, timestampAdjuster, responseHeaders, sniffingExtractorInput, playerId, + ) + if (LiveAiSubsDebug.enabled) { + LiveAiSubsLog.i( + "hls chunk extractor ${extractor.javaClass.simpleName} packedAudio=${extractor.isPackedAudioExtractor} " + + "codecs=${format.codecs} mime=${format.sampleMimeType ?: format.containerMimeType}", + ) + } + return AudioTapHlsChunkExtractor(extractor, sink) + } + + override fun setSubtitleParserFactory(subtitleParserFactory: SubtitleParser.Factory): HlsExtractorFactory { + delegate.setSubtitleParserFactory(subtitleParserFactory) + return this + } + + override fun experimentalParseSubtitlesDuringExtraction(parseSubtitlesDuringExtraction: Boolean): HlsExtractorFactory { + delegate.experimentalParseSubtitlesDuringExtraction(parseSubtitlesDuringExtraction) + return this + } + + override fun experimentalSetCodecsToParseWithinGopSampleDependencies( + codecsToParseWithinGopSampleDependencies: Int, + ): HlsExtractorFactory { + delegate.experimentalSetCodecsToParseWithinGopSampleDependencies(codecsToParseWithinGopSampleDependencies) + return this + } + + override fun getOutputTextFormat(sourceFormat: Format): Format = delegate.getOutputTextFormat(sourceFormat) +} + +private class AudioTapHlsChunkExtractor( + private val delegate: HlsMediaChunkExtractor, + private val sink: AudioSampleSink, +) : HlsMediaChunkExtractor { + override fun init(extractorOutput: ExtractorOutput) = + delegate.init(AudioTapExtractorOutput(TrackLoggingOutput(extractorOutput), sink)) + + override fun read(extractorInput: ExtractorInput): Boolean = delegate.read(extractorInput) + + override fun isPackedAudioExtractor(): Boolean = delegate.isPackedAudioExtractor + + override fun isReusable(): Boolean = delegate.isReusable + + override fun recreate(): HlsMediaChunkExtractor = AudioTapHlsChunkExtractor(delegate.recreate(), sink) + + override fun onTruncatedSegmentParsed() { + // The extractor restarts its state; the next samples may not continue the previous ones. + sink.onDiscontinuity() + delegate.onTruncatedSegmentParsed() + } +} + +/** Phase 0 diagnostics: logs which tracks a segment declares, to explain channels with no tapped audio. */ +private class TrackLoggingOutput(private val delegate: ExtractorOutput) : ExtractorOutput by delegate { + override fun track(id: Int, type: Int): androidx.media3.extractor.TrackOutput { + if (LiveAiSubsDebug.enabled) LiveAiSubsLog.i("hls segment track id=$id type=$type") + return delegate.track(id, type) + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechProbe.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechProbe.kt new file mode 100644 index 0000000000..e1f949372a --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechProbe.kt @@ -0,0 +1,266 @@ +@file:androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) + +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.content.Context +import android.os.SystemClock +import androidx.media3.common.Format +import com.arflix.tv.R +import com.arflix.tv.ui.screens.player.audiosync.AudioSampleSink +import com.arflix.tv.ui.screens.player.audiosync.AudioSyncDecoder +import com.arflix.tv.ui.screens.player.audiosync.PlaybackPcmListener +import com.arflix.tv.ui.screens.player.audiosync.SileroVad +import com.arflix.tv.ui.screens.player.audiosync.SileroVadWeights +import com.arflix.tv.ui.screens.player.audiosync.SpeechAnalyzer +import com.arflix.tv.ui.screens.player.audiosync.SpeechTimeline +import com.arflix.tv.ui.screens.player.audiosync.asr.SpeechSegmenter + +/** + * Phase 0 proof of concept for Live TV AI subtitles: copies the channel's audio at the extractor + * (ahead of playback), decodes it on a side thread, cuts it into speech segments with Silero and + * logs everything needed to judge the design, under tag `LiveAiSubs`: + * + * - tapped sample times next to the player's period position, and how far ahead the tap runs; + * - each segment's start/end in media time, and how long before its playback it was ready; + * - with [LiveAiSubsDebug.verify], the same VAD run on the audio as it is played (its times come + * from the audio renderer); matching segments from both sides must start at the same media + * time if the tap and playback share one time base. + * + * Nothing here talks to an AI service or draws anything. One probe lives as long as the Live TV + * player; [newSink] is called for every prepared channel so stale loads of the previous channel + * can never feed the new one. + */ +internal class LiveSpeechProbe(context: Context) { + private val appContext = context.applicationContext + + @Volatile + private var weights: SileroVadWeights? = null + + @Volatile + private var generation = 0 + + @Volatile + private var decoder: AudioSyncDecoder? = null + + /** Whether the player is fullscreen; the probe only runs there. Set on the UI thread. */ + @Volatile + var fullscreen: Boolean = false + + // Last known player position, written on the UI thread, read by the other threads to estimate + // where playback is right now. + @Volatile private var positionUs = UNKNOWN + @Volatile private var positionAtMs = 0L + @Volatile private var positionPlaying = false + + // Tap statistics. + @Volatile private var lastTapUs = UNKNOWN + @Volatile private var lastPlayedUs = UNKNOWN + @Volatile private var tapSamples = 0L + @Volatile private var droppedSamples = 0L + @Volatile private var tapSegments = 0 + @Volatile private var playedSegments = 0 + private var lastStatsMs = 0L + + /** Recent look-ahead segment starts, for matching the played side's segments. */ + private val recentTapStarts = ArrayDeque() + private val deltasMs = ArrayList() + + init { + LiveAiSubsDebug.start() + Thread({ + weights = runCatching { + appContext.resources.openRawResource(R.raw.silero_vad_v5_16k).use { SileroVadWeights.read(it) } + }.onFailure { LiveAiSubsLog.w("could not load VAD weights: ${it.message}") }.getOrNull() + }, "LiveAiSubsInit").apply { + isDaemon = true + start() + } + } + + private val active: Boolean + get() = LiveAiSubsDebug.enabled && fullscreen && weights != null + + /** A sink for one prepared channel; it goes quiet as soon as another channel is prepared. */ + fun newSink(label: String): AudioSampleSink { + val sinkGeneration = ++generation + synchronized(recentTapStarts) { + recentTapStarts.clear() + deltasMs.clear() + } + lastTapUs = UNKNOWN + lastPlayedUs = UNKNOWN + tapSamples = 0L + droppedSamples = 0L + tapSegments = 0 + playedSegments = 0 + decoder?.discontinuity() + if (LiveAiSubsDebug.enabled) LiveAiSubsLog.i("new channel source gen=$sinkGeneration ($label)") + return ChannelSink(sinkGeneration) + } + + /** Audio as it is played, from the Live TV player's audio sink (debug builds only). */ + val playbackListener: PlaybackPcmListener = object : PlaybackPcmListener { + override fun wantsPlaybackPcm(mediaTimeUs: Long, durationUs: Long): Boolean = + LiveAiSubsDebug.verify && active && decoder != null + + override fun onPlaybackPcm(mono: FloatArray, frames: Int, sampleRate: Int, mediaTimeUs: Long) { + lastPlayedUs = mediaTimeUs + decoder?.offerPlaybackPcm(mono, frames, sampleRate, mediaTimeUs) + } + } + + /** + * Called on the UI thread about every half second. [periodPositionUs] is the playback position + * on the period timeline (the window position plus the window's offset in the period). + */ + fun onPosition(periodPositionUs: Long, bufferedAheadMs: Long, playing: Boolean) { + positionUs = periodPositionUs + positionAtMs = SystemClock.elapsedRealtime() + positionPlaying = playing + if (!active) return + val now = positionAtMs + if (now - lastStatsMs < STATS_INTERVAL_MS) return + lastStatsMs = now + val tap = lastTapUs + val played = lastPlayedUs + val median = synchronized(recentTapStarts) { deltasMs.sorted().let { if (it.isEmpty()) null else it[it.size / 2] } } + LiveAiSubsLog.i( + "stats pos=${periodPositionUs / 1_000}ms playing=$playing bufferedAhead=${bufferedAheadMs}ms " + + "tapAhead=${if (tap == UNKNOWN) "-" else "${(tap - periodPositionUs) / 1_000}ms"} " + + "playedAhead=${if (played == UNKNOWN) "-" else "${(played - periodPositionUs) / 1_000}ms"} " + + "samples=$tapSamples dropped=$droppedSamples segments=$tapSegments played=$playedSegments " + + "matchMedianDelta=${median ?: "-"}ms", + ) + } + + fun release() { + generation++ + decoder?.release() + decoder = null + } + + /** Where playback is right now, extrapolated from the last UI-thread report. */ + private fun estimatedPositionUs(): Long { + val position = positionUs + if (position == UNKNOWN) return UNKNOWN + return if (positionPlaying) position + (SystemClock.elapsedRealtime() - positionAtMs) * 1_000L else position + } + + private fun ensureDecoder(): AudioSyncDecoder? { + decoder?.let { return it } + val vadWeights = weights ?: return null + synchronized(this) { + decoder?.let { return it } + val tapAnalyzer = SpeechAnalyzer(SileroVad(vadWeights), SpeechTimeline()) + val playedAnalyzer = SpeechAnalyzer(SileroVad(vadWeights), SpeechTimeline()) + tapAnalyzer.chunkListener = SpeechSegmenter(LIVE_MAX_SEGMENT_FRAMES) { startFrame, samples -> + onTapSegment(startFrame, samples.size) + } + playedAnalyzer.chunkListener = SpeechSegmenter(LIVE_MAX_SEGMENT_FRAMES) { startFrame, samples -> + onPlayedSegment(startFrame, samples.size) + } + return AudioSyncDecoder(tapAnalyzer, playedAnalyzer) { mime -> + LiveAiSubsLog.w("no look-ahead decoder for $mime: AI subtitles not supported on this channel") + }.also { decoder = it } + } + } + + private fun onTapSegment(startFrame: Int, sampleCount: Int) { + val startUs = startFrame.toLong() * SpeechTimeline.FRAME_DURATION_US + val endUs = startUs + sampleCount * 1_000_000L / SileroVad.SAMPLE_RATE + val position = estimatedPositionUs() + tapSegments++ + synchronized(recentTapStarts) { + recentTapStarts.addLast(startUs) + while (recentTapStarts.size > RECENT_SEGMENTS) recentTapStarts.removeFirst() + } + LiveAiSubsLog.i( + "segment tap start=${startUs / 1_000}ms end=${endUs / 1_000}ms len=${(endUs - startUs) / 1_000}ms " + + "readyBeforeStart=${if (position == UNKNOWN) "-" else "${(startUs - position) / 1_000}ms"} " + + "readyBeforeEnd=${if (position == UNKNOWN) "-" else "${(endUs - position) / 1_000}ms"}", + ) + } + + private fun onPlayedSegment(startFrame: Int, sampleCount: Int) { + val startUs = startFrame.toLong() * SpeechTimeline.FRAME_DURATION_US + val endUs = startUs + sampleCount * 1_000_000L / SileroVad.SAMPLE_RATE + playedSegments++ + val match = synchronized(recentTapStarts) { + recentTapStarts.minByOrNull { kotlin.math.abs(it - startUs) } + ?.takeIf { kotlin.math.abs(it - startUs) <= MATCH_WINDOW_US } + ?.also { deltasMs += (startUs - it) / 1_000; if (deltasMs.size > 200) deltasMs.removeAt(0) } + } + LiveAiSubsLog.i( + "segment played start=${startUs / 1_000}ms end=${endUs / 1_000}ms " + + "tapDelta=${if (match == null) "unmatched" else "${(startUs - match) / 1_000}ms"}", + ) + } + + private inner class ChannelSink(private val sinkGeneration: Int) : AudioSampleSink { + /** A channel can carry several audio tracks; the first one seen is the one analysed. */ + @Volatile private var lockedTrackId: String? = null + private var ignoredLogged = false + + private var formatLogged: Format? = null + + override fun wantsSamples(format: Format): Boolean { + if (sinkGeneration != generation || !active) return false + if (formatLogged !== format) { + formatLogged = format + LiveAiSubsLog.i( + "audio format offered: id=${format.id} mime=${format.sampleMimeType} codecs=${format.codecs} " + + "accepted=${ensureDecoder()?.accepts(format)}", + ) + } + val locked = lockedTrackId + if (locked != null && locked != (format.id ?: NO_ID)) { + if (!ignoredLogged) { + ignoredLogged = true + LiveAiSubsLog.i("ignoring second audio track ${format.id} (${format.sampleMimeType}); analysing $locked") + } + return false + } + val current = ensureDecoder() ?: return false + return current.accepts(format) + } + + override fun onSample(format: Format, timeUs: Long, data: ByteArray, offset: Int, size: Int) { + if (sinkGeneration != generation) return + if (lockedTrackId == null) { + lockedTrackId = format.id ?: NO_ID + LiveAiSubsLog.i( + "tapping track ${format.id}: ${format.sampleMimeType} ${format.channelCount}ch " + + "${format.sampleRate}Hz lang=${format.language}", + ) + } + val current = decoder ?: return + val count = ++tapSamples + lastTapUs = timeUs + if (count == 1L || count % SAMPLE_LOG_EVERY == 0L) { + val position = estimatedPositionUs() + LiveAiSubsLog.i( + "sample #$count t=${timeUs / 1_000}ms pos=${if (position == UNKNOWN) "-" else "${position / 1_000}ms"} " + + "ahead=${if (position == UNKNOWN) "-" else "${(timeUs - position) / 1_000}ms"}", + ) + } + if (!current.offer(format, timeUs, data, offset, size)) droppedSamples++ + } + + override fun onDiscontinuity() { + if (sinkGeneration != generation) return + LiveAiSubsLog.i("discontinuity (extractor seek/reset) gen=$sinkGeneration") + decoder?.discontinuity() + } + } + + companion object { + private const val UNKNOWN = Long.MIN_VALUE + private const val NO_ID = "(no id)" + /** About 5 s: bounds how long after a sentence starts its translation can begin. */ + const val LIVE_MAX_SEGMENT_FRAMES = 156 + private const val SAMPLE_LOG_EVERY = 500L + private const val STATS_INTERVAL_MS = 2_000L + private const val RECENT_SEGMENTS = 64 + private const val MATCH_WINDOW_US = 1_000_000L + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/SubtitleLookaheadLoadControl.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/SubtitleLookaheadLoadControl.kt new file mode 100644 index 0000000000..1a64d587f6 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/SubtitleLookaheadLoadControl.kt @@ -0,0 +1,79 @@ +@file:androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) + +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.os.SystemClock +import androidx.media3.common.Timeline +import androidx.media3.exoplayer.LoadControl +import androidx.media3.exoplayer.analytics.PlayerId +import androidx.media3.exoplayer.source.MediaSource +import androidx.media3.exoplayer.source.TrackGroupArray +import androidx.media3.exoplayer.trackselection.ExoTrackSelection +import androidx.media3.exoplayer.upstream.Allocator + +/** + * Holds live playback back until [requiredLookaheadUs] of media is buffered, at channel start and + * after every rebuffer, so audio reaches the subtitle pipeline that much before it is heard. A + * line can only be translated once it has been spoken in full, so without this hold every + * subtitle would be late. Everything else is [delegate]'s decision. + * + * The hold only applies while [requiredLookaheadUs] is above zero (set from the UI thread), so a + * channel without AI subtitles starts as fast as before. Waiting always converges: while playback + * waits, the live edge keeps moving, so the buffered duration keeps growing. + */ +internal class SubtitleLookaheadLoadControl( + private val delegate: LoadControl, +) : LoadControl { + @Volatile + var requiredLookaheadUs: Long = 0L + + private var lastHoldLogMs = 0L + + override fun onPrepared(playerId: PlayerId) = delegate.onPrepared(playerId) + + override fun onTracksSelected( + parameters: LoadControl.Parameters, + trackGroups: TrackGroupArray, + trackSelections: Array, + ) = delegate.onTracksSelected(parameters, trackGroups, trackSelections) + + override fun onStopped(playerId: PlayerId) = delegate.onStopped(playerId) + + override fun onReleased(playerId: PlayerId) = delegate.onReleased(playerId) + + override fun getAllocator(playerId: PlayerId): Allocator = delegate.getAllocator(playerId) + + override fun getBackBufferDurationUs(playerId: PlayerId): Long = delegate.getBackBufferDurationUs(playerId) + + override fun retainBackBufferFromKeyframe(playerId: PlayerId): Boolean = + delegate.retainBackBufferFromKeyframe(playerId) + + override fun shouldContinueLoading(parameters: LoadControl.Parameters): Boolean = + delegate.shouldContinueLoading(parameters) + + override fun shouldContinuePreloading( + playerId: PlayerId, + timeline: Timeline, + mediaPeriodId: MediaSource.MediaPeriodId, + bufferedDurationUs: Long, + ): Boolean = delegate.shouldContinuePreloading(playerId, timeline, mediaPeriodId, bufferedDurationUs) + + override fun shouldStartPlayback(parameters: LoadControl.Parameters): Boolean { + if (!delegate.shouldStartPlayback(parameters)) return false + val required = requiredLookaheadUs + if (required <= 0L || parameters.bufferedDurationUs >= required) return true + val now = SystemClock.elapsedRealtime() + if (now - lastHoldLogMs >= HOLD_LOG_INTERVAL_MS) { + lastHoldLogMs = now + LiveAiSubsLog.i( + "holding playback: buffered=${parameters.bufferedDurationUs / 1_000}ms " + + "required=${required / 1_000}ms rebuffering=${parameters.rebuffering}", + ) + } + return false + } + + private companion object { + const val HOLD_LOG_INTERVAL_MS = 1_000L + } +} From 8c3cee6234a595ce52d5c318c97c6e917b9a2359 Mon Sep 17 00:00:00 2001 From: silentbil Date: Fri, 6 Nov 2026 11:48:38 +0200 Subject: [PATCH 2/8] Live TV AI subtitles: Gemini transcription and translation, held-back playback Live channels get AI subtitles in the viewer's language that appear on time: the channel's audio is copied as it downloads (extractor tap), cut into speech segments with Silero, and each segment is sent to Gemini, which transcribes and translates it in one call. Lines are timed from the segment's own media time and shown by the player position, while SubtitleLookaheadLoadControl holds playback back so answers arrive first. - Pipeline skips its backlog when analysis falls behind playback (busy boxes) - Segments end at speech pauses after 3.5 s, hard limit 5 s - Up to 2 requests in flight, applied in order; 429 backoff, invalid key, offline and unsupported-audio states with toasts - Delay Auto (default, from measured latency) / 6 / 10 / 15 s; "Preparing subtitles" while held; pause-to-build when turned on mid-channel - Settings (AI subtitles section): enable with key check, delay, activation per channel / always, target language, own Gemini key when the AI subtitles model is not Gemini; cloud synced - Subtitle picker gets an "AI subtitles" entry, remembered per channel - Overlay uses the profile's subtitle style; hidden debug panel via debug.arvio.liveaisubs.overlay - Strings in en, iw, de, es, pt, pt-BR; unit tests for scheduler, delay math, Gemini JSON parsing and the picker option --- .../tv/data/repository/CloudSyncRepository.kt | 16 +- .../player/audiosync/AudioSyncDecoder.kt | 7 +- .../player/audiosync/asr/SpeechSegmenter.kt | 9 +- .../subtitles/SubtitleTranslationService.kt | 112 ++++--- .../settings/LiveAiSubsSettingsRows.kt | 199 +++++++++++ .../tv/ui/screens/settings/SettingsScreen.kt | 7 +- .../ui/screens/settings/SettingsViewModel.kt | 11 + .../arflix/tv/ui/screens/tv/TvViewModel.kt | 35 ++ .../tv/ui/screens/tv/live/LiveSubtitles.kt | 21 ++ .../tv/ui/screens/tv/live/LiveTvScreen.kt | 223 +++++++++--- .../tv/live/aisubs/GeminiSegmentEngine.kt | 317 ++++++++++++++++++ .../screens/tv/live/aisubs/LiveAiSubsDebug.kt | 36 +- .../tv/live/aisubs/LiveAiSubsSettings.kt | 116 +++++++ .../tv/live/aisubs/LiveAiSubtitleOverlay.kt | 108 ++++++ .../screens/tv/live/aisubs/LiveAudioTapHls.kt | 10 +- .../tv/live/aisubs/LiveCueScheduler.kt | 131 ++++++++ .../tv/live/aisubs/LiveSpeechPipeline.kt | 226 +++++++++++++ .../screens/tv/live/aisubs/LiveSpeechProbe.kt | 266 --------------- .../tv/live/aisubs/LiveSubtitleSession.kt | 278 +++++++++++++++ .../aisubs/SubtitleLookaheadLoadControl.kt | 32 +- app/src/main/res/values-de/strings.xml | 30 ++ app/src/main/res/values-es/strings.xml | 30 ++ app/src/main/res/values-iw/strings.xml | 30 ++ app/src/main/res/values-pt-rBR/strings.xml | 30 ++ app/src/main/res/values-pt/strings.xml | 30 ++ app/src/main/res/values/strings.xml | 31 +- .../tv/live/LiveAiSubtitlesOptionTest.kt | 34 ++ .../tv/live/aisubs/GeminiLiveProtocolTest.kt | 130 +++++++ .../tv/live/aisubs/LiveCueSchedulerTest.kt | 135 ++++++++ 29 files changed, 2222 insertions(+), 418 deletions(-) create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/settings/LiveAiSubsSettingsRows.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiSegmentEngine.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsSettings.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueScheduler.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt delete mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechProbe.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt create mode 100644 app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/LiveAiSubtitlesOptionTest.kt create mode 100644 app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiLiveProtocolTest.kt create mode 100644 app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueSchedulerTest.kt diff --git a/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt b/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt index 9c91fd12aa..ac7324ad0d 100644 --- a/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt +++ b/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt @@ -498,7 +498,8 @@ class CloudSyncRepository @Inject constructor( "dnsProvider", "subtitleAiEnabled", "subtitleAiAutoSelect", "subtitleAiFindBestMatch", "subtitlePreloadEnabled", "dolbyVisionCompatEnabled", "subtitleAiApiKey", "subtitleAiModel", "subtitleRemoveHearingImpaired", "telegramSearchOnClickOnly", - "subtitleHearingSyncV2" + "subtitleHearingSyncV2", "liveAiSubsEnabled", "liveAiSubsDelay", "liveAiSubsActivation", + "liveAiSubsTargetLanguage", "liveAiSubsApiKey" ) // Per-profile fields excluded from the generic merge (handled by their own logic / not values). private val profileMergeExclude = setOf("defaultSubtitle", "subtitleSettingsUpdatedAt") @@ -793,6 +794,12 @@ class CloudSyncRepository @Inject constructor( root.put("subtitleAiApiKey", prefs[subtitleAiApiKeyKey] ?: "") root.put("subtitleAiModel", prefs[subtitleAiModelKey] ?: "GROQ_LLAMA_70B") root.put("subtitleRemoveHearingImpaired", prefs[subtitleRemoveHearingImpairedKey] ?: true) + // Live TV AI subtitles (global, like the AI subtitle key they share). + root.put("liveAiSubsEnabled", prefs[com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.ENABLED] ?: false) + root.put("liveAiSubsDelay", prefs[com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.DELAY] ?: "") + root.put("liveAiSubsActivation", prefs[com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.ACTIVATION] ?: "") + root.put("liveAiSubsTargetLanguage", prefs[com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.TARGET_LANGUAGE] ?: "") + root.put("liveAiSubsApiKey", prefs[com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.OWN_API_KEY] ?: "") root.put( "telegramSearchOnClickOnly", prefs[com.arflix.tv.data.telegram.TelegramRepository.KEY_SEARCH_ON_CLICK_ONLY] ?: true @@ -1863,6 +1870,13 @@ class CloudSyncRepository @Inject constructor( val model = root.optString("subtitleAiModel", "GROQ_LLAMA_70B") if (model.isNotBlank()) prefs[subtitleAiModelKey] = model prefs[subtitleRemoveHearingImpairedKey] = root.optBoolean("subtitleRemoveHearingImpaired", true) + // has() guards: backups from versions without Live TV AI subtitles must not reset them. + val live = com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs + if (root.has("liveAiSubsEnabled")) prefs[live.ENABLED] = root.optBoolean("liveAiSubsEnabled", false) + root.optString("liveAiSubsDelay", "").takeIf { root.has("liveAiSubsDelay") && it.isNotBlank() }?.let { prefs[live.DELAY] = it } + root.optString("liveAiSubsActivation", "").takeIf { root.has("liveAiSubsActivation") && it.isNotBlank() }?.let { prefs[live.ACTIVATION] = it } + if (root.has("liveAiSubsTargetLanguage")) prefs[live.TARGET_LANGUAGE] = root.optString("liveAiSubsTargetLanguage", "") + root.optString("liveAiSubsApiKey", "").takeIf { it.isNotBlank() }?.let { prefs[live.OWN_API_KEY] = it } } } if (preservedNewerLocalSubtitle) { diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncDecoder.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncDecoder.kt index 1f6e991038..4ef5ba82dc 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncDecoder.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncDecoder.kt @@ -34,6 +34,11 @@ internal class AudioSyncDecoder( * next to the look-ahead one, which keep to platform software decoders and FFmpeg. */ private val allowVendorDecoders: Boolean = true, + /** + * Priority of the decoding thread. Background by default; Live TV subtitles run a little + * higher, since a starved thread there makes every subtitle late. + */ + private val threadPriority: Int = Process.THREAD_PRIORITY_BACKGROUND, /** Called once per audio format that has no usable decoder. */ private val onUnsupportedFormat: (mimeType: String) -> Unit = {}, ) { @@ -144,7 +149,7 @@ internal class AudioSyncDecoder( } private fun runLoop() { - Process.setThreadPriority(Process.THREAD_PRIORITY_BACKGROUND) + Process.setThreadPriority(threadPriority) try { while (true) { val item = synchronized(lock) { diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/asr/SpeechSegmenter.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/asr/SpeechSegmenter.kt index 611dfa4052..ae8f3ad069 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/asr/SpeechSegmenter.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/asr/SpeechSegmenter.kt @@ -14,6 +14,11 @@ internal class SpeechSegmenter( * be translated once its segment has ended, so it bounds how far ahead playback must be held. */ private val maxSegmentFrames: Int = MAX_SEGMENT_FRAMES, + /** + * Past this length, a segment also ends at the first chunk whose speech probability dips below + * [DIP_THRESHOLD], a pause between words, so long speech is not cut mid-word at the hard limit. + */ + private val softMaxSegmentFrames: Int = maxSegmentFrames, private val onSegment: (startFrame: Int, samples: FloatArray) -> Unit, ) : SpeechAnalyzer.ChunkListener { private val chunk = SileroVad.CHUNK_SAMPLES @@ -47,7 +52,8 @@ internal class SpeechSegmenter( } append(chunk) if (speech) lastSpeechFrame = frame - if (frame - lastSpeechFrame > HANGOVER_FRAMES || bufferFrames >= maxSegmentFrames) finish() + val dip = bufferFrames >= softMaxSegmentFrames && probability < DIP_THRESHOLD + if (frame - lastSpeechFrame > HANGOVER_FRAMES || bufferFrames >= maxSegmentFrames || dip) finish() } @Synchronized @@ -80,6 +86,7 @@ internal class SpeechSegmenter( companion object { private const val SPEECH_THRESHOLD = 0.5f + private const val DIP_THRESHOLD = 0.7f private const val PREROLL_FRAMES = 3 private const val TAIL_FRAMES = 4 private const val HANGOVER_FRAMES = 8 diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/subtitles/SubtitleTranslationService.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/subtitles/SubtitleTranslationService.kt index 20e5475bc0..1b1b5b06e8 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/subtitles/SubtitleTranslationService.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/subtitles/SubtitleTranslationService.kt @@ -14,7 +14,7 @@ import java.util.concurrent.TimeUnit private const val TAG = "SubtitleTranslation" -private val RTL_LANGUAGES = setOf("hebrew", "arabic", "urdu", "persian", "farsi", "yiddish") +internal val RTL_LANGUAGES = setOf("hebrew", "arabic", "urdu", "persian", "farsi", "yiddish") data class TranslationResult( val lines: List, @@ -28,7 +28,7 @@ data class TranslationResult( // it's not actionable and translation keeps working for everything else. const val TRANSLATION_ERROR_CONTENT_BLOCKED = "CONTENT_BLOCKED" -private val BLOCKED_FINISH_REASONS = setOf( +internal val BLOCKED_FINISH_REASONS = setOf( "PROHIBITED_CONTENT", "SAFETY", "RECITATION", "BLOCKLIST", "SPII" ) @@ -54,9 +54,62 @@ private const val GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v // with short exponential backoff before falling back to English. Kept short so subtitles stay in // sync with playback. NOT applied to 429 (that's our own quota — retrying fast makes it worse; // the manager's 5s cooldown handles it) or 4xx/content blocks (retrying won't change the outcome). -private const val GEMINI_MAX_TRANSIENT_RETRIES = 2 // 3 attempts total -private const val GEMINI_RETRY_BASE_DELAY_MS = 400L // 400ms, then 800ms (×2 each retry) -private val GEMINI_RETRYABLE_HTTP = setOf(500, 502, 503) +internal const val GEMINI_MAX_TRANSIENT_RETRIES = 2 // 3 attempts total +internal const val GEMINI_RETRY_BASE_DELAY_MS = 400L // 400ms, then 800ms (×2 each retry) +internal val GEMINI_RETRYABLE_HTTP = setOf(500, 502, 503) + +/** + * The JSON array in a model reply: bare, fenced, wrapped in a one-field object, or with a truncated + * tail. Shared by every Gemini/Groq caller in the app. + */ +internal fun extractJsonArray(text: String): JSONArray? { + val codeBlocks = Regex("```(?:json)?\\s*([\\s\\S]*?)```").findAll(text) + .map { it.groupValues[1].trim() }.toList().reversed() + val stripped = text.replace(Regex("```[^`]*```"), "").trim() + val candidates = codeBlocks + listOf(stripped, text) + + for (candidate in candidates) { + try { return JSONArray(candidate) } catch (_: Exception) {} + // With responseMimeType=application/json the model sometimes wraps the array in an + // object (e.g. {"translations": [...]}) — unwrap a lone array-valued field. + try { + val obj = JSONObject(candidate) + val arrays = obj.keys().asSequence() + .mapNotNull { key -> obj.optJSONArray(key) } + .toList() + if (arrays.size == 1) return arrays[0] + } catch (_: Exception) {} + val start = candidate.indexOf('[') + val end = candidate.lastIndexOf(']') + if (start < 0 || end <= start) { + repairTruncatedArray(candidate)?.let { return it } + continue + } + try { + return JSONArray(candidate.substring(start, end + 1)) + } catch (_: Exception) {} + } + return null +} + +/** + * Gemini's JSON mode intermittently truncates output at the very tail (finishReason STOP but + * the closing bracket — sometimes a trailing element — is missing). The translations + * themselves are intact, so re-terminate the array instead of failing the whole batch. + */ +internal fun repairTruncatedArray(text: String): JSONArray? { + val t = text.trim() + if (!t.startsWith("[") || t.endsWith("]")) return null + val lastQuote = t.lastIndexOf('"') + if (lastQuote <= 0) return null + // Try closing as-is, then after dropping a trailing partial element. + for (cut in listOf(t, t.substring(0, lastQuote + 1))) { + val trimmed = cut.trimEnd().trimEnd(',') + try { return JSONArray("$trimmed]") } catch (_: Exception) {} + try { return JSONArray("$trimmed\"]") } catch (_: Exception) {} + } + return null +} class SubtitleTranslationService( private val apiKeyProvider: () -> String, @@ -68,55 +121,6 @@ class SubtitleTranslationService( .readTimeout(30, TimeUnit.SECONDS) .build() - private fun extractJsonArray(text: String): JSONArray? { - val codeBlocks = Regex("```(?:json)?\\s*([\\s\\S]*?)```").findAll(text) - .map { it.groupValues[1].trim() }.toList().reversed() - val stripped = text.replace(Regex("```[^`]*```"), "").trim() - val candidates = codeBlocks + listOf(stripped, text) - - for (candidate in candidates) { - try { return JSONArray(candidate) } catch (_: Exception) {} - // With responseMimeType=application/json the model sometimes wraps the array in an - // object (e.g. {"translations": [...]}) — unwrap a lone array-valued field. - try { - val obj = JSONObject(candidate) - val arrays = obj.keys().asSequence() - .mapNotNull { key -> obj.optJSONArray(key) } - .toList() - if (arrays.size == 1) return arrays[0] - } catch (_: Exception) {} - val start = candidate.indexOf('[') - val end = candidate.lastIndexOf(']') - if (start < 0 || end <= start) { - repairTruncatedArray(candidate)?.let { return it } - continue - } - try { - return JSONArray(candidate.substring(start, end + 1)) - } catch (_: Exception) {} - } - return null - } - - /** - * Gemini's JSON mode intermittently truncates output at the very tail (finishReason STOP but - * the closing bracket — sometimes a trailing element — is missing). The translations - * themselves are intact, so re-terminate the array instead of failing the whole batch. - */ - private fun repairTruncatedArray(text: String): JSONArray? { - val t = text.trim() - if (!t.startsWith("[") || t.endsWith("]")) return null - val lastQuote = t.lastIndexOf('"') - if (lastQuote <= 0) return null - // Try closing as-is, then after dropping a trailing partial element. - for (cut in listOf(t, t.substring(0, lastQuote + 1))) { - val trimmed = cut.trimEnd().trimEnd(',') - try { return JSONArray("$trimmed]") } catch (_: Exception) {} - try { return JSONArray("$trimmed\"]") } catch (_: Exception) {} - } - return null - } - private fun buildSystemPrompt(targetLanguage: String, NL: String) = "You are a professional subtitle translator. Translate the following JSON array into natural $targetLanguage.\n" + "Rules:\n" + diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/LiveAiSubsSettingsRows.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/LiveAiSubsSettingsRows.kt new file mode 100644 index 0000000000..faf473b928 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/LiveAiSubsSettingsRows.kt @@ -0,0 +1,199 @@ +package com.arflix.tv.ui.screens.settings + +import android.content.Context +import android.widget.Toast +import androidx.compose.material.icons.Icons +import androidx.compose.material.icons.filled.Language +import androidx.compose.material.icons.filled.Schedule +import androidx.compose.material.icons.filled.Tv +import androidx.compose.material.icons.filled.VpnKey +import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.mutableStateOf +import androidx.compose.runtime.remember +import androidx.compose.runtime.setValue +import androidx.compose.ui.Modifier +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource +import com.arflix.tv.R +import com.arflix.tv.ui.components.MobileSettingsRow +import com.arflix.tv.ui.components.SettingsRow +import com.arflix.tv.ui.components.SettingsToggleRow +import com.arflix.tv.ui.screens.player.subtitles.SubtitleAiModel +import com.arflix.tv.ui.screens.tv.live.aisubs.GeminiSegmentEngine +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsActivation +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsDelay +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsSettings +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch +import java.util.Locale + +/* + * Live TV AI subtitles settings, shown in the "AI subtitles" section next to the Gemini key they + * use. The rows read and write their own preferences (LiveAiSubsPrefs) instead of threading new + * state through the settings screen; SettingsViewModel pushes changes to the cloud. + */ + +/** TV settings row indices, after the existing ones (0–50). */ +internal val LIVE_AI_SUBS_ROWS = listOf(51, 52, 53, 54, 55) + +private object LiveAiSubsKeyDialogState { + var open by mutableStateOf(false) +} + +private val actionScope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate) + +/** Handles OK/click on one of [LIVE_AI_SUBS_ROWS]. */ +internal fun onLiveAiSubsRowClick(context: Context, index: Int) { + val appContext = context.applicationContext + actionScope.launch { + val settings = LiveAiSubsPrefs.flow(appContext).first() + when (index) { + 51 -> if (settings.enabled) { + LiveAiSubsPrefs.setEnabled(appContext, false) + } else { + enableAfterKeyCheck(appContext, settings) + } + 52 -> LiveAiSubsPrefs.setDelay(appContext, settings.delay.next()) + 53 -> LiveAiSubsPrefs.setActivation(appContext, settings.activation.next()) + 54 -> { + val choices = LiveAiSubsPrefs.TARGET_LANGUAGE_CHOICES + val next = choices[(choices.indexOf(settings.targetLanguage).coerceAtLeast(0) + 1) % choices.size] + LiveAiSubsPrefs.setTargetLanguage(appContext, next) + } + 55 -> if (settings.usesSharedKey) { + toast(appContext, R.string.live_ai_subs_key_shared) + } else { + LiveAiSubsKeyDialogState.open = true + } + } + } +} + +/** Validates the key with a cheap call before turning the feature on, and says how it went. */ +private suspend fun enableAfterKeyCheck(context: Context, settings: LiveAiSubsSettings) { + if (!settings.hasKey) { + toast(context, R.string.live_ai_subs_needs_key) + if (!settings.usesSharedKey) LiveAiSubsKeyDialogState.open = true + return + } + when (GeminiSegmentEngine(apiKey = { settings.apiKey }).validateKey(settings.apiKey)) { + true -> { + toast(context, R.string.live_ai_subs_key_valid) + LiveAiSubsPrefs.setEnabled(context, true) + } + false -> toast(context, R.string.live_ai_subs_key_rejected) + // Could not check (offline): turn it on; the player reports key problems itself. + null -> { + toast(context, R.string.live_ai_subs_key_unchecked) + LiveAiSubsPrefs.setEnabled(context, true) + } + } +} + +private fun toast(context: Context, resId: Int) { + Toast.makeText(context, context.getString(resId), Toast.LENGTH_SHORT).show() +} + +@Composable +private fun rememberLiveAiSubsSettings(): LiveAiSubsSettings { + val context = LocalContext.current + val flow = remember(context) { LiveAiSubsPrefs.flow(context.applicationContext) } + val settings by flow.collectAsState(initial = LiveAiSubsSettings()) + return settings +} + +@Composable +private fun delayLabel(delay: LiveAiSubsDelay): String = + if (delay == LiveAiSubsDelay.AUTO) stringResource(R.string.auto) else stringResource(R.string.live_ai_subs_seconds, (delay.fixedMs ?: 0L) / 1_000L) + +@Composable +private fun activationLabel(activation: LiveAiSubsActivation): String = stringResource( + if (activation == LiveAiSubsActivation.ALWAYS) R.string.live_ai_subs_activation_always else R.string.live_ai_subs_activation_per_channel, +) + +@Composable +private fun languageLabel(code: String): String = + if (code.isBlank()) { + stringResource(R.string.live_ai_subs_target_profile) + } else { + Locale.forLanguageTag(code).getDisplayLanguage(Locale.getDefault()).replaceFirstChar { it.titlecase(Locale.getDefault()) } + } + +@Composable +private fun keyLabel(settings: LiveAiSubsSettings): String = when { + settings.usesSharedKey -> stringResource(R.string.live_ai_subs_key_shared_short) + settings.ownApiKey.isBlank() -> stringResource(R.string.ai_key_not_set) + else -> "••••" + settings.ownApiKey.takeLast(4) +} + +/** One TV row for [index] in [LIVE_AI_SUBS_ROWS]. */ +@Composable +internal fun LiveAiSubsTvRow(index: Int, isFocused: Boolean, modifier: Modifier) { + val context = LocalContext.current + val settings = rememberLiveAiSubsSettings() + val click = { onLiveAiSubsRowClick(context, index) } + when (index) { + 51 -> SettingsToggleRow( + title = stringResource(R.string.live_ai_subs_title), + subtitle = stringResource(R.string.live_ai_subs_desc), + isEnabled = settings.enabled, + isFocused = isFocused, + onToggle = { click() }, + modifier = modifier, + ) + 52 -> SettingsRow(Icons.Default.Schedule, stringResource(R.string.live_ai_subs_delay_title), stringResource(R.string.live_ai_subs_delay_desc), delayLabel(settings.delay), isFocused, click, modifier) + 53 -> SettingsRow(Icons.Default.Tv, stringResource(R.string.live_ai_subs_activation_title), stringResource(R.string.live_ai_subs_activation_desc), activationLabel(settings.activation), isFocused, click, modifier) + 54 -> SettingsRow(Icons.Default.Language, stringResource(R.string.live_ai_subs_target_title), stringResource(R.string.live_ai_subs_target_desc), languageLabel(settings.targetLanguage), isFocused, click, modifier) + 55 -> SettingsRow(Icons.Default.VpnKey, stringResource(R.string.live_ai_subs_key_title), stringResource(R.string.live_ai_subs_key_desc), keyLabel(settings), isFocused, click, modifier) + } + if (index == LIVE_AI_SUBS_ROWS.last()) LiveAiSubsKeyDialogHost() +} + +/** The same settings as touch rows, for the mobile settings page. */ +@Composable +internal fun LiveAiSubsMobileRows() { + val context = LocalContext.current + val settings = rememberLiveAiSubsSettings() + MobileSettingsRow( + title = stringResource(R.string.live_ai_subs_title), + subtitle = stringResource(R.string.live_ai_subs_desc), + value = stringResource(if (settings.enabled) R.string.on else R.string.off), + toggleChecked = settings.enabled, + onClick = { onLiveAiSubsRowClick(context, 51) }, + ) + MobileSettingsRow(icon = Icons.Default.Schedule, title = stringResource(R.string.live_ai_subs_delay_title), subtitle = stringResource(R.string.live_ai_subs_delay_desc), value = delayLabel(settings.delay), isToggle = false, onClick = { onLiveAiSubsRowClick(context, 52) }) + MobileSettingsRow(icon = Icons.Default.Tv, title = stringResource(R.string.live_ai_subs_activation_title), subtitle = stringResource(R.string.live_ai_subs_activation_desc), value = activationLabel(settings.activation), isToggle = false, onClick = { onLiveAiSubsRowClick(context, 53) }) + MobileSettingsRow(icon = Icons.Default.Language, title = stringResource(R.string.live_ai_subs_target_title), subtitle = stringResource(R.string.live_ai_subs_target_desc), value = languageLabel(settings.targetLanguage), isToggle = false, onClick = { onLiveAiSubsRowClick(context, 54) }) + MobileSettingsRow(icon = Icons.Default.VpnKey, title = stringResource(R.string.live_ai_subs_key_title), subtitle = stringResource(R.string.live_ai_subs_key_desc), value = keyLabel(settings), isToggle = false, showDivider = false, onClick = { onLiveAiSubsRowClick(context, 55) }) + LiveAiSubsKeyDialogHost() +} + +@Composable +private fun LiveAiSubsKeyDialogHost() { + if (!LiveAiSubsKeyDialogState.open) return + val context = LocalContext.current + val settings = rememberLiveAiSubsSettings() + AiApiKeyDialog( + currentKey = settings.ownApiKey, + model = SubtitleAiModel.GEMINI_FLASH_25, + onSave = { key -> + LiveAiSubsKeyDialogState.open = false + val appContext = context.applicationContext + actionScope.launch { + LiveAiSubsPrefs.setOwnApiKey(appContext, key) + when (GeminiSegmentEngine(apiKey = { key }).validateKey(key)) { + true -> toast(appContext, R.string.live_ai_subs_key_valid) + false -> toast(appContext, R.string.live_ai_subs_key_rejected) + null -> toast(appContext, R.string.live_ai_subs_key_unchecked) + } + } + }, + onDismiss = { LiveAiSubsKeyDialogState.open = false }, + ) +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt index 1af66a086a..37366a2687 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt @@ -339,7 +339,7 @@ private fun tvGeneralRowsForSection(section: String): List { "subtitles" -> listOf(4, 5, 6, 7, 42, 8, 38) + (if (BuildConfig.AUDIO_SYNC_AVAILABLE) listOf(48) else emptyList()) + listOf(39, 9, 45) - "ai_subtitles" -> listOf(28, 29, 30, 31, 32, 33) + "ai_subtitles" -> listOf(28, 29, 30, 31, 32, 33) + LIVE_AI_SUBS_ROWS "playback" -> listOf(10, 11, 12, 43, 44, 13, 14, 34, 37, 16, 15, 40, 27) "appearance" -> listOf(17, 49, 50, 18, 20, 21, 24, 23, 22, 41, 46, 36, 47) "profiles" -> listOf(19) @@ -1486,6 +1486,7 @@ fun SettingsScreen( 31 -> viewModel.setSubtitleRemoveHearingImpaired(!uiState.subtitleRemoveHearingImpaired) 32 -> showAiApiKeyDialog = true 33 -> viewModel.startAiKeyServer() + 51, 52, 53, 54, 55 -> onLiveAiSubsRowClick(context, tvGeneralRowsForSection(currentSection)[contentFocusIndex]) 34 -> viewModel.cycleTrailerDelay() 47 -> viewModel.cycleGuideRowCount() 37 -> viewModel.setTrailerInCards(!uiState.trailerInCards) @@ -5521,6 +5522,7 @@ private fun MobileSettingsSubPage( showDivider = false, onClick = onSubtitleAiQrClick ) + LiveAiSubsMobileRows() if (uiState.subtitleAiEnabled) { Text( text = when (uiState.subtitleAiModel) { @@ -6967,6 +6969,7 @@ private fun TvGeneralSettingsRows( 37 -> SettingsToggleRow(stringResource(R.string.trailer_in_cards), stringResource(R.string.trailer_in_cards_desc), trailerInCards, focusedIndex == localIndex, onTrailerInCardsToggle, Modifier.settingsFocusSlot(localIndex)) 49 -> SettingsToggleRow(stringResource(R.string.netflix_home_style), stringResource(R.string.netflix_home_style_desc), netflixHomeStyle, focusedIndex == localIndex, onNetflixHomeStyleToggle, Modifier.settingsFocusSlot(localIndex)) 50 -> SettingsToggleRow(stringResource(R.string.watchlist_on_home), stringResource(R.string.watchlist_on_home_desc), watchlistOnHome, focusedIndex == localIndex, onWatchlistOnHomeToggle, Modifier.settingsFocusSlot(localIndex)) + in LIVE_AI_SUBS_ROWS -> LiveAiSubsTvRow(rowId, focusedIndex == localIndex, Modifier.settingsFocusSlot(localIndex)) } } } @@ -7584,7 +7587,7 @@ private fun AiModelDialog( } @Composable -private fun AiApiKeyDialog( +internal fun AiApiKeyDialog( currentKey: String, onSave: (String) -> Unit, onDismiss: () -> Unit, diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt index 19681de7c7..9202b6ede5 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt @@ -113,6 +113,8 @@ import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.flowOn import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.map import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeoutOrNull @@ -655,6 +657,15 @@ class SettingsViewModel @Inject constructor( } init { + // Live TV AI subtitles settings are written by their own rows (LiveAiSubsSettingsRows); + // push them to the cloud like every other setting, once they settle. + viewModelScope.launch { + com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.flow(context) + .map { listOf(it.enabled, it.delay, it.activation, it.targetLanguage, it.ownApiKey) } + .distinctUntilChanged() + .drop(1) + .collect { syncLocalStateToCloud(silent = true) } + } _uiState.value = _uiState.value.copy( diagnosticsSharingEnabled = DiagnosticsManager.isReportingEnabled(context), usageStatisticsEnabled = DiagnosticsManager.isUsageEnabled(context) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/TvViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/TvViewModel.kt index 7821ec35a6..150ec2448a 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/TvViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/TvViewModel.kt @@ -143,6 +143,41 @@ class TvViewModel @Inject constructor( ) }.getOrDefault(com.arflix.tv.ui.screens.tv.live.LiveSubtitleDefault.Off) + /** + * The language Live TV AI subtitles translate into when settings leave it to the profile: the + * profile's default subtitle language, else the device language. + */ + internal suspend fun liveAiSubsDefaultLanguage(): String = + (liveSubtitleDefault() as? com.arflix.tv.ui.screens.tv.live.LiveSubtitleDefault.Language)?.code + ?: java.util.Locale.getDefault().language + + /** The profile's subtitle look, read the same way the VOD player reads it. */ + internal suspend fun liveSubtitleStyle(): com.arflix.tv.ui.screens.tv.live.aisubs.LiveSubtitleStyle = runCatching { + val prefs = context.settingsDataStore.data.first() + val size = prefs[profileManager.profileStringKey("subtitle_size")] ?: "Medium" + val offset = prefs[profileManager.profileStringKey("subtitle_offset")] ?: "Bottom" + com.arflix.tv.ui.screens.tv.live.aisubs.LiveSubtitleStyle( + sizePreference = size, + sizePercent = prefs[profileManager.profileIntKey("subtitle_size_pct")] ?: when (size.lowercase()) { + "small" -> 80 + "large" -> 120 + "extra large" -> 140 + else -> 100 + }, + verticalPercent = prefs[profileManager.profileIntKey("subtitle_vertical_pct")] ?: when (offset) { + "Low" -> 8 + "Medium" -> 15 + "High" -> 25 + else -> 2 + }, + color = prefs[profileManager.profileStringKey("subtitle_color")] ?: "White", + style = prefs[profileManager.profileStringKey("subtitle_style")] ?: "Bold", + font = com.arflix.tv.ui.screens.player.SubtitleFontOption.fromPreference( + prefs[profileManager.profileStringKey("subtitle_font")], + ).preferenceValue, + ) + }.getOrDefault(com.arflix.tv.ui.screens.tv.live.aisubs.LiveSubtitleStyle()) + suspend fun cachedSportsMetadata() = sportsRepository.cachedMetadata() internal var cachedSportsSchedule: com.arflix.tv.ui.screens.tv.live.SportsScheduleSnapshot? = null private var sportsBroadcasterIndexLoad: Pair>? = null diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt index 443c06a2b2..1195ca0057 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt @@ -65,6 +65,27 @@ internal fun audioPreferenceLanguage(preference: String?): String? { internal fun liveSubtitleOptions(tracks: Tracks, offLabel: String, displayLocale: Locale = Locale.getDefault()) = liveTrackOptions(tracks, C.TRACK_TYPE_TEXT, offLabel, displayLocale) +/** The picker key of the "AI subtitles" entry (Live TV AI subtitles). */ +internal const val LIVE_SUBTITLES_AI = "ai" + +/** + * [streamOptions] (from [liveSubtitleOptions]) plus an "AI subtitles" entry after the stream's own + * tracks, when the feature is on. A channel without subtitle tracks gets "Off" and "AI subtitles". + * While AI subtitles are on for the channel, that entry is the selected one. + */ +internal fun withAiSubtitlesOption( + streamOptions: List, + offLabel: String, + aiLabel: String, + aiAvailable: Boolean, + aiSelected: Boolean, +): List { + if (!aiAvailable) return streamOptions + val base = streamOptions.ifEmpty { listOf(LiveTrackOption(LIVE_SUBTITLES_OFF, offLabel, selected = true)) } + val withSelection = if (aiSelected) base.map { it.copy(selected = false) } else base + return withSelection + LiveTrackOption(LIVE_SUBTITLES_AI, aiLabel, selected = aiSelected) +} + /** The audio choices for what is playing, one per language or track. */ internal fun liveAudioOptions(tracks: Tracks, displayLocale: Locale = Locale.getDefault()) = liveTrackOptions(tracks, C.TRACK_TYPE_AUDIO, offLabel = null, displayLocale) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt index 20d8ae6524..b494828862 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt @@ -80,6 +80,7 @@ import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.produceState import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.collectAsState import androidx.compose.runtime.mutableStateMapOf import androidx.compose.runtime.rememberUpdatedState import androidx.compose.runtime.saveable.rememberSaveable @@ -121,8 +122,17 @@ import androidx.media3.datasource.okhttp.OkHttpDataSource import androidx.media3.exoplayer.DefaultLoadControl import com.arflix.tv.ui.screens.player.audiosync.AudioSyncExtractorsFactory import com.arflix.tv.ui.screens.tv.live.aisubs.AudioTapHlsExtractorFactory +import com.arflix.tv.ui.screens.player.subtitles.RTL_LANGUAGES +import com.arflix.tv.ui.screens.tv.live.aisubs.GeminiSegmentEngine +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsActivation import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsDebug -import com.arflix.tv.ui.screens.tv.live.aisubs.LiveSpeechProbe +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsSettings +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsStatus +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubtitleOverlay +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveSubtitleSession +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveSubtitleStyle +import com.arflix.tv.ui.screens.tv.live.aisubs.liveAiSubsDelayMs import com.arflix.tv.ui.screens.tv.live.aisubs.SubtitleLookaheadLoadControl import androidx.media3.exoplayer.ExoPlayer import androidx.media3.exoplayer.hls.HlsMediaSource @@ -3091,10 +3101,6 @@ fun LiveTvScreen( isLowRamDevice = activityManager?.isLowRamDevice == true ) } - // Live TV AI subtitles, Phase 0 probe: taps the channel audio ahead of playback and logs - // segment timing under "LiveAiSubs". Inert unless switched on over adb in a debug build. - val liveSpeechProbe = remember(context) { LiveSpeechProbe(context) } - DisposableEffect(liveSpeechProbe) { onDispose { liveSpeechProbe.release() } } val liveLoadControl = remember(livePlaybackBufferProfile) { val loadControl = DefaultLoadControl.Builder() .setBufferDurationsMs( @@ -3120,24 +3126,6 @@ fun LiveTvScreen( ExoPlayer.Builder(context) .setMediaSourceFactory(mediaSourceFactory) .setLoadControl(liveLoadControl) - .apply { - // Debug builds only: hear the played audio too, to check the tap's time base. - if (com.arflix.tv.BuildConfig.DEBUG) { - setRenderersFactory(object : androidx.media3.exoplayer.DefaultRenderersFactory(context) { - override fun buildAudioSink( - context: Context, - enableFloatOutput: Boolean, - enableAudioTrackPlaybackParams: Boolean, - ): androidx.media3.exoplayer.audio.AudioSink? { - val sink = super.buildAudioSink(context, enableFloatOutput, enableAudioTrackPlaybackParams) - ?: return null - return com.arflix.tv.ui.screens.player.audiosync.PlaybackAudioTap( - sink, liveSpeechProbe.playbackListener, - ) - } - }) - } - } // A live stream whose buffer stops moving while buffering raises an error after // this long, which the retry below recovers, instead of a spinner for ten minutes. .setStuckBufferingDetectionTimeoutMs(LIVE_STUCK_BUFFERING_TIMEOUT_MS) @@ -3170,27 +3158,6 @@ fun LiveTvScreen( } } - val probeFullscreen by rememberUpdatedState(isFullScreen && !fullscreenGuideOpen && !quickZapOpen) - LaunchedEffect(exoPlayer, liveSpeechProbe) { - val window = androidx.media3.common.Timeline.Window() - while (true) { - liveSpeechProbe.fullscreen = probeFullscreen - if (LiveAiSubsDebug.enabled) { - // currentPosition is relative to the live window; the tapped samples are timed on - // the period, so add the window's offset into the period. - val timeline = exoPlayer.currentTimeline - val windowOffsetUs = if (timeline.isEmpty) 0L else - timeline.getWindow(exoPlayer.currentMediaItemIndex, window).positionInFirstPeriodUs - liveSpeechProbe.onPosition( - periodPositionUs = exoPlayer.currentPosition * 1_000L + windowOffsetUs, - bufferedAheadMs = exoPlayer.bufferedPosition - exoPlayer.currentPosition, - playing = exoPlayer.isPlaying, - ) - } - delay(250L) - } - } - var playbackQuality by remember(exoPlayer) { mutableStateOf(null) } // Subtitles: start every channel on the profile's default subtitle language; a choice made in @@ -3270,6 +3237,119 @@ fun LiveTvScreen( var playerIsPlaying by remember { mutableStateOf(false) } var playerPlayWhenReady by remember { mutableStateOf(true) } var playerIsBuffering by remember { mutableStateOf(false) } + + // -- Live TV AI subtitles ------------------------------------------------------------------ + // The channel's audio is copied as it downloads, cut into speech segments and sent to Gemini; + // playback is held back (liveLoadControl) so each line is ready before it is heard, and lines + // are shown by the player's position, never by when the answer came. + val liveAiSubsSettings by remember(context) { LiveAiSubsPrefs.flow(context.applicationContext) } + .collectAsState(initial = LiveAiSubsSettings()) + val liveAiKey = rememberUpdatedState(liveAiSubsSettings.apiKey) + val liveAiSession = remember(context) { + LiveAiSubsDebug.start() + LiveSubtitleSession(context, GeminiSegmentEngine(apiKey = { liveAiKey.value })) + } + DisposableEffect(liveAiSession) { onDispose { liveAiSession.release() } } + // Per channel, like the audio choice: true/false once picked; unset follows the activation setting. + val liveAiSubsChoices = remember { mutableStateMapOf() } + val liveAiSubsOn = liveAiSubsSettings.enabled && playingChannelId != null && + (liveAiSubsChoices[playingChannelId] ?: (liveAiSubsSettings.activation == LiveAiSubsActivation.ALWAYS)) + val liveAiCue by liveAiSession.cue.collectAsState() + val liveAiStats by liveAiSession.stats.collectAsState() + var liveAiPreparingSeconds by remember { mutableStateOf(null) } + var liveAiDebugVisible by remember { mutableStateOf(false) } + var liveAiWarmupSinceMs by remember { mutableStateOf(null) } + var liveSubtitleStyle by remember { mutableStateOf(LiveSubtitleStyle()) } + LaunchedEffect(Unit) { liveSubtitleStyle = viewModel.liveSubtitleStyle() } + LaunchedEffect(liveAiSubsSettings.targetLanguage) { + val code = liveAiSubsSettings.targetLanguage.ifBlank { viewModel.liveAiSubsDefaultLanguage() } + val name = java.util.Locale.forLanguageTag(code).getDisplayLanguage(java.util.Locale.ENGLISH) + .ifBlank { "English" } + liveAiSession.setTargetLanguage(name, name.lowercase(java.util.Locale.ROOT) in RTL_LANGUAGES) + } + LaunchedEffect(liveAiSubsSettings.apiKey) { liveAiSession.onKeyChanged() } + // Capture costs API calls, so only while it can be seen: fullscreen, and playing or about to. + val liveAiCapture = liveAiSubsOn && isFullScreen && !fullscreenGuideOpen && !quickZapOpen && + (playerPlayWhenReady || liveAiWarmupSinceMs != null) + LaunchedEffect(liveAiSubsOn, liveAiSubsSettings.hasKey, liveAiCapture) { + liveAiSession.update(enabled = liveAiSubsOn, hasKey = liveAiSubsSettings.hasKey, capture = liveAiCapture) + } + // The stream's own subtitles stay off while AI subtitles show. + LaunchedEffect(liveAiSubsOn, liveTracks) { + if (liveAiSubsOn && liveSubtitleOptions(liveTracks, "").any { it.selected && it.key != LIVE_SUBTITLES_OFF }) { + applyLiveSubtitleChoice(exoPlayer, LIVE_SUBTITLES_OFF) + } + } + // Turned on mid-channel: pause until the look-ahead has built up (shown as "Preparing"). + LaunchedEffect(liveAiSubsOn) { + if (liveAiSubsOn && playingCatchupProgram == null && exoPlayer.isPlaying) { + val requiredMs = liveAiSubsDelayMs(liveAiSubsSettings.delay, liveAiSession.latencyP95Ms) + if (exoPlayer.bufferedPosition - exoPlayer.currentPosition < requiredMs) { + liveAiWarmupSinceMs = android.os.SystemClock.elapsedRealtime() + exoPlayer.pause() + } + } else if (!liveAiSubsOn && liveAiWarmupSinceMs != null) { + liveAiWarmupSinceMs = null + exoPlayer.play() + } + } + val liveAiTick by rememberUpdatedState(Triple(liveAiSubsOn, liveAiSubsSettings.delay, playingCatchupProgram == null)) + LaunchedEffect(exoPlayer, liveAiSession) { + val window = androidx.media3.common.Timeline.Window() + while (true) { + val (on, delaySetting, isLive) = liveAiTick + val requiredMs = if (on && isLive) liveAiSubsDelayMs(delaySetting, liveAiSession.latencyP95Ms) else 0L + liveLoadControl.requiredLookaheadUs = requiredMs * 1_000L + liveAiDebugVisible = LiveAiSubsDebug.enabled + if (on) { + // currentPosition is relative to the live window; captured audio is timed on the + // period, so the window offset into the period is added. + val timeline = exoPlayer.currentTimeline + val windowOffsetUs = if (timeline.isEmpty) 0L else + timeline.getWindow(exoPlayer.currentMediaItemIndex, window).positionInFirstPeriodUs + val lookaheadMs = (exoPlayer.bufferedPosition - exoPlayer.currentPosition).coerceAtLeast(0L) + liveAiSession.tick(exoPlayer.currentPosition * 1_000L + windowOffsetUs, exoPlayer.isPlaying, lookaheadMs) + liveAiWarmupSinceMs?.let { since -> + val waited = android.os.SystemClock.elapsedRealtime() - since + if (lookaheadMs >= requiredMs || waited > LIVE_AI_WARMUP_TIMEOUT_MS) { + liveAiWarmupSinceMs = null + exoPlayer.play() + } + } + val held = (liveLoadControl.holding && exoPlayer.playbackState == Player.STATE_BUFFERING) || + liveAiWarmupSinceMs != null + liveAiPreparingSeconds = if (held && requiredMs > 0L) { + ((requiredMs - lookaheadMs + 999L) / 1_000L).toInt().coerceAtLeast(1) + } else { + null + } + if (liveLoadControl.cappedByMemory) { + liveLoadControl.cappedByMemory = false + android.widget.Toast.makeText( + context, context.getString(R.string.live_ai_subs_memory_capped), android.widget.Toast.LENGTH_LONG, + ).show() + } + } else { + liveAiPreparingSeconds = null + } + delay(LIVE_AI_TICK_MS) + } + } + LaunchedEffect(liveAiSession) { + liveAiSession.status.collect { status -> + val message = when (status) { + LiveAiSubsStatus.NoKey -> R.string.live_ai_subs_no_key + LiveAiSubsStatus.InvalidKey -> R.string.live_ai_subs_invalid_key + is LiveAiSubsStatus.QuotaPaused -> R.string.live_ai_subs_quota + LiveAiSubsStatus.Offline -> R.string.live_ai_subs_offline + is LiveAiSubsStatus.Unsupported -> R.string.live_ai_subs_unsupported + else -> null + } + message?.let { + android.widget.Toast.makeText(context, context.getString(it), android.widget.Toast.LENGTH_LONG).show() + } + } + } LaunchedEffect(exoPlayer, playingCatchupProgram, catchupUrlAnchorOffsetMs) { while (true) { val programDuration = playingCatchupProgram @@ -3434,11 +3514,11 @@ fun LiveTvScreen( } // HLS gets its own factory: DefaultMediaSourceFactory cannot set the HLS extractors, // and channels without IDR frames need iptvHlsExtractorFactory() to show video. - // Live TV AI subtitles: every channel source gets a fresh audio tap (Phase 0 probe), so a - // load still running for the previous channel can never feed this one. - val liveAudioSink = liveSpeechProbe.newSink(if (isHls) "hls" else "progressive") - liveLoadControl.requiredLookaheadUs = - if (playingCatchupProgram == null) LiveAiSubsDebug.delayMs * 1_000L else 0L + // Live TV AI subtitles: every channel source gets a fresh audio tap, so a load still + // running for the previous channel can never feed this one. + liveAiSession.onNewSource() + liveAiWarmupSinceMs = null + val liveAudioSink = liveAiSession.pipeline.newSink() val source = if (isHls) { HlsMediaSource.Factory(sourceDataFactory) .setAllowChunklessPreparation(IPTV_HLS_CHUNKLESS_PREPARATION) @@ -4852,6 +4932,14 @@ fun LiveTvScreen( }, modifier = Modifier.fillMaxSize(), ) + if (isFullScreen && !fullscreenGuideOpen && !quickZapOpen && liveAiSubsOn) { + LiveAiSubtitleOverlay( + cue = liveAiCue, + style = liveSubtitleStyle, + preparingSeconds = liveAiPreparingSeconds, + debugStats = liveAiStats.takeIf { liveAiDebugVisible }, + ) + } if (isFullScreen && !fullscreenGuideOpen && !quickZapOpen) { val categoryTitle = playingChannel?.source?.group?.takeIf { it.isNotBlank() } ?: visibleEnrichedState.value.tree.byId(selectedCategoryId)?.label @@ -4885,7 +4973,12 @@ fun LiveTvScreen( }, onGuideClick = { openFullscreenGuide() }, onSubtitlesClick = { subtitlePickerOpen = true } - .takeIf { liveSubtitleOptions(liveTracks, "").size > 1 }, + .takeIf { + withAiSubtitlesOption( + liveSubtitleOptions(liveTracks, ""), "", "", + aiAvailable = liveAiSubsSettings.enabled, aiSelected = false, + ).size > 1 + }, onAudioClick = { audioPickerOpen = true } .takeIf { liveAudioOptions(liveTracks).size > 1 }, onOpenVariants = playingChannel?.takeIf { playingCatchupProgram == null }?.let { channel -> @@ -5153,14 +5246,32 @@ fun LiveTvScreen( title = stringResource(R.string.live_subtitles), icon = Icons.Filled.Subtitles, channelName = playingChannel?.name.orEmpty(), - options = liveSubtitleOptions(liveTracks, stringResource(R.string.live_subtitles_off)), + options = withAiSubtitlesOption( + liveSubtitleOptions(liveTracks, stringResource(R.string.live_subtitles_off)), + offLabel = stringResource(R.string.live_subtitles_off), + aiLabel = stringResource(R.string.live_ai_subtitles), + aiAvailable = liveAiSubsSettings.enabled, + aiSelected = liveAiSubsOn, + ), onDismiss = { subtitlePickerOpen = false returnToHudControls() }, onPick = { key -> - playingChannelId?.let { liveSubtitleChoices[it] = key } - applyLiveSubtitleChoice(exoPlayer, key) + if (key == LIVE_SUBTITLES_AI) { + // AI subtitles replace the stream's own; remembered per channel. + playingChannelId?.let { + liveAiSubsChoices[it] = true + liveSubtitleChoices[it] = LIVE_SUBTITLES_OFF + } + applyLiveSubtitleChoice(exoPlayer, LIVE_SUBTITLES_OFF) + } else { + playingChannelId?.let { + liveAiSubsChoices[it] = false + liveSubtitleChoices[it] = key + } + applyLiveSubtitleChoice(exoPlayer, key) + } }, ) } @@ -5494,6 +5605,12 @@ private fun classifyPlaybackError(error: PlaybackException): String { } } +/** How often Live TV AI subtitles follow the player position (cue display, hold, look-ahead). */ +private const val LIVE_AI_TICK_MS = 100L + +/** Turned on mid-channel, playback waits at most this long for the look-ahead to build up. */ +private const val LIVE_AI_WARMUP_TIMEOUT_MS = 20_000L + internal fun buildLiveTvConfiguration(): MediaItem.LiveConfiguration = MediaItem.LiveConfiguration.Builder() // Respect the manifest's segment/hold-back timing. An arbitrary eight-second diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiSegmentEngine.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiSegmentEngine.kt new file mode 100644 index 0000000000..531dac34f6 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiSegmentEngine.kt @@ -0,0 +1,317 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import com.arflix.tv.ui.screens.player.subtitles.BLOCKED_FINISH_REASONS +import com.arflix.tv.ui.screens.player.subtitles.GEMINI_MAX_TRANSIENT_RETRIES +import com.arflix.tv.ui.screens.player.subtitles.GEMINI_RETRYABLE_HTTP +import com.arflix.tv.ui.screens.player.subtitles.GEMINI_RETRY_BASE_DELAY_MS +import com.arflix.tv.ui.screens.player.subtitles.extractJsonArray +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONArray +import org.json.JSONObject +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.TimeUnit + +/** One stretch of speech cut from the channel's audio, on the media timeline. */ +internal class SpeechSegment( + val startUs: Long, + val endUs: Long, + /** 16 kHz mono samples in [-1, 1]. */ + val pcm16k: FloatArray, +) + +/** What an engine made of one segment. */ +internal sealed interface LiveEngineResult { + data class Cues(val cues: List, val sourceLanguage: String?) : LiveEngineResult + + /** Nothing more will work until the user fixes it (wrong key). */ + data object InvalidKey : LiveEngineResult + + /** Quota or rate limit (HTTP 429): pause and back off. */ + data object QuotaExceeded : LiveEngineResult + + /** No connection. */ + data object Offline : LiveEngineResult + + /** This segment only (content policy, malformed reply, server error). */ + data class Failed(val reason: String) : LiveEngineResult +} + +/** Turns speech segments into timed subtitle lines in the viewer's language. */ +internal interface LiveSubtitleEngine { + /** + * [context] holds the last few lines already shown (in the target language), so names, + * gender and register stay consistent across segments. + */ + suspend fun process(segment: SpeechSegment, context: List, targetLanguageName: String, rtl: Boolean): LiveEngineResult +} + +/** + * Gemini `generateContent` with the segment as inline WAV audio: transcribes and translates in + * one call, and returns lines that are timed from the segment's own media time. Plain HTTP per + * segment, so there is no long-lived session to expire over hours of TV. + * + * Follows the app's Gemini rules (see SubtitleTranslationService): JSON response mode, minimal + * thinking, BLOCK_NONE safety settings, short retries on 500/502/503 and network errors, never on + * 429. The key goes in a header, never in a URL or the log. + */ +internal class GeminiSegmentEngine( + private val apiKey: () -> String, + private val client: OkHttpClient = DEFAULT_CLIENT, + private val baseUrl: String = GEMINI_LIVE_URL, +) : LiveSubtitleEngine { + + override suspend fun process( + segment: SpeechSegment, + context: List, + targetLanguageName: String, + rtl: Boolean, + ): LiveEngineResult = withContext(Dispatchers.IO) { + val key = apiKey() + if (key.isBlank()) return@withContext LiveEngineResult.InvalidKey + val body = GeminiLiveProtocol.requestBody(segment, context, targetLanguageName) + var transient = 0 + while (true) { + val request = Request.Builder() + .url(baseUrl) + .header("x-goog-api-key", key) + .post(body.toRequestBody(JSON_MEDIA_TYPE)) + .build() + try { + client.newCall(request).execute().use { response -> + val text = response.body?.string().orEmpty() + when { + response.isSuccessful -> + return@withContext GeminiLiveProtocol.parseResponse(text, segment, rtl) + response.code == 429 -> return@withContext LiveEngineResult.QuotaExceeded + GeminiLiveProtocol.isInvalidKey(response.code, text) -> return@withContext LiveEngineResult.InvalidKey + response.code in GEMINI_RETRYABLE_HTTP && transient < GEMINI_MAX_TRANSIENT_RETRIES -> Unit + else -> return@withContext LiveEngineResult.Failed("HTTP ${response.code}") + } + } + } catch (error: CancellationException) { + throw error + } catch (error: IOException) { + if (transient >= GEMINI_MAX_TRANSIENT_RETRIES) { + return@withContext if (error is java.net.UnknownHostException || error is java.net.ConnectException) { + LiveEngineResult.Offline + } else { + LiveEngineResult.Failed("network: ${error.javaClass.simpleName}") + } + } + } + // ±30% jittered exponential backoff, as in SubtitleTranslationService. + val jitter = 0.7 + Math.random() * 0.6 + delay((GEMINI_RETRY_BASE_DELAY_MS * (1L shl transient) * jitter).toLong()) + transient++ + } + @Suppress("UNREACHABLE_CODE") + LiveEngineResult.Failed("unreachable") + } + + /** A cheap call that tells whether [key] is accepted: true, false, or null when it could not be checked. */ + suspend fun validateKey(key: String): Boolean? = withContext(Dispatchers.IO) { + val request = Request.Builder() + .url(GEMINI_MODEL_INFO_URL) + .header("x-goog-api-key", key.trim()) + .get() + .build() + try { + client.newCall(request).execute().use { response -> + when { + response.isSuccessful -> true + GeminiLiveProtocol.isInvalidKey(response.code, response.body?.string().orEmpty()) -> false + else -> null + } + } + } catch (error: CancellationException) { + throw error + } catch (_: IOException) { + null + } + } + + companion object { + /** + * Same model as the app's subtitle translation: fast and cheap enough for continuous use, + * and it takes audio input. + */ + const val MODEL_ID = "gemini-3.5-flash-lite" + const val GEMINI_LIVE_URL = "https://generativelanguage.googleapis.com/v1beta/models/$MODEL_ID:generateContent" + private const val GEMINI_MODEL_INFO_URL = "https://generativelanguage.googleapis.com/v1beta/models/$MODEL_ID" + private val JSON_MEDIA_TYPE = "application/json".toMediaType() + private val DEFAULT_CLIENT: OkHttpClient by lazy { + OkHttpClient.Builder() + .connectTimeout(8, TimeUnit.SECONDS) + // A segment's answer is useless once its moment has passed. + .readTimeout(12, TimeUnit.SECONDS) + .callTimeout(15, TimeUnit.SECONDS) + .build() + } + } +} + +/** The request and response format for [GeminiSegmentEngine]; pure, so it is unit tested. */ +internal object GeminiLiveProtocol { + + fun systemPrompt(targetLanguageName: String): String = """ + You write live TV subtitles. The audio is a few seconds of a TV broadcast. + Transcribe what is said and translate it into $targetLanguageName. If the speech is + already in $targetLanguageName, return the transcription. + Write natural, idiomatic $targetLanguageName as a professional subtitler would: short + lines, correct grammatical gender, keep names as they are spoken. + Split the text into subtitle lines of at most about 42 characters, in spoken order. + For each line give startOffsetMs, the time from the start of the audio when it begins. + If there is no intelligible speech (music, noise), return an empty "lines" array. + Never describe sounds, never add notes or explanations. + Reply with JSON only: + {"lines":[{"text":"...","startOffsetMs":0}],"sourceLanguage":"en"} + """.trimIndent() + + fun requestBody(segment: SpeechSegment, context: List, targetLanguageName: String): String { + val wav = android.util.Base64.encodeToString(wav16kMono(segment.pcm16k), android.util.Base64.NO_WRAP) + val userText = buildString { + if (context.isNotEmpty()) { + append("Previous subtitles, for context only (do not repeat them):\n") + context.forEach { append("- ").append(it).append('\n') } + } + append("Subtitle this audio in ").append(targetLanguageName).append('.') + } + return JSONObject().apply { + put("system_instruction", JSONObject().put("parts", JSONArray().put(JSONObject().put("text", systemPrompt(targetLanguageName))))) + put( + "contents", + JSONArray().put( + JSONObject().put( + "parts", + JSONArray() + .put(JSONObject().put("inline_data", JSONObject().put("mime_type", "audio/wav").put("data", wav))) + .put(JSONObject().put("text", userText)), + ), + ), + ) + put( + "generationConfig", + JSONObject() + .put("temperature", 0.2) + .put("responseMimeType", "application/json") + .put("thinkingConfig", JSONObject().put("thinkingLevel", "minimal")), + ) + put( + "safetySettings", + JSONArray().apply { + listOf( + "HARM_CATEGORY_HARASSMENT", + "HARM_CATEGORY_HATE_SPEECH", + "HARM_CATEGORY_SEXUALLY_EXPLICIT", + "HARM_CATEGORY_DANGEROUS_CONTENT", + ).forEach { put(JSONObject().put("category", it).put("threshold", "BLOCK_NONE")) } + }, + ) + }.toString() + } + + /** A Gemini `generateContent` HTTP 200 body → cues on the segment's media time. */ + fun parseResponse(body: String, segment: SpeechSegment, rtl: Boolean): LiveEngineResult { + val json = runCatching { JSONObject(body) }.getOrNull() ?: return LiveEngineResult.Failed("unreadable reply") + val candidate = json.optJSONArray("candidates")?.optJSONObject(0) + ?: return LiveEngineResult.Failed( + if (json.optJSONObject("promptFeedback")?.optString("blockReason").isNullOrBlank()) "no candidates" else "blocked", + ) + val parts = candidate.optJSONObject("content")?.optJSONArray("parts") + val text = buildString { + for (i in 0 until (parts?.length() ?: 0)) { + val part = parts!!.optJSONObject(i) ?: continue + if (part.optBoolean("thought", false)) continue + append(part.optString("text")) + } + }.trim() + if (text.isBlank()) { + return LiveEngineResult.Failed( + if (candidate.optString("finishReason") in BLOCKED_FINISH_REASONS) "blocked" else "empty reply", + ) + } + return parseLines(text, segment, rtl) + } + + /** + * The model's JSON text → cues. Accepts the documented object, a bare array of lines or strings + * (JSON mode sometimes drops the wrapper), and fenced or truncated output via the shared + * [extractJsonArray]. + */ + fun parseLines(text: String, segment: SpeechSegment, rtl: Boolean): LiveEngineResult { + val obj = runCatching { JSONObject(text.trim()) }.getOrNull() + val array: JSONArray = obj?.optJSONArray("lines") + ?: extractJsonArray(text) + ?: return LiveEngineResult.Failed("malformed reply") + val language = obj?.optString("sourceLanguage")?.takeIf { it.isNotBlank() } + val raw = ArrayList>() + for (i in 0 until array.length()) { + val item = array.opt(i) + when (item) { + is JSONObject -> { + val line = cleanLine(item.optString("text")) + val offset = if (item.has("startOffsetMs")) item.optLong("startOffsetMs", -1L).takeIf { it >= 0 } else null + if (line.isNotEmpty()) raw += line to offset + } + is String -> cleanLine(item).takeIf { it.isNotEmpty() }?.let { raw += it to null } + } + } + return LiveEngineResult.Cues(timeLines(raw, segment, rtl), language) + } + + /** + * Lines with offsets start there (kept in order, inside the segment); the rest share the + * segment by text length. Each line ends where the next begins; the last at the segment end. + */ + fun timeLines(lines: List>, segment: SpeechSegment, rtl: Boolean): List { + if (lines.isEmpty()) return emptyList() + val duration = (segment.endUs - segment.startUs).coerceAtLeast(1L) + val totalChars = lines.sumOf { it.first.length }.coerceAtLeast(1) + var proportional = 0L + var previousStart = segment.startUs + val starts = lines.map { (text, offsetMs) -> + val byLength = segment.startUs + duration * proportional / totalChars + proportional += text.length + val start = offsetMs?.let { segment.startUs + it * 1_000L } ?: byLength + start.coerceIn(previousStart, segment.endUs - 1).also { previousStart = it } + } + return lines.mapIndexed { index, (text, _) -> + LiveCue( + startUs = starts[index], + endUs = starts.getOrNull(index + 1)?.takeIf { it > starts[index] } ?: segment.endUs, + text = text, + rtl = rtl, + ) + } + } + + fun isInvalidKey(code: Int, body: String): Boolean = + code == 401 || code == 403 || + (code == 400 && (body.contains("API_KEY_INVALID") || body.contains("API key not valid"))) + + private fun cleanLine(text: String): String = text.replace('\n', ' ').replace(Regex("\\s+"), " ").trim() + + /** 16-bit little-endian mono WAV at 16 kHz. */ + fun wav16kMono(samples: FloatArray): ByteArray { + val dataBytes = samples.size * 2 + val buffer = ByteBuffer.allocate(44 + dataBytes).order(ByteOrder.LITTLE_ENDIAN) + buffer.put("RIFF".toByteArray(Charsets.US_ASCII)).putInt(36 + dataBytes) + buffer.put("WAVE".toByteArray(Charsets.US_ASCII)) + buffer.put("fmt ".toByteArray(Charsets.US_ASCII)).putInt(16) + .putShort(1).putShort(1).putInt(16_000).putInt(32_000).putShort(2).putShort(16) + buffer.put("data".toByteArray(Charsets.US_ASCII)).putInt(dataBytes) + for (sample in samples) { + buffer.putShort((sample.coerceIn(-1f, 1f) * Short.MAX_VALUE).toInt().toShort()) + } + return buffer.array() + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsDebug.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsDebug.kt index c1ef08e87a..18fff2a076 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsDebug.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsDebug.kt @@ -4,31 +4,17 @@ import com.arflix.tv.BuildConfig import java.util.concurrent.TimeUnit /** - * Phase 0 switches for the Live TV AI subtitles proof of concept, set over adb so no UI is - * needed yet (debug builds only; release builds always read "off"): - * - * - `adb shell setprop debug.arvio.liveaisubs 1` taps the channel's audio and logs segments. - * - `adb shell setprop debug.arvio.liveaisubs.delay 10` holds playback until 10 s are buffered - * (0 = no hold). Read when a channel is prepared. - * - `adb shell setprop debug.arvio.liveaisubs.verify 1` also runs the VAD on the audio as it is - * played, to compare its segment times with the look-ahead tap's (doubles the VAD cost). - * - * `debug.*` properties are writable from `adb shell` without root. They are polled on a daemon - * thread, because reading them means running `getprop`, which must not happen on the UI thread. + * Hidden diagnostics for Live TV AI subtitles, debug builds only: + * `adb shell setprop debug.arvio.liveaisubs.overlay 1` shows look-ahead, AI latency p50/p95, + * late cues and segments per minute over the picture, and logs HLS segment details under + * `LiveAiSubs`. `debug.*` properties are writable from `adb shell` without root. They are polled + * on a daemon thread, because reading one means running `getprop`, never on the UI thread. */ internal object LiveAiSubsDebug { @Volatile var enabled: Boolean = false private set - @Volatile - var delayMs: Long = 0L - private set - - @Volatile - var verify: Boolean = false - private set - @Volatile private var started = false @@ -46,15 +32,7 @@ internal object LiveAiSubsDebug { private fun pollLoop() { while (true) { - val nowEnabled = read("debug.arvio.liveaisubs") == "1" - val nowDelay = read("debug.arvio.liveaisubs.delay")?.toLongOrNull()?.coerceIn(0L, 30L) ?: 0L - val nowVerify = read("debug.arvio.liveaisubs.verify") == "1" - if (nowEnabled != enabled || nowDelay * 1_000L != delayMs || nowVerify != verify) { - LiveAiSubsLog.i("debug switches: enabled=$nowEnabled delay=${nowDelay}s verify=$nowVerify") - } - enabled = nowEnabled - delayMs = if (nowEnabled) nowDelay * 1_000L else 0L - verify = nowEnabled && nowVerify + enabled = read("debug.arvio.liveaisubs.overlay") == "1" try { Thread.sleep(POLL_MS) } catch (_: InterruptedException) { @@ -70,5 +48,5 @@ internal object LiveAiSubsDebug { value.ifEmpty { null } }.getOrNull() - private const val POLL_MS = 2_000L + private const val POLL_MS = 3_000L } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsSettings.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsSettings.kt new file mode 100644 index 0000000000..8536e90553 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsSettings.kt @@ -0,0 +1,116 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.content.Context +import androidx.datastore.preferences.core.Preferences +import androidx.datastore.preferences.core.booleanPreferencesKey +import androidx.datastore.preferences.core.edit +import androidx.datastore.preferences.core.stringPreferencesKey +import com.arflix.tv.data.repository.effectiveSettingsDataStore as settingsDataStore +import com.arflix.tv.ui.screens.player.subtitles.SubtitleAiModel +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.map + +/** How long live playback is held back so subtitles arrive before their line is heard. */ +internal enum class LiveAiSubsDelay(val prefValue: String, val fixedMs: Long?) { + /** Uses the measured AI latency; see [autoLiveAiSubsDelayMs]. */ + AUTO("auto", null), + SIX("6", 6_000L), + TEN("10", 10_000L), + FIFTEEN("15", 15_000L); + + fun next(): LiveAiSubsDelay = entries[(ordinal + 1) % entries.size] + + companion object { + /** + * Auto: Gemini's answer time for a few seconds of audio was measured at 5-9 s on a real + * box, so no fixed value suits every connection; Auto follows the measurement. + */ + val DEFAULT = AUTO + fun fromPref(value: String?): LiveAiSubsDelay = entries.firstOrNull { it.prefValue == value } ?: DEFAULT + } +} + +/** + * Per channel by default: with the feature on, every channel change waits for the delay, so only + * channels the viewer picked "AI subtitles" on pay that cost. + */ +internal enum class LiveAiSubsActivation(val prefValue: String) { + PER_CHANNEL("per_channel"), + ALWAYS("always"); + + fun next(): LiveAiSubsActivation = if (this == PER_CHANNEL) ALWAYS else PER_CHANNEL + + companion object { + fun fromPref(value: String?): LiveAiSubsActivation = entries.firstOrNull { it.prefValue == value } ?: PER_CHANNEL + } +} + +internal data class LiveAiSubsSettings( + val enabled: Boolean = false, + val delay: LiveAiSubsDelay = LiveAiSubsDelay.DEFAULT, + val activation: LiveAiSubsActivation = LiveAiSubsActivation.PER_CHANNEL, + /** A language code, or blank for the profile's subtitle language. */ + val targetLanguage: String = "", + /** The Gemini key in effect: the AI subtitles key when that is a Gemini key, else [ownApiKey]. */ + val apiKey: String = "", + /** A Gemini key just for Live TV, used while the AI subtitles model is not Gemini. */ + val ownApiKey: String = "", + /** True while the shared AI subtitles key is a Gemini key (and so is the one used here). */ + val usesSharedKey: Boolean = true, +) { + val hasKey: Boolean get() = apiKey.isNotBlank() +} + +/** + * Global (not per-profile) settings for Live TV AI subtitles, next to the existing AI subtitle + * key and model in the same DataStore, and synced to the cloud with them (CloudSyncRepository). + */ +internal object LiveAiSubsPrefs { + val ENABLED = booleanPreferencesKey("live_ai_subs_enabled") + val DELAY = stringPreferencesKey("live_ai_subs_delay") + val ACTIVATION = stringPreferencesKey("live_ai_subs_activation") + val TARGET_LANGUAGE = stringPreferencesKey("live_ai_subs_target_language") + val OWN_API_KEY = stringPreferencesKey("live_ai_subs_api_key") + private val SHARED_API_KEY = stringPreferencesKey("subtitle_ai_api_key") + private val SHARED_MODEL = stringPreferencesKey("subtitle_ai_model") + + /** Target language choices offered in settings, cycled in this order ("" = profile default). */ + val TARGET_LANGUAGE_CHOICES = listOf("", "he", "en", "ar", "ru", "es", "fr", "de", "pt", "it") + + fun read(prefs: Preferences): LiveAiSubsSettings { + val sharedIsGemini = prefs[SHARED_MODEL] == SubtitleAiModel.GEMINI_FLASH_25.name + val shared = prefs[SHARED_API_KEY].orEmpty().trim() + val own = prefs[OWN_API_KEY].orEmpty().trim() + return LiveAiSubsSettings( + enabled = prefs[ENABLED] ?: false, + delay = LiveAiSubsDelay.fromPref(prefs[DELAY]), + activation = LiveAiSubsActivation.fromPref(prefs[ACTIVATION]), + targetLanguage = prefs[TARGET_LANGUAGE].orEmpty(), + apiKey = if (sharedIsGemini && shared.isNotBlank()) shared else own, + ownApiKey = own, + usesSharedKey = sharedIsGemini && shared.isNotBlank(), + ) + } + + fun flow(context: Context): Flow = context.settingsDataStore.data.map(::read) + + suspend fun setEnabled(context: Context, enabled: Boolean) { + context.settingsDataStore.edit { it[ENABLED] = enabled } + } + + suspend fun setDelay(context: Context, delay: LiveAiSubsDelay) { + context.settingsDataStore.edit { it[DELAY] = delay.prefValue } + } + + suspend fun setActivation(context: Context, activation: LiveAiSubsActivation) { + context.settingsDataStore.edit { it[ACTIVATION] = activation.prefValue } + } + + suspend fun setTargetLanguage(context: Context, code: String) { + context.settingsDataStore.edit { it[TARGET_LANGUAGE] = code } + } + + suspend fun setOwnApiKey(context: Context, key: String) { + context.settingsDataStore.edit { it[OWN_API_KEY] = key.trim() } + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt new file mode 100644 index 0000000000..ef1e2c1259 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt @@ -0,0 +1,108 @@ +@file:androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) + +package com.arflix.tv.ui.screens.tv.live.aisubs + +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.res.stringResource +import androidx.compose.ui.text.font.FontFamily +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.compose.ui.viewinterop.AndroidView +import androidx.media3.common.text.Cue +import androidx.media3.ui.SubtitleView +import androidx.tv.material3.Text +import com.arflix.tv.R +import com.arflix.tv.ui.screens.player.applySubtitleAppearance + +/** The profile's subtitle look, as the VOD player uses it. */ +internal data class LiveSubtitleStyle( + val sizePreference: String = "Medium", + val sizePercent: Int = 100, + val verticalPercent: Int = 2, + val color: String = "White", + val style: String = "Bold", + val font: String = "", +) + +/** + * Fullscreen layer over the Live TV picture: the current AI subtitle (drawn by Media3's + * SubtitleView with the profile's subtitle style, so right-to-left languages lay out natively), + * the "Preparing subtitles… N s" note while playback is held, and the hidden debug panel. + */ +@Composable +internal fun LiveAiSubtitleOverlay( + cue: LiveCue?, + style: LiveSubtitleStyle, + preparingSeconds: Int?, + debugStats: LiveAiSubsStats?, + modifier: Modifier = Modifier, +) { + Box(modifier = modifier.fillMaxSize()) { + AndroidView( + factory = { context -> + SubtitleView(context).apply { + isFocusable = false + applyStyle(this, style) + } + }, + update = { view -> + applyStyle(view, style) + view.setCues(cue?.let { listOf(Cue.Builder().setText(it.text).build()) } ?: emptyList()) + }, + modifier = Modifier.fillMaxSize(), + ) + if (preparingSeconds != null) { + Text( + text = stringResource(R.string.live_ai_subs_preparing, preparingSeconds.coerceAtLeast(1)), + color = Color.White, + fontSize = 16.sp, + modifier = Modifier + .align(Alignment.TopCenter) + .padding(top = 32.dp) + .background(Color.Black.copy(alpha = 0.65f), RoundedCornerShape(18.dp)) + .padding(horizontal = 18.dp, vertical = 8.dp), + ) + } + if (debugStats != null) { + Text( + text = with(debugStats) { + "look-ahead ${lookaheadMs?.let { "%.1f s".format(it / 1000f) } ?: "-"}\n" + + "AI p50 ${latencyP50Ms ?: "-"} ms p95 ${latencyP95Ms ?: "-"} ms\n" + + "segments/min $segmentsPerMinute late dropped $lateDropped\n" + + "skipped $skippedSegments backlog skips $backlogSkips failures $failures" + }, + color = Color(0xFFB9F6CA), + fontSize = 12.sp, + fontFamily = FontFamily.Monospace, + modifier = Modifier + // Left in every layout direction, clear of the channel title (top end in RTL). + .align(androidx.compose.ui.AbsoluteAlignment.TopLeft) + .padding(top = 120.dp, start = 24.dp) + .background(Color.Black.copy(alpha = 0.7f), RoundedCornerShape(6.dp)) + .padding(10.dp), + ) + } + } +} + +private fun applyStyle(view: SubtitleView, style: LiveSubtitleStyle) { + view.applySubtitleAppearance( + context = view.context, + sizePreference = style.sizePreference, + sizePercent = style.sizePercent, + verticalPercent = style.verticalPercent, + colorPreference = style.color, + stylePreference = style.style, + fontPreference = style.font, + preserveEmbeddedStyles = false, + inPictureInPicture = false, + ) +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt index 3c042f6c34..4f1beec8e0 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt @@ -73,7 +73,7 @@ private class AudioTapHlsChunkExtractor( private val sink: AudioSampleSink, ) : HlsMediaChunkExtractor { override fun init(extractorOutput: ExtractorOutput) = - delegate.init(AudioTapExtractorOutput(TrackLoggingOutput(extractorOutput), sink)) + delegate.init(AudioTapExtractorOutput(extractorOutput, sink)) override fun read(extractorInput: ExtractorInput): Boolean = delegate.read(extractorInput) @@ -89,11 +89,3 @@ private class AudioTapHlsChunkExtractor( delegate.onTruncatedSegmentParsed() } } - -/** Phase 0 diagnostics: logs which tracks a segment declares, to explain channels with no tapped audio. */ -private class TrackLoggingOutput(private val delegate: ExtractorOutput) : ExtractorOutput by delegate { - override fun track(id: Int, type: Int): androidx.media3.extractor.TrackOutput { - if (LiveAiSubsDebug.enabled) LiveAiSubsLog.i("hls segment track id=$id type=$type") - return delegate.track(id, type) - } -} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueScheduler.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueScheduler.kt new file mode 100644 index 0000000000..3b8bc3027d --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueScheduler.kt @@ -0,0 +1,131 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +/** One subtitle line on the media timeline (the player's period time, in microseconds). */ +internal data class LiveCue( + val startUs: Long, + val endUs: Long, + val text: String, + val rtl: Boolean = false, +) + +/** + * Keeps AI subtitle cues in media time and answers "what is on screen at this position". Cues are + * shown by where playback is, never by when the AI answered, so pausing, rebuffering and fast + * dialogue cannot push lines out of step with the picture. + * + * Display rules: each cue stays up at least [MIN_DISPLAY_US] and at most [MAX_DISPLAY_US], and is + * cut short by the next cue. Text longer than [SPLIT_CHARS] is split into two cues, timed by + * length. A cue that arrives after its start still shows while it is inside its display window; + * one that arrives after the window has closed is dropped and counted in [lateDropped]. + * + * Not thread-safe; the session calls it from one thread (the main thread). + */ +internal class LiveCueScheduler { + private val cues = ArrayList() + + var lateDropped: Int = 0 + private set + + var shown: Int = 0 + private set + + private var lastShown: LiveCue? = null + + /** Adds cues from one segment. [positionUs] is where playback is now, for the late-cue rule. */ + fun add(newCues: List, positionUs: Long) { + for (cue in newCues.flatMap(::split)) { + if (cue.text.isBlank()) continue + val provisionalEnd = displayEnd(cue, next = null) + if (positionUs >= provisionalEnd) { + lateDropped++ + continue + } + val index = cues.indexOfFirst { it.startUs > cue.startUs }.let { if (it < 0) cues.size else it } + cues.add(index, cue) + } + } + + /** The cue on screen at [positionUs], or null. Forgets cues that are over. */ + fun cueAt(positionUs: Long): LiveCue? { + var result: LiveCue? = null + for (i in cues.indices) { + val cue = cues[i] + if (cue.startUs > positionUs) break + val end = displayEnd(cue, cues.getOrNull(i + 1)) + if (positionUs < end) result = cue + } + // Drop cues whose window closed well before the position (bounded memory). + cues.removeAll { displayEnd(it, null) < positionUs - FORGET_AFTER_US } + if (result != null && result !== lastShown) shown++ + lastShown = result + return result + } + + fun clear() { + cues.clear() + lastShown = null + } + + val pendingCount: Int get() = cues.size + + private fun displayEnd(cue: LiveCue, next: LiveCue?): Long { + val natural = cue.endUs.coerceAtLeast(cue.startUs + MIN_DISPLAY_US) + val capped = natural.coerceAtMost(cue.startUs + MAX_DISPLAY_US) + return if (next != null && next.startUs > cue.startUs) minOf(capped, next.startUs) else capped + } + + companion object { + const val MIN_DISPLAY_US = 1_200_000L + const val MAX_DISPLAY_US = 6_000_000L + const val SPLIT_CHARS = 84 + private const val FORGET_AFTER_US = 30_000_000L + + /** + * Splits a long line into two cues at the word boundary nearest its middle, each getting a + * share of the time proportional to its length. + */ + fun split(cue: LiveCue): List { + val text = cue.text.trim() + if (text.length <= SPLIT_CHARS) return listOf(cue.copy(text = text)) + val middle = text.length / 2 + val spaces = text.indices.filter { text[it] == ' ' } + val cut = spaces.minByOrNull { kotlin.math.abs(it - middle) } ?: return listOf(cue.copy(text = text)) + val first = text.substring(0, cut).trim() + val second = text.substring(cut + 1).trim() + if (first.isEmpty() || second.isEmpty()) return listOf(cue.copy(text = text)) + val duration = (cue.endUs - cue.startUs).coerceAtLeast(0L) + val splitAt = cue.startUs + duration * first.length / (first.length + second.length) + return listOf( + cue.copy(endUs = splitAt, text = first), + cue.copy(startUs = splitAt, text = second), + ) + } + } +} + +/** + * The hold ("Auto" delay) that lets a whole segment be heard, sent and answered before its first + * word plays: the longest segment, plus the AI's p95 latency, plus a margin. Before any latency has + * been measured, it assumes [DEFAULT_LATENCY_P95_MS]. Clamped to 6–15 s, the range offered in settings. + */ +internal fun autoLiveAiSubsDelayMs(latencyP95Ms: Long?): Long { + val latency = latencyP95Ms ?: DEFAULT_LATENCY_P95_MS + return (LIVE_MAX_SEGMENT_MS + latency + DELAY_MARGIN_MS).coerceIn(6_000L, 15_000L) +} + +/** The hold for [delay], given the measured latency for Auto. */ +internal fun liveAiSubsDelayMs(delay: LiveAiSubsDelay, latencyP95Ms: Long?): Long = + delay.fixedMs ?: autoLiveAiSubsDelayMs(latencyP95Ms) + +/** The 95th percentile of [samples], or null with fewer than [minSamples]. */ +internal fun percentile95(samples: List, minSamples: Int = 3): Long? { + if (samples.size < minSamples) return null + val sorted = samples.sorted() + val index = ((sorted.size - 1) * 0.95).let { kotlin.math.ceil(it).toInt() }.coerceIn(0, sorted.size - 1) + return sorted[index] +} + +internal const val LIVE_MAX_SEGMENT_MS = 5_000L +/** Measured on a Mi Box 4 with gemini-3.5-flash-lite: p50 about 5 s, p95 about 8.5 s. */ +private const val DEFAULT_LATENCY_P95_MS = 5_000L +private const val DELAY_MARGIN_MS = 1_000L diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt new file mode 100644 index 0000000000..a82d63a4f3 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt @@ -0,0 +1,226 @@ +@file:androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) + +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.content.Context +import android.os.Process +import android.os.SystemClock +import androidx.media3.common.Format +import com.arflix.tv.R +import com.arflix.tv.ui.screens.player.audiosync.AudioSampleSink +import com.arflix.tv.ui.screens.player.audiosync.AudioSyncDecoder +import com.arflix.tv.ui.screens.player.audiosync.SileroVad +import com.arflix.tv.ui.screens.player.audiosync.SileroVadWeights +import com.arflix.tv.ui.screens.player.audiosync.SpeechAnalyzer +import com.arflix.tv.ui.screens.player.audiosync.SpeechTimeline +import com.arflix.tv.ui.screens.player.audiosync.asr.SpeechSegmenter + +/** + * Live TV speech capture: copies the channel's compressed audio at the extractor, as it is + * downloaded and before it is played; decodes it on a side thread; and cuts it into speech + * segments with Silero. Each [SpeechSegment] carries its own media time, which the player's + * period position shares (measured in Phase 0: the same 32 ms frame), so subtitles can be timed + * exactly however late the AI answers. + * + * One pipeline lives as long as the Live TV player. [newSink] is called for every prepared channel + * source; a sink from an earlier channel goes quiet at once, so a load still running for the + * previous channel can never feed the new one. Nothing is captured unless [active]. + * + * Threads: sinks run on the player's loading threads, segments are delivered on the decoder + * thread, and [onPosition] comes from the UI thread. + */ +internal class LiveSpeechPipeline( + context: Context, + private val onSegment: (SpeechSegment) -> Unit, + /** The channel's audio format cannot be decoded ahead of playback on this device. */ + private val onUnsupportedAudio: (mimeType: String) -> Unit, +) { + private val appContext = context.applicationContext + + @Volatile private var weights: SileroVadWeights? = null + @Volatile private var generation = 0 + @Volatile private var decoder: AudioSyncDecoder? = null + + /** Capture only while AI subtitles are on for the channel and it is playing fullscreen. */ + @Volatile + var active: Boolean = false + + // Where playback is, from the UI thread; extrapolated on other threads. + @Volatile private var positionUs = UNKNOWN + @Volatile private var positionAtMs = 0L + @Volatile private var positionPlaying = false + + /** Media time of the newest analysed audio, to detect when analysis falls behind playback. */ + @Volatile private var analysedUs = UNKNOWN + @Volatile private var lastTapUs = UNKNOWN + + /** Decoder thread only. */ + private var skipping = false + + @Volatile var backlogSkips: Int = 0 + private set + @Volatile var segmentsEmitted: Int = 0 + private set + + init { + Thread({ + weights = runCatching { + appContext.resources.openRawResource(R.raw.silero_vad_v5_16k).use { SileroVadWeights.read(it) } + }.onFailure { LiveAiSubsLog.w("could not load VAD weights: ${it.message}") }.getOrNull() + }, "LiveAiSubsInit").apply { + isDaemon = true + start() + } + } + + /** True once the VAD model is loaded. */ + val ready: Boolean get() = weights != null + + /** How far ahead of playback the newest copied audio is, or null when unknown. */ + fun tapAheadUs(): Long? { + val tap = lastTapUs + val position = estimatedPositionUs() + return if (tap == UNKNOWN || position == UNKNOWN) null else tap - position + } + + /** A sink for one prepared channel source. */ + fun newSink(): AudioSampleSink { + val sinkGeneration = ++generation + reset() + return ChannelSink(sinkGeneration) + } + + /** Forget everything in flight (channel change, seek, resume). */ + fun reset() { + analysedUs = UNKNOWN + lastTapUs = UNKNOWN + decoder?.discontinuity() + } + + /** From the UI thread, about every 100–250 ms: the player's period position. */ + fun onPosition(periodPositionUs: Long, playing: Boolean) { + positionUs = periodPositionUs + positionAtMs = SystemClock.elapsedRealtime() + positionPlaying = playing + } + + fun estimatedPositionUs(): Long { + val position = positionUs + if (position == UNKNOWN) return UNKNOWN + return if (positionPlaying) position + (SystemClock.elapsedRealtime() - positionAtMs) * 1_000L else position + } + + fun release() { + generation++ + active = false + decoder?.release() + decoder = null + } + + private fun ensureDecoder(): AudioSyncDecoder? { + decoder?.let { return it } + val vadWeights = weights ?: return null + synchronized(this) { + decoder?.let { return it } + val analyzer = SpeechAnalyzer(SileroVad(vadWeights), SpeechTimeline()) + val segmenter = SpeechSegmenter( + maxSegmentFrames = MAX_SEGMENT_FRAMES, + softMaxSegmentFrames = SOFT_MAX_SEGMENT_FRAMES, + ) { startFrame, samples -> + val startUs = startFrame.toLong() * SpeechTimeline.FRAME_DURATION_US + val endUs = startUs + samples.size * 1_000_000L / SileroVad.SAMPLE_RATE + segmentsEmitted++ + onSegment(SpeechSegment(startUs, endUs, samples)) + } + analyzer.chunkListener = object : SpeechAnalyzer.ChunkListener { + override fun onChunk(frame: Int, chunk: FloatArray, probability: Float) { + val chunkUs = frame.toLong() * SpeechTimeline.FRAME_DURATION_US + analysedUs = chunkUs + if (isBehindPlayback(chunkUs)) { + // Once per episode: the chunks still in the decoder are behind too. + if (!skipping) { + skipping = true + skipBacklog() + } + return + } + skipping = false + segmenter.onChunk(frame, chunk, probability) + } + + override fun onReset() = segmenter.onReset() + } + return AudioSyncDecoder( + analyzer = analyzer, + liveAnalyzer = analyzer, + onUnsupportedFormat = { mime -> + LiveAiSubsLog.w("no look-ahead decoder for $mime") + onUnsupportedAudio(mime) + }, + threadPriority = Process.THREAD_PRIORITY_LESS_FAVORABLE, + ).also { decoder = it } + } + } + + /** + * Audio that will be heard within [MIN_USEFUL_AHEAD_US] cannot be subtitled in time. On a + * busy box (measured: a 1080p channel on a Mi Box 4) analysis can fall 10–20 s behind; going + * on would only make every later subtitle late too, so the backlog is dropped and analysis + * restarts on the newest downloaded audio. + */ + private fun isBehindPlayback(chunkUs: Long): Boolean { + val position = estimatedPositionUs() + return position != UNKNOWN && positionPlaying && chunkUs < position + MIN_USEFUL_AHEAD_US + } + + private fun skipBacklog() { + backlogSkips++ + analysedUs = UNKNOWN + LiveAiSubsLog.i("analysis fell behind playback; skipping to the newest audio (skip #$backlogSkips)") + decoder?.discontinuity() + } + + private inner class ChannelSink(private val sinkGeneration: Int) : AudioSampleSink { + /** A channel can carry several audio tracks; the first one seen is the one analysed. */ + @Volatile private var lockedTrackId: String? = null + + override fun wantsSamples(format: Format): Boolean { + if (sinkGeneration != generation || !active) return false + val locked = lockedTrackId + if (locked != null && locked != (format.id ?: NO_ID)) return false + val current = ensureDecoder() ?: return false + return current.accepts(format) + } + + override fun onSample(format: Format, timeUs: Long, data: ByteArray, offset: Int, size: Int) { + if (sinkGeneration != generation) return + if (lockedTrackId == null) { + lockedTrackId = format.id ?: NO_ID + LiveAiSubsLog.i( + "tapping ${format.sampleMimeType} ${format.channelCount}ch ${format.sampleRate}Hz lang=${format.language}", + ) + } + // Audio already about to play is useless; don't spend the decoder on it. + val position = estimatedPositionUs() + if (position != UNKNOWN && positionPlaying && timeUs < position + MIN_USEFUL_AHEAD_US) return + lastTapUs = timeUs + decoder?.offer(format, timeUs, data, offset, size) + } + + override fun onDiscontinuity() { + if (sinkGeneration != generation) return + decoder?.discontinuity() + } + } + + companion object { + private const val UNKNOWN = Long.MIN_VALUE + private const val NO_ID = "(no id)" + /** 5 s hard limit: a line can only be sent once its segment has ended. */ + const val MAX_SEGMENT_FRAMES = 156 + /** From 3.5 s on, a pause between words ends the segment. */ + const val SOFT_MAX_SEGMENT_FRAMES = 110 + /** Audio closer than this to the playhead cannot be subtitled before it is heard. */ + const val MIN_USEFUL_AHEAD_US = 2_500_000L + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechProbe.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechProbe.kt deleted file mode 100644 index e1f949372a..0000000000 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechProbe.kt +++ /dev/null @@ -1,266 +0,0 @@ -@file:androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) - -package com.arflix.tv.ui.screens.tv.live.aisubs - -import android.content.Context -import android.os.SystemClock -import androidx.media3.common.Format -import com.arflix.tv.R -import com.arflix.tv.ui.screens.player.audiosync.AudioSampleSink -import com.arflix.tv.ui.screens.player.audiosync.AudioSyncDecoder -import com.arflix.tv.ui.screens.player.audiosync.PlaybackPcmListener -import com.arflix.tv.ui.screens.player.audiosync.SileroVad -import com.arflix.tv.ui.screens.player.audiosync.SileroVadWeights -import com.arflix.tv.ui.screens.player.audiosync.SpeechAnalyzer -import com.arflix.tv.ui.screens.player.audiosync.SpeechTimeline -import com.arflix.tv.ui.screens.player.audiosync.asr.SpeechSegmenter - -/** - * Phase 0 proof of concept for Live TV AI subtitles: copies the channel's audio at the extractor - * (ahead of playback), decodes it on a side thread, cuts it into speech segments with Silero and - * logs everything needed to judge the design, under tag `LiveAiSubs`: - * - * - tapped sample times next to the player's period position, and how far ahead the tap runs; - * - each segment's start/end in media time, and how long before its playback it was ready; - * - with [LiveAiSubsDebug.verify], the same VAD run on the audio as it is played (its times come - * from the audio renderer); matching segments from both sides must start at the same media - * time if the tap and playback share one time base. - * - * Nothing here talks to an AI service or draws anything. One probe lives as long as the Live TV - * player; [newSink] is called for every prepared channel so stale loads of the previous channel - * can never feed the new one. - */ -internal class LiveSpeechProbe(context: Context) { - private val appContext = context.applicationContext - - @Volatile - private var weights: SileroVadWeights? = null - - @Volatile - private var generation = 0 - - @Volatile - private var decoder: AudioSyncDecoder? = null - - /** Whether the player is fullscreen; the probe only runs there. Set on the UI thread. */ - @Volatile - var fullscreen: Boolean = false - - // Last known player position, written on the UI thread, read by the other threads to estimate - // where playback is right now. - @Volatile private var positionUs = UNKNOWN - @Volatile private var positionAtMs = 0L - @Volatile private var positionPlaying = false - - // Tap statistics. - @Volatile private var lastTapUs = UNKNOWN - @Volatile private var lastPlayedUs = UNKNOWN - @Volatile private var tapSamples = 0L - @Volatile private var droppedSamples = 0L - @Volatile private var tapSegments = 0 - @Volatile private var playedSegments = 0 - private var lastStatsMs = 0L - - /** Recent look-ahead segment starts, for matching the played side's segments. */ - private val recentTapStarts = ArrayDeque() - private val deltasMs = ArrayList() - - init { - LiveAiSubsDebug.start() - Thread({ - weights = runCatching { - appContext.resources.openRawResource(R.raw.silero_vad_v5_16k).use { SileroVadWeights.read(it) } - }.onFailure { LiveAiSubsLog.w("could not load VAD weights: ${it.message}") }.getOrNull() - }, "LiveAiSubsInit").apply { - isDaemon = true - start() - } - } - - private val active: Boolean - get() = LiveAiSubsDebug.enabled && fullscreen && weights != null - - /** A sink for one prepared channel; it goes quiet as soon as another channel is prepared. */ - fun newSink(label: String): AudioSampleSink { - val sinkGeneration = ++generation - synchronized(recentTapStarts) { - recentTapStarts.clear() - deltasMs.clear() - } - lastTapUs = UNKNOWN - lastPlayedUs = UNKNOWN - tapSamples = 0L - droppedSamples = 0L - tapSegments = 0 - playedSegments = 0 - decoder?.discontinuity() - if (LiveAiSubsDebug.enabled) LiveAiSubsLog.i("new channel source gen=$sinkGeneration ($label)") - return ChannelSink(sinkGeneration) - } - - /** Audio as it is played, from the Live TV player's audio sink (debug builds only). */ - val playbackListener: PlaybackPcmListener = object : PlaybackPcmListener { - override fun wantsPlaybackPcm(mediaTimeUs: Long, durationUs: Long): Boolean = - LiveAiSubsDebug.verify && active && decoder != null - - override fun onPlaybackPcm(mono: FloatArray, frames: Int, sampleRate: Int, mediaTimeUs: Long) { - lastPlayedUs = mediaTimeUs - decoder?.offerPlaybackPcm(mono, frames, sampleRate, mediaTimeUs) - } - } - - /** - * Called on the UI thread about every half second. [periodPositionUs] is the playback position - * on the period timeline (the window position plus the window's offset in the period). - */ - fun onPosition(periodPositionUs: Long, bufferedAheadMs: Long, playing: Boolean) { - positionUs = periodPositionUs - positionAtMs = SystemClock.elapsedRealtime() - positionPlaying = playing - if (!active) return - val now = positionAtMs - if (now - lastStatsMs < STATS_INTERVAL_MS) return - lastStatsMs = now - val tap = lastTapUs - val played = lastPlayedUs - val median = synchronized(recentTapStarts) { deltasMs.sorted().let { if (it.isEmpty()) null else it[it.size / 2] } } - LiveAiSubsLog.i( - "stats pos=${periodPositionUs / 1_000}ms playing=$playing bufferedAhead=${bufferedAheadMs}ms " + - "tapAhead=${if (tap == UNKNOWN) "-" else "${(tap - periodPositionUs) / 1_000}ms"} " + - "playedAhead=${if (played == UNKNOWN) "-" else "${(played - periodPositionUs) / 1_000}ms"} " + - "samples=$tapSamples dropped=$droppedSamples segments=$tapSegments played=$playedSegments " + - "matchMedianDelta=${median ?: "-"}ms", - ) - } - - fun release() { - generation++ - decoder?.release() - decoder = null - } - - /** Where playback is right now, extrapolated from the last UI-thread report. */ - private fun estimatedPositionUs(): Long { - val position = positionUs - if (position == UNKNOWN) return UNKNOWN - return if (positionPlaying) position + (SystemClock.elapsedRealtime() - positionAtMs) * 1_000L else position - } - - private fun ensureDecoder(): AudioSyncDecoder? { - decoder?.let { return it } - val vadWeights = weights ?: return null - synchronized(this) { - decoder?.let { return it } - val tapAnalyzer = SpeechAnalyzer(SileroVad(vadWeights), SpeechTimeline()) - val playedAnalyzer = SpeechAnalyzer(SileroVad(vadWeights), SpeechTimeline()) - tapAnalyzer.chunkListener = SpeechSegmenter(LIVE_MAX_SEGMENT_FRAMES) { startFrame, samples -> - onTapSegment(startFrame, samples.size) - } - playedAnalyzer.chunkListener = SpeechSegmenter(LIVE_MAX_SEGMENT_FRAMES) { startFrame, samples -> - onPlayedSegment(startFrame, samples.size) - } - return AudioSyncDecoder(tapAnalyzer, playedAnalyzer) { mime -> - LiveAiSubsLog.w("no look-ahead decoder for $mime: AI subtitles not supported on this channel") - }.also { decoder = it } - } - } - - private fun onTapSegment(startFrame: Int, sampleCount: Int) { - val startUs = startFrame.toLong() * SpeechTimeline.FRAME_DURATION_US - val endUs = startUs + sampleCount * 1_000_000L / SileroVad.SAMPLE_RATE - val position = estimatedPositionUs() - tapSegments++ - synchronized(recentTapStarts) { - recentTapStarts.addLast(startUs) - while (recentTapStarts.size > RECENT_SEGMENTS) recentTapStarts.removeFirst() - } - LiveAiSubsLog.i( - "segment tap start=${startUs / 1_000}ms end=${endUs / 1_000}ms len=${(endUs - startUs) / 1_000}ms " + - "readyBeforeStart=${if (position == UNKNOWN) "-" else "${(startUs - position) / 1_000}ms"} " + - "readyBeforeEnd=${if (position == UNKNOWN) "-" else "${(endUs - position) / 1_000}ms"}", - ) - } - - private fun onPlayedSegment(startFrame: Int, sampleCount: Int) { - val startUs = startFrame.toLong() * SpeechTimeline.FRAME_DURATION_US - val endUs = startUs + sampleCount * 1_000_000L / SileroVad.SAMPLE_RATE - playedSegments++ - val match = synchronized(recentTapStarts) { - recentTapStarts.minByOrNull { kotlin.math.abs(it - startUs) } - ?.takeIf { kotlin.math.abs(it - startUs) <= MATCH_WINDOW_US } - ?.also { deltasMs += (startUs - it) / 1_000; if (deltasMs.size > 200) deltasMs.removeAt(0) } - } - LiveAiSubsLog.i( - "segment played start=${startUs / 1_000}ms end=${endUs / 1_000}ms " + - "tapDelta=${if (match == null) "unmatched" else "${(startUs - match) / 1_000}ms"}", - ) - } - - private inner class ChannelSink(private val sinkGeneration: Int) : AudioSampleSink { - /** A channel can carry several audio tracks; the first one seen is the one analysed. */ - @Volatile private var lockedTrackId: String? = null - private var ignoredLogged = false - - private var formatLogged: Format? = null - - override fun wantsSamples(format: Format): Boolean { - if (sinkGeneration != generation || !active) return false - if (formatLogged !== format) { - formatLogged = format - LiveAiSubsLog.i( - "audio format offered: id=${format.id} mime=${format.sampleMimeType} codecs=${format.codecs} " + - "accepted=${ensureDecoder()?.accepts(format)}", - ) - } - val locked = lockedTrackId - if (locked != null && locked != (format.id ?: NO_ID)) { - if (!ignoredLogged) { - ignoredLogged = true - LiveAiSubsLog.i("ignoring second audio track ${format.id} (${format.sampleMimeType}); analysing $locked") - } - return false - } - val current = ensureDecoder() ?: return false - return current.accepts(format) - } - - override fun onSample(format: Format, timeUs: Long, data: ByteArray, offset: Int, size: Int) { - if (sinkGeneration != generation) return - if (lockedTrackId == null) { - lockedTrackId = format.id ?: NO_ID - LiveAiSubsLog.i( - "tapping track ${format.id}: ${format.sampleMimeType} ${format.channelCount}ch " + - "${format.sampleRate}Hz lang=${format.language}", - ) - } - val current = decoder ?: return - val count = ++tapSamples - lastTapUs = timeUs - if (count == 1L || count % SAMPLE_LOG_EVERY == 0L) { - val position = estimatedPositionUs() - LiveAiSubsLog.i( - "sample #$count t=${timeUs / 1_000}ms pos=${if (position == UNKNOWN) "-" else "${position / 1_000}ms"} " + - "ahead=${if (position == UNKNOWN) "-" else "${(timeUs - position) / 1_000}ms"}", - ) - } - if (!current.offer(format, timeUs, data, offset, size)) droppedSamples++ - } - - override fun onDiscontinuity() { - if (sinkGeneration != generation) return - LiveAiSubsLog.i("discontinuity (extractor seek/reset) gen=$sinkGeneration") - decoder?.discontinuity() - } - } - - companion object { - private const val UNKNOWN = Long.MIN_VALUE - private const val NO_ID = "(no id)" - /** About 5 s: bounds how long after a sentence starts its translation can begin. */ - const val LIVE_MAX_SEGMENT_FRAMES = 156 - private const val SAMPLE_LOG_EVERY = 500L - private const val STATS_INTERVAL_MS = 2_000L - private const val RECENT_SEGMENTS = 64 - private const val MATCH_WINDOW_US = 1_000_000L - } -} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt new file mode 100644 index 0000000000..fb0182fb13 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt @@ -0,0 +1,278 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.content.Context +import android.os.SystemClock +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.async +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.BufferOverflow +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit + +/** What the viewer should be told about Live TV AI subtitles. */ +internal sealed interface LiveAiSubsStatus { + data object Off : LiveAiSubsStatus + data object Running : LiveAiSubsStatus + data object NoKey : LiveAiSubsStatus + data object InvalidKey : LiveAiSubsStatus + /** HTTP 429: sending pauses until [untilMs] (elapsed realtime). */ + data class QuotaPaused(val untilMs: Long) : LiveAiSubsStatus + data object Offline : LiveAiSubsStatus + data class Unsupported(val mimeType: String) : LiveAiSubsStatus +} + +/** Counters for the hidden debug overlay. */ +internal data class LiveAiSubsStats( + val lookaheadMs: Long? = null, + val latencyP50Ms: Long? = null, + val latencyP95Ms: Long? = null, + val lateDropped: Int = 0, + val skippedSegments: Int = 0, + val backlogSkips: Int = 0, + val segmentsPerMinute: Int = 0, + val failures: Int = 0, +) + +/** + * Live TV AI subtitles for the one Live TV player: [pipeline] cuts speech segments from the + * channel's audio ahead of playback, [engine] turns each into timed lines, and [scheduler] shows + * them when playback reaches them. The UI calls [tick] with the player position about every + * 100 ms and observes [cue] and [status]. + * + * Requests: at most [MAX_IN_FLIGHT] at once, results applied in segment order. A segment that + * could no longer be shown in time is not sent (cost). Main-thread confined except for segment + * hand-off from the decoder thread. + */ +internal class LiveSubtitleSession( + context: Context, + private val engine: LiveSubtitleEngine, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate) + private val segments = Channel(capacity = 8, onBufferOverflow = BufferOverflow.DROP_OLDEST) + private val inFlight = Semaphore(MAX_IN_FLIGHT) + + val pipeline = LiveSpeechPipeline( + context = context, + onSegment = { segment -> segments.trySend(segment) }, + onUnsupportedAudio = { mime -> scope.launch { setStatus(LiveAiSubsStatus.Unsupported(mime)) } }, + ) + private val scheduler = LiveCueScheduler() + + private val _cue = MutableStateFlow(null) + val cue: StateFlow = _cue.asStateFlow() + + private val _status = MutableStateFlow(LiveAiSubsStatus.Off) + val status: StateFlow = _status.asStateFlow() + + private val _stats = MutableStateFlow(LiveAiSubsStats()) + val stats: StateFlow = _stats.asStateFlow() + + private var languageName = "English" + private var rtl = false + private var running = false + private var epoch = 0 + private var positionUs = 0L + private var quotaBackoffMs = QUOTA_BACKOFF_START_MS + private var pausedUntilMs = 0L + private val recentLines = ArrayDeque() + private val latencies = ArrayDeque() + private val segmentTimes = ArrayDeque() + private var skippedSegments = 0 + private var failures = 0 + private var worker: Job? = null + + /** Measured p95 of the AI's answer time, for the Auto delay. */ + val latencyP95Ms: Long? get() = percentile95(latencies.toList()) + + fun setTargetLanguage(name: String, isRtl: Boolean) { + if (name != languageName) recentLines.clear() + languageName = name + rtl = isRtl + } + + /** + * Starts or stops subtitling. [hasKey] false reports [LiveAiSubsStatus.NoKey]. Capture only + * runs while [capture] (fullscreen and playing), to keep cost down. + */ + fun update(enabled: Boolean, hasKey: Boolean, capture: Boolean) { + if (!enabled) { + stop() + return + } + if (!hasKey) { + stop() + setStatus(LiveAiSubsStatus.NoKey) + return + } + if (!running) { + running = true + if (_status.value is LiveAiSubsStatus.Off || _status.value is LiveAiSubsStatus.NoKey) { + setStatus(LiveAiSubsStatus.Running) + } + startWorker() + } + pipeline.active = capture && _status.value !is LiveAiSubsStatus.InvalidKey && + _status.value !is LiveAiSubsStatus.Unsupported + } + + /** A new channel (or a re-prepared source): nothing from before may show. */ + fun onNewSource() { + epoch++ + scheduler.clear() + recentLines.clear() + _cue.value = null + // A channel that could not be decoded says nothing about the next one. + if (_status.value is LiveAiSubsStatus.Unsupported) setStatus(if (running) LiveAiSubsStatus.Running else LiveAiSubsStatus.Off) + } + + /** The key was changed in settings: try again. */ + fun onKeyChanged() { + if (_status.value is LiveAiSubsStatus.InvalidKey || _status.value is LiveAiSubsStatus.NoKey) { + setStatus(if (running) LiveAiSubsStatus.Running else LiveAiSubsStatus.Off) + } + } + + /** UI thread, about every 100 ms. */ + fun tick(periodPositionUs: Long, playing: Boolean, lookaheadMs: Long?) { + positionUs = periodPositionUs + pipeline.onPosition(periodPositionUs, playing) + val now = SystemClock.elapsedRealtime() + (_status.value as? LiveAiSubsStatus.QuotaPaused)?.let { if (now >= it.untilMs) setStatus(LiveAiSubsStatus.Running) } + _cue.value = if (running) scheduler.cueAt(periodPositionUs) else null + while (segmentTimes.isNotEmpty() && now - segmentTimes.first() > 60_000L) segmentTimes.removeFirst() + _stats.value = LiveAiSubsStats( + lookaheadMs = lookaheadMs, + latencyP50Ms = latencies.sorted().let { if (it.isEmpty()) null else it[it.size / 2] }, + latencyP95Ms = latencyP95Ms, + lateDropped = scheduler.lateDropped, + skippedSegments = skippedSegments, + backlogSkips = pipeline.backlogSkips, + segmentsPerMinute = segmentTimes.size, + failures = failures, + ) + } + + fun release() { + stop() + pipeline.release() + scope.cancel() + } + + private fun stop() { + running = false + pipeline.active = false + worker?.cancel() + worker = null + epoch++ + scheduler.clear() + _cue.value = null + if (_status.value !is LiveAiSubsStatus.Off) setStatus(LiveAiSubsStatus.Off) + } + + private fun setStatus(status: LiveAiSubsStatus) { + if (_status.value != status) LiveAiSubsLog.i("status: $status") + _status.value = status + } + + private fun startWorker() { + worker?.cancel() + // Drop anything queued while stopped. + while (segments.tryReceive().isSuccess) Unit + worker = scope.launch { + val ordered = Channel>>(capacity = MAX_IN_FLIGHT) + launch { + for ((resultEpoch, deferred) in ordered) { + val result = deferred.await() ?: continue + if (resultEpoch == epoch) apply(result) + } + } + for (segment in segments) { + if (!shouldSend(segment)) { + skippedSegments++ + continue + } + segmentTimes.addLast(SystemClock.elapsedRealtime()) + val sentEpoch = epoch + val context = recentLines.toList() + val name = languageName + val isRtl = rtl + inFlight.acquire() + val deferred = async { + try { + val started = SystemClock.elapsedRealtime() + val result = engine.process(segment, context, name, isRtl) + if (result is LiveEngineResult.Cues) recordLatency(SystemClock.elapsedRealtime() - started) + result + } finally { + inFlight.release() + } + } + ordered.send(sentEpoch to deferred) + } + } + } + + private fun shouldSend(segment: SpeechSegment): Boolean { + val status = _status.value + if (status is LiveAiSubsStatus.InvalidKey || status is LiveAiSubsStatus.Unsupported) return false + val now = SystemClock.elapsedRealtime() + if (status is LiveAiSubsStatus.QuotaPaused && now < status.untilMs) return false + if (status is LiveAiSubsStatus.Offline && now < pausedUntilMs) return false + // Its window would already be over when an answer could arrive. + return segment.endUs + LiveCueScheduler.MIN_DISPLAY_US > pipeline.estimatedPositionUs() + MIN_ANSWER_US + } + + private fun apply(result: LiveEngineResult) { + when (result) { + is LiveEngineResult.Cues -> { + quotaBackoffMs = QUOTA_BACKOFF_START_MS + if (_status.value is LiveAiSubsStatus.Offline || _status.value is LiveAiSubsStatus.QuotaPaused) { + setStatus(LiveAiSubsStatus.Running) + } + scheduler.add(result.cues, positionUs) + result.cues.forEach { + recentLines.addLast(it.text) + while (recentLines.size > CONTEXT_LINES) recentLines.removeFirst() + } + } + LiveEngineResult.InvalidKey -> setStatus(LiveAiSubsStatus.InvalidKey) + LiveEngineResult.QuotaExceeded -> { + setStatus(LiveAiSubsStatus.QuotaPaused(SystemClock.elapsedRealtime() + quotaBackoffMs)) + quotaBackoffMs = (quotaBackoffMs * 2).coerceAtMost(QUOTA_BACKOFF_MAX_MS) + } + LiveEngineResult.Offline -> { + pausedUntilMs = SystemClock.elapsedRealtime() + OFFLINE_RETRY_MS + setStatus(LiveAiSubsStatus.Offline) + } + is LiveEngineResult.Failed -> { + failures++ + LiveAiSubsLog.w("segment failed: ${result.reason}") + } + } + } + + private fun recordLatency(ms: Long) { + latencies.addLast(ms) + while (latencies.size > LATENCY_SAMPLES) latencies.removeFirst() + } + + private companion object { + const val MAX_IN_FLIGHT = 2 + const val CONTEXT_LINES = 3 + const val LATENCY_SAMPLES = 30 + /** The fastest a segment can realistically come back. */ + const val MIN_ANSWER_US = 800_000L + const val QUOTA_BACKOFF_START_MS = 30_000L + const val QUOTA_BACKOFF_MAX_MS = 240_000L + const val OFFLINE_RETRY_MS = 10_000L + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/SubtitleLookaheadLoadControl.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/SubtitleLookaheadLoadControl.kt index 1a64d587f6..438cc7daea 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/SubtitleLookaheadLoadControl.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/SubtitleLookaheadLoadControl.kt @@ -27,6 +27,23 @@ internal class SubtitleLookaheadLoadControl( @Volatile var requiredLookaheadUs: Long = 0L + /** True while playback is being held for the look-ahead (read by the UI). */ + @Volatile + var holding: Boolean = false + private set + + /** Buffered media when the hold was last checked, for "Preparing subtitles… N s". */ + @Volatile + var lastBufferedUs: Long = 0L + private set + + /** + * The buffer's memory cap was reached before the look-ahead (a high-bitrate channel on a + * low-memory box): playback started with less, so some subtitles may be late. + */ + @Volatile + var cappedByMemory: Boolean = false + private var lastHoldLogMs = 0L override fun onPrepared(playerId: PlayerId) = delegate.onPrepared(playerId) @@ -61,7 +78,20 @@ internal class SubtitleLookaheadLoadControl( override fun shouldStartPlayback(parameters: LoadControl.Parameters): Boolean { if (!delegate.shouldStartPlayback(parameters)) return false val required = requiredLookaheadUs - if (required <= 0L || parameters.bufferedDurationUs >= required) return true + lastBufferedUs = parameters.bufferedDurationUs + if (required <= 0L || parameters.bufferedDurationUs >= required) { + holding = false + return true + } + // Loading has stopped (memory target reached): waiting longer would never get there and + // would end in the stuck-buffering error, so play with what there is. + if (!delegate.shouldContinueLoading(parameters)) { + if (!cappedByMemory) LiveAiSubsLog.w("look-ahead capped by the buffer memory target at ${parameters.bufferedDurationUs / 1_000}ms") + cappedByMemory = true + holding = false + return true + } + holding = true val now = SystemClock.elapsedRealtime() if (now - lastHoldLogMs >= HOLD_LOG_INTERVAL_MS) { lastHoldLogMs = now diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index 29aa1b2066..0dd32d84d8 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -1919,4 +1919,34 @@ 1. QR-Code mit dem Telefon scannen\n2. In Discord auf \'Autorisieren\' tippen\n3. Verbindet sich automatisch In dieser Version nicht enthalten Zeige deinen Wiedergabestatus in deinem Discord-Profil. + + + KI-Untertitel + Untertitel werden vorbereitet… %1$d s + KI-Untertitel brauchen einen Google-Gemini-API-Schlüssel (Einstellungen → KI-Untertitel) + KI-Untertitel: Der Gemini-API-Schlüssel wurde abgelehnt + KI-Untertitel pausiert: Gemini-Kontingent erreicht + KI-Untertitel: keine Internetverbindung + KI-Untertitel werden auf diesem Sender nicht unterstützt + Dieser Sender kann nicht genug verzögert werden; manche Untertitel kommen evtl. zu spät + KI-Untertitel für Live-TV + Live-Sender mit Gemini transkribieren und übersetzen. Live-TV wird einige Sekunden verzögert, damit die Zeilen rechtzeitig erscheinen. + Untertitel-Verzögerung (Live-TV) + Wie weit Live-TV zurückgehalten wird. Auto nutzt die gemessene KI-Geschwindigkeit. + %1$d s + Aktivierung der KI-Untertitel (Live-TV) + Jeder Senderwechsel wartet die Verzögerung ab; pro Sender verzögert nur Sender, auf denen du sie einschaltest + Pro Sender (Untertitelmenü) + Immer an + Untertitelsprache (Live-TV) + Die Sprache, in die Live-Sender übersetzt werden + Untertitelsprache des Profils + Gemini-API-Schlüssel für Live-TV + Live-TV nutzt den KI-Untertitel-Schlüssel, wenn es ein Gemini-Schlüssel ist, sonst diesen + Live-TV nutzt den Gemini-Schlüssel der KI-Untertitel oben + Nutzt den KI-Untertitel-Schlüssel + Gemini-API-Schlüssel funktioniert + Gemini hat diesen API-Schlüssel abgelehnt + Schlüssel konnte nicht geprüft werden (offline?) + Füge zuerst einen Google-Gemini-API-Schlüssel hinzu diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index f758d8e75a..5a68cc959f 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -1639,4 +1639,34 @@ Muestra tu estado de visionado en tu perfil de Discord. No se encontraron otras fuentes para este canal. No se pudieron cargar las fuentes. Por favor, inténtalo de nuevo. + + + Subtítulos IA + Preparando subtítulos… %1$d s + Los subtítulos IA necesitan una clave de API de Google Gemini (Ajustes → Subtítulos IA) + Subtítulos IA: se rechazó la clave de API de Gemini + Subtítulos IA en pausa: se alcanzó la cuota de Gemini + Subtítulos IA: sin conexión a internet + Subtítulos IA no disponibles en este canal + Este canal no se puede retrasar lo suficiente; algunos subtítulos pueden llegar tarde + Subtítulos IA en TV en directo + Transcribe y traduce canales en directo con Gemini. La TV en directo se retrasa unos segundos para que las líneas lleguen a tiempo. + Retraso de subtítulos en directo + Cuánto se retrasa la TV en directo. Auto usa la velocidad medida de la IA. + %1$d s + Activación de subtítulos IA en directo + Cada cambio de canal espera el retraso; por canal solo retrasa los canales donde los activas + Por canal (menú de subtítulos) + Siempre activados + Idioma de subtítulos en directo + El idioma al que se traducen los canales en directo + Idioma de subtítulos del perfil + Clave de API de Gemini para TV en directo + Usa la clave de subtítulos IA si es de Gemini; si no, esta + La TV en directo usa la clave de Gemini de subtítulos IA de arriba + Usa la clave de subtítulos IA + La clave de API de Gemini funciona + Gemini rechazó esta clave de API + No se pudo comprobar la clave (¿sin conexión?) + Primero añade una clave de API de Google Gemini diff --git a/app/src/main/res/values-iw/strings.xml b/app/src/main/res/values-iw/strings.xml index a97d45d7e3..b5a0f62b19 100644 --- a/app/src/main/res/values-iw/strings.xml +++ b/app/src/main/res/values-iw/strings.xml @@ -376,4 +376,34 @@ אין חיבור ל-MDBList, ולכן הסנכרון מושהה. ARVIO ממשיך לנסות; אם זה נמשך, חבר את MDBList מחדש. אין חיבור ל-MDBList כרגע. נסה שוב בעוד כמה דקות. לא ניתן לטעון את הרשימה הזו מ-MDBList. נסה שוב. + + + כתוביות AI + מכין כתוביות… %1$d שנ׳ + כתוביות AI דורשות מפתח Google Gemini API (הגדרות ← כתוביות AI) + כתוביות AI: מפתח ה-Gemini API נדחה + כתוביות AI הושהו: הגעת למכסת Gemini + כתוביות AI: אין חיבור לאינטרנט + כתוביות AI אינן נתמכות בערוץ הזה + אי אפשר לעכב את הערוץ הזה מספיק; חלק מהכתוביות עלולות לאחר + כתוביות AI לטלוויזיה חיה + תמלול ותרגום של ערוצים חיים עם Gemini. השידור מתעכב כמה שניות כדי שהכתוביות יופיעו בזמן. + עיכוב כתוביות בטלוויזיה חיה + בכמה לעכב את השידור החי. אוטומטי מתאים את עצמו למהירות ה-AI שנמדדה. + %1$d שנ׳ + הפעלת כתוביות AI בטלוויזיה חיה + כל החלפת ערוץ ממתינה לעיכוב, לכן \"לפי ערוץ\" מעכב רק ערוצים שבהם הפעלת + לפי ערוץ (תפריט כתוביות) + תמיד פעיל + שפת הכתוביות בטלוויזיה חיה + השפה שאליה מתורגמים הערוצים החיים + שפת הכתוביות של הפרופיל + מפתח Gemini API לטלוויזיה חיה + משתמש במפתח של כתוביות AI כשהוא מפתח Gemini, אחרת במפתח הזה + הטלוויזיה החיה משתמשת במפתח Gemini של כתוביות AI שלמעלה + משתמש במפתח של כתוביות AI + מפתח ה-Gemini API תקין + Gemini דחה את המפתח הזה + לא ניתן לבדוק את המפתח (אין חיבור?) + יש להוסיף קודם מפתח Google Gemini API diff --git a/app/src/main/res/values-pt-rBR/strings.xml b/app/src/main/res/values-pt-rBR/strings.xml index a94d5ea804..4a8f257fd5 100644 --- a/app/src/main/res/values-pt-rBR/strings.xml +++ b/app/src/main/res/values-pt-rBR/strings.xml @@ -1551,4 +1551,34 @@ Portal Ministra / Stalker com autenticação por endereço MAC. Conteúdo a importar URL da lista M3U + + + Legendas IA + Preparando legendas… %1$d s + As legendas IA precisam de uma chave de API do Google Gemini (Configurações → Legendas IA) + Legendas IA: a chave de API do Gemini foi rejeitada + Legendas IA em pausa: quota do Gemini atingida + Legendas IA: sem conexão com a internet + Legendas IA não suportadas neste canal + Não dá para atrasar este canal o suficiente; algumas legendas podem chegar atrasadas + Legendas IA na TV ao vivo + Transcreve e traduz canais ao vivo com o Gemini. A TV ao vivo é atrasada alguns segundos para as linhas aparecerem na hora certa. + Atraso das legendas ao vivo + Quanto a TV ao vivo é atrasada. Auto usa a velocidade medida da IA. + %1$d s + Ativação das legendas IA ao vivo + Cada troca de canal espera o atraso; por canal só atrasa os canais em que você ativar + Por canal (menu de legendas) + Sempre ativas + Idioma das legendas ao vivo + O idioma para o qual os canais ao vivo são traduzidos + Idioma de legendas do perfil + Chave de API do Gemini para TV ao vivo + Usa a chave das legendas IA quando é do Gemini; caso contrário, esta + A TV ao vivo usa a chave do Gemini das legendas IA acima + Usa a chave das legendas IA + A chave de API do Gemini funciona + O Gemini rejeitou esta chave de API + Não foi possível verificar a chave (sem conexão?) + Adicione primeiro uma chave de API do Google Gemini diff --git a/app/src/main/res/values-pt/strings.xml b/app/src/main/res/values-pt/strings.xml index 96c16dc7e7..092e7c2807 100644 --- a/app/src/main/res/values-pt/strings.xml +++ b/app/src/main/res/values-pt/strings.xml @@ -1551,4 +1551,34 @@ Portal Ministra / Stalker com autenticação por endereço MAC. Conteúdo a importar URL da lista M3U + + + Legendas IA + A preparar legendas… %1$d s + As legendas IA precisam de uma chave de API do Google Gemini (Definições → Legendas IA) + Legendas IA: a chave de API do Gemini foi rejeitada + Legendas IA em pausa: quota do Gemini atingida + Legendas IA: sem ligação à internet + Legendas IA não suportadas neste canal + Não é possível atrasar este canal o suficiente; algumas legendas podem chegar tarde + Legendas IA na TV em direto + Transcreve e traduz canais em direto com o Gemini. A TV em direto é atrasada alguns segundos para as linhas aparecerem a tempo. + Atraso das legendas em direto + Quanto a TV em direto é atrasada. Auto usa a velocidade medida da IA. + %1$d s + Ativação das legendas IA em direto + Cada mudança de canal espera pelo atraso; por canal só atrasa os canais onde as ativa + Por canal (menu de legendas) + Sempre ativas + Idioma das legendas em direto + O idioma para o qual os canais em direto são traduzidos + Idioma de legendas do perfil + Chave de API do Gemini para TV em direto + Usa a chave das legendas IA quando é do Gemini; caso contrário, esta + A TV em direto usa a chave do Gemini das legendas IA acima + Usa a chave das legendas IA + A chave de API do Gemini funciona + O Gemini rejeitou esta chave de API + Não foi possível verificar a chave (sem ligação?) + Adicione primeiro uma chave de API do Google Gemini diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index a2ea370cc8..a8c5aeb8d1 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2083,5 +2083,34 @@ Connect WeTrakr Copy code and open WeTrakr Maximum of 10 IPTV playlists reached - + + AI subtitles + Preparing subtitles… %1$d s + AI subtitles need a Google Gemini API key (Settings → AI subtitles) + AI subtitles: the Gemini API key was rejected + AI subtitles paused: Gemini quota reached + AI subtitles: no internet connection + AI subtitles not supported on this channel + This channel can\'t be delayed enough; some subtitles may be late + Live TV AI subtitles + Transcribe and translate live channels with Gemini. Live TV is delayed a few seconds so lines appear on time. + Live TV subtitle delay + How far live TV is held back. Auto uses the measured AI speed. + %1$d s + Live TV AI subtitles activation + Every channel change waits for the delay, so per channel only delays channels you turn it on for + Per channel (subtitle menu) + Always on + Live TV subtitle language + The language live channels are translated into + Profile subtitle language + Gemini API key for Live TV + Live TV subtitles use the AI subtitles key when it is a Gemini key, otherwise this one + Live TV uses the AI subtitles Gemini key above + Uses the AI subtitles key + Gemini API key works + Gemini rejected this API key + Couldn\'t check the API key (offline?) + Add a Google Gemini API key first + diff --git a/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/LiveAiSubtitlesOptionTest.kt b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/LiveAiSubtitlesOptionTest.kt new file mode 100644 index 0000000000..7b8dad8470 --- /dev/null +++ b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/LiveAiSubtitlesOptionTest.kt @@ -0,0 +1,34 @@ +package com.arflix.tv.ui.screens.tv.live + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertSame +import org.junit.Test + +class LiveAiSubtitlesOptionTest { + private val streamOptions = listOf( + LiveTrackOption(LIVE_SUBTITLES_OFF, "Off", selected = false), + LiveTrackOption("en", "English", selected = true), + ) + + @Test fun featureOffLeavesTheStreamChoicesAlone() { + val options = withAiSubtitlesOption(streamOptions, "Off", "AI subtitles", aiAvailable = false, aiSelected = false) + assertSame(streamOptions, options) + } + + @Test fun aiEntryComesAfterTheStreamTracks() { + val options = withAiSubtitlesOption(streamOptions, "Off", "AI subtitles", aiAvailable = true, aiSelected = false) + assertEquals(listOf(LIVE_SUBTITLES_OFF, "en", LIVE_SUBTITLES_AI), options.map { it.key }) + assertEquals(listOf(false, true, false), options.map { it.selected }) + } + + @Test fun selectedAiEntryIsTheOnlySelection() { + val options = withAiSubtitlesOption(streamOptions, "Off", "AI subtitles", aiAvailable = true, aiSelected = true) + assertEquals(listOf(false, false, true), options.map { it.selected }) + } + + @Test fun channelWithoutSubtitleTracksGetsOffAndAi() { + val options = withAiSubtitlesOption(emptyList(), "Off", "AI subtitles", aiAvailable = true, aiSelected = false) + assertEquals(listOf(LIVE_SUBTITLES_OFF, LIVE_SUBTITLES_AI), options.map { it.key }) + assertEquals(listOf(true, false), options.map { it.selected }) + } +} diff --git a/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiLiveProtocolTest.kt b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiLiveProtocolTest.kt new file mode 100644 index 0000000000..06adc1ce77 --- /dev/null +++ b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiLiveProtocolTest.kt @@ -0,0 +1,130 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.robolectric.annotation.ConscryptMode + +/** org.json is part of the Android framework, so these run under Robolectric. */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [28], application = android.app.Application::class, manifest = Config.NONE) +@ConscryptMode(ConscryptMode.Mode.OFF) +class GeminiLiveProtocolTest { + private val segment = SpeechSegment(startUs = 100_000_000, endUs = 104_000_000, pcm16k = FloatArray(16)) + + private fun cues(result: LiveEngineResult) = (result as LiveEngineResult.Cues).cues + + @Test fun documentedObjectIsTimedFromItsOffsets() { + val result = GeminiLiveProtocol.parseLines( + """{"lines":[{"text":"שלום","startOffsetMs":0},{"text":"מה שלומך?","startOffsetMs":1500}],"sourceLanguage":"en"}""", + segment, rtl = true, + ) + val cues = cues(result) + assertEquals(listOf("שלום", "מה שלומך?"), cues.map { it.text }) + assertEquals(100_000_000L, cues[0].startUs) + assertEquals(101_500_000L, cues[0].endUs) + assertEquals(101_500_000L, cues[1].startUs) + assertEquals(104_000_000L, cues[1].endUs) + assertTrue(cues.all { it.rtl }) + assertEquals("en", (result as LiveEngineResult.Cues).sourceLanguage) + } + + @Test fun linesWithoutOffsetsShareTheSegmentByLength() { + val cues = cues(GeminiLiveProtocol.parseLines("""{"lines":[{"text":"aaaa"},{"text":"bbbb"}]}""", segment, rtl = false)) + assertEquals(102_000_000L, cues[1].startUs) + } + + @Test fun offsetsAreKeptInOrderAndInsideTheSegment() { + val cues = cues( + GeminiLiveProtocol.parseLines( + """{"lines":[{"text":"a","startOffsetMs":3000},{"text":"b","startOffsetMs":1000},{"text":"c","startOffsetMs":99000}]}""", + segment, rtl = false, + ), + ) + assertTrue(cues.zipWithNext().all { (x, y) -> y.startUs >= x.startUs }) + assertTrue(cues.all { it.startUs in segment.startUs until segment.endUs }) + } + + @Test fun emptyLinesMeanNoSpeech() { + assertTrue(cues(GeminiLiveProtocol.parseLines("""{"lines":[]}""", segment, rtl = false)).isEmpty()) + } + + @Test fun bareArrayOfStringsIsAccepted() { + val cues = cues(GeminiLiveProtocol.parseLines("""["Hello there", "General Kenobi"]""", segment, rtl = false)) + assertEquals(listOf("Hello there", "General Kenobi"), cues.map { it.text }) + } + + @Test fun fencedReplyIsAccepted() { + val text = "```json\n[{\"text\":\"One\"},{\"text\":\"Two\"}]\n```" + assertEquals(2, cues(GeminiLiveProtocol.parseLines(text, segment, rtl = false)).size) + } + + @Test fun newlinesAndExtraSpacesAreCollapsed() { + val cues = cues(GeminiLiveProtocol.parseLines("""{"lines":[{"text":" two\nlines here "}]}""", segment, rtl = false)) + assertEquals("two lines here", cues.single().text) + } + + @Test fun malformedReplyFails() { + assertTrue(GeminiLiveProtocol.parseLines("not json at all", segment, rtl = false) is LiveEngineResult.Failed) + } + + @Test fun responseSkipsThoughtPartsAndReadsTheText() { + val body = JSONObject().put( + "candidates", + JSONArray().put( + JSONObject().put( + "content", + JSONObject().put( + "parts", + JSONArray() + .put(JSONObject().put("thought", true).put("text", "thinking…")) + .put(JSONObject().put("text", """{"lines":[{"text":"Hi"}]}""")), + ), + ).put("finishReason", "STOP"), + ), + ).toString() + assertEquals("Hi", cues(GeminiLiveProtocol.parseResponse(body, segment, rtl = false)).single().text) + } + + @Test fun responseWithoutCandidatesFails() { + val blocked = """{"promptFeedback":{"blockReason":"SAFETY"}}""" + assertEquals(LiveEngineResult.Failed("blocked"), GeminiLiveProtocol.parseResponse(blocked, segment, rtl = false)) + assertTrue(GeminiLiveProtocol.parseResponse("", segment, rtl = false) is LiveEngineResult.Failed) + } + + @Test fun invalidKeyIsRecognised() { + assertTrue(GeminiLiveProtocol.isInvalidKey(400, """{"error":{"status":"INVALID_ARGUMENT","details":[{"reason":"API_KEY_INVALID"}]}}""")) + assertTrue(GeminiLiveProtocol.isInvalidKey(403, "")) + assertFalse(GeminiLiveProtocol.isInvalidKey(400, """{"error":{"message":"Request too large"}}""")) + assertFalse(GeminiLiveProtocol.isInvalidKey(500, "")) + } + + @Test fun requestCarriesAudioContextAndJsonMode() { + val body = JSONObject(GeminiLiveProtocol.requestBody(segment, listOf("previous line"), "Hebrew")) + val parts = body.getJSONArray("contents").getJSONObject(0).getJSONArray("parts") + assertEquals("audio/wav", parts.getJSONObject(0).getJSONObject("inline_data").getString("mime_type")) + assertTrue(parts.getJSONObject(1).getString("text").contains("previous line")) + val config = body.getJSONObject("generationConfig") + assertEquals("application/json", config.getString("responseMimeType")) + assertEquals("minimal", config.getJSONObject("thinkingConfig").getString("thinkingLevel")) + assertTrue(body.getJSONObject("system_instruction").toString().contains("Hebrew")) + } + + @Test fun wavHeaderDescribes16kMonoPcm16() { + val wav = GeminiLiveProtocol.wav16kMono(floatArrayOf(0f, 1f, -1f)) + assertEquals(44 + 6, wav.size) + assertEquals("RIFF", String(wav, 0, 4, Charsets.US_ASCII)) + assertEquals("WAVE", String(wav, 8, 4, Charsets.US_ASCII)) + val rate = (wav[24].toInt() and 0xFF) or ((wav[25].toInt() and 0xFF) shl 8) or ((wav[26].toInt() and 0xFF) shl 16) + assertEquals(16_000, rate) + assertEquals(1, wav[22].toInt()) + val loud = (wav[46].toInt() and 0xFF) or (wav[47].toInt() shl 8) + assertEquals(Short.MAX_VALUE.toInt(), loud) + } +} diff --git a/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueSchedulerTest.kt b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueSchedulerTest.kt new file mode 100644 index 0000000000..b7647304ff --- /dev/null +++ b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueSchedulerTest.kt @@ -0,0 +1,135 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class LiveCueSchedulerTest { + private fun cue(startMs: Long, endMs: Long, text: String = "line") = LiveCue(startMs * 1_000, endMs * 1_000, text) + private fun LiveCueScheduler.at(ms: Long) = cueAt(ms * 1_000)?.text + + @Test fun cueShowsFromItsStartUntilItsEnd() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 12_500, "hello")), positionUs = 0) + assertNull(scheduler.at(9_999)) + assertEquals("hello", scheduler.at(10_000)) + assertEquals("hello", scheduler.at(12_499)) + assertNull(scheduler.at(12_500)) + } + + @Test fun shortCueStaysUpForTheMinimum() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 10_300, "hi")), positionUs = 0) + assertEquals("hi", scheduler.at(11_100)) + assertNull(scheduler.at(11_200)) + } + + @Test fun longCueIsCappedAtTheMaximum() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 30_000, "long")), positionUs = 0) + assertEquals("long", scheduler.at(15_900)) + assertNull(scheduler.at(16_000)) + } + + @Test fun nextCueCutsThePreviousOne() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 13_000, "one"), cue(11_000, 13_000, "two")), positionUs = 0) + assertEquals("one", scheduler.at(10_500)) + assertEquals("two", scheduler.at(11_000)) + } + + @Test fun cuesFromSegmentsArrivingOutOfOrderAreSorted() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(20_000, 22_000, "later")), positionUs = 0) + scheduler.add(listOf(cue(10_000, 12_000, "earlier")), positionUs = 0) + assertEquals("earlier", scheduler.at(11_000)) + assertEquals("later", scheduler.at(21_000)) + } + + @Test fun lateCueStillInsideItsWindowShowsImmediately() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 13_000, "late")), positionUs = 11_000_000) + assertEquals("late", scheduler.at(11_000)) + assertEquals(0, scheduler.lateDropped) + } + + @Test fun cueArrivingAfterItsWindowIsDroppedAndCounted() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 11_000, "gone")), positionUs = 12_000_000) + assertNull(scheduler.at(12_000)) + assertEquals(1, scheduler.lateDropped) + assertEquals(0, scheduler.pendingCount) + } + + @Test fun clearForgetsEverything() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 12_000)), positionUs = 0) + scheduler.clear() + assertNull(scheduler.at(11_000)) + assertEquals(0, scheduler.pendingCount) + } + + @Test fun blankLinesAreIgnored() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 12_000, " ")), positionUs = 0) + assertEquals(0, scheduler.pendingCount) + } + + @Test fun shortTextIsNotSplit() { + val parts = LiveCueScheduler.split(cue(0, 4_000, "A short line")) + assertEquals(1, parts.size) + } + + @Test fun longTextSplitsAtAWordNearTheMiddleWithProportionalTime() { + val text = "This is a rather long subtitle line that keeps going " + + "and going until it no longer fits on the screen at once" + val parts = LiveCueScheduler.split(cue(0, 6_000, text)) + assertEquals(2, parts.size) + assertEquals(text, parts[0].text + " " + parts[1].text) + assertEquals(parts[0].endUs, parts[1].startUs) + val expectedSplit = 6_000_000L * parts[0].text.length / (parts[0].text.length + parts[1].text.length) + assertEquals(expectedSplit, parts[0].endUs) + assertTrue(parts.all { it.text.length <= LiveCueScheduler.SPLIT_CHARS }) + } + + @Test fun longTextWithoutSpacesStaysWhole() { + val parts = LiveCueScheduler.split(cue(0, 6_000, "x".repeat(120))) + assertEquals(1, parts.size) + } +} + +class LiveAiSubsDelayTest { + @Test fun autoAssumesFiveSecondsBeforeAnyMeasurement() { + assertEquals(11_000L, autoLiveAiSubsDelayMs(null)) + } + + @Test fun autoAddsMeasuredLatencyToSegmentLengthAndMargin() { + assertEquals(8_200L, autoLiveAiSubsDelayMs(2_200L)) + } + + @Test fun autoIsClampedToTheOfferedRange() { + assertEquals(6_000L, autoLiveAiSubsDelayMs(0L)) + assertEquals(15_000L, autoLiveAiSubsDelayMs(30_000L)) + } + + @Test fun fixedDelaysIgnoreLatency() { + assertEquals(6_000L, liveAiSubsDelayMs(LiveAiSubsDelay.SIX, 9_000L)) + assertEquals(10_000L, liveAiSubsDelayMs(LiveAiSubsDelay.TEN, null)) + assertEquals(15_000L, liveAiSubsDelayMs(LiveAiSubsDelay.FIFTEEN, 100L)) + assertEquals(11_000L, liveAiSubsDelayMs(LiveAiSubsDelay.AUTO, null)) + } + + @Test fun percentileNeedsAFewSamples() { + assertNull(percentile95(listOf(1L, 2L))) + assertEquals(96L, percentile95((1L..100L).toList())) + assertEquals(1_900L, percentile95(listOf(1_000L, 1_200L, 1_900L, 1_100L))) + } + + @Test fun delayChoicesCycleAndUnknownValuesFallBackToAuto() { + assertEquals(LiveAiSubsDelay.SIX, LiveAiSubsDelay.AUTO.next()) + assertEquals(LiveAiSubsDelay.AUTO, LiveAiSubsDelay.FIFTEEN.next()) + assertEquals(LiveAiSubsDelay.AUTO, LiveAiSubsDelay.fromPref("garbage")) + assertEquals(LiveAiSubsActivation.PER_CHANNEL, LiveAiSubsActivation.fromPref(null)) + } +} From 80a9e84f34a6cb15cac6bd87abb105e4b658a0b6 Mon Sep 17 00:00:00 2001 From: silentbil Date: Fri, 6 Nov 2026 12:20:37 +0200 Subject: [PATCH 3/8] Live TV AI subtitles: always offer them in the player's subtitle menu Picking "AI subtitles" turns them on for that channel even when the settings switch is off; the switch now only makes channels start with them (together with Always on). --- .../com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt | 12 +++++++----- app/src/main/res/values-iw/strings.xml | 2 +- app/src/main/res/values/strings.xml | 2 +- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt index b494828862..f0da97559b 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt @@ -3250,10 +3250,12 @@ fun LiveTvScreen( LiveSubtitleSession(context, GeminiSegmentEngine(apiKey = { liveAiKey.value })) } DisposableEffect(liveAiSession) { onDispose { liveAiSession.release() } } - // Per channel, like the audio choice: true/false once picked; unset follows the activation setting. + // Per channel, like the audio choice: picking "AI subtitles" in the player always works for + // that stream; the settings switch only decides whether channels start with them on (Always). val liveAiSubsChoices = remember { mutableStateMapOf() } - val liveAiSubsOn = liveAiSubsSettings.enabled && playingChannelId != null && - (liveAiSubsChoices[playingChannelId] ?: (liveAiSubsSettings.activation == LiveAiSubsActivation.ALWAYS)) + val liveAiSubsOn = playingChannelId != null && + (liveAiSubsChoices[playingChannelId] + ?: (liveAiSubsSettings.enabled && liveAiSubsSettings.activation == LiveAiSubsActivation.ALWAYS)) val liveAiCue by liveAiSession.cue.collectAsState() val liveAiStats by liveAiSession.stats.collectAsState() var liveAiPreparingSeconds by remember { mutableStateOf(null) } @@ -4976,7 +4978,7 @@ fun LiveTvScreen( .takeIf { withAiSubtitlesOption( liveSubtitleOptions(liveTracks, ""), "", "", - aiAvailable = liveAiSubsSettings.enabled, aiSelected = false, + aiAvailable = true, aiSelected = false, ).size > 1 }, onAudioClick = { audioPickerOpen = true } @@ -5250,7 +5252,7 @@ fun LiveTvScreen( liveSubtitleOptions(liveTracks, stringResource(R.string.live_subtitles_off)), offLabel = stringResource(R.string.live_subtitles_off), aiLabel = stringResource(R.string.live_ai_subtitles), - aiAvailable = liveAiSubsSettings.enabled, + aiAvailable = true, aiSelected = liveAiSubsOn, ), onDismiss = { diff --git a/app/src/main/res/values-iw/strings.xml b/app/src/main/res/values-iw/strings.xml index b5a0f62b19..a29ff88d21 100644 --- a/app/src/main/res/values-iw/strings.xml +++ b/app/src/main/res/values-iw/strings.xml @@ -387,7 +387,7 @@ כתוביות AI אינן נתמכות בערוץ הזה אי אפשר לעכב את הערוץ הזה מספיק; חלק מהכתוביות עלולות לאחר כתוביות AI לטלוויזיה חיה - תמלול ותרגום של ערוצים חיים עם Gemini. השידור מתעכב כמה שניות כדי שהכתוביות יופיעו בזמן. + אפשר להפעיל כתוביות AI מתפריט הכתוביות בנגן בכל ערוץ. עם ״תמיד פעיל״ למטה, כל ערוץ מתחיל איתן. השידור מתעכב כמה שניות כדי שהכתוביות יופיעו בזמן. עיכוב כתוביות בטלוויזיה חיה בכמה לעכב את השידור החי. אוטומטי מתאים את עצמו למהירות ה-AI שנמדדה. %1$d שנ׳ diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index a8c5aeb8d1..5cc55a7ea4 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2094,7 +2094,7 @@ AI subtitles not supported on this channel This channel can\'t be delayed enough; some subtitles may be late Live TV AI subtitles - Transcribe and translate live channels with Gemini. Live TV is delayed a few seconds so lines appear on time. + Turn on AI subtitles from the subtitle menu in the player, on any channel. With Always on below, every channel starts with them. Live TV is delayed a few seconds so lines appear on time. Live TV subtitle delay How far live TV is held back. Auto uses the measured AI speed. %1$d s From c30f6e473e994cac95a4de334d479b7fa12fdf13 Mon Sep 17 00:00:00 2001 From: silentbil Date: Fri, 6 Nov 2026 14:12:39 +0200 Subject: [PATCH 4/8] Live TV AI subtitles: two settings, restart on switch-on, capture from the start - Settings are now just the feature switch and "auto-translate when a channel starts"; the delay is always measured, the language is the profile's subtitle language and the key is the AI subtitles Gemini key. - Switching to AI subtitles while a channel plays restarts the stream, so the audio that was already buffered is subtitled too; "Preparing subtitles" shows during the hold and until the first line arrives. - Audio is captured whenever AI subtitles are on for the channel (mini player included), so the lines for the start-up hold are not lost. --- .../tv/data/repository/CloudSyncRepository.kt | 13 +- .../settings/LiveAiSubsSettingsRows.kt | 117 ++++-------------- .../tv/ui/screens/settings/SettingsScreen.kt | 4 +- .../ui/screens/settings/SettingsViewModel.kt | 2 +- .../tv/ui/screens/tv/live/LiveTvScreen.kt | 96 ++++++++------ .../tv/live/aisubs/LiveAiSubsSettings.kt | 90 +++----------- .../tv/live/aisubs/LiveAiSubtitleOverlay.kt | 9 +- .../tv/live/aisubs/LiveCueScheduler.kt | 5 +- app/src/main/res/values-de/strings.xml | 19 +-- app/src/main/res/values-es/strings.xml | 19 +-- app/src/main/res/values-iw/strings.xml | 19 +-- app/src/main/res/values-pt-rBR/strings.xml | 19 +-- app/src/main/res/values-pt/strings.xml | 19 +-- app/src/main/res/values/strings.xml | 19 +-- .../tv/live/aisubs/LiveCueSchedulerTest.kt | 14 --- 15 files changed, 128 insertions(+), 336 deletions(-) diff --git a/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt b/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt index ac7324ad0d..5a00e1a1fc 100644 --- a/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt +++ b/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt @@ -498,8 +498,7 @@ class CloudSyncRepository @Inject constructor( "dnsProvider", "subtitleAiEnabled", "subtitleAiAutoSelect", "subtitleAiFindBestMatch", "subtitlePreloadEnabled", "dolbyVisionCompatEnabled", "subtitleAiApiKey", "subtitleAiModel", "subtitleRemoveHearingImpaired", "telegramSearchOnClickOnly", - "subtitleHearingSyncV2", "liveAiSubsEnabled", "liveAiSubsDelay", "liveAiSubsActivation", - "liveAiSubsTargetLanguage", "liveAiSubsApiKey" + "subtitleHearingSyncV2", "liveAiSubsEnabled", "liveAiSubsAutoStart" ) // Per-profile fields excluded from the generic merge (handled by their own logic / not values). private val profileMergeExclude = setOf("defaultSubtitle", "subtitleSettingsUpdatedAt") @@ -796,10 +795,7 @@ class CloudSyncRepository @Inject constructor( root.put("subtitleRemoveHearingImpaired", prefs[subtitleRemoveHearingImpairedKey] ?: true) // Live TV AI subtitles (global, like the AI subtitle key they share). root.put("liveAiSubsEnabled", prefs[com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.ENABLED] ?: false) - root.put("liveAiSubsDelay", prefs[com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.DELAY] ?: "") - root.put("liveAiSubsActivation", prefs[com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.ACTIVATION] ?: "") - root.put("liveAiSubsTargetLanguage", prefs[com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.TARGET_LANGUAGE] ?: "") - root.put("liveAiSubsApiKey", prefs[com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.OWN_API_KEY] ?: "") + root.put("liveAiSubsAutoStart", prefs[com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.AUTO_START] ?: false) root.put( "telegramSearchOnClickOnly", prefs[com.arflix.tv.data.telegram.TelegramRepository.KEY_SEARCH_ON_CLICK_ONLY] ?: true @@ -1873,10 +1869,7 @@ class CloudSyncRepository @Inject constructor( // has() guards: backups from versions without Live TV AI subtitles must not reset them. val live = com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs if (root.has("liveAiSubsEnabled")) prefs[live.ENABLED] = root.optBoolean("liveAiSubsEnabled", false) - root.optString("liveAiSubsDelay", "").takeIf { root.has("liveAiSubsDelay") && it.isNotBlank() }?.let { prefs[live.DELAY] = it } - root.optString("liveAiSubsActivation", "").takeIf { root.has("liveAiSubsActivation") && it.isNotBlank() }?.let { prefs[live.ACTIVATION] = it } - if (root.has("liveAiSubsTargetLanguage")) prefs[live.TARGET_LANGUAGE] = root.optString("liveAiSubsTargetLanguage", "") - root.optString("liveAiSubsApiKey", "").takeIf { it.isNotBlank() }?.let { prefs[live.OWN_API_KEY] = it } + if (root.has("liveAiSubsAutoStart")) prefs[live.AUTO_START] = root.optBoolean("liveAiSubsAutoStart", false) } } if (preservedNewerLocalSubtitle) { diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/LiveAiSubsSettingsRows.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/LiveAiSubsSettingsRows.kt index faf473b928..56ec74ef62 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/LiveAiSubsSettingsRows.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/LiveAiSubsSettingsRows.kt @@ -2,28 +2,18 @@ package com.arflix.tv.ui.screens.settings import android.content.Context import android.widget.Toast -import androidx.compose.material.icons.Icons -import androidx.compose.material.icons.filled.Language -import androidx.compose.material.icons.filled.Schedule -import androidx.compose.material.icons.filled.Tv -import androidx.compose.material.icons.filled.VpnKey import androidx.compose.runtime.Composable import androidx.compose.runtime.collectAsState import androidx.compose.runtime.getValue -import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.remember -import androidx.compose.runtime.setValue import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha import androidx.compose.ui.platform.LocalContext import androidx.compose.ui.res.stringResource import com.arflix.tv.R import com.arflix.tv.ui.components.MobileSettingsRow -import com.arflix.tv.ui.components.SettingsRow import com.arflix.tv.ui.components.SettingsToggleRow -import com.arflix.tv.ui.screens.player.subtitles.SubtitleAiModel import com.arflix.tv.ui.screens.tv.live.aisubs.GeminiSegmentEngine -import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsActivation -import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsDelay import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsSettings import kotlinx.coroutines.CoroutineScope @@ -31,20 +21,15 @@ import kotlinx.coroutines.Dispatchers import kotlinx.coroutines.SupervisorJob import kotlinx.coroutines.flow.first import kotlinx.coroutines.launch -import java.util.Locale /* - * Live TV AI subtitles settings, shown in the "AI subtitles" section next to the Gemini key they - * use. The rows read and write their own preferences (LiveAiSubsPrefs) instead of threading new - * state through the settings screen; SettingsViewModel pushes changes to the cloud. + * Live TV AI subtitles settings, in the "AI subtitles" section next to the Gemini key they use: + * the feature switch and "auto-translate when a channel starts". The rows read and write their + * own preferences (LiveAiSubsPrefs); SettingsViewModel pushes changes to the cloud. */ /** TV settings row indices, after the existing ones (0–50). */ -internal val LIVE_AI_SUBS_ROWS = listOf(51, 52, 53, 54, 55) - -private object LiveAiSubsKeyDialogState { - var open by mutableStateOf(false) -} +internal val LIVE_AI_SUBS_ROWS = listOf(51, 52) private val actionScope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate) @@ -59,27 +44,15 @@ internal fun onLiveAiSubsRowClick(context: Context, index: Int) { } else { enableAfterKeyCheck(appContext, settings) } - 52 -> LiveAiSubsPrefs.setDelay(appContext, settings.delay.next()) - 53 -> LiveAiSubsPrefs.setActivation(appContext, settings.activation.next()) - 54 -> { - val choices = LiveAiSubsPrefs.TARGET_LANGUAGE_CHOICES - val next = choices[(choices.indexOf(settings.targetLanguage).coerceAtLeast(0) + 1) % choices.size] - LiveAiSubsPrefs.setTargetLanguage(appContext, next) - } - 55 -> if (settings.usesSharedKey) { - toast(appContext, R.string.live_ai_subs_key_shared) - } else { - LiveAiSubsKeyDialogState.open = true - } + 52 -> if (settings.enabled) LiveAiSubsPrefs.setAutoStart(appContext, !settings.autoStart) } } } -/** Validates the key with a cheap call before turning the feature on, and says how it went. */ +/** Validates the AI subtitles Gemini key with a cheap call before turning the feature on. */ private suspend fun enableAfterKeyCheck(context: Context, settings: LiveAiSubsSettings) { if (!settings.hasKey) { toast(context, R.string.live_ai_subs_needs_key) - if (!settings.usesSharedKey) LiveAiSubsKeyDialogState.open = true return } when (GeminiSegmentEngine(apiKey = { settings.apiKey }).validateKey(settings.apiKey)) { @@ -108,51 +81,29 @@ private fun rememberLiveAiSubsSettings(): LiveAiSubsSettings { return settings } -@Composable -private fun delayLabel(delay: LiveAiSubsDelay): String = - if (delay == LiveAiSubsDelay.AUTO) stringResource(R.string.auto) else stringResource(R.string.live_ai_subs_seconds, (delay.fixedMs ?: 0L) / 1_000L) - -@Composable -private fun activationLabel(activation: LiveAiSubsActivation): String = stringResource( - if (activation == LiveAiSubsActivation.ALWAYS) R.string.live_ai_subs_activation_always else R.string.live_ai_subs_activation_per_channel, -) - -@Composable -private fun languageLabel(code: String): String = - if (code.isBlank()) { - stringResource(R.string.live_ai_subs_target_profile) - } else { - Locale.forLanguageTag(code).getDisplayLanguage(Locale.getDefault()).replaceFirstChar { it.titlecase(Locale.getDefault()) } - } - -@Composable -private fun keyLabel(settings: LiveAiSubsSettings): String = when { - settings.usesSharedKey -> stringResource(R.string.live_ai_subs_key_shared_short) - settings.ownApiKey.isBlank() -> stringResource(R.string.ai_key_not_set) - else -> "••••" + settings.ownApiKey.takeLast(4) -} - /** One TV row for [index] in [LIVE_AI_SUBS_ROWS]. */ @Composable internal fun LiveAiSubsTvRow(index: Int, isFocused: Boolean, modifier: Modifier) { val context = LocalContext.current val settings = rememberLiveAiSubsSettings() - val click = { onLiveAiSubsRowClick(context, index) } when (index) { 51 -> SettingsToggleRow( title = stringResource(R.string.live_ai_subs_title), subtitle = stringResource(R.string.live_ai_subs_desc), isEnabled = settings.enabled, isFocused = isFocused, - onToggle = { click() }, + onToggle = { onLiveAiSubsRowClick(context, 51) }, modifier = modifier, ) - 52 -> SettingsRow(Icons.Default.Schedule, stringResource(R.string.live_ai_subs_delay_title), stringResource(R.string.live_ai_subs_delay_desc), delayLabel(settings.delay), isFocused, click, modifier) - 53 -> SettingsRow(Icons.Default.Tv, stringResource(R.string.live_ai_subs_activation_title), stringResource(R.string.live_ai_subs_activation_desc), activationLabel(settings.activation), isFocused, click, modifier) - 54 -> SettingsRow(Icons.Default.Language, stringResource(R.string.live_ai_subs_target_title), stringResource(R.string.live_ai_subs_target_desc), languageLabel(settings.targetLanguage), isFocused, click, modifier) - 55 -> SettingsRow(Icons.Default.VpnKey, stringResource(R.string.live_ai_subs_key_title), stringResource(R.string.live_ai_subs_key_desc), keyLabel(settings), isFocused, click, modifier) + 52 -> SettingsToggleRow( + title = stringResource(R.string.live_ai_subs_auto_title), + subtitle = stringResource(R.string.live_ai_subs_auto_desc), + isEnabled = settings.enabled && settings.autoStart, + isFocused = isFocused, + onToggle = { onLiveAiSubsRowClick(context, 52) }, + modifier = if (settings.enabled) modifier else modifier.alpha(0.4f), + ) } - if (index == LIVE_AI_SUBS_ROWS.last()) LiveAiSubsKeyDialogHost() } /** The same settings as touch rows, for the mobile settings page. */ @@ -167,33 +118,13 @@ internal fun LiveAiSubsMobileRows() { toggleChecked = settings.enabled, onClick = { onLiveAiSubsRowClick(context, 51) }, ) - MobileSettingsRow(icon = Icons.Default.Schedule, title = stringResource(R.string.live_ai_subs_delay_title), subtitle = stringResource(R.string.live_ai_subs_delay_desc), value = delayLabel(settings.delay), isToggle = false, onClick = { onLiveAiSubsRowClick(context, 52) }) - MobileSettingsRow(icon = Icons.Default.Tv, title = stringResource(R.string.live_ai_subs_activation_title), subtitle = stringResource(R.string.live_ai_subs_activation_desc), value = activationLabel(settings.activation), isToggle = false, onClick = { onLiveAiSubsRowClick(context, 53) }) - MobileSettingsRow(icon = Icons.Default.Language, title = stringResource(R.string.live_ai_subs_target_title), subtitle = stringResource(R.string.live_ai_subs_target_desc), value = languageLabel(settings.targetLanguage), isToggle = false, onClick = { onLiveAiSubsRowClick(context, 54) }) - MobileSettingsRow(icon = Icons.Default.VpnKey, title = stringResource(R.string.live_ai_subs_key_title), subtitle = stringResource(R.string.live_ai_subs_key_desc), value = keyLabel(settings), isToggle = false, showDivider = false, onClick = { onLiveAiSubsRowClick(context, 55) }) - LiveAiSubsKeyDialogHost() -} - -@Composable -private fun LiveAiSubsKeyDialogHost() { - if (!LiveAiSubsKeyDialogState.open) return - val context = LocalContext.current - val settings = rememberLiveAiSubsSettings() - AiApiKeyDialog( - currentKey = settings.ownApiKey, - model = SubtitleAiModel.GEMINI_FLASH_25, - onSave = { key -> - LiveAiSubsKeyDialogState.open = false - val appContext = context.applicationContext - actionScope.launch { - LiveAiSubsPrefs.setOwnApiKey(appContext, key) - when (GeminiSegmentEngine(apiKey = { key }).validateKey(key)) { - true -> toast(appContext, R.string.live_ai_subs_key_valid) - false -> toast(appContext, R.string.live_ai_subs_key_rejected) - null -> toast(appContext, R.string.live_ai_subs_key_unchecked) - } - } - }, - onDismiss = { LiveAiSubsKeyDialogState.open = false }, + MobileSettingsRow( + title = stringResource(R.string.live_ai_subs_auto_title), + subtitle = stringResource(R.string.live_ai_subs_auto_desc), + value = stringResource(if (settings.enabled && settings.autoStart) R.string.on else R.string.off), + toggleChecked = settings.enabled && settings.autoStart, + enabled = settings.enabled, + showDivider = false, + onClick = { onLiveAiSubsRowClick(context, 52) }, ) } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt index 37366a2687..f5990ff1bb 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt @@ -1486,7 +1486,7 @@ fun SettingsScreen( 31 -> viewModel.setSubtitleRemoveHearingImpaired(!uiState.subtitleRemoveHearingImpaired) 32 -> showAiApiKeyDialog = true 33 -> viewModel.startAiKeyServer() - 51, 52, 53, 54, 55 -> onLiveAiSubsRowClick(context, tvGeneralRowsForSection(currentSection)[contentFocusIndex]) + 51, 52 -> onLiveAiSubsRowClick(context, tvGeneralRowsForSection(currentSection)[contentFocusIndex]) 34 -> viewModel.cycleTrailerDelay() 47 -> viewModel.cycleGuideRowCount() 37 -> viewModel.setTrailerInCards(!uiState.trailerInCards) @@ -7587,7 +7587,7 @@ private fun AiModelDialog( } @Composable -internal fun AiApiKeyDialog( +private fun AiApiKeyDialog( currentKey: String, onSave: (String) -> Unit, onDismiss: () -> Unit, diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt index 9202b6ede5..1edab8a861 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt @@ -661,7 +661,7 @@ class SettingsViewModel @Inject constructor( // push them to the cloud like every other setting, once they settle. viewModelScope.launch { com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.flow(context) - .map { listOf(it.enabled, it.delay, it.activation, it.targetLanguage, it.ownApiKey) } + .map { listOf(it.enabled, it.autoStart) } .distinctUntilChanged() .drop(1) .collect { syncLocalStateToCloud(silent = true) } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt index f0da97559b..87068af01e 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt @@ -124,7 +124,6 @@ import com.arflix.tv.ui.screens.player.audiosync.AudioSyncExtractorsFactory import com.arflix.tv.ui.screens.tv.live.aisubs.AudioTapHlsExtractorFactory import com.arflix.tv.ui.screens.player.subtitles.RTL_LANGUAGES import com.arflix.tv.ui.screens.tv.live.aisubs.GeminiSegmentEngine -import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsActivation import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsDebug import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsSettings @@ -132,7 +131,7 @@ import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsStatus import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubtitleOverlay import com.arflix.tv.ui.screens.tv.live.aisubs.LiveSubtitleSession import com.arflix.tv.ui.screens.tv.live.aisubs.LiveSubtitleStyle -import com.arflix.tv.ui.screens.tv.live.aisubs.liveAiSubsDelayMs +import com.arflix.tv.ui.screens.tv.live.aisubs.autoLiveAiSubsDelayMs import com.arflix.tv.ui.screens.tv.live.aisubs.SubtitleLookaheadLoadControl import androidx.media3.exoplayer.ExoPlayer import androidx.media3.exoplayer.hls.HlsMediaSource @@ -3250,29 +3249,33 @@ fun LiveTvScreen( LiveSubtitleSession(context, GeminiSegmentEngine(apiKey = { liveAiKey.value })) } DisposableEffect(liveAiSession) { onDispose { liveAiSession.release() } } - // Per channel, like the audio choice: picking "AI subtitles" in the player always works for - // that stream; the settings switch only decides whether channels start with them on (Always). + // Per channel, like the audio choice: picked in the subtitle menu, or on by default when + // "auto-translate when a channel starts" is set. val liveAiSubsChoices = remember { mutableStateMapOf() } - val liveAiSubsOn = playingChannelId != null && - (liveAiSubsChoices[playingChannelId] - ?: (liveAiSubsSettings.enabled && liveAiSubsSettings.activation == LiveAiSubsActivation.ALWAYS)) + val liveAiSubsOn = liveAiSubsSettings.enabled && playingChannelId != null && + (liveAiSubsChoices[playingChannelId] ?: liveAiSubsSettings.autoStart) val liveAiCue by liveAiSession.cue.collectAsState() val liveAiStats by liveAiSession.stats.collectAsState() var liveAiPreparingSeconds by remember { mutableStateOf(null) } var liveAiDebugVisible by remember { mutableStateOf(false) } - var liveAiWarmupSinceMs by remember { mutableStateOf(null) } + // Since when AI subtitles have been waiting for their first line (new source or switched on). + var liveAiAwaitingSinceMs by remember { mutableStateOf(null) } + var liveAiAwaitingFirstLine by remember { mutableStateOf(false) } + // Bumped to restart the stream so audio already buffered gets subtitled too. + var liveAiRestartRequest by remember { mutableIntStateOf(0) } var liveSubtitleStyle by remember { mutableStateOf(LiveSubtitleStyle()) } LaunchedEffect(Unit) { liveSubtitleStyle = viewModel.liveSubtitleStyle() } - LaunchedEffect(liveAiSubsSettings.targetLanguage) { - val code = liveAiSubsSettings.targetLanguage.ifBlank { viewModel.liveAiSubsDefaultLanguage() } + // Always the profile's subtitle language. + LaunchedEffect(Unit) { + val code = viewModel.liveAiSubsDefaultLanguage() val name = java.util.Locale.forLanguageTag(code).getDisplayLanguage(java.util.Locale.ENGLISH) .ifBlank { "English" } liveAiSession.setTargetLanguage(name, name.lowercase(java.util.Locale.ROOT) in RTL_LANGUAGES) } LaunchedEffect(liveAiSubsSettings.apiKey) { liveAiSession.onKeyChanged() } - // Capture costs API calls, so only while it can be seen: fullscreen, and playing or about to. - val liveAiCapture = liveAiSubsOn && isFullScreen && !fullscreenGuideOpen && !quickZapOpen && - (playerPlayWhenReady || liveAiWarmupSinceMs != null) + // Capture whenever AI subtitles are on for the channel, the mini player included: the audio + // buffered during the start-up hold is what the first lines are made from. + val liveAiCapture = liveAiSubsOn && playerPlayWhenReady LaunchedEffect(liveAiSubsOn, liveAiSubsSettings.hasKey, liveAiCapture) { liveAiSession.update(enabled = liveAiSubsOn, hasKey = liveAiSubsSettings.hasKey, capture = liveAiCapture) } @@ -3282,25 +3285,26 @@ fun LiveTvScreen( applyLiveSubtitleChoice(exoPlayer, LIVE_SUBTITLES_OFF) } } - // Turned on mid-channel: pause until the look-ahead has built up (shown as "Preparing"). + // Turned on while the channel plays: the seconds already buffered were downloaded before + // capture started, so they would play without subtitles. Restart the stream instead; the + // start-up hold then shows "Preparing subtitles" and the first line is covered. LaunchedEffect(liveAiSubsOn) { - if (liveAiSubsOn && playingCatchupProgram == null && exoPlayer.isPlaying) { - val requiredMs = liveAiSubsDelayMs(liveAiSubsSettings.delay, liveAiSession.latencyP95Ms) - if (exoPlayer.bufferedPosition - exoPlayer.currentPosition < requiredMs) { - liveAiWarmupSinceMs = android.os.SystemClock.elapsedRealtime() - exoPlayer.pause() - } - } else if (!liveAiSubsOn && liveAiWarmupSinceMs != null) { - liveAiWarmupSinceMs = null - exoPlayer.play() + if (!liveAiSubsOn) { + liveAiAwaitingSinceMs = null + return@LaunchedEffect + } + liveAiAwaitingSinceMs = android.os.SystemClock.elapsedRealtime() + if (exoPlayer.isPlaying && exoPlayer.currentMediaItem?.mediaId == playingChannelId.orEmpty()) { + liveAiRestartRequest++ } } - val liveAiTick by rememberUpdatedState(Triple(liveAiSubsOn, liveAiSubsSettings.delay, playingCatchupProgram == null)) + LaunchedEffect(liveAiCue) { if (liveAiCue != null) liveAiAwaitingSinceMs = null } + val liveAiTick by rememberUpdatedState(liveAiSubsOn to (playingCatchupProgram == null)) LaunchedEffect(exoPlayer, liveAiSession) { val window = androidx.media3.common.Timeline.Window() while (true) { - val (on, delaySetting, isLive) = liveAiTick - val requiredMs = if (on && isLive) liveAiSubsDelayMs(delaySetting, liveAiSession.latencyP95Ms) else 0L + val (on, isLive) = liveAiTick + val requiredMs = if (on && isLive) autoLiveAiSubsDelayMs(liveAiSession.latencyP95Ms) else 0L liveLoadControl.requiredLookaheadUs = requiredMs * 1_000L liveAiDebugVisible = LiveAiSubsDebug.enabled if (on) { @@ -3311,15 +3315,10 @@ fun LiveTvScreen( timeline.getWindow(exoPlayer.currentMediaItemIndex, window).positionInFirstPeriodUs val lookaheadMs = (exoPlayer.bufferedPosition - exoPlayer.currentPosition).coerceAtLeast(0L) liveAiSession.tick(exoPlayer.currentPosition * 1_000L + windowOffsetUs, exoPlayer.isPlaying, lookaheadMs) - liveAiWarmupSinceMs?.let { since -> - val waited = android.os.SystemClock.elapsedRealtime() - since - if (lookaheadMs >= requiredMs || waited > LIVE_AI_WARMUP_TIMEOUT_MS) { - liveAiWarmupSinceMs = null - exoPlayer.play() - } - } - val held = (liveLoadControl.holding && exoPlayer.playbackState == Player.STATE_BUFFERING) || - liveAiWarmupSinceMs != null + val held = liveLoadControl.holding && exoPlayer.playbackState == Player.STATE_BUFFERING + liveAiAwaitingFirstLine = !held && liveAiAwaitingSinceMs?.let { since -> + android.os.SystemClock.elapsedRealtime() - since < LIVE_AI_FIRST_LINE_NOTE_MS + } == true liveAiPreparingSeconds = if (held && requiredMs > 0L) { ((requiredMs - lookaheadMs + 999L) / 1_000L).toInt().coerceAtLeast(1) } else { @@ -3333,6 +3332,7 @@ fun LiveTvScreen( } } else { liveAiPreparingSeconds = null + liveAiAwaitingFirstLine = false } delay(LIVE_AI_TICK_MS) } @@ -3519,7 +3519,7 @@ fun LiveTvScreen( // Live TV AI subtitles: every channel source gets a fresh audio tap, so a load still // running for the previous channel can never feed this one. liveAiSession.onNewSource() - liveAiWarmupSinceMs = null + liveAiAwaitingSinceMs = android.os.SystemClock.elapsedRealtime() val liveAudioSink = liveAiSession.pipeline.newSink() val source = if (isHls) { HlsMediaSource.Factory(sourceDataFactory) @@ -3567,6 +3567,19 @@ fun LiveTvScreen( ) } + // AI subtitles switched on mid-channel: restart the stream (see liveAiRestartRequest). + LaunchedEffect(liveAiRestartRequest) { + if (liveAiRestartRequest == 0) return@LaunchedEffect + val prepared = lastPreparedStreamUrl ?: return@LaunchedEffect + val restart = { + prepareStream(prepared, isHls = lastPreparedIsHls, headers = lastPreparedHeaders, resetRetry = false, + drmInfo = playingChannel?.source?.drmInfo, forcePrepare = true, + resolvedMimeType = lastPreparedMimeType) + } + val restartChannelId = playingChannelId + liveStreamGate.reopen(coroutineScope, { playingChannelId == restartChannelId }, { restart() }, restart) + } + fun toggleCatchupPlayback() { if (playingCatchupProgram == null) return if (exoPlayer.isPlaying) { @@ -4938,7 +4951,8 @@ fun LiveTvScreen( LiveAiSubtitleOverlay( cue = liveAiCue, style = liveSubtitleStyle, - preparingSeconds = liveAiPreparingSeconds, + preparingText = liveAiPreparingSeconds?.let { stringResource(R.string.live_ai_subs_preparing, it) } + ?: stringResource(R.string.live_ai_subs_preparing_short).takeIf { liveAiAwaitingFirstLine }, debugStats = liveAiStats.takeIf { liveAiDebugVisible }, ) } @@ -4978,7 +4992,7 @@ fun LiveTvScreen( .takeIf { withAiSubtitlesOption( liveSubtitleOptions(liveTracks, ""), "", "", - aiAvailable = true, aiSelected = false, + aiAvailable = liveAiSubsSettings.enabled, aiSelected = false, ).size > 1 }, onAudioClick = { audioPickerOpen = true } @@ -5252,7 +5266,7 @@ fun LiveTvScreen( liveSubtitleOptions(liveTracks, stringResource(R.string.live_subtitles_off)), offLabel = stringResource(R.string.live_subtitles_off), aiLabel = stringResource(R.string.live_ai_subtitles), - aiAvailable = true, + aiAvailable = liveAiSubsSettings.enabled, aiSelected = liveAiSubsOn, ), onDismiss = { @@ -5610,8 +5624,8 @@ private fun classifyPlaybackError(error: PlaybackException): String { /** How often Live TV AI subtitles follow the player position (cue display, hold, look-ahead). */ private const val LIVE_AI_TICK_MS = 100L -/** Turned on mid-channel, playback waits at most this long for the look-ahead to build up. */ -private const val LIVE_AI_WARMUP_TIMEOUT_MS = 20_000L +/** How long "Preparing subtitles…" may wait for the first line before giving up (silence, music). */ +private const val LIVE_AI_FIRST_LINE_NOTE_MS = 25_000L internal fun buildLiveTvConfiguration(): MediaItem.LiveConfiguration = MediaItem.LiveConfiguration.Builder() diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsSettings.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsSettings.kt index 8536e90553..ffae563b6d 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsSettings.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsSettings.kt @@ -10,85 +10,35 @@ import com.arflix.tv.ui.screens.player.subtitles.SubtitleAiModel import kotlinx.coroutines.flow.Flow import kotlinx.coroutines.flow.map -/** How long live playback is held back so subtitles arrive before their line is heard. */ -internal enum class LiveAiSubsDelay(val prefValue: String, val fixedMs: Long?) { - /** Uses the measured AI latency; see [autoLiveAiSubsDelayMs]. */ - AUTO("auto", null), - SIX("6", 6_000L), - TEN("10", 10_000L), - FIFTEEN("15", 15_000L); - - fun next(): LiveAiSubsDelay = entries[(ordinal + 1) % entries.size] - - companion object { - /** - * Auto: Gemini's answer time for a few seconds of audio was measured at 5-9 s on a real - * box, so no fixed value suits every connection; Auto follows the measurement. - */ - val DEFAULT = AUTO - fun fromPref(value: String?): LiveAiSubsDelay = entries.firstOrNull { it.prefValue == value } ?: DEFAULT - } -} - /** - * Per channel by default: with the feature on, every channel change waits for the delay, so only - * channels the viewer picked "AI subtitles" on pay that cost. + * Live TV AI subtitles settings, kept deliberately small: the feature switch, and whether + * channels start with AI subtitles on. The delay is always measured (Auto), the language is the + * profile's subtitle language, and the key is the AI subtitles Gemini key. */ -internal enum class LiveAiSubsActivation(val prefValue: String) { - PER_CHANNEL("per_channel"), - ALWAYS("always"); - - fun next(): LiveAiSubsActivation = if (this == PER_CHANNEL) ALWAYS else PER_CHANNEL - - companion object { - fun fromPref(value: String?): LiveAiSubsActivation = entries.firstOrNull { it.prefValue == value } ?: PER_CHANNEL - } -} - internal data class LiveAiSubsSettings( + /** Offers "AI subtitles" in the live player's subtitle menu. */ val enabled: Boolean = false, - val delay: LiveAiSubsDelay = LiveAiSubsDelay.DEFAULT, - val activation: LiveAiSubsActivation = LiveAiSubsActivation.PER_CHANNEL, - /** A language code, or blank for the profile's subtitle language. */ - val targetLanguage: String = "", - /** The Gemini key in effect: the AI subtitles key when that is a Gemini key, else [ownApiKey]. */ + /** Every channel starts with AI subtitles on, without picking them in the menu. */ + val autoStart: Boolean = false, + /** The AI subtitles key when that model is Gemini; blank otherwise. */ val apiKey: String = "", - /** A Gemini key just for Live TV, used while the AI subtitles model is not Gemini. */ - val ownApiKey: String = "", - /** True while the shared AI subtitles key is a Gemini key (and so is the one used here). */ - val usesSharedKey: Boolean = true, ) { val hasKey: Boolean get() = apiKey.isNotBlank() } -/** - * Global (not per-profile) settings for Live TV AI subtitles, next to the existing AI subtitle - * key and model in the same DataStore, and synced to the cloud with them (CloudSyncRepository). - */ +/** Global (not per-profile) settings, next to the AI subtitle key and model, synced to the cloud. */ internal object LiveAiSubsPrefs { val ENABLED = booleanPreferencesKey("live_ai_subs_enabled") - val DELAY = stringPreferencesKey("live_ai_subs_delay") - val ACTIVATION = stringPreferencesKey("live_ai_subs_activation") - val TARGET_LANGUAGE = stringPreferencesKey("live_ai_subs_target_language") - val OWN_API_KEY = stringPreferencesKey("live_ai_subs_api_key") + val AUTO_START = booleanPreferencesKey("live_ai_subs_auto_start") private val SHARED_API_KEY = stringPreferencesKey("subtitle_ai_api_key") private val SHARED_MODEL = stringPreferencesKey("subtitle_ai_model") - /** Target language choices offered in settings, cycled in this order ("" = profile default). */ - val TARGET_LANGUAGE_CHOICES = listOf("", "he", "en", "ar", "ru", "es", "fr", "de", "pt", "it") - fun read(prefs: Preferences): LiveAiSubsSettings { - val sharedIsGemini = prefs[SHARED_MODEL] == SubtitleAiModel.GEMINI_FLASH_25.name - val shared = prefs[SHARED_API_KEY].orEmpty().trim() - val own = prefs[OWN_API_KEY].orEmpty().trim() + val isGemini = prefs[SHARED_MODEL] == SubtitleAiModel.GEMINI_FLASH_25.name return LiveAiSubsSettings( enabled = prefs[ENABLED] ?: false, - delay = LiveAiSubsDelay.fromPref(prefs[DELAY]), - activation = LiveAiSubsActivation.fromPref(prefs[ACTIVATION]), - targetLanguage = prefs[TARGET_LANGUAGE].orEmpty(), - apiKey = if (sharedIsGemini && shared.isNotBlank()) shared else own, - ownApiKey = own, - usesSharedKey = sharedIsGemini && shared.isNotBlank(), + autoStart = prefs[AUTO_START] ?: false, + apiKey = if (isGemini) prefs[SHARED_API_KEY].orEmpty().trim() else "", ) } @@ -98,19 +48,7 @@ internal object LiveAiSubsPrefs { context.settingsDataStore.edit { it[ENABLED] = enabled } } - suspend fun setDelay(context: Context, delay: LiveAiSubsDelay) { - context.settingsDataStore.edit { it[DELAY] = delay.prefValue } - } - - suspend fun setActivation(context: Context, activation: LiveAiSubsActivation) { - context.settingsDataStore.edit { it[ACTIVATION] = activation.prefValue } - } - - suspend fun setTargetLanguage(context: Context, code: String) { - context.settingsDataStore.edit { it[TARGET_LANGUAGE] = code } - } - - suspend fun setOwnApiKey(context: Context, key: String) { - context.settingsDataStore.edit { it[OWN_API_KEY] = key.trim() } + suspend fun setAutoStart(context: Context, autoStart: Boolean) { + context.settingsDataStore.edit { it[AUTO_START] = autoStart } } } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt index ef1e2c1259..d6b3627c80 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt @@ -11,7 +11,6 @@ import androidx.compose.runtime.Composable import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color -import androidx.compose.ui.res.stringResource import androidx.compose.ui.text.font.FontFamily import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp @@ -19,7 +18,6 @@ import androidx.compose.ui.viewinterop.AndroidView import androidx.media3.common.text.Cue import androidx.media3.ui.SubtitleView import androidx.tv.material3.Text -import com.arflix.tv.R import com.arflix.tv.ui.screens.player.applySubtitleAppearance /** The profile's subtitle look, as the VOD player uses it. */ @@ -41,7 +39,8 @@ internal data class LiveSubtitleStyle( internal fun LiveAiSubtitleOverlay( cue: LiveCue?, style: LiveSubtitleStyle, - preparingSeconds: Int?, + /** "Preparing subtitles…" while the first lines are not ready yet, or null. */ + preparingText: String?, debugStats: LiveAiSubsStats?, modifier: Modifier = Modifier, ) { @@ -59,9 +58,9 @@ internal fun LiveAiSubtitleOverlay( }, modifier = Modifier.fillMaxSize(), ) - if (preparingSeconds != null) { + if (preparingText != null) { Text( - text = stringResource(R.string.live_ai_subs_preparing, preparingSeconds.coerceAtLeast(1)), + text = preparingText, color = Color.White, fontSize = 16.sp, modifier = Modifier diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueScheduler.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueScheduler.kt index 3b8bc3027d..dde4288383 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueScheduler.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueScheduler.kt @@ -106,16 +106,13 @@ internal class LiveCueScheduler { /** * The hold ("Auto" delay) that lets a whole segment be heard, sent and answered before its first * word plays: the longest segment, plus the AI's p95 latency, plus a margin. Before any latency has - * been measured, it assumes [DEFAULT_LATENCY_P95_MS]. Clamped to 6–15 s, the range offered in settings. + * been measured, it assumes [DEFAULT_LATENCY_P95_MS]. Clamped to 6–15 s. */ internal fun autoLiveAiSubsDelayMs(latencyP95Ms: Long?): Long { val latency = latencyP95Ms ?: DEFAULT_LATENCY_P95_MS return (LIVE_MAX_SEGMENT_MS + latency + DELAY_MARGIN_MS).coerceIn(6_000L, 15_000L) } -/** The hold for [delay], given the measured latency for Auto. */ -internal fun liveAiSubsDelayMs(delay: LiveAiSubsDelay, latencyP95Ms: Long?): Long = - delay.fixedMs ?: autoLiveAiSubsDelayMs(latencyP95Ms) /** The 95th percentile of [samples], or null with fewer than [minSamples]. */ internal fun percentile95(samples: List, minSamples: Int = 3): Long? { diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index 0dd32d84d8..9d243b84ef 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -1930,23 +1930,12 @@ KI-Untertitel werden auf diesem Sender nicht unterstützt Dieser Sender kann nicht genug verzögert werden; manche Untertitel kommen evtl. zu spät KI-Untertitel für Live-TV - Live-Sender mit Gemini transkribieren und übersetzen. Live-TV wird einige Sekunden verzögert, damit die Zeilen rechtzeitig erscheinen. - Untertitel-Verzögerung (Live-TV) - Wie weit Live-TV zurückgehalten wird. Auto nutzt die gemessene KI-Geschwindigkeit. - %1$d s - Aktivierung der KI-Untertitel (Live-TV) - Jeder Senderwechsel wartet die Verzögerung ab; pro Sender verzögert nur Sender, auf denen du sie einschaltest - Pro Sender (Untertitelmenü) - Immer an - Untertitelsprache (Live-TV) - Die Sprache, in die Live-Sender übersetzt werden - Untertitelsprache des Profils - Gemini-API-Schlüssel für Live-TV - Live-TV nutzt den KI-Untertitel-Schlüssel, wenn es ein Gemini-Schlüssel ist, sonst diesen - Live-TV nutzt den Gemini-Schlüssel der KI-Untertitel oben - Nutzt den KI-Untertitel-Schlüssel + Fügt KI-Untertitel zum Untertitelmenü von Live-Sendern hinzu: Gemini transkribiert und übersetzt in die Untertitelsprache des Profils. Live-TV wird einige Sekunden verzögert, damit die Zeilen rechtzeitig erscheinen. Gemini-API-Schlüssel funktioniert Gemini hat diesen API-Schlüssel abgelehnt Schlüssel konnte nicht geprüft werden (offline?) Füge zuerst einen Google-Gemini-API-Schlüssel hinzu + Automatisch übersetzen, wenn ein Sender startet + Jeder Sender startet mit KI-Untertiteln, ohne sie im Untertitelmenü auszuwählen + Untertitel werden vorbereitet… diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index 5a68cc959f..ec3f505311 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -1650,23 +1650,12 @@ Subtítulos IA no disponibles en este canal Este canal no se puede retrasar lo suficiente; algunos subtítulos pueden llegar tarde Subtítulos IA en TV en directo - Transcribe y traduce canales en directo con Gemini. La TV en directo se retrasa unos segundos para que las líneas lleguen a tiempo. - Retraso de subtítulos en directo - Cuánto se retrasa la TV en directo. Auto usa la velocidad medida de la IA. - %1$d s - Activación de subtítulos IA en directo - Cada cambio de canal espera el retraso; por canal solo retrasa los canales donde los activas - Por canal (menú de subtítulos) - Siempre activados - Idioma de subtítulos en directo - El idioma al que se traducen los canales en directo - Idioma de subtítulos del perfil - Clave de API de Gemini para TV en directo - Usa la clave de subtítulos IA si es de Gemini; si no, esta - La TV en directo usa la clave de Gemini de subtítulos IA de arriba - Usa la clave de subtítulos IA + Añade subtítulos IA al menú de subtítulos de los canales en directo: Gemini transcribe y traduce al idioma de subtítulos del perfil. La TV en directo se retrasa unos segundos para que las líneas lleguen a tiempo. La clave de API de Gemini funciona Gemini rechazó esta clave de API No se pudo comprobar la clave (¿sin conexión?) Primero añade una clave de API de Google Gemini + Traducir automáticamente al iniciar un canal + Cada canal empieza con subtítulos IA, sin elegirlos en el menú de subtítulos + Preparando subtítulos… diff --git a/app/src/main/res/values-iw/strings.xml b/app/src/main/res/values-iw/strings.xml index a29ff88d21..46cd397fee 100644 --- a/app/src/main/res/values-iw/strings.xml +++ b/app/src/main/res/values-iw/strings.xml @@ -387,23 +387,12 @@ כתוביות AI אינן נתמכות בערוץ הזה אי אפשר לעכב את הערוץ הזה מספיק; חלק מהכתוביות עלולות לאחר כתוביות AI לטלוויזיה חיה - אפשר להפעיל כתוביות AI מתפריט הכתוביות בנגן בכל ערוץ. עם ״תמיד פעיל״ למטה, כל ערוץ מתחיל איתן. השידור מתעכב כמה שניות כדי שהכתוביות יופיעו בזמן. - עיכוב כתוביות בטלוויזיה חיה - בכמה לעכב את השידור החי. אוטומטי מתאים את עצמו למהירות ה-AI שנמדדה. - %1$d שנ׳ - הפעלת כתוביות AI בטלוויזיה חיה - כל החלפת ערוץ ממתינה לעיכוב, לכן \"לפי ערוץ\" מעכב רק ערוצים שבהם הפעלת - לפי ערוץ (תפריט כתוביות) - תמיד פעיל - שפת הכתוביות בטלוויזיה חיה - השפה שאליה מתורגמים הערוצים החיים - שפת הכתוביות של הפרופיל - מפתח Gemini API לטלוויזיה חיה - משתמש במפתח של כתוביות AI כשהוא מפתח Gemini, אחרת במפתח הזה - הטלוויזיה החיה משתמשת במפתח Gemini של כתוביות AI שלמעלה - משתמש במפתח של כתוביות AI + מוסיף כתוביות AI לתפריט הכתוביות בערוצים חיים: Gemini מתמלל ומתרגם לשפת הכתוביות של הפרופיל. השידור מתעכב כמה שניות כדי שהכתוביות יופיעו בזמן. מפתח ה-Gemini API תקין Gemini דחה את המפתח הזה לא ניתן לבדוק את המפתח (אין חיבור?) יש להוסיף קודם מפתח Google Gemini API + תרגום אוטומטי כשערוץ מתחיל + כל ערוץ מתחיל עם כתוביות AI, בלי לבחור אותן בתפריט הכתוביות + מכין כתוביות… diff --git a/app/src/main/res/values-pt-rBR/strings.xml b/app/src/main/res/values-pt-rBR/strings.xml index 4a8f257fd5..8b4df4e3a5 100644 --- a/app/src/main/res/values-pt-rBR/strings.xml +++ b/app/src/main/res/values-pt-rBR/strings.xml @@ -1562,23 +1562,12 @@ Legendas IA não suportadas neste canal Não dá para atrasar este canal o suficiente; algumas legendas podem chegar atrasadas Legendas IA na TV ao vivo - Transcreve e traduz canais ao vivo com o Gemini. A TV ao vivo é atrasada alguns segundos para as linhas aparecerem na hora certa. - Atraso das legendas ao vivo - Quanto a TV ao vivo é atrasada. Auto usa a velocidade medida da IA. - %1$d s - Ativação das legendas IA ao vivo - Cada troca de canal espera o atraso; por canal só atrasa os canais em que você ativar - Por canal (menu de legendas) - Sempre ativas - Idioma das legendas ao vivo - O idioma para o qual os canais ao vivo são traduzidos - Idioma de legendas do perfil - Chave de API do Gemini para TV ao vivo - Usa a chave das legendas IA quando é do Gemini; caso contrário, esta - A TV ao vivo usa a chave do Gemini das legendas IA acima - Usa a chave das legendas IA + Adiciona legendas IA ao menu de legendas dos canais ao vivo: o Gemini transcreve e traduz para o idioma de legendas do perfil. A TV ao vivo é atrasada alguns segundos para as linhas aparecerem na hora certa. A chave de API do Gemini funciona O Gemini rejeitou esta chave de API Não foi possível verificar a chave (sem conexão?) Adicione primeiro uma chave de API do Google Gemini + Traduzir automaticamente ao iniciar um canal + Cada canal começa com legendas IA, sem escolher no menu de legendas + Preparando legendas… diff --git a/app/src/main/res/values-pt/strings.xml b/app/src/main/res/values-pt/strings.xml index 092e7c2807..a6f9e10645 100644 --- a/app/src/main/res/values-pt/strings.xml +++ b/app/src/main/res/values-pt/strings.xml @@ -1562,23 +1562,12 @@ Legendas IA não suportadas neste canal Não é possível atrasar este canal o suficiente; algumas legendas podem chegar tarde Legendas IA na TV em direto - Transcreve e traduz canais em direto com o Gemini. A TV em direto é atrasada alguns segundos para as linhas aparecerem a tempo. - Atraso das legendas em direto - Quanto a TV em direto é atrasada. Auto usa a velocidade medida da IA. - %1$d s - Ativação das legendas IA em direto - Cada mudança de canal espera pelo atraso; por canal só atrasa os canais onde as ativa - Por canal (menu de legendas) - Sempre ativas - Idioma das legendas em direto - O idioma para o qual os canais em direto são traduzidos - Idioma de legendas do perfil - Chave de API do Gemini para TV em direto - Usa a chave das legendas IA quando é do Gemini; caso contrário, esta - A TV em direto usa a chave do Gemini das legendas IA acima - Usa a chave das legendas IA + Adiciona legendas IA ao menu de legendas dos canais em direto: o Gemini transcreve e traduz para o idioma de legendas do perfil. A TV em direto é atrasada alguns segundos para as linhas aparecerem a tempo. A chave de API do Gemini funciona O Gemini rejeitou esta chave de API Não foi possível verificar a chave (sem ligação?) Adicione primeiro uma chave de API do Google Gemini + Traduzir automaticamente ao iniciar um canal + Cada canal começa com legendas IA, sem as escolher no menu de legendas + A preparar legendas… diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index 5cc55a7ea4..b410f18a62 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2094,23 +2094,12 @@ AI subtitles not supported on this channel This channel can\'t be delayed enough; some subtitles may be late Live TV AI subtitles - Turn on AI subtitles from the subtitle menu in the player, on any channel. With Always on below, every channel starts with them. Live TV is delayed a few seconds so lines appear on time. - Live TV subtitle delay - How far live TV is held back. Auto uses the measured AI speed. - %1$d s - Live TV AI subtitles activation - Every channel change waits for the delay, so per channel only delays channels you turn it on for - Per channel (subtitle menu) - Always on - Live TV subtitle language - The language live channels are translated into - Profile subtitle language - Gemini API key for Live TV - Live TV subtitles use the AI subtitles key when it is a Gemini key, otherwise this one - Live TV uses the AI subtitles Gemini key above - Uses the AI subtitles key + Adds AI subtitles to the subtitle menu of live channels: Gemini transcribes and translates into your profile\'s subtitle language. Live TV is delayed a few seconds so lines appear on time. Gemini API key works Gemini rejected this API key Couldn\'t check the API key (offline?) Add a Google Gemini API key first + Auto-translate when a channel starts + Every channel starts with AI subtitles on, without picking them in the subtitle menu + Preparing subtitles… diff --git a/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueSchedulerTest.kt b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueSchedulerTest.kt index b7647304ff..716a4f28f8 100644 --- a/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueSchedulerTest.kt +++ b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueSchedulerTest.kt @@ -113,23 +113,9 @@ class LiveAiSubsDelayTest { assertEquals(15_000L, autoLiveAiSubsDelayMs(30_000L)) } - @Test fun fixedDelaysIgnoreLatency() { - assertEquals(6_000L, liveAiSubsDelayMs(LiveAiSubsDelay.SIX, 9_000L)) - assertEquals(10_000L, liveAiSubsDelayMs(LiveAiSubsDelay.TEN, null)) - assertEquals(15_000L, liveAiSubsDelayMs(LiveAiSubsDelay.FIFTEEN, 100L)) - assertEquals(11_000L, liveAiSubsDelayMs(LiveAiSubsDelay.AUTO, null)) - } - @Test fun percentileNeedsAFewSamples() { assertNull(percentile95(listOf(1L, 2L))) assertEquals(96L, percentile95((1L..100L).toList())) assertEquals(1_900L, percentile95(listOf(1_000L, 1_200L, 1_900L, 1_100L))) } - - @Test fun delayChoicesCycleAndUnknownValuesFallBackToAuto() { - assertEquals(LiveAiSubsDelay.SIX, LiveAiSubsDelay.AUTO.next()) - assertEquals(LiveAiSubsDelay.AUTO, LiveAiSubsDelay.FIFTEEN.next()) - assertEquals(LiveAiSubsDelay.AUTO, LiveAiSubsDelay.fromPref("garbage")) - assertEquals(LiveAiSubsActivation.PER_CHANNEL, LiveAiSubsActivation.fromPref(null)) - } } From b6c7e079ee6b9470bdf13f4da21452ef2bb88add Mon Sep 17 00:00:00 2001 From: silentbil Date: Fri, 6 Nov 2026 14:37:29 +0200 Subject: [PATCH 5/8] Live TV AI subtitles: show "Preparing subtitles" only during the hold Once playback is held back, every coming line is translated in time, so the extra note after the hold only misled during quiet stretches. --- .../tv/ui/screens/tv/live/LiveTvScreen.kt | 22 ++----------------- .../tv/live/aisubs/LiveAiSubtitleOverlay.kt | 2 +- app/src/main/res/values-de/strings.xml | 1 - app/src/main/res/values-es/strings.xml | 1 - app/src/main/res/values-iw/strings.xml | 1 - app/src/main/res/values-pt-rBR/strings.xml | 1 - app/src/main/res/values-pt/strings.xml | 1 - app/src/main/res/values/strings.xml | 1 - 8 files changed, 3 insertions(+), 27 deletions(-) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt index 87068af01e..bb72cf56d7 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt @@ -3258,9 +3258,6 @@ fun LiveTvScreen( val liveAiStats by liveAiSession.stats.collectAsState() var liveAiPreparingSeconds by remember { mutableStateOf(null) } var liveAiDebugVisible by remember { mutableStateOf(false) } - // Since when AI subtitles have been waiting for their first line (new source or switched on). - var liveAiAwaitingSinceMs by remember { mutableStateOf(null) } - var liveAiAwaitingFirstLine by remember { mutableStateOf(false) } // Bumped to restart the stream so audio already buffered gets subtitled too. var liveAiRestartRequest by remember { mutableIntStateOf(0) } var liveSubtitleStyle by remember { mutableStateOf(LiveSubtitleStyle()) } @@ -3289,16 +3286,10 @@ fun LiveTvScreen( // capture started, so they would play without subtitles. Restart the stream instead; the // start-up hold then shows "Preparing subtitles" and the first line is covered. LaunchedEffect(liveAiSubsOn) { - if (!liveAiSubsOn) { - liveAiAwaitingSinceMs = null - return@LaunchedEffect - } - liveAiAwaitingSinceMs = android.os.SystemClock.elapsedRealtime() - if (exoPlayer.isPlaying && exoPlayer.currentMediaItem?.mediaId == playingChannelId.orEmpty()) { + if (liveAiSubsOn && exoPlayer.isPlaying && exoPlayer.currentMediaItem?.mediaId == playingChannelId.orEmpty()) { liveAiRestartRequest++ } } - LaunchedEffect(liveAiCue) { if (liveAiCue != null) liveAiAwaitingSinceMs = null } val liveAiTick by rememberUpdatedState(liveAiSubsOn to (playingCatchupProgram == null)) LaunchedEffect(exoPlayer, liveAiSession) { val window = androidx.media3.common.Timeline.Window() @@ -3316,9 +3307,6 @@ fun LiveTvScreen( val lookaheadMs = (exoPlayer.bufferedPosition - exoPlayer.currentPosition).coerceAtLeast(0L) liveAiSession.tick(exoPlayer.currentPosition * 1_000L + windowOffsetUs, exoPlayer.isPlaying, lookaheadMs) val held = liveLoadControl.holding && exoPlayer.playbackState == Player.STATE_BUFFERING - liveAiAwaitingFirstLine = !held && liveAiAwaitingSinceMs?.let { since -> - android.os.SystemClock.elapsedRealtime() - since < LIVE_AI_FIRST_LINE_NOTE_MS - } == true liveAiPreparingSeconds = if (held && requiredMs > 0L) { ((requiredMs - lookaheadMs + 999L) / 1_000L).toInt().coerceAtLeast(1) } else { @@ -3332,7 +3320,6 @@ fun LiveTvScreen( } } else { liveAiPreparingSeconds = null - liveAiAwaitingFirstLine = false } delay(LIVE_AI_TICK_MS) } @@ -3519,7 +3506,6 @@ fun LiveTvScreen( // Live TV AI subtitles: every channel source gets a fresh audio tap, so a load still // running for the previous channel can never feed this one. liveAiSession.onNewSource() - liveAiAwaitingSinceMs = android.os.SystemClock.elapsedRealtime() val liveAudioSink = liveAiSession.pipeline.newSink() val source = if (isHls) { HlsMediaSource.Factory(sourceDataFactory) @@ -4951,8 +4937,7 @@ fun LiveTvScreen( LiveAiSubtitleOverlay( cue = liveAiCue, style = liveSubtitleStyle, - preparingText = liveAiPreparingSeconds?.let { stringResource(R.string.live_ai_subs_preparing, it) } - ?: stringResource(R.string.live_ai_subs_preparing_short).takeIf { liveAiAwaitingFirstLine }, + preparingText = liveAiPreparingSeconds?.let { stringResource(R.string.live_ai_subs_preparing, it) }, debugStats = liveAiStats.takeIf { liveAiDebugVisible }, ) } @@ -5624,9 +5609,6 @@ private fun classifyPlaybackError(error: PlaybackException): String { /** How often Live TV AI subtitles follow the player position (cue display, hold, look-ahead). */ private const val LIVE_AI_TICK_MS = 100L -/** How long "Preparing subtitles…" may wait for the first line before giving up (silence, music). */ -private const val LIVE_AI_FIRST_LINE_NOTE_MS = 25_000L - internal fun buildLiveTvConfiguration(): MediaItem.LiveConfiguration = MediaItem.LiveConfiguration.Builder() // Respect the manifest's segment/hold-back timing. An arbitrary eight-second diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt index d6b3627c80..4001856c8c 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt @@ -39,7 +39,7 @@ internal data class LiveSubtitleStyle( internal fun LiveAiSubtitleOverlay( cue: LiveCue?, style: LiveSubtitleStyle, - /** "Preparing subtitles…" while the first lines are not ready yet, or null. */ + /** "Preparing subtitles… N s" while playback is held for the delay, or null. */ preparingText: String?, debugStats: LiveAiSubsStats?, modifier: Modifier = Modifier, diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index 9d243b84ef..0f69f8c245 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -1937,5 +1937,4 @@ Füge zuerst einen Google-Gemini-API-Schlüssel hinzu Automatisch übersetzen, wenn ein Sender startet Jeder Sender startet mit KI-Untertiteln, ohne sie im Untertitelmenü auszuwählen - Untertitel werden vorbereitet… diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index ec3f505311..f50bedea6b 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -1657,5 +1657,4 @@ Primero añade una clave de API de Google Gemini Traducir automáticamente al iniciar un canal Cada canal empieza con subtítulos IA, sin elegirlos en el menú de subtítulos - Preparando subtítulos… diff --git a/app/src/main/res/values-iw/strings.xml b/app/src/main/res/values-iw/strings.xml index 46cd397fee..9026a29dfb 100644 --- a/app/src/main/res/values-iw/strings.xml +++ b/app/src/main/res/values-iw/strings.xml @@ -394,5 +394,4 @@ יש להוסיף קודם מפתח Google Gemini API תרגום אוטומטי כשערוץ מתחיל כל ערוץ מתחיל עם כתוביות AI, בלי לבחור אותן בתפריט הכתוביות - מכין כתוביות… diff --git a/app/src/main/res/values-pt-rBR/strings.xml b/app/src/main/res/values-pt-rBR/strings.xml index 8b4df4e3a5..ce8b9a7f66 100644 --- a/app/src/main/res/values-pt-rBR/strings.xml +++ b/app/src/main/res/values-pt-rBR/strings.xml @@ -1569,5 +1569,4 @@ Adicione primeiro uma chave de API do Google Gemini Traduzir automaticamente ao iniciar um canal Cada canal começa com legendas IA, sem escolher no menu de legendas - Preparando legendas… diff --git a/app/src/main/res/values-pt/strings.xml b/app/src/main/res/values-pt/strings.xml index a6f9e10645..64c0cdcc79 100644 --- a/app/src/main/res/values-pt/strings.xml +++ b/app/src/main/res/values-pt/strings.xml @@ -1569,5 +1569,4 @@ Adicione primeiro uma chave de API do Google Gemini Traduzir automaticamente ao iniciar um canal Cada canal começa com legendas IA, sem as escolher no menu de legendas - A preparar legendas… diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index b410f18a62..ffe6915935 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2101,5 +2101,4 @@ Add a Google Gemini API key first Auto-translate when a channel starts Every channel starts with AI subtitles on, without picking them in the subtitle menu - Preparing subtitles… From e5ca2c65125455e7a432dcebb70c2dc498c39800 Mon Sep 17 00:00:00 2001 From: silentbil Date: Fri, 6 Nov 2026 15:47:47 +0200 Subject: [PATCH 6/8] Live TV AI subtitles: translate the channel's text subtitles first, audio as fallback - Tap text tracks next to audio: CEA-608 closed captions and parsed text renditions (WebVTT etc.) become sentences that are translated as text; audio is only sent for stretches the captions do not cover. - Picture subtitles (DVB, PGS, VobSub) are never used, including ones Media3 parses into cues during extraction (original type in Format.codecs). - Auto-translate does not start on a channel that already has subtitles in the profile language. --- .../arflix/tv/ui/screens/tv/TvViewModel.kt | 3 + .../tv/ui/screens/tv/live/LiveSubtitles.kt | 30 ++++ .../tv/ui/screens/tv/live/LiveTvScreen.kt | 49 ++++-- .../tv/live/aisubs/CaptionCueReader.kt | 43 +++++ .../tv/live/aisubs/Cea608TextAssembler.kt | 157 ++++++++++++++++++ .../tv/live/aisubs/GeminiCaptionTranslator.kt | 45 +++++ .../tv/live/aisubs/LiveAiSubtitleOverlay.kt | 3 +- .../screens/tv/live/aisubs/LiveAudioTapHls.kt | 17 +- .../tv/live/aisubs/LiveSpeechPipeline.kt | 53 +++++- .../tv/live/aisubs/LiveSubtitleSession.kt | 48 +++++- .../tv/live/aisubs/LiveTapExtractors.kt | 81 +++++++++ .../tv/live/aisubs/Cea608TextAssemblerTest.kt | 96 +++++++++++ 12 files changed, 591 insertions(+), 34 deletions(-) create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/CaptionCueReader.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/Cea608TextAssembler.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiCaptionTranslator.kt create mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveTapExtractors.kt create mode 100644 app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/Cea608TextAssemblerTest.kt diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/TvViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/TvViewModel.kt index 150ec2448a..4abcbc0019 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/TvViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/TvViewModel.kt @@ -149,6 +149,9 @@ class TvViewModel @Inject constructor( */ internal suspend fun liveAiSubsDefaultLanguage(): String = (liveSubtitleDefault() as? com.arflix.tv.ui.screens.tv.live.LiveSubtitleDefault.Language)?.code + // The device language, in the same form as track languages ("iw" -> "he"). + ?: (com.arflix.tv.ui.screens.tv.live.liveSubtitleDefault(java.util.Locale.getDefault().language) + as? com.arflix.tv.ui.screens.tv.live.LiveSubtitleDefault.Language)?.code ?: java.util.Locale.getDefault().language /** The profile's subtitle look, read the same way the VOD player reads it. */ diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt index 1195ca0057..26901b3489 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt @@ -86,6 +86,36 @@ internal fun withAiSubtitlesOption( return withSelection + LiveTrackOption(LIVE_SUBTITLES_AI, aiLabel, selected = aiSelected) } +/** + * The subtitle track AI subtitles read text from: a text track (not a bitmap one, which carries + * no text), English first, never one already in [avoidLanguage] (that one needs no AI). Null when + * the channel has none; it is then subtitled from its audio. + */ +internal fun liveCaptionSourceKey(tracks: Tracks, avoidLanguage: String?): String? { + val textTracks = typedTracks(tracks, C.TRACK_TYPE_TEXT).filter { track -> + !isBitmapSubtitle(track.group.getTrackFormat(track.trackIndex)) && + (avoidLanguage == null || track.language != avoidLanguage) + } + return (textTracks.firstOrNull { it.language == "en" } ?: textTracks.firstOrNull())?.key +} + +/** + * True for picture subtitles (DVB, PGS, VobSub), which carry no text. Subtitles the player parses + * during extraction arrive as `application/x-media3-cues`, with the original type kept in codecs. + */ +internal fun isBitmapSubtitle(format: androidx.media3.common.Format): Boolean = + format.sampleMimeType in BITMAP_SUBTITLE_MIMES || format.codecs in BITMAP_SUBTITLE_MIMES + +/** The key of the subtitle track selected now, or null when subtitles are off. */ +internal fun liveSelectedSubtitleKey(tracks: Tracks): String? = + typedTracks(tracks, C.TRACK_TYPE_TEXT).firstOrNull { it.selected }?.key + +private val BITMAP_SUBTITLE_MIMES = setOf( + androidx.media3.common.MimeTypes.APPLICATION_DVBSUBS, + androidx.media3.common.MimeTypes.APPLICATION_PGS, + androidx.media3.common.MimeTypes.APPLICATION_VOBSUB, +) + /** The audio choices for what is playing, one per language or track. */ internal fun liveAudioOptions(tracks: Tracks, displayLocale: Locale = Locale.getDefault()) = liveTrackOptions(tracks, C.TRACK_TYPE_AUDIO, offLabel = null, displayLocale) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt index bb72cf56d7..f7c4e31876 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt @@ -120,10 +120,11 @@ import androidx.media3.common.Player import androidx.media3.datasource.HttpDataSource import androidx.media3.datasource.okhttp.OkHttpDataSource import androidx.media3.exoplayer.DefaultLoadControl -import com.arflix.tv.ui.screens.player.audiosync.AudioSyncExtractorsFactory import com.arflix.tv.ui.screens.tv.live.aisubs.AudioTapHlsExtractorFactory import com.arflix.tv.ui.screens.player.subtitles.RTL_LANGUAGES +import com.arflix.tv.ui.screens.tv.live.aisubs.GeminiCaptionTranslator import com.arflix.tv.ui.screens.tv.live.aisubs.GeminiSegmentEngine +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveTapExtractorsFactory import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsDebug import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsSettings @@ -3246,14 +3247,24 @@ fun LiveTvScreen( val liveAiKey = rememberUpdatedState(liveAiSubsSettings.apiKey) val liveAiSession = remember(context) { LiveAiSubsDebug.start() - LiveSubtitleSession(context, GeminiSegmentEngine(apiKey = { liveAiKey.value })) + LiveSubtitleSession( + context, + GeminiSegmentEngine(apiKey = { liveAiKey.value }), + GeminiCaptionTranslator(apiKey = { liveAiKey.value }), + ) } DisposableEffect(liveAiSession) { onDispose { liveAiSession.release() } } + // The profile's subtitle language: what AI subtitles translate into. + var liveAiTargetCode by remember { mutableStateOf(null) } + // A channel that already carries subtitles in that language needs no AI to start with. + val liveChannelHasTargetSubs = liveAiTargetCode?.let { code -> + liveSubtitleOptions(liveTracks, "").any { it.key == code } + } == true // Per channel, like the audio choice: picked in the subtitle menu, or on by default when - // "auto-translate when a channel starts" is set. + // "auto-translate when a channel starts" is set (unless the channel has the language already). val liveAiSubsChoices = remember { mutableStateMapOf() } val liveAiSubsOn = liveAiSubsSettings.enabled && playingChannelId != null && - (liveAiSubsChoices[playingChannelId] ?: liveAiSubsSettings.autoStart) + (liveAiSubsChoices[playingChannelId] ?: (liveAiSubsSettings.autoStart && !liveChannelHasTargetSubs)) val liveAiCue by liveAiSession.cue.collectAsState() val liveAiStats by liveAiSession.stats.collectAsState() var liveAiPreparingSeconds by remember { mutableStateOf(null) } @@ -3265,6 +3276,7 @@ fun LiveTvScreen( // Always the profile's subtitle language. LaunchedEffect(Unit) { val code = viewModel.liveAiSubsDefaultLanguage() + liveAiTargetCode = code val name = java.util.Locale.forLanguageTag(code).getDisplayLanguage(java.util.Locale.ENGLISH) .ifBlank { "English" } liveAiSession.setTargetLanguage(name, name.lowercase(java.util.Locale.ROOT) in RTL_LANGUAGES) @@ -3276,10 +3288,16 @@ fun LiveTvScreen( LaunchedEffect(liveAiSubsOn, liveAiSubsSettings.hasKey, liveAiCapture) { liveAiSession.update(enabled = liveAiSubsOn, hasKey = liveAiSubsSettings.hasKey, capture = liveAiCapture) } - // The stream's own subtitles stay off while AI subtitles show. - LaunchedEffect(liveAiSubsOn, liveTracks) { - if (liveAiSubsOn && liveSubtitleOptions(liveTracks, "").any { it.selected && it.key != LIVE_SUBTITLES_OFF }) { - applyLiveSubtitleChoice(exoPlayer, LIVE_SUBTITLES_OFF) + // While AI subtitles show, the stream's own text subtitles stay selected but hidden: they are + // read ahead of playback and translated (a stream that is not selected is not even + // downloaded). Without a text track, AI subtitles come from the audio. + LaunchedEffect(liveAiSubsOn, liveTracks, liveAiTargetCode) { + if (!liveAiSubsOn) return@LaunchedEffect + val source = liveCaptionSourceKey(liveTracks, liveAiTargetCode) + val selected = liveSelectedSubtitleKey(liveTracks) + when { + source != null && selected != source -> applyLiveSubtitleChoice(exoPlayer, source) + source == null && selected != null -> applyLiveSubtitleChoice(exoPlayer, LIVE_SUBTITLES_OFF) } } // Turned on while the channel plays: the seconds already buffered were downloaded before @@ -3506,15 +3524,15 @@ fun LiveTvScreen( // Live TV AI subtitles: every channel source gets a fresh audio tap, so a load still // running for the previous channel can never feed this one. liveAiSession.onNewSource() - val liveAudioSink = liveAiSession.pipeline.newSink() + val liveTapSinks = liveAiSession.pipeline.newSinks() val source = if (isHls) { HlsMediaSource.Factory(sourceDataFactory) .setAllowChunklessPreparation(IPTV_HLS_CHUNKLESS_PREPARATION) - .setExtractorFactory(AudioTapHlsExtractorFactory(iptvHlsExtractorFactory(), liveAudioSink)) + .setExtractorFactory(AudioTapHlsExtractorFactory(iptvHlsExtractorFactory(), liveTapSinks)) .setLoadErrorHandlingPolicy(IptvLoadErrorHandlingPolicy()) .createMediaSource(mediaItem) } else { - DefaultMediaSourceFactory(context, AudioSyncExtractorsFactory(iptvExtractorsFactory(), liveAudioSink)) + DefaultMediaSourceFactory(context, LiveTapExtractorsFactory(iptvExtractorsFactory(), liveTapSinks)) .setDataSourceFactory(sourceDataFactory) .setLoadErrorHandlingPolicy( IptvLoadErrorHandlingPolicy( @@ -4930,6 +4948,9 @@ fun LiveTvScreen( if (view.player !== exoPlayer) { view.player = exoPlayer } + // With AI subtitles the stream's own subtitles are only read, not shown. + view.subtitleView?.visibility = + if (liveAiSubsOn) android.view.View.INVISIBLE else android.view.View.VISIBLE }, modifier = Modifier.fillMaxSize(), ) @@ -5260,12 +5281,12 @@ fun LiveTvScreen( }, onPick = { key -> if (key == LIVE_SUBTITLES_AI) { - // AI subtitles replace the stream's own; remembered per channel. + // AI subtitles replace the stream's own (which keep feeding them, hidden); + // remembered per channel. playingChannelId?.let { liveAiSubsChoices[it] = true - liveSubtitleChoices[it] = LIVE_SUBTITLES_OFF + liveSubtitleChoices.remove(it) } - applyLiveSubtitleChoice(exoPlayer, LIVE_SUBTITLES_OFF) } else { playingChannelId?.let { liveAiSubsChoices[it] = false diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/CaptionCueReader.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/CaptionCueReader.kt new file mode 100644 index 0000000000..417d4940ad --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/CaptionCueReader.kt @@ -0,0 +1,43 @@ +@file:androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) + +package com.arflix.tv.ui.screens.tv.live.aisubs + +import androidx.media3.extractor.text.CueDecoder + +/** + * Reads subtitle streams the player already parsed during extraction (WebVTT and other text + * renditions arrive as Media3's own cue samples, `application/x-media3-cues`) into sentences on + * the media timeline. Live caption streams usually roll up: each cue repeats the previous line + * and adds one, so lines seen in the last few cues are dropped and only new ones are emitted. + * + * Not thread-safe: one instance per channel source, fed from the loading thread. + */ +internal class CaptionCueReader(private val onSentence: (CaptionSentence) -> Unit) { + private val decoder = CueDecoder() + private val recentLines = ArrayDeque() + + fun feed(timeUs: Long, data: ByteArray, offset: Int, size: Int) { + val cues = runCatching { decoder.decode(timeUs, data, offset, size) }.getOrNull() ?: return + val lines = cues.cues.flatMap { cue -> cue.text?.toString().orEmpty().split('\n') } + .map { it.replace(WHITESPACE, " ").trim() } + .filter { it.isNotEmpty() } + val fresh = lines.filter { it !in recentLines } + lines.forEach { line -> + if (line !in recentLines) recentLines.addLast(line) + while (recentLines.size > RECENT_LINES) recentLines.removeFirst() + } + if (fresh.isEmpty()) return + val endUs = if (cues.endTimeUs > cues.startTimeUs) cues.endTimeUs else cues.startTimeUs + DEFAULT_DURATION_US + onSentence(CaptionSentence(cues.startTimeUs, endUs, fresh.joinToString(" "))) + } + + fun reset() { + recentLines.clear() + } + + private companion object { + const val RECENT_LINES = 6 + const val DEFAULT_DURATION_US = 3_000_000L + val WHITESPACE = Regex("\\s+") + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/Cea608TextAssembler.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/Cea608TextAssembler.kt new file mode 100644 index 0000000000..9c223dbcba --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/Cea608TextAssembler.kt @@ -0,0 +1,157 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +/** One caption sentence on the media timeline: from its first character to its last. */ +internal data class CaptionSentence(val startUs: Long, val endUs: Long, val text: String) + +/** + * Turns a channel's CEA-608 closed captions (field 1, caption channel CC1) into plain sentences + * with media times, so they can be translated ahead of playback instead of transcribing audio. + * + * Input is the `cc_data` the TS extractor copies out of the video's SEI messages: triplets of + * (marker/valid/type, byte 1, byte 2), as the player's own CEA-608 decoder receives them. Display + * layout (roll-up, pop-on, positions, colours) is ignored: only the spoken words matter here. + * Control codes are sent twice by broadcasters; the repeat is dropped. A sentence ends at . ? ! + * (or after a pause of [GAP_US], or at [MAX_CHARS]). + * + * Not thread-safe: one instance per channel source, fed from the loading thread. + */ +internal class Cea608TextAssembler(private val onSentence: (CaptionSentence) -> Unit) { + private val text = StringBuilder() + private var firstCharUs = UNSET + private var lastCharUs = UNSET + private var lastControl = -1 + private var pendingBoundary = false + + /** True once any caption character has been seen for this source. */ + var sawText: Boolean = false + private set + + fun feed(timeUs: Long, data: ByteArray, offset: Int, size: Int) { + if (lastCharUs != UNSET && timeUs - lastCharUs > GAP_US) flush() + var i = offset + val end = offset + size - 2 + while (i < end) { + val header = data[i].toInt() and 0xFF + val b1 = data[i + 1].toInt() and 0x7F + val b2 = data[i + 2].toInt() and 0x7F + i += 3 + val valid = header and 0x04 != 0 + val fieldOne = header and 0x03 == 0 + if (!valid || !fieldOne || (b1 == 0 && b2 == 0)) continue + handlePair(timeUs, b1, b2) + } + } + + /** The stream jumped (seek, reload): what was collected no longer joins what comes next. */ + fun reset() { + text.setLength(0) + firstCharUs = UNSET + lastCharUs = UNSET + lastControl = -1 + pendingBoundary = false + } + + /** Emits whatever is collected as a sentence. */ + fun flush() { + val sentence = text.toString().replace(WHITESPACE, " ").trim() + if (sentence.length >= MIN_CHARS && firstCharUs != UNSET) { + onSentence(CaptionSentence(firstCharUs, lastCharUs, sentence)) + } + text.setLength(0) + firstCharUs = UNSET + lastCharUs = UNSET + pendingBoundary = false + } + + private fun handlePair(timeUs: Long, b1: Int, b2: Int) { + if (b1 in 0x10..0x1F) { + val code = (b1 shl 8) or b2 + // Broadcasters send every control code twice; act on it once. + if (code == lastControl) { + lastControl = -1 + return + } + lastControl = code + // The second caption channel (CC2) uses 0x18-0x1F. + if (b1 >= 0x18) return + handleControl(timeUs, b1, b2) + return + } + lastControl = -1 + if (b1 >= 0x20) appendChar(timeUs, basicChar(b1)) + if (b2 >= 0x20) appendChar(timeUs, basicChar(b2)) + } + + private fun handleControl(timeUs: Long, b1: Int, b2: Int) { + when { + // Miscellaneous control codes (RCL, BS, RU2-4, EDM, CR, ENM, EOC...). + (b1 == 0x14 || b1 == 0x15) && b2 in 0x20..0x2F -> when (b2) { + 0x21 -> backspace() + else -> boundary() + } + // Special characters (♪, ®, ½...): only letters with meaning are kept. + b1 == 0x11 && b2 in 0x30..0x3F -> if (b2 == 0x37) boundary() else appendChar(timeUs, SPECIAL[b2 - 0x30]) + // Extended characters (accented capitals, symbols) follow a plain fallback letter + // that a basic decoder shows instead; keeping that letter is good enough here. + (b1 == 0x12 || b1 == 0x13) && b2 in 0x20..0x3F -> Unit + // Preamble address codes and mid-row codes move the cursor: a word boundary. + else -> boundary() + } + } + + private fun boundary() { + pendingBoundary = true + } + + private fun backspace() { + if (text.isNotEmpty()) text.setLength(text.length - 1) + } + + private fun appendChar(timeUs: Long, c: Char) { + if (c == '\u0000') return + if (pendingBoundary && text.isNotEmpty() && text.last() != ' ') text.append(' ') + pendingBoundary = false + if (c == ' ' && (text.isEmpty() || text.last() == ' ')) return + if (c != ' ') { + sawText = true + if (firstCharUs == UNSET) firstCharUs = timeUs + lastCharUs = timeUs + } + text.append(c) + if (c in SENTENCE_END && text.length >= MIN_CHARS) { + flush() + } else if (c == ' ' && (text.length >= MAX_CHARS || lastCharUs - firstCharUs >= MAX_DURATION_US)) { + // Long run-on captions end at a word boundary, like audio segments end at 5 s, so a + // line is never held back longer than the playback delay allows for. + flush() + } + } + + private fun basicChar(b: Int): Char = when (b) { + 0x2A -> 'á' + 0x5C -> 'é' + 0x5E -> 'í' + 0x5F -> 'ó' + 0x60 -> 'ú' + 0x7B -> 'ç' + 0x7C -> '÷' + 0x7D -> 'Ñ' + 0x7E -> 'ñ' + 0x7F -> '\u0000' + else -> b.toChar() + } + + companion object { + private const val UNSET = Long.MIN_VALUE + /** A pause in the captions this long ends the sentence. */ + const val GAP_US = 2_500_000L + const val MAX_CHARS = 160 + const val MAX_DURATION_US = 5_000_000L + private const val MIN_CHARS = 2 + private val SENTENCE_END = setOf('.', '?', '!') + private val WHITESPACE = Regex("\\s+") + private val SPECIAL = charArrayOf( + '®', '°', '½', '¿', '™', '¢', '£', '\u0000', 'à', ' ', 'è', 'â', 'ê', 'î', 'ô', 'û', + ) + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiCaptionTranslator.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiCaptionTranslator.kt new file mode 100644 index 0000000000..1a8da8cf86 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiCaptionTranslator.kt @@ -0,0 +1,45 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import com.arflix.tv.ui.screens.player.subtitles.SubtitleAiModel +import com.arflix.tv.ui.screens.player.subtitles.SubtitleTranslationService + +/** Translates one sentence of a channel's own captions into timed lines. */ +internal interface LiveCaptionTranslator { + suspend fun translate(sentence: CaptionSentence, targetLanguageName: String, rtl: Boolean): LiveEngineResult +} + +/** + * Caption sentences go through the app's existing Gemini subtitle translation (the same model, + * prompt and retry rules as AI subtitles for films), as one-line batches: the text is already + * there, so this is faster and cheaper than sending audio, and makes no listening mistakes. + */ +internal class GeminiCaptionTranslator(private val apiKey: () -> String) : LiveCaptionTranslator { + private val service = SubtitleTranslationService( + apiKeyProvider = apiKey, + modelProvider = { SubtitleAiModel.GEMINI_FLASH_25 }, + ) + + override suspend fun translate(sentence: CaptionSentence, targetLanguageName: String, rtl: Boolean): LiveEngineResult { + if (apiKey().isBlank()) return LiveEngineResult.InvalidKey + val result = service.translateBatch(listOf(sentence.text), targetLanguageName) + if (!result.success) return errorResult(result.errorMessage.orEmpty()) + val line = result.lines.firstOrNull()?.trim().orEmpty() + if (line.isEmpty()) return LiveEngineResult.Cues(emptyList(), null) + val endUs = maxOf(sentence.endUs, sentence.startUs) + CAPTION_TAIL_US + return LiveEngineResult.Cues(listOf(LiveCue(sentence.startUs, endUs, line, rtl)), null) + } + + companion object { + /** Captions are typed as they are spoken; the line stays a little after its last word. */ + const val CAPTION_TAIL_US = 1_500_000L + + fun errorResult(message: String): LiveEngineResult = when { + message == "RATE_LIMITED" -> LiveEngineResult.QuotaExceeded + message.startsWith("HTTP 401") || message.startsWith("HTTP 403") || + message.contains("API_KEY_INVALID") || message.contains("API key not valid") -> LiveEngineResult.InvalidKey + message.contains("Unable to resolve host", ignoreCase = true) || + message.contains("Failed to connect", ignoreCase = true) -> LiveEngineResult.Offline + else -> LiveEngineResult.Failed(message.take(80)) + } + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt index 4001856c8c..1105c6452f 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt @@ -76,7 +76,8 @@ internal fun LiveAiSubtitleOverlay( "look-ahead ${lookaheadMs?.let { "%.1f s".format(it / 1000f) } ?: "-"}\n" + "AI p50 ${latencyP50Ms ?: "-"} ms p95 ${latencyP95Ms ?: "-"} ms\n" + "segments/min $segmentsPerMinute late dropped $lateDropped\n" + - "skipped $skippedSegments backlog skips $backlogSkips failures $failures" + "skipped $skippedSegments backlog skips $backlogSkips failures $failures\n" + + "lines from captions $captionLines" }, color = Color(0xFFB9F6CA), fontSize = 12.sp, diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt index 4f1beec8e0..5c50f437ad 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt @@ -11,11 +11,9 @@ import androidx.media3.exoplayer.hls.HlsMediaChunkExtractor import androidx.media3.extractor.ExtractorInput import androidx.media3.extractor.ExtractorOutput import androidx.media3.extractor.text.SubtitleParser -import com.arflix.tv.ui.screens.player.audiosync.AudioSampleSink -import com.arflix.tv.ui.screens.player.audiosync.AudioTapExtractorOutput /** - * HLS counterpart of `AudioSyncExtractorsFactory`: HLS segments are not read through an + * HLS counterpart of [LiveTapExtractorsFactory]: HLS segments are not read through an * `ExtractorsFactory`, so the audio copy is made here, around whatever [delegate] returns. Hooking * the chunk extractor's output covers TS segments (ARVIO's reused TS extractor) and every other * segment type (packed audio, fMP4) in one place. @@ -25,7 +23,7 @@ import com.arflix.tv.ui.screens.player.audiosync.AudioTapExtractorOutput */ internal class AudioTapHlsExtractorFactory( private val delegate: HlsExtractorFactory, - private val sink: AudioSampleSink, + private val sinks: LiveTapSinks, ) : HlsExtractorFactory { override fun createExtractor( uri: Uri, @@ -45,7 +43,7 @@ internal class AudioTapHlsExtractorFactory( "codecs=${format.codecs} mime=${format.sampleMimeType ?: format.containerMimeType}", ) } - return AudioTapHlsChunkExtractor(extractor, sink) + return AudioTapHlsChunkExtractor(extractor, sinks) } override fun setSubtitleParserFactory(subtitleParserFactory: SubtitleParser.Factory): HlsExtractorFactory { @@ -70,10 +68,10 @@ internal class AudioTapHlsExtractorFactory( private class AudioTapHlsChunkExtractor( private val delegate: HlsMediaChunkExtractor, - private val sink: AudioSampleSink, + private val sinks: LiveTapSinks, ) : HlsMediaChunkExtractor { override fun init(extractorOutput: ExtractorOutput) = - delegate.init(AudioTapExtractorOutput(extractorOutput, sink)) + delegate.init(LiveTapExtractorOutput(extractorOutput, sinks)) override fun read(extractorInput: ExtractorInput): Boolean = delegate.read(extractorInput) @@ -81,11 +79,12 @@ private class AudioTapHlsChunkExtractor( override fun isReusable(): Boolean = delegate.isReusable - override fun recreate(): HlsMediaChunkExtractor = AudioTapHlsChunkExtractor(delegate.recreate(), sink) + override fun recreate(): HlsMediaChunkExtractor = AudioTapHlsChunkExtractor(delegate.recreate(), sinks) override fun onTruncatedSegmentParsed() { // The extractor restarts its state; the next samples may not continue the previous ones. - sink.onDiscontinuity() + sinks.audio.onDiscontinuity() + sinks.captions.onDiscontinuity() delegate.onTruncatedSegmentParsed() } } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt index a82d63a4f3..74b4371257 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt @@ -6,6 +6,7 @@ import android.content.Context import android.os.Process import android.os.SystemClock import androidx.media3.common.Format +import androidx.media3.common.MimeTypes import com.arflix.tv.R import com.arflix.tv.ui.screens.player.audiosync.AudioSampleSink import com.arflix.tv.ui.screens.player.audiosync.AudioSyncDecoder @@ -14,6 +15,7 @@ import com.arflix.tv.ui.screens.player.audiosync.SileroVadWeights import com.arflix.tv.ui.screens.player.audiosync.SpeechAnalyzer import com.arflix.tv.ui.screens.player.audiosync.SpeechTimeline import com.arflix.tv.ui.screens.player.audiosync.asr.SpeechSegmenter +import com.arflix.tv.ui.screens.tv.live.isBitmapSubtitle /** * Live TV speech capture: copies the channel's compressed audio at the extractor, as it is @@ -32,6 +34,8 @@ import com.arflix.tv.ui.screens.player.audiosync.asr.SpeechSegmenter internal class LiveSpeechPipeline( context: Context, private val onSegment: (SpeechSegment) -> Unit, + /** A sentence of the channel's own closed captions, read ahead of playback. */ + private val onCaption: (CaptionSentence) -> Unit, /** The channel's audio format cannot be decoded ahead of playback on this device. */ private val onUnsupportedAudio: (mimeType: String) -> Unit, ) { @@ -83,11 +87,11 @@ internal class LiveSpeechPipeline( return if (tap == UNKNOWN || position == UNKNOWN) null else tap - position } - /** A sink for one prepared channel source. */ - fun newSink(): AudioSampleSink { + /** The audio and caption sinks for one prepared channel source. */ + fun newSinks(): LiveTapSinks { val sinkGeneration = ++generation reset() - return ChannelSink(sinkGeneration) + return LiveTapSinks(ChannelSink(sinkGeneration), CaptionSink(sinkGeneration)) } /** Forget everything in flight (channel change, seek, resume). */ @@ -213,6 +217,49 @@ internal class LiveSpeechPipeline( } } + /** + * Reads the channel's own text subtitles into sentences: CEA-608 closed captions carried in + * the video, and text renditions (WebVTT and the like) the player parsed during extraction. + * Bitmap subtitles (DVB) carry no text; those channels are subtitled from their audio. + */ + private inner class CaptionSink(private val sinkGeneration: Int) : AudioSampleSink { + private val assembler = Cea608TextAssembler { sentence -> + if (sinkGeneration == generation) onCaption(sentence) + } + private val cueReader = CaptionCueReader { sentence -> + if (sinkGeneration == generation) onCaption(sentence) + } + private val loggedMimes = HashSet() + + override fun wantsSamples(format: Format): Boolean { + if (sinkGeneration != generation || !active) return false + val mime = format.sampleMimeType ?: return false + if (synchronized(loggedMimes) { loggedMimes.add("$mime ${format.codecs} ${format.language}") }) { + LiveAiSubsLog.i("text track offered: $mime codecs=${format.codecs} lang=${format.language}") + } + if (isBitmapSubtitle(format)) return false + return mime == MimeTypes.APPLICATION_CEA608 || mime == MimeTypes.APPLICATION_MEDIA3_CUES + } + + override fun onSample(format: Format, timeUs: Long, data: ByteArray, offset: Int, size: Int) { + if (sinkGeneration != generation) return + synchronized(assembler) { + if (format.sampleMimeType == MimeTypes.APPLICATION_MEDIA3_CUES) { + cueReader.feed(timeUs, data, offset, size) + } else { + assembler.feed(timeUs, data, offset, size) + } + } + } + + override fun onDiscontinuity() { + synchronized(assembler) { + assembler.reset() + cueReader.reset() + } + } + } + companion object { private const val UNKNOWN = Long.MIN_VALUE private const val NO_ID = "(no id)" diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt index fb0182fb13..cda4860b42 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt @@ -30,6 +30,12 @@ internal sealed interface LiveAiSubsStatus { data class Unsupported(val mimeType: String) : LiveAiSubsStatus } +/** One piece of work for the AI: a stretch of speech, or a sentence of the channel's captions. */ +internal sealed interface LiveWork { + class Audio(val segment: SpeechSegment) : LiveWork + class Caption(val sentence: CaptionSentence) : LiveWork +} + /** Counters for the hidden debug overlay. */ internal data class LiveAiSubsStats( val lookaheadMs: Long? = null, @@ -40,6 +46,8 @@ internal data class LiveAiSubsStats( val backlogSkips: Int = 0, val segmentsPerMinute: Int = 0, val failures: Int = 0, + /** Lines made from the channel's own captions rather than its audio. */ + val captionLines: Int = 0, ) /** @@ -55,14 +63,25 @@ internal data class LiveAiSubsStats( internal class LiveSubtitleSession( context: Context, private val engine: LiveSubtitleEngine, + private val captionTranslator: LiveCaptionTranslator, ) { private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate) - private val segments = Channel(capacity = 8, onBufferOverflow = BufferOverflow.DROP_OLDEST) + private val segments = Channel(capacity = 8, onBufferOverflow = BufferOverflow.DROP_OLDEST) + + /** + * Captions first: while the channel's own captions keep coming (up to [CAPTION_HOLDOVER_US] + * after the last one), its audio is not sent. When they stop, the audio takes over again. + */ + @Volatile private var captionsUntilUs = Long.MIN_VALUE private val inFlight = Semaphore(MAX_IN_FLIGHT) val pipeline = LiveSpeechPipeline( context = context, - onSegment = { segment -> segments.trySend(segment) }, + onSegment = { segment -> segments.trySend(LiveWork.Audio(segment)) }, + onCaption = { sentence -> + captionsUntilUs = sentence.endUs + CAPTION_HOLDOVER_US + segments.trySend(LiveWork.Caption(sentence)) + }, onUnsupportedAudio = { mime -> scope.launch { setStatus(LiveAiSubsStatus.Unsupported(mime)) } }, ) private val scheduler = LiveCueScheduler() @@ -88,6 +107,7 @@ internal class LiveSubtitleSession( private val segmentTimes = ArrayDeque() private var skippedSegments = 0 private var failures = 0 + private var captionLines = 0 private var worker: Job? = null /** Measured p95 of the AI's answer time, for the Auto delay. */ @@ -127,6 +147,7 @@ internal class LiveSubtitleSession( /** A new channel (or a re-prepared source): nothing from before may show. */ fun onNewSource() { epoch++ + captionsUntilUs = Long.MIN_VALUE scheduler.clear() recentLines.clear() _cue.value = null @@ -158,6 +179,7 @@ internal class LiveSubtitleSession( backlogSkips = pipeline.backlogSkips, segmentsPerMinute = segmentTimes.size, failures = failures, + captionLines = captionLines, ) } @@ -195,8 +217,8 @@ internal class LiveSubtitleSession( if (resultEpoch == epoch) apply(result) } } - for (segment in segments) { - if (!shouldSend(segment)) { + for (work in segments) { + if (!shouldSend(work)) { skippedSegments++ continue } @@ -209,7 +231,11 @@ internal class LiveSubtitleSession( val deferred = async { try { val started = SystemClock.elapsedRealtime() - val result = engine.process(segment, context, name, isRtl) + val result = when (work) { + is LiveWork.Audio -> engine.process(work.segment, context, name, isRtl) + is LiveWork.Caption -> captionTranslator.translate(work.sentence, name, isRtl).also { + if (it is LiveEngineResult.Cues && it.cues.isNotEmpty()) captionLines++ } + } if (result is LiveEngineResult.Cues) recordLatency(SystemClock.elapsedRealtime() - started) result } finally { @@ -221,14 +247,20 @@ internal class LiveSubtitleSession( } } - private fun shouldSend(segment: SpeechSegment): Boolean { + private fun shouldSend(work: LiveWork): Boolean { + val (startUs, endUs) = when (work) { + is LiveWork.Audio -> work.segment.startUs to work.segment.endUs + is LiveWork.Caption -> work.sentence.startUs to work.sentence.endUs + } + // The channel's captions cover this stretch: no need to listen to its audio. + if (work is LiveWork.Audio && startUs <= captionsUntilUs) return false val status = _status.value if (status is LiveAiSubsStatus.InvalidKey || status is LiveAiSubsStatus.Unsupported) return false val now = SystemClock.elapsedRealtime() if (status is LiveAiSubsStatus.QuotaPaused && now < status.untilMs) return false if (status is LiveAiSubsStatus.Offline && now < pausedUntilMs) return false // Its window would already be over when an answer could arrive. - return segment.endUs + LiveCueScheduler.MIN_DISPLAY_US > pipeline.estimatedPositionUs() + MIN_ANSWER_US + return endUs + LiveCueScheduler.MIN_DISPLAY_US > pipeline.estimatedPositionUs() + MIN_ANSWER_US } private fun apply(result: LiveEngineResult) { @@ -274,5 +306,7 @@ internal class LiveSubtitleSession( const val QUOTA_BACKOFF_START_MS = 30_000L const val QUOTA_BACKOFF_MAX_MS = 240_000L const val OFFLINE_RETRY_MS = 10_000L + /** How long after the last caption the audio stays unused (captions pause between lines). */ + const val CAPTION_HOLDOVER_US = 20_000_000L } } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveTapExtractors.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveTapExtractors.kt new file mode 100644 index 0000000000..0e927bbcb8 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveTapExtractors.kt @@ -0,0 +1,81 @@ +@file:androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) + +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.net.Uri +import androidx.media3.common.C +import androidx.media3.extractor.Extractor +import androidx.media3.extractor.ExtractorInput +import androidx.media3.extractor.ExtractorOutput +import androidx.media3.extractor.ExtractorsFactory +import androidx.media3.extractor.PositionHolder +import androidx.media3.extractor.SeekMap +import androidx.media3.extractor.SniffFailure +import androidx.media3.extractor.TrackOutput +import com.arflix.tv.ui.screens.player.audiosync.AudioSampleSink +import com.arflix.tv.ui.screens.player.audiosync.AudioTapTrackOutput + +/** The two copies Live TV AI subtitles take of a channel as it downloads. */ +internal class LiveTapSinks(val audio: AudioSampleSink, val captions: AudioSampleSink) + +/** + * Copies audio samples to [sinks].audio and text samples (closed captions carried in the video, + * subtitle streams) to [sinks].captions, while forwarding everything unchanged to the player. + * The copy happens at load time, so both arrive as far ahead of playback as the player buffers. + */ +internal class LiveTapExtractorOutput( + private val delegate: ExtractorOutput, + private val sinks: LiveTapSinks, +) : ExtractorOutput { + private val outputs = HashMap() + + override fun track(id: Int, type: Int): TrackOutput { + val output = delegate.track(id, type) + return when (type) { + C.TRACK_TYPE_AUDIO -> outputs.getOrPut(id) { AudioTapTrackOutput(output, sinks.audio) } + C.TRACK_TYPE_TEXT -> outputs.getOrPut(id) { AudioTapTrackOutput(output, sinks.captions) } + else -> output + } + } + + override fun endTracks() = delegate.endTracks() + + override fun seekMap(seekMap: SeekMap) = delegate.seekMap(seekMap) +} + +/** Progressive (raw MPEG-TS) streams: every extractor's output goes through [LiveTapExtractorOutput]. */ +internal class LiveTapExtractorsFactory( + private val delegate: ExtractorsFactory, + private val sinks: LiveTapSinks, +) : ExtractorsFactory { + override fun createExtractors(): Array = wrap(delegate.createExtractors()) + + override fun createExtractors(uri: Uri, responseHeaders: Map>): Array = + wrap(delegate.createExtractors(uri, responseHeaders)) + + private fun wrap(extractors: Array): Array = + Array(extractors.size) { LiveTapExtractor(extractors[it], sinks) } +} + +private class LiveTapExtractor( + private val delegate: Extractor, + private val sinks: LiveTapSinks, +) : Extractor { + override fun sniff(input: ExtractorInput): Boolean = delegate.sniff(input) + + override fun getSniffFailureDetails(): List = delegate.sniffFailureDetails + + override fun init(output: ExtractorOutput) = delegate.init(LiveTapExtractorOutput(output, sinks)) + + override fun read(input: ExtractorInput, seekPosition: PositionHolder): Int = delegate.read(input, seekPosition) + + override fun seek(position: Long, timeUs: Long) { + sinks.audio.onDiscontinuity() + sinks.captions.onDiscontinuity() + delegate.seek(position, timeUs) + } + + override fun release() = delegate.release() + + override fun getUnderlyingImplementation(): Extractor = delegate.underlyingImplementation +} diff --git a/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/Cea608TextAssemblerTest.kt b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/Cea608TextAssemblerTest.kt new file mode 100644 index 0000000000..b836efa143 --- /dev/null +++ b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/Cea608TextAssemblerTest.kt @@ -0,0 +1,96 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class Cea608TextAssemblerTest { + private val sentences = ArrayList() + private val assembler = Cea608TextAssembler { sentences += it } + + /** cc_data triplets on field 1 (header 0xFC = valid, NTSC field 1). */ + private fun cc(vararg pairs: Pair, header: Int = 0xFC): ByteArray = + pairs.flatMap { (a, b) -> listOf(header.toByte(), a.toByte(), b.toByte()) }.toByteArray() + + private fun text(s: String): ByteArray { + val padded = if (s.length % 2 == 0) s else "$s\u0000" + return cc(*padded.chunked(2).map { it[0].code to it[1].code }.toTypedArray()) + } + + private fun feed(ms: Long, data: ByteArray) = assembler.feed(ms * 1_000, data, 0, data.size) + + private val carriageReturn = cc(0x14 to 0x2D) + + @Test fun charactersBecomeASentenceEndingAtAFullStop() { + feed(1_000, text("HELLO ")) + feed(1_500, text("WORLD.")) + assertEquals(listOf(CaptionSentence(1_000_000, 1_500_000, "HELLO WORLD.")), sentences) + } + + @Test fun repeatedControlCodesActOnce() { + feed(1_000, text("ONE")) + // Control codes come twice in a row; the pair must give one word boundary, not two. + feed(1_100, cc(0x14 to 0x2D, 0x14 to 0x2D)) + feed(1_200, text("TWO.")) + assertEquals("ONE TWO.", sentences.single().text) + } + + @Test fun carriageReturnSeparatesWords() { + feed(1_000, text("ROLL")) + feed(1_100, carriageReturn) + feed(1_200, text("UP.")) + assertEquals("ROLL UP.", sentences.single().text) + } + + @Test fun backspaceRemovesTheLastCharacter() { + feed(1_000, text("CATS")) + feed(1_100, cc(0x14 to 0x21)) + feed(1_200, text("!")) + assertEquals("CAT!", sentences.single().text) + } + + @Test fun paddingAndSecondFieldAreIgnored() { + feed(1_000, cc(0x80 to 0x80)) + feed(1_100, text("XY").also { it[0] = 0xFD.toByte() }) + feed(1_200, text("OK.")) + assertEquals("OK.", sentences.single().text) + } + + @Test fun secondCaptionChannelIsIgnored() { + feed(1_000, cc(0x1C to 0x2D)) + feed(1_100, text("A.")) + assertEquals("A.", sentences.single().text) + } + + @Test fun aPauseEndsTheSentence() { + feed(1_000, text("NO STOP HERE")) + assertTrue(sentences.isEmpty()) + feed(1_000 + Cea608TextAssembler.GAP_US / 1_000 + 100, cc(0x80 to 0x80)) + assertEquals("NO STOP HERE", sentences.single().text) + } + + @Test fun longRunOnCaptionsAreCutAtAWordAfterFiveSeconds() { + // Roll-up captions arrive a word at a time, without pauses. + listOf("THIS ", "GOES ", "ON ", "AND ", "ON ", "AND ").forEachIndexed { second, word -> + feed(second * 1_000L, text(word)) + } + assertEquals("THIS GOES ON AND ON AND", sentences.single().text) + assertEquals(0L, sentences.single().startUs) + assertEquals(5_000_000L, sentences.single().endUs) + } + + @Test fun resetDropsWhatWasCollected() { + feed(1_000, text("HALF")) + assembler.reset() + feed(1_100, text("NEW.")) + assertEquals("NEW.", sentences.single().text) + } + + @Test fun musicNoteIsAWordBoundaryNotText() { + feed(1_000, cc(0x11 to 0x37)) + assertFalse(assembler.sawText) + feed(1_100, text("SONG.")) + assertEquals("SONG.", sentences.single().text) + } +} From 9cae38a34e6a7862d8e0006302d37bae21cb4aa8 Mon Sep 17 00:00:00 2001 From: silentbil Date: Fri, 6 Nov 2026 16:13:38 +0200 Subject: [PATCH 7/8] Live TV AI subtitles: remove the debug stats panel The hidden look-ahead/latency panel (debug.arvio.liveaisubs.overlay) and its counters were for measuring on a device; the measured AI latency still drives the Auto delay. --- .../tv/ui/screens/tv/live/LiveTvScreen.kt | 8 +-- .../screens/tv/live/aisubs/LiveAiSubsDebug.kt | 52 ------------------- .../tv/live/aisubs/LiveAiSubtitleOverlay.kt | 24 +-------- .../screens/tv/live/aisubs/LiveAudioTapHls.kt | 6 --- .../tv/live/aisubs/LiveSubtitleSession.kt | 49 ++--------------- 5 files changed, 6 insertions(+), 133 deletions(-) delete mode 100644 app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsDebug.kt diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt index f7c4e31876..bbb5eb6bad 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt @@ -125,7 +125,6 @@ import com.arflix.tv.ui.screens.player.subtitles.RTL_LANGUAGES import com.arflix.tv.ui.screens.tv.live.aisubs.GeminiCaptionTranslator import com.arflix.tv.ui.screens.tv.live.aisubs.GeminiSegmentEngine import com.arflix.tv.ui.screens.tv.live.aisubs.LiveTapExtractorsFactory -import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsDebug import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsSettings import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsStatus @@ -3246,7 +3245,6 @@ fun LiveTvScreen( .collectAsState(initial = LiveAiSubsSettings()) val liveAiKey = rememberUpdatedState(liveAiSubsSettings.apiKey) val liveAiSession = remember(context) { - LiveAiSubsDebug.start() LiveSubtitleSession( context, GeminiSegmentEngine(apiKey = { liveAiKey.value }), @@ -3266,9 +3264,7 @@ fun LiveTvScreen( val liveAiSubsOn = liveAiSubsSettings.enabled && playingChannelId != null && (liveAiSubsChoices[playingChannelId] ?: (liveAiSubsSettings.autoStart && !liveChannelHasTargetSubs)) val liveAiCue by liveAiSession.cue.collectAsState() - val liveAiStats by liveAiSession.stats.collectAsState() var liveAiPreparingSeconds by remember { mutableStateOf(null) } - var liveAiDebugVisible by remember { mutableStateOf(false) } // Bumped to restart the stream so audio already buffered gets subtitled too. var liveAiRestartRequest by remember { mutableIntStateOf(0) } var liveSubtitleStyle by remember { mutableStateOf(LiveSubtitleStyle()) } @@ -3315,7 +3311,6 @@ fun LiveTvScreen( val (on, isLive) = liveAiTick val requiredMs = if (on && isLive) autoLiveAiSubsDelayMs(liveAiSession.latencyP95Ms) else 0L liveLoadControl.requiredLookaheadUs = requiredMs * 1_000L - liveAiDebugVisible = LiveAiSubsDebug.enabled if (on) { // currentPosition is relative to the live window; captured audio is timed on the // period, so the window offset into the period is added. @@ -3323,7 +3318,7 @@ fun LiveTvScreen( val windowOffsetUs = if (timeline.isEmpty) 0L else timeline.getWindow(exoPlayer.currentMediaItemIndex, window).positionInFirstPeriodUs val lookaheadMs = (exoPlayer.bufferedPosition - exoPlayer.currentPosition).coerceAtLeast(0L) - liveAiSession.tick(exoPlayer.currentPosition * 1_000L + windowOffsetUs, exoPlayer.isPlaying, lookaheadMs) + liveAiSession.tick(exoPlayer.currentPosition * 1_000L + windowOffsetUs, exoPlayer.isPlaying) val held = liveLoadControl.holding && exoPlayer.playbackState == Player.STATE_BUFFERING liveAiPreparingSeconds = if (held && requiredMs > 0L) { ((requiredMs - lookaheadMs + 999L) / 1_000L).toInt().coerceAtLeast(1) @@ -4959,7 +4954,6 @@ fun LiveTvScreen( cue = liveAiCue, style = liveSubtitleStyle, preparingText = liveAiPreparingSeconds?.let { stringResource(R.string.live_ai_subs_preparing, it) }, - debugStats = liveAiStats.takeIf { liveAiDebugVisible }, ) } if (isFullScreen && !fullscreenGuideOpen && !quickZapOpen) { diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsDebug.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsDebug.kt deleted file mode 100644 index 18fff2a076..0000000000 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsDebug.kt +++ /dev/null @@ -1,52 +0,0 @@ -package com.arflix.tv.ui.screens.tv.live.aisubs - -import com.arflix.tv.BuildConfig -import java.util.concurrent.TimeUnit - -/** - * Hidden diagnostics for Live TV AI subtitles, debug builds only: - * `adb shell setprop debug.arvio.liveaisubs.overlay 1` shows look-ahead, AI latency p50/p95, - * late cues and segments per minute over the picture, and logs HLS segment details under - * `LiveAiSubs`. `debug.*` properties are writable from `adb shell` without root. They are polled - * on a daemon thread, because reading one means running `getprop`, never on the UI thread. - */ -internal object LiveAiSubsDebug { - @Volatile - var enabled: Boolean = false - private set - - @Volatile - private var started = false - - fun start() { - if (!BuildConfig.DEBUG || started) return - synchronized(this) { - if (started) return - started = true - Thread({ pollLoop() }, "LiveAiSubsProps").apply { - isDaemon = true - start() - } - } - } - - private fun pollLoop() { - while (true) { - enabled = read("debug.arvio.liveaisubs.overlay") == "1" - try { - Thread.sleep(POLL_MS) - } catch (_: InterruptedException) { - return - } - } - } - - private fun read(name: String): String? = runCatching { - val process = ProcessBuilder("getprop", name).redirectErrorStream(true).start() - val value = process.inputStream.bufferedReader().use { it.readText().trim() } - process.waitFor(1, TimeUnit.SECONDS) - value.ifEmpty { null } - }.getOrNull() - - private const val POLL_MS = 3_000L -} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt index 1105c6452f..2d22cd8a94 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt @@ -11,7 +11,6 @@ import androidx.compose.runtime.Composable import androidx.compose.ui.Alignment import androidx.compose.ui.Modifier import androidx.compose.ui.graphics.Color -import androidx.compose.ui.text.font.FontFamily import androidx.compose.ui.unit.dp import androidx.compose.ui.unit.sp import androidx.compose.ui.viewinterop.AndroidView @@ -33,7 +32,7 @@ internal data class LiveSubtitleStyle( /** * Fullscreen layer over the Live TV picture: the current AI subtitle (drawn by Media3's * SubtitleView with the profile's subtitle style, so right-to-left languages lay out natively), - * the "Preparing subtitles… N s" note while playback is held, and the hidden debug panel. + * the "Preparing subtitles… N s" note while playback is held. */ @Composable internal fun LiveAiSubtitleOverlay( @@ -41,7 +40,6 @@ internal fun LiveAiSubtitleOverlay( style: LiveSubtitleStyle, /** "Preparing subtitles… N s" while playback is held for the delay, or null. */ preparingText: String?, - debugStats: LiveAiSubsStats?, modifier: Modifier = Modifier, ) { Box(modifier = modifier.fillMaxSize()) { @@ -70,26 +68,6 @@ internal fun LiveAiSubtitleOverlay( .padding(horizontal = 18.dp, vertical = 8.dp), ) } - if (debugStats != null) { - Text( - text = with(debugStats) { - "look-ahead ${lookaheadMs?.let { "%.1f s".format(it / 1000f) } ?: "-"}\n" + - "AI p50 ${latencyP50Ms ?: "-"} ms p95 ${latencyP95Ms ?: "-"} ms\n" + - "segments/min $segmentsPerMinute late dropped $lateDropped\n" + - "skipped $skippedSegments backlog skips $backlogSkips failures $failures\n" + - "lines from captions $captionLines" - }, - color = Color(0xFFB9F6CA), - fontSize = 12.sp, - fontFamily = FontFamily.Monospace, - modifier = Modifier - // Left in every layout direction, clear of the channel title (top end in RTL). - .align(androidx.compose.ui.AbsoluteAlignment.TopLeft) - .padding(top = 120.dp, start = 24.dp) - .background(Color.Black.copy(alpha = 0.7f), RoundedCornerShape(6.dp)) - .padding(10.dp), - ) - } } } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt index 5c50f437ad..992368b30c 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt @@ -37,12 +37,6 @@ internal class AudioTapHlsExtractorFactory( val extractor = delegate.createExtractor( uri, format, muxedCaptionFormats, timestampAdjuster, responseHeaders, sniffingExtractorInput, playerId, ) - if (LiveAiSubsDebug.enabled) { - LiveAiSubsLog.i( - "hls chunk extractor ${extractor.javaClass.simpleName} packedAudio=${extractor.isPackedAudioExtractor} " + - "codecs=${format.codecs} mime=${format.sampleMimeType ?: format.containerMimeType}", - ) - } return AudioTapHlsChunkExtractor(extractor, sinks) } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt index cda4860b42..14c1c71507 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt @@ -36,20 +36,6 @@ internal sealed interface LiveWork { class Caption(val sentence: CaptionSentence) : LiveWork } -/** Counters for the hidden debug overlay. */ -internal data class LiveAiSubsStats( - val lookaheadMs: Long? = null, - val latencyP50Ms: Long? = null, - val latencyP95Ms: Long? = null, - val lateDropped: Int = 0, - val skippedSegments: Int = 0, - val backlogSkips: Int = 0, - val segmentsPerMinute: Int = 0, - val failures: Int = 0, - /** Lines made from the channel's own captions rather than its audio. */ - val captionLines: Int = 0, -) - /** * Live TV AI subtitles for the one Live TV player: [pipeline] cuts speech segments from the * channel's audio ahead of playback, [engine] turns each into timed lines, and [scheduler] shows @@ -92,9 +78,6 @@ internal class LiveSubtitleSession( private val _status = MutableStateFlow(LiveAiSubsStatus.Off) val status: StateFlow = _status.asStateFlow() - private val _stats = MutableStateFlow(LiveAiSubsStats()) - val stats: StateFlow = _stats.asStateFlow() - private var languageName = "English" private var rtl = false private var running = false @@ -104,10 +87,6 @@ internal class LiveSubtitleSession( private var pausedUntilMs = 0L private val recentLines = ArrayDeque() private val latencies = ArrayDeque() - private val segmentTimes = ArrayDeque() - private var skippedSegments = 0 - private var failures = 0 - private var captionLines = 0 private var worker: Job? = null /** Measured p95 of the AI's answer time, for the Auto delay. */ @@ -163,24 +142,12 @@ internal class LiveSubtitleSession( } /** UI thread, about every 100 ms. */ - fun tick(periodPositionUs: Long, playing: Boolean, lookaheadMs: Long?) { + fun tick(periodPositionUs: Long, playing: Boolean) { positionUs = periodPositionUs pipeline.onPosition(periodPositionUs, playing) val now = SystemClock.elapsedRealtime() (_status.value as? LiveAiSubsStatus.QuotaPaused)?.let { if (now >= it.untilMs) setStatus(LiveAiSubsStatus.Running) } _cue.value = if (running) scheduler.cueAt(periodPositionUs) else null - while (segmentTimes.isNotEmpty() && now - segmentTimes.first() > 60_000L) segmentTimes.removeFirst() - _stats.value = LiveAiSubsStats( - lookaheadMs = lookaheadMs, - latencyP50Ms = latencies.sorted().let { if (it.isEmpty()) null else it[it.size / 2] }, - latencyP95Ms = latencyP95Ms, - lateDropped = scheduler.lateDropped, - skippedSegments = skippedSegments, - backlogSkips = pipeline.backlogSkips, - segmentsPerMinute = segmentTimes.size, - failures = failures, - captionLines = captionLines, - ) } fun release() { @@ -218,11 +185,7 @@ internal class LiveSubtitleSession( } } for (work in segments) { - if (!shouldSend(work)) { - skippedSegments++ - continue - } - segmentTimes.addLast(SystemClock.elapsedRealtime()) + if (!shouldSend(work)) continue val sentEpoch = epoch val context = recentLines.toList() val name = languageName @@ -233,8 +196,7 @@ internal class LiveSubtitleSession( val started = SystemClock.elapsedRealtime() val result = when (work) { is LiveWork.Audio -> engine.process(work.segment, context, name, isRtl) - is LiveWork.Caption -> captionTranslator.translate(work.sentence, name, isRtl).also { - if (it is LiveEngineResult.Cues && it.cues.isNotEmpty()) captionLines++ } + is LiveWork.Caption -> captionTranslator.translate(work.sentence, name, isRtl) } if (result is LiveEngineResult.Cues) recordLatency(SystemClock.elapsedRealtime() - started) result @@ -285,10 +247,7 @@ internal class LiveSubtitleSession( pausedUntilMs = SystemClock.elapsedRealtime() + OFFLINE_RETRY_MS setStatus(LiveAiSubsStatus.Offline) } - is LiveEngineResult.Failed -> { - failures++ - LiveAiSubsLog.w("segment failed: ${result.reason}") - } + is LiveEngineResult.Failed -> LiveAiSubsLog.w("segment failed: ${result.reason}") } } From be00f98fa21a0a3d32bb4034a21d63f19aada180 Mon Sep 17 00:00:00 2001 From: silentbil Date: Fri, 6 Nov 2026 22:23:31 +0200 Subject: [PATCH 8/8] Live TV AI subtitles: drop stale channel work, fall back when AI can't run, keep catch-up position - Tag audio and caption work with the channel source it was captured from and drop stale work before sending, so an earlier channel's lines never reach the next one. - When AI cannot run (no or rejected key, undecodable audio) skip the look-ahead hold and show the channel's own subtitles; during a quota or offline pause show them too. - Do not restart the stream when AI is switched on during catch-up. --- .../tv/ui/screens/tv/live/LiveSubtitles.kt | 13 +++++++ .../tv/ui/screens/tv/live/LiveTvScreen.kt | 35 ++++++++++++++----- .../tv/live/aisubs/LiveSpeechPipeline.kt | 26 ++++++++++---- .../tv/live/aisubs/LiveSubtitleSession.kt | 25 ++++++++----- 4 files changed, 75 insertions(+), 24 deletions(-) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt index 26901b3489..66e5fab67e 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt @@ -99,6 +99,19 @@ internal fun liveCaptionSourceKey(tracks: Tracks, avoidLanguage: String?): Strin return (textTracks.firstOrNull { it.language == "en" } ?: textTracks.firstOrNull())?.key } +/** + * The channel's own subtitles to show while AI subtitles are on but cannot run (no or rejected + * key, audio that cannot be decoded, quota or connection pause): [preferredLanguage] first, then + * English, then any track (picture subtitles included: these are only shown). Null when the + * channel has none. + */ +internal fun liveAiFallbackSubtitleKey(tracks: Tracks, preferredLanguage: String?): String? { + val all = typedTracks(tracks, C.TRACK_TYPE_TEXT) + return (all.firstOrNull { preferredLanguage != null && it.language == preferredLanguage } + ?: all.firstOrNull { it.language == "en" } + ?: all.firstOrNull())?.key +} + /** * True for picture subtitles (DVB, PGS, VobSub), which carry no text. Subtitles the player parses * during extraction arrive as `application/x-media3-cues`, with the original type kept in codecs. diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt index bbb5eb6bad..194e0d33ce 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt @@ -3264,6 +3264,17 @@ fun LiveTvScreen( val liveAiSubsOn = liveAiSubsSettings.enabled && playingChannelId != null && (liveAiSubsChoices[playingChannelId] ?: (liveAiSubsSettings.autoStart && !liveChannelHasTargetSubs)) val liveAiCue by liveAiSession.cue.collectAsState() + val liveAiStatus by liveAiSession.status.collectAsState() + // On, but unable to run on this channel (no or rejected key, audio that cannot be decoded): + // no hold, and the channel's own subtitles show instead. + val liveAiBlocked = !liveAiSubsSettings.hasKey || liveAiStatus is LiveAiSubsStatus.NoKey || + liveAiStatus is LiveAiSubsStatus.InvalidKey || liveAiStatus is LiveAiSubsStatus.Unsupported + val liveAiHolds = liveAiSubsOn && !liveAiBlocked + // A quota or connection pause is temporary: the hold stays (so playback does not stall when + // AI comes back), but the channel's own subtitles fill in meanwhile. + val liveAiShowsNative = liveAiSubsOn && (liveAiBlocked || + liveAiStatus is LiveAiSubsStatus.QuotaPaused || liveAiStatus is LiveAiSubsStatus.Offline) + val liveAiShows = liveAiSubsOn && !liveAiShowsNative var liveAiPreparingSeconds by remember { mutableStateOf(null) } // Bumped to restart the stream so audio already buffered gets subtitled too. var liveAiRestartRequest by remember { mutableIntStateOf(0) } @@ -3286,10 +3297,12 @@ fun LiveTvScreen( } // While AI subtitles show, the stream's own text subtitles stay selected but hidden: they are // read ahead of playback and translated (a stream that is not selected is not even - // downloaded). Without a text track, AI subtitles come from the audio. - LaunchedEffect(liveAiSubsOn, liveTracks, liveAiTargetCode) { + // downloaded). Without a text track, AI subtitles come from the audio. While AI cannot run, + // the channel's own subtitles are selected and shown instead. + LaunchedEffect(liveAiSubsOn, liveAiShowsNative, liveTracks, liveAiTargetCode) { if (!liveAiSubsOn) return@LaunchedEffect - val source = liveCaptionSourceKey(liveTracks, liveAiTargetCode) + val source = if (liveAiShowsNative) liveAiFallbackSubtitleKey(liveTracks, liveAiTargetCode) + else liveCaptionSourceKey(liveTracks, liveAiTargetCode) val selected = liveSelectedSubtitleKey(liveTracks) when { source != null && selected != source -> applyLiveSubtitleChoice(exoPlayer, source) @@ -3298,13 +3311,17 @@ fun LiveTvScreen( } // Turned on while the channel plays: the seconds already buffered were downloaded before // capture started, so they would play without subtitles. Restart the stream instead; the - // start-up hold then shows "Preparing subtitles" and the first line is covered. - LaunchedEffect(liveAiSubsOn) { - if (liveAiSubsOn && exoPlayer.isPlaying && exoPlayer.currentMediaItem?.mediaId == playingChannelId.orEmpty()) { + // start-up hold then shows "Preparing subtitles" and the first line is covered. Not when AI + // cannot run (nothing to prepare), and not in catch-up: a recording has no hold and reads + // far enough ahead on its own, and a restart would lose the position in it. + LaunchedEffect(liveAiHolds) { + if (liveAiHolds && playingCatchupProgram == null && exoPlayer.isPlaying && + exoPlayer.currentMediaItem?.mediaId == playingChannelId.orEmpty() + ) { liveAiRestartRequest++ } } - val liveAiTick by rememberUpdatedState(liveAiSubsOn to (playingCatchupProgram == null)) + val liveAiTick by rememberUpdatedState(liveAiHolds to (playingCatchupProgram == null)) LaunchedEffect(exoPlayer, liveAiSession) { val window = androidx.media3.common.Timeline.Window() while (true) { @@ -4945,11 +4962,11 @@ fun LiveTvScreen( } // With AI subtitles the stream's own subtitles are only read, not shown. view.subtitleView?.visibility = - if (liveAiSubsOn) android.view.View.INVISIBLE else android.view.View.VISIBLE + if (liveAiShows) android.view.View.INVISIBLE else android.view.View.VISIBLE }, modifier = Modifier.fillMaxSize(), ) - if (isFullScreen && !fullscreenGuideOpen && !quickZapOpen && liveAiSubsOn) { + if (isFullScreen && !fullscreenGuideOpen && !quickZapOpen && liveAiShows) { LiveAiSubtitleOverlay( cue = liveAiCue, style = liveSubtitleStyle, diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt index 74b4371257..ae20abd0d4 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt @@ -33,16 +33,24 @@ import com.arflix.tv.ui.screens.tv.live.isBitmapSubtitle */ internal class LiveSpeechPipeline( context: Context, - private val onSegment: (SpeechSegment) -> Unit, + /** A speech segment, with the [generation] of the channel source its audio came from. */ + private val onSegment: (SpeechSegment, generation: Int) -> Unit, /** A sentence of the channel's own closed captions, read ahead of playback. */ - private val onCaption: (CaptionSentence) -> Unit, + private val onCaption: (CaptionSentence, generation: Int) -> Unit, /** The channel's audio format cannot be decoded ahead of playback on this device. */ private val onUnsupportedAudio: (mimeType: String) -> Unit, ) { private val appContext = context.applicationContext @Volatile private var weights: SileroVadWeights? = null - @Volatile private var generation = 0 + /** Bumped for every prepared channel source; work tagged with an older one is stale. */ + @Volatile var generation = 0 + private set + /** + * The source the decoder's audio belongs to. Decoder thread: it moves to [generation] when + * the decoder resets, which drops everything queued from before ([newSinks] queues a reset). + */ + @Volatile private var audioGeneration = 0 @Volatile private var decoder: AudioSyncDecoder? = null /** Capture only while AI subtitles are on for the channel and it is playing fullscreen. */ @@ -126,6 +134,7 @@ internal class LiveSpeechPipeline( val vadWeights = weights ?: return null synchronized(this) { decoder?.let { return it } + audioGeneration = generation val analyzer = SpeechAnalyzer(SileroVad(vadWeights), SpeechTimeline()) val segmenter = SpeechSegmenter( maxSegmentFrames = MAX_SEGMENT_FRAMES, @@ -134,7 +143,7 @@ internal class LiveSpeechPipeline( val startUs = startFrame.toLong() * SpeechTimeline.FRAME_DURATION_US val endUs = startUs + samples.size * 1_000_000L / SileroVad.SAMPLE_RATE segmentsEmitted++ - onSegment(SpeechSegment(startUs, endUs, samples)) + onSegment(SpeechSegment(startUs, endUs, samples), audioGeneration) } analyzer.chunkListener = object : SpeechAnalyzer.ChunkListener { override fun onChunk(frame: Int, chunk: FloatArray, probability: Float) { @@ -152,7 +161,10 @@ internal class LiveSpeechPipeline( segmenter.onChunk(frame, chunk, probability) } - override fun onReset() = segmenter.onReset() + override fun onReset() { + audioGeneration = generation + segmenter.onReset() + } } return AudioSyncDecoder( analyzer = analyzer, @@ -224,10 +236,10 @@ internal class LiveSpeechPipeline( */ private inner class CaptionSink(private val sinkGeneration: Int) : AudioSampleSink { private val assembler = Cea608TextAssembler { sentence -> - if (sinkGeneration == generation) onCaption(sentence) + if (sinkGeneration == generation) onCaption(sentence, sinkGeneration) } private val cueReader = CaptionCueReader { sentence -> - if (sinkGeneration == generation) onCaption(sentence) + if (sinkGeneration == generation) onCaption(sentence, sinkGeneration) } private val loggedMimes = HashSet() diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt index 14c1c71507..af398008ea 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt @@ -30,10 +30,14 @@ internal sealed interface LiveAiSubsStatus { data class Unsupported(val mimeType: String) : LiveAiSubsStatus } -/** One piece of work for the AI: a stretch of speech, or a sentence of the channel's captions. */ +/** + * One piece of work for the AI: a stretch of speech, or a sentence of the channel's captions. + * [generation] is the channel source it was captured from (see [LiveSpeechPipeline.generation]). + */ internal sealed interface LiveWork { - class Audio(val segment: SpeechSegment) : LiveWork - class Caption(val sentence: CaptionSentence) : LiveWork + val generation: Int + class Audio(val segment: SpeechSegment, override val generation: Int) : LiveWork + class Caption(val sentence: CaptionSentence, override val generation: Int) : LiveWork } /** @@ -63,10 +67,10 @@ internal class LiveSubtitleSession( val pipeline = LiveSpeechPipeline( context = context, - onSegment = { segment -> segments.trySend(LiveWork.Audio(segment)) }, - onCaption = { sentence -> + onSegment = { segment, generation -> segments.trySend(LiveWork.Audio(segment, generation)) }, + onCaption = { sentence, generation -> captionsUntilUs = sentence.endUs + CAPTION_HOLDOVER_US - segments.trySend(LiveWork.Caption(sentence)) + segments.trySend(LiveWork.Caption(sentence, generation)) }, onUnsupportedAudio = { mime -> scope.launch { setStatus(LiveAiSubsStatus.Unsupported(mime)) } }, ) @@ -185,12 +189,17 @@ internal class LiveSubtitleSession( } } for (work in segments) { - if (!shouldSend(work)) continue + // Waits for a free request slot first: the channel can change while waiting. + inFlight.acquire() + // Captured from an earlier channel (or before a restart): never sent, never shown. + if (work.generation != pipeline.generation || !shouldSend(work)) { + inFlight.release() + continue + } val sentEpoch = epoch val context = recentLines.toList() val name = languageName val isRtl = rtl - inFlight.acquire() val deferred = async { try { val started = SystemClock.elapsedRealtime()