diff --git a/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt b/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt index 9c91fd12aa..5a00e1a1fc 100644 --- a/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt +++ b/app/src/main/kotlin/com/arflix/tv/data/repository/CloudSyncRepository.kt @@ -498,7 +498,7 @@ class CloudSyncRepository @Inject constructor( "dnsProvider", "subtitleAiEnabled", "subtitleAiAutoSelect", "subtitleAiFindBestMatch", "subtitlePreloadEnabled", "dolbyVisionCompatEnabled", "subtitleAiApiKey", "subtitleAiModel", "subtitleRemoveHearingImpaired", "telegramSearchOnClickOnly", - "subtitleHearingSyncV2" + "subtitleHearingSyncV2", "liveAiSubsEnabled", "liveAiSubsAutoStart" ) // Per-profile fields excluded from the generic merge (handled by their own logic / not values). private val profileMergeExclude = setOf("defaultSubtitle", "subtitleSettingsUpdatedAt") @@ -793,6 +793,9 @@ class CloudSyncRepository @Inject constructor( root.put("subtitleAiApiKey", prefs[subtitleAiApiKeyKey] ?: "") root.put("subtitleAiModel", prefs[subtitleAiModelKey] ?: "GROQ_LLAMA_70B") root.put("subtitleRemoveHearingImpaired", prefs[subtitleRemoveHearingImpairedKey] ?: true) + // Live TV AI subtitles (global, like the AI subtitle key they share). + root.put("liveAiSubsEnabled", prefs[com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.ENABLED] ?: false) + root.put("liveAiSubsAutoStart", prefs[com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.AUTO_START] ?: false) root.put( "telegramSearchOnClickOnly", prefs[com.arflix.tv.data.telegram.TelegramRepository.KEY_SEARCH_ON_CLICK_ONLY] ?: true @@ -1863,6 +1866,10 @@ class CloudSyncRepository @Inject constructor( val model = root.optString("subtitleAiModel", "GROQ_LLAMA_70B") if (model.isNotBlank()) prefs[subtitleAiModelKey] = model prefs[subtitleRemoveHearingImpairedKey] = root.optBoolean("subtitleRemoveHearingImpaired", true) + // has() guards: backups from versions without Live TV AI subtitles must not reset them. + val live = com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs + if (root.has("liveAiSubsEnabled")) prefs[live.ENABLED] = root.optBoolean("liveAiSubsEnabled", false) + if (root.has("liveAiSubsAutoStart")) prefs[live.AUTO_START] = root.optBoolean("liveAiSubsAutoStart", false) } } if (preservedNewerLocalSubtitle) { diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSubtitleSyncController.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSubtitleSyncController.kt index 9e5f92070f..80216f237d 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSubtitleSyncController.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSubtitleSyncController.kt @@ -1052,8 +1052,8 @@ internal class AudioSubtitleSyncController( ) recognizer?.let(engine::setRecognizer) asr = engine - analyzer.chunkListener = SpeechSegmenter(engine::offerSegment) - liveAnalyzer.chunkListener = SpeechSegmenter(engine::offerSegment) + analyzer.chunkListener = SpeechSegmenter(onSegment = engine::offerSegment) + liveAnalyzer.chunkListener = SpeechSegmenter(onSegment = engine::offerSegment) return AudioSyncDecoder(analyzer, liveAnalyzer) { mime -> SyncLog.i("look-ahead capture unavailable for $mime; syncing from playback audio instead") liveOnly = true diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncDecoder.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncDecoder.kt index 1f6e991038..4ef5ba82dc 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncDecoder.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncDecoder.kt @@ -34,6 +34,11 @@ internal class AudioSyncDecoder( * next to the look-ahead one, which keep to platform software decoders and FFmpeg. */ private val allowVendorDecoders: Boolean = true, + /** + * Priority of the decoding thread. Background by default; Live TV subtitles run a little + * higher, since a starved thread there makes every subtitle late. + */ + private val threadPriority: Int = Process.THREAD_PRIORITY_BACKGROUND, /** Called once per audio format that has no usable decoder. */ private val onUnsupportedFormat: (mimeType: String) -> Unit = {}, ) { @@ -144,7 +149,7 @@ internal class AudioSyncDecoder( } private fun runLoop() { - Process.setThreadPriority(Process.THREAD_PRIORITY_BACKGROUND) + Process.setThreadPriority(threadPriority) try { while (true) { val item = synchronized(lock) { diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncExtractors.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncExtractors.kt index 808d4ca3d8..cb816fca80 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncExtractors.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/AudioSyncExtractors.kt @@ -69,7 +69,7 @@ private class AudioTapExtractor( override fun getUnderlyingImplementation(): Extractor = delegate.underlyingImplementation } -private class AudioTapExtractorOutput( +internal class AudioTapExtractorOutput( private val delegate: ExtractorOutput, private val sink: AudioSampleSink, ) : ExtractorOutput { @@ -91,7 +91,7 @@ private class AudioTapExtractorOutput( * report its metadata later (with [offset] bytes of following samples already written), so the * pending bytes are kept until [sampleMetadata] identifies the sample. */ -private class AudioTapTrackOutput( +internal class AudioTapTrackOutput( private val delegate: TrackOutput, private val sink: AudioSampleSink, ) : TrackOutput { diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/asr/SpeechSegmenter.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/asr/SpeechSegmenter.kt index 03c5d7ce5e..ae8f3ad069 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/asr/SpeechSegmenter.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/audiosync/asr/SpeechSegmenter.kt @@ -6,9 +6,19 @@ import com.arflix.tv.ui.screens.player.audiosync.SpeechAnalyzer /** * Cuts the analysed 16 kHz audio into speech segments using the Silero probabilities that are * already being computed, so the recogniser only ever sees dialogue: no silence, music-only - * stretches or effects. Segments are at most [MAX_SEGMENT_FRAMES] long so words come back quickly. + * stretches or effects. Segments are at most [maxSegmentFrames] long so words come back quickly. */ internal class SpeechSegmenter( + /** + * Longest segment in 32 ms frames. Live subtitles keep this short (about 5 s): a line can only + * be translated once its segment has ended, so it bounds how far ahead playback must be held. + */ + private val maxSegmentFrames: Int = MAX_SEGMENT_FRAMES, + /** + * Past this length, a segment also ends at the first chunk whose speech probability dips below + * [DIP_THRESHOLD], a pause between words, so long speech is not cut mid-word at the hard limit. + */ + private val softMaxSegmentFrames: Int = maxSegmentFrames, private val onSegment: (startFrame: Int, samples: FloatArray) -> Unit, ) : SpeechAnalyzer.ChunkListener { private val chunk = SileroVad.CHUNK_SAMPLES @@ -42,7 +52,8 @@ internal class SpeechSegmenter( } append(chunk) if (speech) lastSpeechFrame = frame - if (frame - lastSpeechFrame > HANGOVER_FRAMES || bufferFrames >= MAX_SEGMENT_FRAMES) finish() + val dip = bufferFrames >= softMaxSegmentFrames && probability < DIP_THRESHOLD + if (frame - lastSpeechFrame > HANGOVER_FRAMES || bufferFrames >= maxSegmentFrames || dip) finish() } @Synchronized @@ -75,11 +86,12 @@ internal class SpeechSegmenter( companion object { private const val SPEECH_THRESHOLD = 0.5f + private const val DIP_THRESHOLD = 0.7f private const val PREROLL_FRAMES = 3 private const val TAIL_FRAMES = 4 private const val HANGOVER_FRAMES = 8 private const val MIN_SPEECH_FRAMES = 6 /** 12 s. */ - private const val MAX_SEGMENT_FRAMES = 375 + const val MAX_SEGMENT_FRAMES = 375 } } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/subtitles/SubtitleTranslationService.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/subtitles/SubtitleTranslationService.kt index 20e5475bc0..1b1b5b06e8 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/player/subtitles/SubtitleTranslationService.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/player/subtitles/SubtitleTranslationService.kt @@ -14,7 +14,7 @@ import java.util.concurrent.TimeUnit private const val TAG = "SubtitleTranslation" -private val RTL_LANGUAGES = setOf("hebrew", "arabic", "urdu", "persian", "farsi", "yiddish") +internal val RTL_LANGUAGES = setOf("hebrew", "arabic", "urdu", "persian", "farsi", "yiddish") data class TranslationResult( val lines: List, @@ -28,7 +28,7 @@ data class TranslationResult( // it's not actionable and translation keeps working for everything else. const val TRANSLATION_ERROR_CONTENT_BLOCKED = "CONTENT_BLOCKED" -private val BLOCKED_FINISH_REASONS = setOf( +internal val BLOCKED_FINISH_REASONS = setOf( "PROHIBITED_CONTENT", "SAFETY", "RECITATION", "BLOCKLIST", "SPII" ) @@ -54,9 +54,62 @@ private const val GEMINI_BASE_URL = "https://generativelanguage.googleapis.com/v // with short exponential backoff before falling back to English. Kept short so subtitles stay in // sync with playback. NOT applied to 429 (that's our own quota — retrying fast makes it worse; // the manager's 5s cooldown handles it) or 4xx/content blocks (retrying won't change the outcome). -private const val GEMINI_MAX_TRANSIENT_RETRIES = 2 // 3 attempts total -private const val GEMINI_RETRY_BASE_DELAY_MS = 400L // 400ms, then 800ms (×2 each retry) -private val GEMINI_RETRYABLE_HTTP = setOf(500, 502, 503) +internal const val GEMINI_MAX_TRANSIENT_RETRIES = 2 // 3 attempts total +internal const val GEMINI_RETRY_BASE_DELAY_MS = 400L // 400ms, then 800ms (×2 each retry) +internal val GEMINI_RETRYABLE_HTTP = setOf(500, 502, 503) + +/** + * The JSON array in a model reply: bare, fenced, wrapped in a one-field object, or with a truncated + * tail. Shared by every Gemini/Groq caller in the app. + */ +internal fun extractJsonArray(text: String): JSONArray? { + val codeBlocks = Regex("```(?:json)?\\s*([\\s\\S]*?)```").findAll(text) + .map { it.groupValues[1].trim() }.toList().reversed() + val stripped = text.replace(Regex("```[^`]*```"), "").trim() + val candidates = codeBlocks + listOf(stripped, text) + + for (candidate in candidates) { + try { return JSONArray(candidate) } catch (_: Exception) {} + // With responseMimeType=application/json the model sometimes wraps the array in an + // object (e.g. {"translations": [...]}) — unwrap a lone array-valued field. + try { + val obj = JSONObject(candidate) + val arrays = obj.keys().asSequence() + .mapNotNull { key -> obj.optJSONArray(key) } + .toList() + if (arrays.size == 1) return arrays[0] + } catch (_: Exception) {} + val start = candidate.indexOf('[') + val end = candidate.lastIndexOf(']') + if (start < 0 || end <= start) { + repairTruncatedArray(candidate)?.let { return it } + continue + } + try { + return JSONArray(candidate.substring(start, end + 1)) + } catch (_: Exception) {} + } + return null +} + +/** + * Gemini's JSON mode intermittently truncates output at the very tail (finishReason STOP but + * the closing bracket — sometimes a trailing element — is missing). The translations + * themselves are intact, so re-terminate the array instead of failing the whole batch. + */ +internal fun repairTruncatedArray(text: String): JSONArray? { + val t = text.trim() + if (!t.startsWith("[") || t.endsWith("]")) return null + val lastQuote = t.lastIndexOf('"') + if (lastQuote <= 0) return null + // Try closing as-is, then after dropping a trailing partial element. + for (cut in listOf(t, t.substring(0, lastQuote + 1))) { + val trimmed = cut.trimEnd().trimEnd(',') + try { return JSONArray("$trimmed]") } catch (_: Exception) {} + try { return JSONArray("$trimmed\"]") } catch (_: Exception) {} + } + return null +} class SubtitleTranslationService( private val apiKeyProvider: () -> String, @@ -68,55 +121,6 @@ class SubtitleTranslationService( .readTimeout(30, TimeUnit.SECONDS) .build() - private fun extractJsonArray(text: String): JSONArray? { - val codeBlocks = Regex("```(?:json)?\\s*([\\s\\S]*?)```").findAll(text) - .map { it.groupValues[1].trim() }.toList().reversed() - val stripped = text.replace(Regex("```[^`]*```"), "").trim() - val candidates = codeBlocks + listOf(stripped, text) - - for (candidate in candidates) { - try { return JSONArray(candidate) } catch (_: Exception) {} - // With responseMimeType=application/json the model sometimes wraps the array in an - // object (e.g. {"translations": [...]}) — unwrap a lone array-valued field. - try { - val obj = JSONObject(candidate) - val arrays = obj.keys().asSequence() - .mapNotNull { key -> obj.optJSONArray(key) } - .toList() - if (arrays.size == 1) return arrays[0] - } catch (_: Exception) {} - val start = candidate.indexOf('[') - val end = candidate.lastIndexOf(']') - if (start < 0 || end <= start) { - repairTruncatedArray(candidate)?.let { return it } - continue - } - try { - return JSONArray(candidate.substring(start, end + 1)) - } catch (_: Exception) {} - } - return null - } - - /** - * Gemini's JSON mode intermittently truncates output at the very tail (finishReason STOP but - * the closing bracket — sometimes a trailing element — is missing). The translations - * themselves are intact, so re-terminate the array instead of failing the whole batch. - */ - private fun repairTruncatedArray(text: String): JSONArray? { - val t = text.trim() - if (!t.startsWith("[") || t.endsWith("]")) return null - val lastQuote = t.lastIndexOf('"') - if (lastQuote <= 0) return null - // Try closing as-is, then after dropping a trailing partial element. - for (cut in listOf(t, t.substring(0, lastQuote + 1))) { - val trimmed = cut.trimEnd().trimEnd(',') - try { return JSONArray("$trimmed]") } catch (_: Exception) {} - try { return JSONArray("$trimmed\"]") } catch (_: Exception) {} - } - return null - } - private fun buildSystemPrompt(targetLanguage: String, NL: String) = "You are a professional subtitle translator. Translate the following JSON array into natural $targetLanguage.\n" + "Rules:\n" + diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/LiveAiSubsSettingsRows.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/LiveAiSubsSettingsRows.kt new file mode 100644 index 0000000000..56ec74ef62 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/LiveAiSubsSettingsRows.kt @@ -0,0 +1,130 @@ +package com.arflix.tv.ui.screens.settings + +import android.content.Context +import android.widget.Toast +import androidx.compose.runtime.Composable +import androidx.compose.runtime.collectAsState +import androidx.compose.runtime.getValue +import androidx.compose.runtime.remember +import androidx.compose.ui.Modifier +import androidx.compose.ui.draw.alpha +import androidx.compose.ui.platform.LocalContext +import androidx.compose.ui.res.stringResource +import com.arflix.tv.R +import com.arflix.tv.ui.components.MobileSettingsRow +import com.arflix.tv.ui.components.SettingsToggleRow +import com.arflix.tv.ui.screens.tv.live.aisubs.GeminiSegmentEngine +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsSettings +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.flow.first +import kotlinx.coroutines.launch + +/* + * Live TV AI subtitles settings, in the "AI subtitles" section next to the Gemini key they use: + * the feature switch and "auto-translate when a channel starts". The rows read and write their + * own preferences (LiveAiSubsPrefs); SettingsViewModel pushes changes to the cloud. + */ + +/** TV settings row indices, after the existing ones (0–50). */ +internal val LIVE_AI_SUBS_ROWS = listOf(51, 52) + +private val actionScope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate) + +/** Handles OK/click on one of [LIVE_AI_SUBS_ROWS]. */ +internal fun onLiveAiSubsRowClick(context: Context, index: Int) { + val appContext = context.applicationContext + actionScope.launch { + val settings = LiveAiSubsPrefs.flow(appContext).first() + when (index) { + 51 -> if (settings.enabled) { + LiveAiSubsPrefs.setEnabled(appContext, false) + } else { + enableAfterKeyCheck(appContext, settings) + } + 52 -> if (settings.enabled) LiveAiSubsPrefs.setAutoStart(appContext, !settings.autoStart) + } + } +} + +/** Validates the AI subtitles Gemini key with a cheap call before turning the feature on. */ +private suspend fun enableAfterKeyCheck(context: Context, settings: LiveAiSubsSettings) { + if (!settings.hasKey) { + toast(context, R.string.live_ai_subs_needs_key) + return + } + when (GeminiSegmentEngine(apiKey = { settings.apiKey }).validateKey(settings.apiKey)) { + true -> { + toast(context, R.string.live_ai_subs_key_valid) + LiveAiSubsPrefs.setEnabled(context, true) + } + false -> toast(context, R.string.live_ai_subs_key_rejected) + // Could not check (offline): turn it on; the player reports key problems itself. + null -> { + toast(context, R.string.live_ai_subs_key_unchecked) + LiveAiSubsPrefs.setEnabled(context, true) + } + } +} + +private fun toast(context: Context, resId: Int) { + Toast.makeText(context, context.getString(resId), Toast.LENGTH_SHORT).show() +} + +@Composable +private fun rememberLiveAiSubsSettings(): LiveAiSubsSettings { + val context = LocalContext.current + val flow = remember(context) { LiveAiSubsPrefs.flow(context.applicationContext) } + val settings by flow.collectAsState(initial = LiveAiSubsSettings()) + return settings +} + +/** One TV row for [index] in [LIVE_AI_SUBS_ROWS]. */ +@Composable +internal fun LiveAiSubsTvRow(index: Int, isFocused: Boolean, modifier: Modifier) { + val context = LocalContext.current + val settings = rememberLiveAiSubsSettings() + when (index) { + 51 -> SettingsToggleRow( + title = stringResource(R.string.live_ai_subs_title), + subtitle = stringResource(R.string.live_ai_subs_desc), + isEnabled = settings.enabled, + isFocused = isFocused, + onToggle = { onLiveAiSubsRowClick(context, 51) }, + modifier = modifier, + ) + 52 -> SettingsToggleRow( + title = stringResource(R.string.live_ai_subs_auto_title), + subtitle = stringResource(R.string.live_ai_subs_auto_desc), + isEnabled = settings.enabled && settings.autoStart, + isFocused = isFocused, + onToggle = { onLiveAiSubsRowClick(context, 52) }, + modifier = if (settings.enabled) modifier else modifier.alpha(0.4f), + ) + } +} + +/** The same settings as touch rows, for the mobile settings page. */ +@Composable +internal fun LiveAiSubsMobileRows() { + val context = LocalContext.current + val settings = rememberLiveAiSubsSettings() + MobileSettingsRow( + title = stringResource(R.string.live_ai_subs_title), + subtitle = stringResource(R.string.live_ai_subs_desc), + value = stringResource(if (settings.enabled) R.string.on else R.string.off), + toggleChecked = settings.enabled, + onClick = { onLiveAiSubsRowClick(context, 51) }, + ) + MobileSettingsRow( + title = stringResource(R.string.live_ai_subs_auto_title), + subtitle = stringResource(R.string.live_ai_subs_auto_desc), + value = stringResource(if (settings.enabled && settings.autoStart) R.string.on else R.string.off), + toggleChecked = settings.enabled && settings.autoStart, + enabled = settings.enabled, + showDivider = false, + onClick = { onLiveAiSubsRowClick(context, 52) }, + ) +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt index 1af66a086a..f5990ff1bb 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsScreen.kt @@ -339,7 +339,7 @@ private fun tvGeneralRowsForSection(section: String): List { "subtitles" -> listOf(4, 5, 6, 7, 42, 8, 38) + (if (BuildConfig.AUDIO_SYNC_AVAILABLE) listOf(48) else emptyList()) + listOf(39, 9, 45) - "ai_subtitles" -> listOf(28, 29, 30, 31, 32, 33) + "ai_subtitles" -> listOf(28, 29, 30, 31, 32, 33) + LIVE_AI_SUBS_ROWS "playback" -> listOf(10, 11, 12, 43, 44, 13, 14, 34, 37, 16, 15, 40, 27) "appearance" -> listOf(17, 49, 50, 18, 20, 21, 24, 23, 22, 41, 46, 36, 47) "profiles" -> listOf(19) @@ -1486,6 +1486,7 @@ fun SettingsScreen( 31 -> viewModel.setSubtitleRemoveHearingImpaired(!uiState.subtitleRemoveHearingImpaired) 32 -> showAiApiKeyDialog = true 33 -> viewModel.startAiKeyServer() + 51, 52 -> onLiveAiSubsRowClick(context, tvGeneralRowsForSection(currentSection)[contentFocusIndex]) 34 -> viewModel.cycleTrailerDelay() 47 -> viewModel.cycleGuideRowCount() 37 -> viewModel.setTrailerInCards(!uiState.trailerInCards) @@ -5521,6 +5522,7 @@ private fun MobileSettingsSubPage( showDivider = false, onClick = onSubtitleAiQrClick ) + LiveAiSubsMobileRows() if (uiState.subtitleAiEnabled) { Text( text = when (uiState.subtitleAiModel) { @@ -6967,6 +6969,7 @@ private fun TvGeneralSettingsRows( 37 -> SettingsToggleRow(stringResource(R.string.trailer_in_cards), stringResource(R.string.trailer_in_cards_desc), trailerInCards, focusedIndex == localIndex, onTrailerInCardsToggle, Modifier.settingsFocusSlot(localIndex)) 49 -> SettingsToggleRow(stringResource(R.string.netflix_home_style), stringResource(R.string.netflix_home_style_desc), netflixHomeStyle, focusedIndex == localIndex, onNetflixHomeStyleToggle, Modifier.settingsFocusSlot(localIndex)) 50 -> SettingsToggleRow(stringResource(R.string.watchlist_on_home), stringResource(R.string.watchlist_on_home_desc), watchlistOnHome, focusedIndex == localIndex, onWatchlistOnHomeToggle, Modifier.settingsFocusSlot(localIndex)) + in LIVE_AI_SUBS_ROWS -> LiveAiSubsTvRow(rowId, focusedIndex == localIndex, Modifier.settingsFocusSlot(localIndex)) } } } diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt index 19681de7c7..1edab8a861 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/settings/SettingsViewModel.kt @@ -113,6 +113,8 @@ import kotlinx.coroutines.flow.asStateFlow import kotlinx.coroutines.flow.first import kotlinx.coroutines.flow.flowOn import kotlinx.coroutines.flow.distinctUntilChanged +import kotlinx.coroutines.flow.drop +import kotlinx.coroutines.flow.map import kotlinx.coroutines.launch import kotlinx.coroutines.withContext import kotlinx.coroutines.withTimeoutOrNull @@ -655,6 +657,15 @@ class SettingsViewModel @Inject constructor( } init { + // Live TV AI subtitles settings are written by their own rows (LiveAiSubsSettingsRows); + // push them to the cloud like every other setting, once they settle. + viewModelScope.launch { + com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs.flow(context) + .map { listOf(it.enabled, it.autoStart) } + .distinctUntilChanged() + .drop(1) + .collect { syncLocalStateToCloud(silent = true) } + } _uiState.value = _uiState.value.copy( diagnosticsSharingEnabled = DiagnosticsManager.isReportingEnabled(context), usageStatisticsEnabled = DiagnosticsManager.isUsageEnabled(context) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/TvViewModel.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/TvViewModel.kt index 7821ec35a6..4abcbc0019 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/TvViewModel.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/TvViewModel.kt @@ -143,6 +143,44 @@ class TvViewModel @Inject constructor( ) }.getOrDefault(com.arflix.tv.ui.screens.tv.live.LiveSubtitleDefault.Off) + /** + * The language Live TV AI subtitles translate into when settings leave it to the profile: the + * profile's default subtitle language, else the device language. + */ + internal suspend fun liveAiSubsDefaultLanguage(): String = + (liveSubtitleDefault() as? com.arflix.tv.ui.screens.tv.live.LiveSubtitleDefault.Language)?.code + // The device language, in the same form as track languages ("iw" -> "he"). + ?: (com.arflix.tv.ui.screens.tv.live.liveSubtitleDefault(java.util.Locale.getDefault().language) + as? com.arflix.tv.ui.screens.tv.live.LiveSubtitleDefault.Language)?.code + ?: java.util.Locale.getDefault().language + + /** The profile's subtitle look, read the same way the VOD player reads it. */ + internal suspend fun liveSubtitleStyle(): com.arflix.tv.ui.screens.tv.live.aisubs.LiveSubtitleStyle = runCatching { + val prefs = context.settingsDataStore.data.first() + val size = prefs[profileManager.profileStringKey("subtitle_size")] ?: "Medium" + val offset = prefs[profileManager.profileStringKey("subtitle_offset")] ?: "Bottom" + com.arflix.tv.ui.screens.tv.live.aisubs.LiveSubtitleStyle( + sizePreference = size, + sizePercent = prefs[profileManager.profileIntKey("subtitle_size_pct")] ?: when (size.lowercase()) { + "small" -> 80 + "large" -> 120 + "extra large" -> 140 + else -> 100 + }, + verticalPercent = prefs[profileManager.profileIntKey("subtitle_vertical_pct")] ?: when (offset) { + "Low" -> 8 + "Medium" -> 15 + "High" -> 25 + else -> 2 + }, + color = prefs[profileManager.profileStringKey("subtitle_color")] ?: "White", + style = prefs[profileManager.profileStringKey("subtitle_style")] ?: "Bold", + font = com.arflix.tv.ui.screens.player.SubtitleFontOption.fromPreference( + prefs[profileManager.profileStringKey("subtitle_font")], + ).preferenceValue, + ) + }.getOrDefault(com.arflix.tv.ui.screens.tv.live.aisubs.LiveSubtitleStyle()) + suspend fun cachedSportsMetadata() = sportsRepository.cachedMetadata() internal var cachedSportsSchedule: com.arflix.tv.ui.screens.tv.live.SportsScheduleSnapshot? = null private var sportsBroadcasterIndexLoad: Pair>? = null diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt index 443c06a2b2..66e5fab67e 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveSubtitles.kt @@ -65,6 +65,70 @@ internal fun audioPreferenceLanguage(preference: String?): String? { internal fun liveSubtitleOptions(tracks: Tracks, offLabel: String, displayLocale: Locale = Locale.getDefault()) = liveTrackOptions(tracks, C.TRACK_TYPE_TEXT, offLabel, displayLocale) +/** The picker key of the "AI subtitles" entry (Live TV AI subtitles). */ +internal const val LIVE_SUBTITLES_AI = "ai" + +/** + * [streamOptions] (from [liveSubtitleOptions]) plus an "AI subtitles" entry after the stream's own + * tracks, when the feature is on. A channel without subtitle tracks gets "Off" and "AI subtitles". + * While AI subtitles are on for the channel, that entry is the selected one. + */ +internal fun withAiSubtitlesOption( + streamOptions: List, + offLabel: String, + aiLabel: String, + aiAvailable: Boolean, + aiSelected: Boolean, +): List { + if (!aiAvailable) return streamOptions + val base = streamOptions.ifEmpty { listOf(LiveTrackOption(LIVE_SUBTITLES_OFF, offLabel, selected = true)) } + val withSelection = if (aiSelected) base.map { it.copy(selected = false) } else base + return withSelection + LiveTrackOption(LIVE_SUBTITLES_AI, aiLabel, selected = aiSelected) +} + +/** + * The subtitle track AI subtitles read text from: a text track (not a bitmap one, which carries + * no text), English first, never one already in [avoidLanguage] (that one needs no AI). Null when + * the channel has none; it is then subtitled from its audio. + */ +internal fun liveCaptionSourceKey(tracks: Tracks, avoidLanguage: String?): String? { + val textTracks = typedTracks(tracks, C.TRACK_TYPE_TEXT).filter { track -> + !isBitmapSubtitle(track.group.getTrackFormat(track.trackIndex)) && + (avoidLanguage == null || track.language != avoidLanguage) + } + return (textTracks.firstOrNull { it.language == "en" } ?: textTracks.firstOrNull())?.key +} + +/** + * The channel's own subtitles to show while AI subtitles are on but cannot run (no or rejected + * key, audio that cannot be decoded, quota or connection pause): [preferredLanguage] first, then + * English, then any track (picture subtitles included: these are only shown). Null when the + * channel has none. + */ +internal fun liveAiFallbackSubtitleKey(tracks: Tracks, preferredLanguage: String?): String? { + val all = typedTracks(tracks, C.TRACK_TYPE_TEXT) + return (all.firstOrNull { preferredLanguage != null && it.language == preferredLanguage } + ?: all.firstOrNull { it.language == "en" } + ?: all.firstOrNull())?.key +} + +/** + * True for picture subtitles (DVB, PGS, VobSub), which carry no text. Subtitles the player parses + * during extraction arrive as `application/x-media3-cues`, with the original type kept in codecs. + */ +internal fun isBitmapSubtitle(format: androidx.media3.common.Format): Boolean = + format.sampleMimeType in BITMAP_SUBTITLE_MIMES || format.codecs in BITMAP_SUBTITLE_MIMES + +/** The key of the subtitle track selected now, or null when subtitles are off. */ +internal fun liveSelectedSubtitleKey(tracks: Tracks): String? = + typedTracks(tracks, C.TRACK_TYPE_TEXT).firstOrNull { it.selected }?.key + +private val BITMAP_SUBTITLE_MIMES = setOf( + androidx.media3.common.MimeTypes.APPLICATION_DVBSUBS, + androidx.media3.common.MimeTypes.APPLICATION_PGS, + androidx.media3.common.MimeTypes.APPLICATION_VOBSUB, +) + /** The audio choices for what is playing, one per language or track. */ internal fun liveAudioOptions(tracks: Tracks, displayLocale: Locale = Locale.getDefault()) = liveTrackOptions(tracks, C.TRACK_TYPE_AUDIO, offLabel = null, displayLocale) diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt index 8224314b3f..194e0d33ce 100644 --- a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/LiveTvScreen.kt @@ -80,6 +80,7 @@ import androidx.compose.runtime.mutableStateOf import androidx.compose.runtime.produceState import androidx.compose.runtime.remember import androidx.compose.runtime.rememberCoroutineScope +import androidx.compose.runtime.collectAsState import androidx.compose.runtime.mutableStateMapOf import androidx.compose.runtime.rememberUpdatedState import androidx.compose.runtime.saveable.rememberSaveable @@ -119,6 +120,19 @@ import androidx.media3.common.Player import androidx.media3.datasource.HttpDataSource import androidx.media3.datasource.okhttp.OkHttpDataSource import androidx.media3.exoplayer.DefaultLoadControl +import com.arflix.tv.ui.screens.tv.live.aisubs.AudioTapHlsExtractorFactory +import com.arflix.tv.ui.screens.player.subtitles.RTL_LANGUAGES +import com.arflix.tv.ui.screens.tv.live.aisubs.GeminiCaptionTranslator +import com.arflix.tv.ui.screens.tv.live.aisubs.GeminiSegmentEngine +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveTapExtractorsFactory +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsPrefs +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsSettings +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubsStatus +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveAiSubtitleOverlay +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveSubtitleSession +import com.arflix.tv.ui.screens.tv.live.aisubs.LiveSubtitleStyle +import com.arflix.tv.ui.screens.tv.live.aisubs.autoLiveAiSubsDelayMs +import com.arflix.tv.ui.screens.tv.live.aisubs.SubtitleLookaheadLoadControl import androidx.media3.exoplayer.ExoPlayer import androidx.media3.exoplayer.hls.HlsMediaSource import androidx.media3.exoplayer.source.DefaultMediaSourceFactory @@ -3086,7 +3100,7 @@ fun LiveTvScreen( isLowRamDevice = activityManager?.isLowRamDevice == true ) } - val exoPlayer = remember(livePlaybackBufferProfile) { + val liveLoadControl = remember(livePlaybackBufferProfile) { val loadControl = DefaultLoadControl.Builder() .setBufferDurationsMs( livePlaybackBufferProfile.minBufferMs, @@ -3104,9 +3118,13 @@ fun LiveTvScreen( .setPrioritizeTimeOverSizeThresholds(false) .setBackBuffer(livePlaybackBufferProfile.backBufferMs, true) .build() + // Holds playback back so AI subtitles get audio ahead of time; a no-op until asked to. + SubtitleLookaheadLoadControl(loadControl) + } + val exoPlayer = remember(liveLoadControl) { ExoPlayer.Builder(context) .setMediaSourceFactory(mediaSourceFactory) - .setLoadControl(loadControl) + .setLoadControl(liveLoadControl) // A live stream whose buffer stops moving while buffering raises an error after // this long, which the retry below recovers, instead of a spinner for ten minutes. .setStuckBufferingDetectionTimeoutMs(LIVE_STUCK_BUFFERING_TIMEOUT_MS) @@ -3218,6 +3236,139 @@ fun LiveTvScreen( var playerIsPlaying by remember { mutableStateOf(false) } var playerPlayWhenReady by remember { mutableStateOf(true) } var playerIsBuffering by remember { mutableStateOf(false) } + + // -- Live TV AI subtitles ------------------------------------------------------------------ + // The channel's audio is copied as it downloads, cut into speech segments and sent to Gemini; + // playback is held back (liveLoadControl) so each line is ready before it is heard, and lines + // are shown by the player's position, never by when the answer came. + val liveAiSubsSettings by remember(context) { LiveAiSubsPrefs.flow(context.applicationContext) } + .collectAsState(initial = LiveAiSubsSettings()) + val liveAiKey = rememberUpdatedState(liveAiSubsSettings.apiKey) + val liveAiSession = remember(context) { + LiveSubtitleSession( + context, + GeminiSegmentEngine(apiKey = { liveAiKey.value }), + GeminiCaptionTranslator(apiKey = { liveAiKey.value }), + ) + } + DisposableEffect(liveAiSession) { onDispose { liveAiSession.release() } } + // The profile's subtitle language: what AI subtitles translate into. + var liveAiTargetCode by remember { mutableStateOf(null) } + // A channel that already carries subtitles in that language needs no AI to start with. + val liveChannelHasTargetSubs = liveAiTargetCode?.let { code -> + liveSubtitleOptions(liveTracks, "").any { it.key == code } + } == true + // Per channel, like the audio choice: picked in the subtitle menu, or on by default when + // "auto-translate when a channel starts" is set (unless the channel has the language already). + val liveAiSubsChoices = remember { mutableStateMapOf() } + val liveAiSubsOn = liveAiSubsSettings.enabled && playingChannelId != null && + (liveAiSubsChoices[playingChannelId] ?: (liveAiSubsSettings.autoStart && !liveChannelHasTargetSubs)) + val liveAiCue by liveAiSession.cue.collectAsState() + val liveAiStatus by liveAiSession.status.collectAsState() + // On, but unable to run on this channel (no or rejected key, audio that cannot be decoded): + // no hold, and the channel's own subtitles show instead. + val liveAiBlocked = !liveAiSubsSettings.hasKey || liveAiStatus is LiveAiSubsStatus.NoKey || + liveAiStatus is LiveAiSubsStatus.InvalidKey || liveAiStatus is LiveAiSubsStatus.Unsupported + val liveAiHolds = liveAiSubsOn && !liveAiBlocked + // A quota or connection pause is temporary: the hold stays (so playback does not stall when + // AI comes back), but the channel's own subtitles fill in meanwhile. + val liveAiShowsNative = liveAiSubsOn && (liveAiBlocked || + liveAiStatus is LiveAiSubsStatus.QuotaPaused || liveAiStatus is LiveAiSubsStatus.Offline) + val liveAiShows = liveAiSubsOn && !liveAiShowsNative + var liveAiPreparingSeconds by remember { mutableStateOf(null) } + // Bumped to restart the stream so audio already buffered gets subtitled too. + var liveAiRestartRequest by remember { mutableIntStateOf(0) } + var liveSubtitleStyle by remember { mutableStateOf(LiveSubtitleStyle()) } + LaunchedEffect(Unit) { liveSubtitleStyle = viewModel.liveSubtitleStyle() } + // Always the profile's subtitle language. + LaunchedEffect(Unit) { + val code = viewModel.liveAiSubsDefaultLanguage() + liveAiTargetCode = code + val name = java.util.Locale.forLanguageTag(code).getDisplayLanguage(java.util.Locale.ENGLISH) + .ifBlank { "English" } + liveAiSession.setTargetLanguage(name, name.lowercase(java.util.Locale.ROOT) in RTL_LANGUAGES) + } + LaunchedEffect(liveAiSubsSettings.apiKey) { liveAiSession.onKeyChanged() } + // Capture whenever AI subtitles are on for the channel, the mini player included: the audio + // buffered during the start-up hold is what the first lines are made from. + val liveAiCapture = liveAiSubsOn && playerPlayWhenReady + LaunchedEffect(liveAiSubsOn, liveAiSubsSettings.hasKey, liveAiCapture) { + liveAiSession.update(enabled = liveAiSubsOn, hasKey = liveAiSubsSettings.hasKey, capture = liveAiCapture) + } + // While AI subtitles show, the stream's own text subtitles stay selected but hidden: they are + // read ahead of playback and translated (a stream that is not selected is not even + // downloaded). Without a text track, AI subtitles come from the audio. While AI cannot run, + // the channel's own subtitles are selected and shown instead. + LaunchedEffect(liveAiSubsOn, liveAiShowsNative, liveTracks, liveAiTargetCode) { + if (!liveAiSubsOn) return@LaunchedEffect + val source = if (liveAiShowsNative) liveAiFallbackSubtitleKey(liveTracks, liveAiTargetCode) + else liveCaptionSourceKey(liveTracks, liveAiTargetCode) + val selected = liveSelectedSubtitleKey(liveTracks) + when { + source != null && selected != source -> applyLiveSubtitleChoice(exoPlayer, source) + source == null && selected != null -> applyLiveSubtitleChoice(exoPlayer, LIVE_SUBTITLES_OFF) + } + } + // Turned on while the channel plays: the seconds already buffered were downloaded before + // capture started, so they would play without subtitles. Restart the stream instead; the + // start-up hold then shows "Preparing subtitles" and the first line is covered. Not when AI + // cannot run (nothing to prepare), and not in catch-up: a recording has no hold and reads + // far enough ahead on its own, and a restart would lose the position in it. + LaunchedEffect(liveAiHolds) { + if (liveAiHolds && playingCatchupProgram == null && exoPlayer.isPlaying && + exoPlayer.currentMediaItem?.mediaId == playingChannelId.orEmpty() + ) { + liveAiRestartRequest++ + } + } + val liveAiTick by rememberUpdatedState(liveAiHolds to (playingCatchupProgram == null)) + LaunchedEffect(exoPlayer, liveAiSession) { + val window = androidx.media3.common.Timeline.Window() + while (true) { + val (on, isLive) = liveAiTick + val requiredMs = if (on && isLive) autoLiveAiSubsDelayMs(liveAiSession.latencyP95Ms) else 0L + liveLoadControl.requiredLookaheadUs = requiredMs * 1_000L + if (on) { + // currentPosition is relative to the live window; captured audio is timed on the + // period, so the window offset into the period is added. + val timeline = exoPlayer.currentTimeline + val windowOffsetUs = if (timeline.isEmpty) 0L else + timeline.getWindow(exoPlayer.currentMediaItemIndex, window).positionInFirstPeriodUs + val lookaheadMs = (exoPlayer.bufferedPosition - exoPlayer.currentPosition).coerceAtLeast(0L) + liveAiSession.tick(exoPlayer.currentPosition * 1_000L + windowOffsetUs, exoPlayer.isPlaying) + val held = liveLoadControl.holding && exoPlayer.playbackState == Player.STATE_BUFFERING + liveAiPreparingSeconds = if (held && requiredMs > 0L) { + ((requiredMs - lookaheadMs + 999L) / 1_000L).toInt().coerceAtLeast(1) + } else { + null + } + if (liveLoadControl.cappedByMemory) { + liveLoadControl.cappedByMemory = false + android.widget.Toast.makeText( + context, context.getString(R.string.live_ai_subs_memory_capped), android.widget.Toast.LENGTH_LONG, + ).show() + } + } else { + liveAiPreparingSeconds = null + } + delay(LIVE_AI_TICK_MS) + } + } + LaunchedEffect(liveAiSession) { + liveAiSession.status.collect { status -> + val message = when (status) { + LiveAiSubsStatus.NoKey -> R.string.live_ai_subs_no_key + LiveAiSubsStatus.InvalidKey -> R.string.live_ai_subs_invalid_key + is LiveAiSubsStatus.QuotaPaused -> R.string.live_ai_subs_quota + LiveAiSubsStatus.Offline -> R.string.live_ai_subs_offline + is LiveAiSubsStatus.Unsupported -> R.string.live_ai_subs_unsupported + else -> null + } + message?.let { + android.widget.Toast.makeText(context, context.getString(it), android.widget.Toast.LENGTH_LONG).show() + } + } + } LaunchedEffect(exoPlayer, playingCatchupProgram, catchupUrlAnchorOffsetMs) { while (true) { val programDuration = playingCatchupProgram @@ -3382,14 +3533,19 @@ fun LiveTvScreen( } // HLS gets its own factory: DefaultMediaSourceFactory cannot set the HLS extractors, // and channels without IDR frames need iptvHlsExtractorFactory() to show video. + // Live TV AI subtitles: every channel source gets a fresh audio tap, so a load still + // running for the previous channel can never feed this one. + liveAiSession.onNewSource() + val liveTapSinks = liveAiSession.pipeline.newSinks() val source = if (isHls) { HlsMediaSource.Factory(sourceDataFactory) .setAllowChunklessPreparation(IPTV_HLS_CHUNKLESS_PREPARATION) - .setExtractorFactory(iptvHlsExtractorFactory()) + .setExtractorFactory(AudioTapHlsExtractorFactory(iptvHlsExtractorFactory(), liveTapSinks)) .setLoadErrorHandlingPolicy(IptvLoadErrorHandlingPolicy()) .createMediaSource(mediaItem) } else { - DefaultMediaSourceFactory(context, iptvExtractorsFactory()).setDataSourceFactory(sourceDataFactory) + DefaultMediaSourceFactory(context, LiveTapExtractorsFactory(iptvExtractorsFactory(), liveTapSinks)) + .setDataSourceFactory(sourceDataFactory) .setLoadErrorHandlingPolicy( IptvLoadErrorHandlingPolicy( liveMidStreamDrop( @@ -3427,6 +3583,19 @@ fun LiveTvScreen( ) } + // AI subtitles switched on mid-channel: restart the stream (see liveAiRestartRequest). + LaunchedEffect(liveAiRestartRequest) { + if (liveAiRestartRequest == 0) return@LaunchedEffect + val prepared = lastPreparedStreamUrl ?: return@LaunchedEffect + val restart = { + prepareStream(prepared, isHls = lastPreparedIsHls, headers = lastPreparedHeaders, resetRetry = false, + drmInfo = playingChannel?.source?.drmInfo, forcePrepare = true, + resolvedMimeType = lastPreparedMimeType) + } + val restartChannelId = playingChannelId + liveStreamGate.reopen(coroutineScope, { playingChannelId == restartChannelId }, { restart() }, restart) + } + fun toggleCatchupPlayback() { if (playingCatchupProgram == null) return if (exoPlayer.isPlaying) { @@ -4791,9 +4960,19 @@ fun LiveTvScreen( if (view.player !== exoPlayer) { view.player = exoPlayer } + // With AI subtitles the stream's own subtitles are only read, not shown. + view.subtitleView?.visibility = + if (liveAiShows) android.view.View.INVISIBLE else android.view.View.VISIBLE }, modifier = Modifier.fillMaxSize(), ) + if (isFullScreen && !fullscreenGuideOpen && !quickZapOpen && liveAiShows) { + LiveAiSubtitleOverlay( + cue = liveAiCue, + style = liveSubtitleStyle, + preparingText = liveAiPreparingSeconds?.let { stringResource(R.string.live_ai_subs_preparing, it) }, + ) + } if (isFullScreen && !fullscreenGuideOpen && !quickZapOpen) { val categoryTitle = playingChannel?.source?.group?.takeIf { it.isNotBlank() } ?: visibleEnrichedState.value.tree.byId(selectedCategoryId)?.label @@ -4827,7 +5006,12 @@ fun LiveTvScreen( }, onGuideClick = { openFullscreenGuide() }, onSubtitlesClick = { subtitlePickerOpen = true } - .takeIf { liveSubtitleOptions(liveTracks, "").size > 1 }, + .takeIf { + withAiSubtitlesOption( + liveSubtitleOptions(liveTracks, ""), "", "", + aiAvailable = liveAiSubsSettings.enabled, aiSelected = false, + ).size > 1 + }, onAudioClick = { audioPickerOpen = true } .takeIf { liveAudioOptions(liveTracks).size > 1 }, onOpenVariants = playingChannel?.takeIf { playingCatchupProgram == null }?.let { channel -> @@ -5095,14 +5279,32 @@ fun LiveTvScreen( title = stringResource(R.string.live_subtitles), icon = Icons.Filled.Subtitles, channelName = playingChannel?.name.orEmpty(), - options = liveSubtitleOptions(liveTracks, stringResource(R.string.live_subtitles_off)), + options = withAiSubtitlesOption( + liveSubtitleOptions(liveTracks, stringResource(R.string.live_subtitles_off)), + offLabel = stringResource(R.string.live_subtitles_off), + aiLabel = stringResource(R.string.live_ai_subtitles), + aiAvailable = liveAiSubsSettings.enabled, + aiSelected = liveAiSubsOn, + ), onDismiss = { subtitlePickerOpen = false returnToHudControls() }, onPick = { key -> - playingChannelId?.let { liveSubtitleChoices[it] = key } - applyLiveSubtitleChoice(exoPlayer, key) + if (key == LIVE_SUBTITLES_AI) { + // AI subtitles replace the stream's own (which keep feeding them, hidden); + // remembered per channel. + playingChannelId?.let { + liveAiSubsChoices[it] = true + liveSubtitleChoices.remove(it) + } + } else { + playingChannelId?.let { + liveAiSubsChoices[it] = false + liveSubtitleChoices[it] = key + } + applyLiveSubtitleChoice(exoPlayer, key) + } }, ) } @@ -5436,6 +5638,9 @@ private fun classifyPlaybackError(error: PlaybackException): String { } } +/** How often Live TV AI subtitles follow the player position (cue display, hold, look-ahead). */ +private const val LIVE_AI_TICK_MS = 100L + internal fun buildLiveTvConfiguration(): MediaItem.LiveConfiguration = MediaItem.LiveConfiguration.Builder() // Respect the manifest's segment/hold-back timing. An arbitrary eight-second diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/CaptionCueReader.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/CaptionCueReader.kt new file mode 100644 index 0000000000..417d4940ad --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/CaptionCueReader.kt @@ -0,0 +1,43 @@ +@file:androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) + +package com.arflix.tv.ui.screens.tv.live.aisubs + +import androidx.media3.extractor.text.CueDecoder + +/** + * Reads subtitle streams the player already parsed during extraction (WebVTT and other text + * renditions arrive as Media3's own cue samples, `application/x-media3-cues`) into sentences on + * the media timeline. Live caption streams usually roll up: each cue repeats the previous line + * and adds one, so lines seen in the last few cues are dropped and only new ones are emitted. + * + * Not thread-safe: one instance per channel source, fed from the loading thread. + */ +internal class CaptionCueReader(private val onSentence: (CaptionSentence) -> Unit) { + private val decoder = CueDecoder() + private val recentLines = ArrayDeque() + + fun feed(timeUs: Long, data: ByteArray, offset: Int, size: Int) { + val cues = runCatching { decoder.decode(timeUs, data, offset, size) }.getOrNull() ?: return + val lines = cues.cues.flatMap { cue -> cue.text?.toString().orEmpty().split('\n') } + .map { it.replace(WHITESPACE, " ").trim() } + .filter { it.isNotEmpty() } + val fresh = lines.filter { it !in recentLines } + lines.forEach { line -> + if (line !in recentLines) recentLines.addLast(line) + while (recentLines.size > RECENT_LINES) recentLines.removeFirst() + } + if (fresh.isEmpty()) return + val endUs = if (cues.endTimeUs > cues.startTimeUs) cues.endTimeUs else cues.startTimeUs + DEFAULT_DURATION_US + onSentence(CaptionSentence(cues.startTimeUs, endUs, fresh.joinToString(" "))) + } + + fun reset() { + recentLines.clear() + } + + private companion object { + const val RECENT_LINES = 6 + const val DEFAULT_DURATION_US = 3_000_000L + val WHITESPACE = Regex("\\s+") + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/Cea608TextAssembler.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/Cea608TextAssembler.kt new file mode 100644 index 0000000000..9c223dbcba --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/Cea608TextAssembler.kt @@ -0,0 +1,157 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +/** One caption sentence on the media timeline: from its first character to its last. */ +internal data class CaptionSentence(val startUs: Long, val endUs: Long, val text: String) + +/** + * Turns a channel's CEA-608 closed captions (field 1, caption channel CC1) into plain sentences + * with media times, so they can be translated ahead of playback instead of transcribing audio. + * + * Input is the `cc_data` the TS extractor copies out of the video's SEI messages: triplets of + * (marker/valid/type, byte 1, byte 2), as the player's own CEA-608 decoder receives them. Display + * layout (roll-up, pop-on, positions, colours) is ignored: only the spoken words matter here. + * Control codes are sent twice by broadcasters; the repeat is dropped. A sentence ends at . ? ! + * (or after a pause of [GAP_US], or at [MAX_CHARS]). + * + * Not thread-safe: one instance per channel source, fed from the loading thread. + */ +internal class Cea608TextAssembler(private val onSentence: (CaptionSentence) -> Unit) { + private val text = StringBuilder() + private var firstCharUs = UNSET + private var lastCharUs = UNSET + private var lastControl = -1 + private var pendingBoundary = false + + /** True once any caption character has been seen for this source. */ + var sawText: Boolean = false + private set + + fun feed(timeUs: Long, data: ByteArray, offset: Int, size: Int) { + if (lastCharUs != UNSET && timeUs - lastCharUs > GAP_US) flush() + var i = offset + val end = offset + size - 2 + while (i < end) { + val header = data[i].toInt() and 0xFF + val b1 = data[i + 1].toInt() and 0x7F + val b2 = data[i + 2].toInt() and 0x7F + i += 3 + val valid = header and 0x04 != 0 + val fieldOne = header and 0x03 == 0 + if (!valid || !fieldOne || (b1 == 0 && b2 == 0)) continue + handlePair(timeUs, b1, b2) + } + } + + /** The stream jumped (seek, reload): what was collected no longer joins what comes next. */ + fun reset() { + text.setLength(0) + firstCharUs = UNSET + lastCharUs = UNSET + lastControl = -1 + pendingBoundary = false + } + + /** Emits whatever is collected as a sentence. */ + fun flush() { + val sentence = text.toString().replace(WHITESPACE, " ").trim() + if (sentence.length >= MIN_CHARS && firstCharUs != UNSET) { + onSentence(CaptionSentence(firstCharUs, lastCharUs, sentence)) + } + text.setLength(0) + firstCharUs = UNSET + lastCharUs = UNSET + pendingBoundary = false + } + + private fun handlePair(timeUs: Long, b1: Int, b2: Int) { + if (b1 in 0x10..0x1F) { + val code = (b1 shl 8) or b2 + // Broadcasters send every control code twice; act on it once. + if (code == lastControl) { + lastControl = -1 + return + } + lastControl = code + // The second caption channel (CC2) uses 0x18-0x1F. + if (b1 >= 0x18) return + handleControl(timeUs, b1, b2) + return + } + lastControl = -1 + if (b1 >= 0x20) appendChar(timeUs, basicChar(b1)) + if (b2 >= 0x20) appendChar(timeUs, basicChar(b2)) + } + + private fun handleControl(timeUs: Long, b1: Int, b2: Int) { + when { + // Miscellaneous control codes (RCL, BS, RU2-4, EDM, CR, ENM, EOC...). + (b1 == 0x14 || b1 == 0x15) && b2 in 0x20..0x2F -> when (b2) { + 0x21 -> backspace() + else -> boundary() + } + // Special characters (♪, ®, ½...): only letters with meaning are kept. + b1 == 0x11 && b2 in 0x30..0x3F -> if (b2 == 0x37) boundary() else appendChar(timeUs, SPECIAL[b2 - 0x30]) + // Extended characters (accented capitals, symbols) follow a plain fallback letter + // that a basic decoder shows instead; keeping that letter is good enough here. + (b1 == 0x12 || b1 == 0x13) && b2 in 0x20..0x3F -> Unit + // Preamble address codes and mid-row codes move the cursor: a word boundary. + else -> boundary() + } + } + + private fun boundary() { + pendingBoundary = true + } + + private fun backspace() { + if (text.isNotEmpty()) text.setLength(text.length - 1) + } + + private fun appendChar(timeUs: Long, c: Char) { + if (c == '\u0000') return + if (pendingBoundary && text.isNotEmpty() && text.last() != ' ') text.append(' ') + pendingBoundary = false + if (c == ' ' && (text.isEmpty() || text.last() == ' ')) return + if (c != ' ') { + sawText = true + if (firstCharUs == UNSET) firstCharUs = timeUs + lastCharUs = timeUs + } + text.append(c) + if (c in SENTENCE_END && text.length >= MIN_CHARS) { + flush() + } else if (c == ' ' && (text.length >= MAX_CHARS || lastCharUs - firstCharUs >= MAX_DURATION_US)) { + // Long run-on captions end at a word boundary, like audio segments end at 5 s, so a + // line is never held back longer than the playback delay allows for. + flush() + } + } + + private fun basicChar(b: Int): Char = when (b) { + 0x2A -> 'á' + 0x5C -> 'é' + 0x5E -> 'í' + 0x5F -> 'ó' + 0x60 -> 'ú' + 0x7B -> 'ç' + 0x7C -> '÷' + 0x7D -> 'Ñ' + 0x7E -> 'ñ' + 0x7F -> '\u0000' + else -> b.toChar() + } + + companion object { + private const val UNSET = Long.MIN_VALUE + /** A pause in the captions this long ends the sentence. */ + const val GAP_US = 2_500_000L + const val MAX_CHARS = 160 + const val MAX_DURATION_US = 5_000_000L + private const val MIN_CHARS = 2 + private val SENTENCE_END = setOf('.', '?', '!') + private val WHITESPACE = Regex("\\s+") + private val SPECIAL = charArrayOf( + '®', '°', '½', '¿', '™', '¢', '£', '\u0000', 'à', ' ', 'è', 'â', 'ê', 'î', 'ô', 'û', + ) + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiCaptionTranslator.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiCaptionTranslator.kt new file mode 100644 index 0000000000..1a8da8cf86 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiCaptionTranslator.kt @@ -0,0 +1,45 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import com.arflix.tv.ui.screens.player.subtitles.SubtitleAiModel +import com.arflix.tv.ui.screens.player.subtitles.SubtitleTranslationService + +/** Translates one sentence of a channel's own captions into timed lines. */ +internal interface LiveCaptionTranslator { + suspend fun translate(sentence: CaptionSentence, targetLanguageName: String, rtl: Boolean): LiveEngineResult +} + +/** + * Caption sentences go through the app's existing Gemini subtitle translation (the same model, + * prompt and retry rules as AI subtitles for films), as one-line batches: the text is already + * there, so this is faster and cheaper than sending audio, and makes no listening mistakes. + */ +internal class GeminiCaptionTranslator(private val apiKey: () -> String) : LiveCaptionTranslator { + private val service = SubtitleTranslationService( + apiKeyProvider = apiKey, + modelProvider = { SubtitleAiModel.GEMINI_FLASH_25 }, + ) + + override suspend fun translate(sentence: CaptionSentence, targetLanguageName: String, rtl: Boolean): LiveEngineResult { + if (apiKey().isBlank()) return LiveEngineResult.InvalidKey + val result = service.translateBatch(listOf(sentence.text), targetLanguageName) + if (!result.success) return errorResult(result.errorMessage.orEmpty()) + val line = result.lines.firstOrNull()?.trim().orEmpty() + if (line.isEmpty()) return LiveEngineResult.Cues(emptyList(), null) + val endUs = maxOf(sentence.endUs, sentence.startUs) + CAPTION_TAIL_US + return LiveEngineResult.Cues(listOf(LiveCue(sentence.startUs, endUs, line, rtl)), null) + } + + companion object { + /** Captions are typed as they are spoken; the line stays a little after its last word. */ + const val CAPTION_TAIL_US = 1_500_000L + + fun errorResult(message: String): LiveEngineResult = when { + message == "RATE_LIMITED" -> LiveEngineResult.QuotaExceeded + message.startsWith("HTTP 401") || message.startsWith("HTTP 403") || + message.contains("API_KEY_INVALID") || message.contains("API key not valid") -> LiveEngineResult.InvalidKey + message.contains("Unable to resolve host", ignoreCase = true) || + message.contains("Failed to connect", ignoreCase = true) -> LiveEngineResult.Offline + else -> LiveEngineResult.Failed(message.take(80)) + } + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiSegmentEngine.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiSegmentEngine.kt new file mode 100644 index 0000000000..531dac34f6 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiSegmentEngine.kt @@ -0,0 +1,317 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import com.arflix.tv.ui.screens.player.subtitles.BLOCKED_FINISH_REASONS +import com.arflix.tv.ui.screens.player.subtitles.GEMINI_MAX_TRANSIENT_RETRIES +import com.arflix.tv.ui.screens.player.subtitles.GEMINI_RETRYABLE_HTTP +import com.arflix.tv.ui.screens.player.subtitles.GEMINI_RETRY_BASE_DELAY_MS +import com.arflix.tv.ui.screens.player.subtitles.extractJsonArray +import kotlinx.coroutines.CancellationException +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.delay +import kotlinx.coroutines.withContext +import okhttp3.MediaType.Companion.toMediaType +import okhttp3.OkHttpClient +import okhttp3.Request +import okhttp3.RequestBody.Companion.toRequestBody +import org.json.JSONArray +import org.json.JSONObject +import java.io.IOException +import java.nio.ByteBuffer +import java.nio.ByteOrder +import java.util.concurrent.TimeUnit + +/** One stretch of speech cut from the channel's audio, on the media timeline. */ +internal class SpeechSegment( + val startUs: Long, + val endUs: Long, + /** 16 kHz mono samples in [-1, 1]. */ + val pcm16k: FloatArray, +) + +/** What an engine made of one segment. */ +internal sealed interface LiveEngineResult { + data class Cues(val cues: List, val sourceLanguage: String?) : LiveEngineResult + + /** Nothing more will work until the user fixes it (wrong key). */ + data object InvalidKey : LiveEngineResult + + /** Quota or rate limit (HTTP 429): pause and back off. */ + data object QuotaExceeded : LiveEngineResult + + /** No connection. */ + data object Offline : LiveEngineResult + + /** This segment only (content policy, malformed reply, server error). */ + data class Failed(val reason: String) : LiveEngineResult +} + +/** Turns speech segments into timed subtitle lines in the viewer's language. */ +internal interface LiveSubtitleEngine { + /** + * [context] holds the last few lines already shown (in the target language), so names, + * gender and register stay consistent across segments. + */ + suspend fun process(segment: SpeechSegment, context: List, targetLanguageName: String, rtl: Boolean): LiveEngineResult +} + +/** + * Gemini `generateContent` with the segment as inline WAV audio: transcribes and translates in + * one call, and returns lines that are timed from the segment's own media time. Plain HTTP per + * segment, so there is no long-lived session to expire over hours of TV. + * + * Follows the app's Gemini rules (see SubtitleTranslationService): JSON response mode, minimal + * thinking, BLOCK_NONE safety settings, short retries on 500/502/503 and network errors, never on + * 429. The key goes in a header, never in a URL or the log. + */ +internal class GeminiSegmentEngine( + private val apiKey: () -> String, + private val client: OkHttpClient = DEFAULT_CLIENT, + private val baseUrl: String = GEMINI_LIVE_URL, +) : LiveSubtitleEngine { + + override suspend fun process( + segment: SpeechSegment, + context: List, + targetLanguageName: String, + rtl: Boolean, + ): LiveEngineResult = withContext(Dispatchers.IO) { + val key = apiKey() + if (key.isBlank()) return@withContext LiveEngineResult.InvalidKey + val body = GeminiLiveProtocol.requestBody(segment, context, targetLanguageName) + var transient = 0 + while (true) { + val request = Request.Builder() + .url(baseUrl) + .header("x-goog-api-key", key) + .post(body.toRequestBody(JSON_MEDIA_TYPE)) + .build() + try { + client.newCall(request).execute().use { response -> + val text = response.body?.string().orEmpty() + when { + response.isSuccessful -> + return@withContext GeminiLiveProtocol.parseResponse(text, segment, rtl) + response.code == 429 -> return@withContext LiveEngineResult.QuotaExceeded + GeminiLiveProtocol.isInvalidKey(response.code, text) -> return@withContext LiveEngineResult.InvalidKey + response.code in GEMINI_RETRYABLE_HTTP && transient < GEMINI_MAX_TRANSIENT_RETRIES -> Unit + else -> return@withContext LiveEngineResult.Failed("HTTP ${response.code}") + } + } + } catch (error: CancellationException) { + throw error + } catch (error: IOException) { + if (transient >= GEMINI_MAX_TRANSIENT_RETRIES) { + return@withContext if (error is java.net.UnknownHostException || error is java.net.ConnectException) { + LiveEngineResult.Offline + } else { + LiveEngineResult.Failed("network: ${error.javaClass.simpleName}") + } + } + } + // ±30% jittered exponential backoff, as in SubtitleTranslationService. + val jitter = 0.7 + Math.random() * 0.6 + delay((GEMINI_RETRY_BASE_DELAY_MS * (1L shl transient) * jitter).toLong()) + transient++ + } + @Suppress("UNREACHABLE_CODE") + LiveEngineResult.Failed("unreachable") + } + + /** A cheap call that tells whether [key] is accepted: true, false, or null when it could not be checked. */ + suspend fun validateKey(key: String): Boolean? = withContext(Dispatchers.IO) { + val request = Request.Builder() + .url(GEMINI_MODEL_INFO_URL) + .header("x-goog-api-key", key.trim()) + .get() + .build() + try { + client.newCall(request).execute().use { response -> + when { + response.isSuccessful -> true + GeminiLiveProtocol.isInvalidKey(response.code, response.body?.string().orEmpty()) -> false + else -> null + } + } + } catch (error: CancellationException) { + throw error + } catch (_: IOException) { + null + } + } + + companion object { + /** + * Same model as the app's subtitle translation: fast and cheap enough for continuous use, + * and it takes audio input. + */ + const val MODEL_ID = "gemini-3.5-flash-lite" + const val GEMINI_LIVE_URL = "https://generativelanguage.googleapis.com/v1beta/models/$MODEL_ID:generateContent" + private const val GEMINI_MODEL_INFO_URL = "https://generativelanguage.googleapis.com/v1beta/models/$MODEL_ID" + private val JSON_MEDIA_TYPE = "application/json".toMediaType() + private val DEFAULT_CLIENT: OkHttpClient by lazy { + OkHttpClient.Builder() + .connectTimeout(8, TimeUnit.SECONDS) + // A segment's answer is useless once its moment has passed. + .readTimeout(12, TimeUnit.SECONDS) + .callTimeout(15, TimeUnit.SECONDS) + .build() + } + } +} + +/** The request and response format for [GeminiSegmentEngine]; pure, so it is unit tested. */ +internal object GeminiLiveProtocol { + + fun systemPrompt(targetLanguageName: String): String = """ + You write live TV subtitles. The audio is a few seconds of a TV broadcast. + Transcribe what is said and translate it into $targetLanguageName. If the speech is + already in $targetLanguageName, return the transcription. + Write natural, idiomatic $targetLanguageName as a professional subtitler would: short + lines, correct grammatical gender, keep names as they are spoken. + Split the text into subtitle lines of at most about 42 characters, in spoken order. + For each line give startOffsetMs, the time from the start of the audio when it begins. + If there is no intelligible speech (music, noise), return an empty "lines" array. + Never describe sounds, never add notes or explanations. + Reply with JSON only: + {"lines":[{"text":"...","startOffsetMs":0}],"sourceLanguage":"en"} + """.trimIndent() + + fun requestBody(segment: SpeechSegment, context: List, targetLanguageName: String): String { + val wav = android.util.Base64.encodeToString(wav16kMono(segment.pcm16k), android.util.Base64.NO_WRAP) + val userText = buildString { + if (context.isNotEmpty()) { + append("Previous subtitles, for context only (do not repeat them):\n") + context.forEach { append("- ").append(it).append('\n') } + } + append("Subtitle this audio in ").append(targetLanguageName).append('.') + } + return JSONObject().apply { + put("system_instruction", JSONObject().put("parts", JSONArray().put(JSONObject().put("text", systemPrompt(targetLanguageName))))) + put( + "contents", + JSONArray().put( + JSONObject().put( + "parts", + JSONArray() + .put(JSONObject().put("inline_data", JSONObject().put("mime_type", "audio/wav").put("data", wav))) + .put(JSONObject().put("text", userText)), + ), + ), + ) + put( + "generationConfig", + JSONObject() + .put("temperature", 0.2) + .put("responseMimeType", "application/json") + .put("thinkingConfig", JSONObject().put("thinkingLevel", "minimal")), + ) + put( + "safetySettings", + JSONArray().apply { + listOf( + "HARM_CATEGORY_HARASSMENT", + "HARM_CATEGORY_HATE_SPEECH", + "HARM_CATEGORY_SEXUALLY_EXPLICIT", + "HARM_CATEGORY_DANGEROUS_CONTENT", + ).forEach { put(JSONObject().put("category", it).put("threshold", "BLOCK_NONE")) } + }, + ) + }.toString() + } + + /** A Gemini `generateContent` HTTP 200 body → cues on the segment's media time. */ + fun parseResponse(body: String, segment: SpeechSegment, rtl: Boolean): LiveEngineResult { + val json = runCatching { JSONObject(body) }.getOrNull() ?: return LiveEngineResult.Failed("unreadable reply") + val candidate = json.optJSONArray("candidates")?.optJSONObject(0) + ?: return LiveEngineResult.Failed( + if (json.optJSONObject("promptFeedback")?.optString("blockReason").isNullOrBlank()) "no candidates" else "blocked", + ) + val parts = candidate.optJSONObject("content")?.optJSONArray("parts") + val text = buildString { + for (i in 0 until (parts?.length() ?: 0)) { + val part = parts!!.optJSONObject(i) ?: continue + if (part.optBoolean("thought", false)) continue + append(part.optString("text")) + } + }.trim() + if (text.isBlank()) { + return LiveEngineResult.Failed( + if (candidate.optString("finishReason") in BLOCKED_FINISH_REASONS) "blocked" else "empty reply", + ) + } + return parseLines(text, segment, rtl) + } + + /** + * The model's JSON text → cues. Accepts the documented object, a bare array of lines or strings + * (JSON mode sometimes drops the wrapper), and fenced or truncated output via the shared + * [extractJsonArray]. + */ + fun parseLines(text: String, segment: SpeechSegment, rtl: Boolean): LiveEngineResult { + val obj = runCatching { JSONObject(text.trim()) }.getOrNull() + val array: JSONArray = obj?.optJSONArray("lines") + ?: extractJsonArray(text) + ?: return LiveEngineResult.Failed("malformed reply") + val language = obj?.optString("sourceLanguage")?.takeIf { it.isNotBlank() } + val raw = ArrayList>() + for (i in 0 until array.length()) { + val item = array.opt(i) + when (item) { + is JSONObject -> { + val line = cleanLine(item.optString("text")) + val offset = if (item.has("startOffsetMs")) item.optLong("startOffsetMs", -1L).takeIf { it >= 0 } else null + if (line.isNotEmpty()) raw += line to offset + } + is String -> cleanLine(item).takeIf { it.isNotEmpty() }?.let { raw += it to null } + } + } + return LiveEngineResult.Cues(timeLines(raw, segment, rtl), language) + } + + /** + * Lines with offsets start there (kept in order, inside the segment); the rest share the + * segment by text length. Each line ends where the next begins; the last at the segment end. + */ + fun timeLines(lines: List>, segment: SpeechSegment, rtl: Boolean): List { + if (lines.isEmpty()) return emptyList() + val duration = (segment.endUs - segment.startUs).coerceAtLeast(1L) + val totalChars = lines.sumOf { it.first.length }.coerceAtLeast(1) + var proportional = 0L + var previousStart = segment.startUs + val starts = lines.map { (text, offsetMs) -> + val byLength = segment.startUs + duration * proportional / totalChars + proportional += text.length + val start = offsetMs?.let { segment.startUs + it * 1_000L } ?: byLength + start.coerceIn(previousStart, segment.endUs - 1).also { previousStart = it } + } + return lines.mapIndexed { index, (text, _) -> + LiveCue( + startUs = starts[index], + endUs = starts.getOrNull(index + 1)?.takeIf { it > starts[index] } ?: segment.endUs, + text = text, + rtl = rtl, + ) + } + } + + fun isInvalidKey(code: Int, body: String): Boolean = + code == 401 || code == 403 || + (code == 400 && (body.contains("API_KEY_INVALID") || body.contains("API key not valid"))) + + private fun cleanLine(text: String): String = text.replace('\n', ' ').replace(Regex("\\s+"), " ").trim() + + /** 16-bit little-endian mono WAV at 16 kHz. */ + fun wav16kMono(samples: FloatArray): ByteArray { + val dataBytes = samples.size * 2 + val buffer = ByteBuffer.allocate(44 + dataBytes).order(ByteOrder.LITTLE_ENDIAN) + buffer.put("RIFF".toByteArray(Charsets.US_ASCII)).putInt(36 + dataBytes) + buffer.put("WAVE".toByteArray(Charsets.US_ASCII)) + buffer.put("fmt ".toByteArray(Charsets.US_ASCII)).putInt(16) + .putShort(1).putShort(1).putInt(16_000).putInt(32_000).putShort(2).putShort(16) + buffer.put("data".toByteArray(Charsets.US_ASCII)).putInt(dataBytes) + for (sample in samples) { + buffer.putShort((sample.coerceIn(-1f, 1f) * Short.MAX_VALUE).toInt().toShort()) + } + return buffer.array() + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsLog.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsLog.kt new file mode 100644 index 0000000000..3d17f5864a --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsLog.kt @@ -0,0 +1,22 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.util.Log + +/** + * Live TV AI subtitles' log (tag `LiveAiSubs`). Links are reduced to their host, since IPTV stream + * URLs carry account credentials; the log holds timings and decisions, never audio or text. + */ +internal object LiveAiSubsLog { + private const val TAG = "LiveAiSubs" + private val url = Regex("""https?://([^/\s?#]+)[^\s]*""") + + fun i(message: String) { + Log.i(TAG, redact(message)) + } + + fun w(message: String) { + Log.w(TAG, redact(message)) + } + + private fun redact(message: String): String = url.replace(message) { "<${it.groupValues[1]}>" } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsSettings.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsSettings.kt new file mode 100644 index 0000000000..ffae563b6d --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubsSettings.kt @@ -0,0 +1,54 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.content.Context +import androidx.datastore.preferences.core.Preferences +import androidx.datastore.preferences.core.booleanPreferencesKey +import androidx.datastore.preferences.core.edit +import androidx.datastore.preferences.core.stringPreferencesKey +import com.arflix.tv.data.repository.effectiveSettingsDataStore as settingsDataStore +import com.arflix.tv.ui.screens.player.subtitles.SubtitleAiModel +import kotlinx.coroutines.flow.Flow +import kotlinx.coroutines.flow.map + +/** + * Live TV AI subtitles settings, kept deliberately small: the feature switch, and whether + * channels start with AI subtitles on. The delay is always measured (Auto), the language is the + * profile's subtitle language, and the key is the AI subtitles Gemini key. + */ +internal data class LiveAiSubsSettings( + /** Offers "AI subtitles" in the live player's subtitle menu. */ + val enabled: Boolean = false, + /** Every channel starts with AI subtitles on, without picking them in the menu. */ + val autoStart: Boolean = false, + /** The AI subtitles key when that model is Gemini; blank otherwise. */ + val apiKey: String = "", +) { + val hasKey: Boolean get() = apiKey.isNotBlank() +} + +/** Global (not per-profile) settings, next to the AI subtitle key and model, synced to the cloud. */ +internal object LiveAiSubsPrefs { + val ENABLED = booleanPreferencesKey("live_ai_subs_enabled") + val AUTO_START = booleanPreferencesKey("live_ai_subs_auto_start") + private val SHARED_API_KEY = stringPreferencesKey("subtitle_ai_api_key") + private val SHARED_MODEL = stringPreferencesKey("subtitle_ai_model") + + fun read(prefs: Preferences): LiveAiSubsSettings { + val isGemini = prefs[SHARED_MODEL] == SubtitleAiModel.GEMINI_FLASH_25.name + return LiveAiSubsSettings( + enabled = prefs[ENABLED] ?: false, + autoStart = prefs[AUTO_START] ?: false, + apiKey = if (isGemini) prefs[SHARED_API_KEY].orEmpty().trim() else "", + ) + } + + fun flow(context: Context): Flow = context.settingsDataStore.data.map(::read) + + suspend fun setEnabled(context: Context, enabled: Boolean) { + context.settingsDataStore.edit { it[ENABLED] = enabled } + } + + suspend fun setAutoStart(context: Context, autoStart: Boolean) { + context.settingsDataStore.edit { it[AUTO_START] = autoStart } + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt new file mode 100644 index 0000000000..2d22cd8a94 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAiSubtitleOverlay.kt @@ -0,0 +1,86 @@ +@file:androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) + +package com.arflix.tv.ui.screens.tv.live.aisubs + +import androidx.compose.foundation.background +import androidx.compose.foundation.layout.Box +import androidx.compose.foundation.layout.fillMaxSize +import androidx.compose.foundation.layout.padding +import androidx.compose.foundation.shape.RoundedCornerShape +import androidx.compose.runtime.Composable +import androidx.compose.ui.Alignment +import androidx.compose.ui.Modifier +import androidx.compose.ui.graphics.Color +import androidx.compose.ui.unit.dp +import androidx.compose.ui.unit.sp +import androidx.compose.ui.viewinterop.AndroidView +import androidx.media3.common.text.Cue +import androidx.media3.ui.SubtitleView +import androidx.tv.material3.Text +import com.arflix.tv.ui.screens.player.applySubtitleAppearance + +/** The profile's subtitle look, as the VOD player uses it. */ +internal data class LiveSubtitleStyle( + val sizePreference: String = "Medium", + val sizePercent: Int = 100, + val verticalPercent: Int = 2, + val color: String = "White", + val style: String = "Bold", + val font: String = "", +) + +/** + * Fullscreen layer over the Live TV picture: the current AI subtitle (drawn by Media3's + * SubtitleView with the profile's subtitle style, so right-to-left languages lay out natively), + * the "Preparing subtitles… N s" note while playback is held. + */ +@Composable +internal fun LiveAiSubtitleOverlay( + cue: LiveCue?, + style: LiveSubtitleStyle, + /** "Preparing subtitles… N s" while playback is held for the delay, or null. */ + preparingText: String?, + modifier: Modifier = Modifier, +) { + Box(modifier = modifier.fillMaxSize()) { + AndroidView( + factory = { context -> + SubtitleView(context).apply { + isFocusable = false + applyStyle(this, style) + } + }, + update = { view -> + applyStyle(view, style) + view.setCues(cue?.let { listOf(Cue.Builder().setText(it.text).build()) } ?: emptyList()) + }, + modifier = Modifier.fillMaxSize(), + ) + if (preparingText != null) { + Text( + text = preparingText, + color = Color.White, + fontSize = 16.sp, + modifier = Modifier + .align(Alignment.TopCenter) + .padding(top = 32.dp) + .background(Color.Black.copy(alpha = 0.65f), RoundedCornerShape(18.dp)) + .padding(horizontal = 18.dp, vertical = 8.dp), + ) + } + } +} + +private fun applyStyle(view: SubtitleView, style: LiveSubtitleStyle) { + view.applySubtitleAppearance( + context = view.context, + sizePreference = style.sizePreference, + sizePercent = style.sizePercent, + verticalPercent = style.verticalPercent, + colorPreference = style.color, + stylePreference = style.style, + fontPreference = style.font, + preserveEmbeddedStyles = false, + inPictureInPicture = false, + ) +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt new file mode 100644 index 0000000000..992368b30c --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveAudioTapHls.kt @@ -0,0 +1,84 @@ +@file:androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) + +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.net.Uri +import androidx.media3.common.Format +import androidx.media3.common.util.TimestampAdjuster +import androidx.media3.exoplayer.analytics.PlayerId +import androidx.media3.exoplayer.hls.HlsExtractorFactory +import androidx.media3.exoplayer.hls.HlsMediaChunkExtractor +import androidx.media3.extractor.ExtractorInput +import androidx.media3.extractor.ExtractorOutput +import androidx.media3.extractor.text.SubtitleParser + +/** + * HLS counterpart of [LiveTapExtractorsFactory]: HLS segments are not read through an + * `ExtractorsFactory`, so the audio copy is made here, around whatever [delegate] returns. Hooking + * the chunk extractor's output covers TS segments (ARVIO's reused TS extractor) and every other + * segment type (packed audio, fMP4) in one place. + * + * Samples reach the sink with the timestamps the extractor wrote, after the HLS timestamp + * adjuster, i.e. on the period timeline the player's position is measured in. + */ +internal class AudioTapHlsExtractorFactory( + private val delegate: HlsExtractorFactory, + private val sinks: LiveTapSinks, +) : HlsExtractorFactory { + override fun createExtractor( + uri: Uri, + format: Format, + muxedCaptionFormats: List?, + timestampAdjuster: TimestampAdjuster, + responseHeaders: Map>, + sniffingExtractorInput: ExtractorInput, + playerId: PlayerId, + ): HlsMediaChunkExtractor { + val extractor = delegate.createExtractor( + uri, format, muxedCaptionFormats, timestampAdjuster, responseHeaders, sniffingExtractorInput, playerId, + ) + return AudioTapHlsChunkExtractor(extractor, sinks) + } + + override fun setSubtitleParserFactory(subtitleParserFactory: SubtitleParser.Factory): HlsExtractorFactory { + delegate.setSubtitleParserFactory(subtitleParserFactory) + return this + } + + override fun experimentalParseSubtitlesDuringExtraction(parseSubtitlesDuringExtraction: Boolean): HlsExtractorFactory { + delegate.experimentalParseSubtitlesDuringExtraction(parseSubtitlesDuringExtraction) + return this + } + + override fun experimentalSetCodecsToParseWithinGopSampleDependencies( + codecsToParseWithinGopSampleDependencies: Int, + ): HlsExtractorFactory { + delegate.experimentalSetCodecsToParseWithinGopSampleDependencies(codecsToParseWithinGopSampleDependencies) + return this + } + + override fun getOutputTextFormat(sourceFormat: Format): Format = delegate.getOutputTextFormat(sourceFormat) +} + +private class AudioTapHlsChunkExtractor( + private val delegate: HlsMediaChunkExtractor, + private val sinks: LiveTapSinks, +) : HlsMediaChunkExtractor { + override fun init(extractorOutput: ExtractorOutput) = + delegate.init(LiveTapExtractorOutput(extractorOutput, sinks)) + + override fun read(extractorInput: ExtractorInput): Boolean = delegate.read(extractorInput) + + override fun isPackedAudioExtractor(): Boolean = delegate.isPackedAudioExtractor + + override fun isReusable(): Boolean = delegate.isReusable + + override fun recreate(): HlsMediaChunkExtractor = AudioTapHlsChunkExtractor(delegate.recreate(), sinks) + + override fun onTruncatedSegmentParsed() { + // The extractor restarts its state; the next samples may not continue the previous ones. + sinks.audio.onDiscontinuity() + sinks.captions.onDiscontinuity() + delegate.onTruncatedSegmentParsed() + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueScheduler.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueScheduler.kt new file mode 100644 index 0000000000..dde4288383 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueScheduler.kt @@ -0,0 +1,128 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +/** One subtitle line on the media timeline (the player's period time, in microseconds). */ +internal data class LiveCue( + val startUs: Long, + val endUs: Long, + val text: String, + val rtl: Boolean = false, +) + +/** + * Keeps AI subtitle cues in media time and answers "what is on screen at this position". Cues are + * shown by where playback is, never by when the AI answered, so pausing, rebuffering and fast + * dialogue cannot push lines out of step with the picture. + * + * Display rules: each cue stays up at least [MIN_DISPLAY_US] and at most [MAX_DISPLAY_US], and is + * cut short by the next cue. Text longer than [SPLIT_CHARS] is split into two cues, timed by + * length. A cue that arrives after its start still shows while it is inside its display window; + * one that arrives after the window has closed is dropped and counted in [lateDropped]. + * + * Not thread-safe; the session calls it from one thread (the main thread). + */ +internal class LiveCueScheduler { + private val cues = ArrayList() + + var lateDropped: Int = 0 + private set + + var shown: Int = 0 + private set + + private var lastShown: LiveCue? = null + + /** Adds cues from one segment. [positionUs] is where playback is now, for the late-cue rule. */ + fun add(newCues: List, positionUs: Long) { + for (cue in newCues.flatMap(::split)) { + if (cue.text.isBlank()) continue + val provisionalEnd = displayEnd(cue, next = null) + if (positionUs >= provisionalEnd) { + lateDropped++ + continue + } + val index = cues.indexOfFirst { it.startUs > cue.startUs }.let { if (it < 0) cues.size else it } + cues.add(index, cue) + } + } + + /** The cue on screen at [positionUs], or null. Forgets cues that are over. */ + fun cueAt(positionUs: Long): LiveCue? { + var result: LiveCue? = null + for (i in cues.indices) { + val cue = cues[i] + if (cue.startUs > positionUs) break + val end = displayEnd(cue, cues.getOrNull(i + 1)) + if (positionUs < end) result = cue + } + // Drop cues whose window closed well before the position (bounded memory). + cues.removeAll { displayEnd(it, null) < positionUs - FORGET_AFTER_US } + if (result != null && result !== lastShown) shown++ + lastShown = result + return result + } + + fun clear() { + cues.clear() + lastShown = null + } + + val pendingCount: Int get() = cues.size + + private fun displayEnd(cue: LiveCue, next: LiveCue?): Long { + val natural = cue.endUs.coerceAtLeast(cue.startUs + MIN_DISPLAY_US) + val capped = natural.coerceAtMost(cue.startUs + MAX_DISPLAY_US) + return if (next != null && next.startUs > cue.startUs) minOf(capped, next.startUs) else capped + } + + companion object { + const val MIN_DISPLAY_US = 1_200_000L + const val MAX_DISPLAY_US = 6_000_000L + const val SPLIT_CHARS = 84 + private const val FORGET_AFTER_US = 30_000_000L + + /** + * Splits a long line into two cues at the word boundary nearest its middle, each getting a + * share of the time proportional to its length. + */ + fun split(cue: LiveCue): List { + val text = cue.text.trim() + if (text.length <= SPLIT_CHARS) return listOf(cue.copy(text = text)) + val middle = text.length / 2 + val spaces = text.indices.filter { text[it] == ' ' } + val cut = spaces.minByOrNull { kotlin.math.abs(it - middle) } ?: return listOf(cue.copy(text = text)) + val first = text.substring(0, cut).trim() + val second = text.substring(cut + 1).trim() + if (first.isEmpty() || second.isEmpty()) return listOf(cue.copy(text = text)) + val duration = (cue.endUs - cue.startUs).coerceAtLeast(0L) + val splitAt = cue.startUs + duration * first.length / (first.length + second.length) + return listOf( + cue.copy(endUs = splitAt, text = first), + cue.copy(startUs = splitAt, text = second), + ) + } + } +} + +/** + * The hold ("Auto" delay) that lets a whole segment be heard, sent and answered before its first + * word plays: the longest segment, plus the AI's p95 latency, plus a margin. Before any latency has + * been measured, it assumes [DEFAULT_LATENCY_P95_MS]. Clamped to 6–15 s. + */ +internal fun autoLiveAiSubsDelayMs(latencyP95Ms: Long?): Long { + val latency = latencyP95Ms ?: DEFAULT_LATENCY_P95_MS + return (LIVE_MAX_SEGMENT_MS + latency + DELAY_MARGIN_MS).coerceIn(6_000L, 15_000L) +} + + +/** The 95th percentile of [samples], or null with fewer than [minSamples]. */ +internal fun percentile95(samples: List, minSamples: Int = 3): Long? { + if (samples.size < minSamples) return null + val sorted = samples.sorted() + val index = ((sorted.size - 1) * 0.95).let { kotlin.math.ceil(it).toInt() }.coerceIn(0, sorted.size - 1) + return sorted[index] +} + +internal const val LIVE_MAX_SEGMENT_MS = 5_000L +/** Measured on a Mi Box 4 with gemini-3.5-flash-lite: p50 about 5 s, p95 about 8.5 s. */ +private const val DEFAULT_LATENCY_P95_MS = 5_000L +private const val DELAY_MARGIN_MS = 1_000L diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt new file mode 100644 index 0000000000..ae20abd0d4 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSpeechPipeline.kt @@ -0,0 +1,285 @@ +@file:androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) + +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.content.Context +import android.os.Process +import android.os.SystemClock +import androidx.media3.common.Format +import androidx.media3.common.MimeTypes +import com.arflix.tv.R +import com.arflix.tv.ui.screens.player.audiosync.AudioSampleSink +import com.arflix.tv.ui.screens.player.audiosync.AudioSyncDecoder +import com.arflix.tv.ui.screens.player.audiosync.SileroVad +import com.arflix.tv.ui.screens.player.audiosync.SileroVadWeights +import com.arflix.tv.ui.screens.player.audiosync.SpeechAnalyzer +import com.arflix.tv.ui.screens.player.audiosync.SpeechTimeline +import com.arflix.tv.ui.screens.player.audiosync.asr.SpeechSegmenter +import com.arflix.tv.ui.screens.tv.live.isBitmapSubtitle + +/** + * Live TV speech capture: copies the channel's compressed audio at the extractor, as it is + * downloaded and before it is played; decodes it on a side thread; and cuts it into speech + * segments with Silero. Each [SpeechSegment] carries its own media time, which the player's + * period position shares (measured in Phase 0: the same 32 ms frame), so subtitles can be timed + * exactly however late the AI answers. + * + * One pipeline lives as long as the Live TV player. [newSink] is called for every prepared channel + * source; a sink from an earlier channel goes quiet at once, so a load still running for the + * previous channel can never feed the new one. Nothing is captured unless [active]. + * + * Threads: sinks run on the player's loading threads, segments are delivered on the decoder + * thread, and [onPosition] comes from the UI thread. + */ +internal class LiveSpeechPipeline( + context: Context, + /** A speech segment, with the [generation] of the channel source its audio came from. */ + private val onSegment: (SpeechSegment, generation: Int) -> Unit, + /** A sentence of the channel's own closed captions, read ahead of playback. */ + private val onCaption: (CaptionSentence, generation: Int) -> Unit, + /** The channel's audio format cannot be decoded ahead of playback on this device. */ + private val onUnsupportedAudio: (mimeType: String) -> Unit, +) { + private val appContext = context.applicationContext + + @Volatile private var weights: SileroVadWeights? = null + /** Bumped for every prepared channel source; work tagged with an older one is stale. */ + @Volatile var generation = 0 + private set + /** + * The source the decoder's audio belongs to. Decoder thread: it moves to [generation] when + * the decoder resets, which drops everything queued from before ([newSinks] queues a reset). + */ + @Volatile private var audioGeneration = 0 + @Volatile private var decoder: AudioSyncDecoder? = null + + /** Capture only while AI subtitles are on for the channel and it is playing fullscreen. */ + @Volatile + var active: Boolean = false + + // Where playback is, from the UI thread; extrapolated on other threads. + @Volatile private var positionUs = UNKNOWN + @Volatile private var positionAtMs = 0L + @Volatile private var positionPlaying = false + + /** Media time of the newest analysed audio, to detect when analysis falls behind playback. */ + @Volatile private var analysedUs = UNKNOWN + @Volatile private var lastTapUs = UNKNOWN + + /** Decoder thread only. */ + private var skipping = false + + @Volatile var backlogSkips: Int = 0 + private set + @Volatile var segmentsEmitted: Int = 0 + private set + + init { + Thread({ + weights = runCatching { + appContext.resources.openRawResource(R.raw.silero_vad_v5_16k).use { SileroVadWeights.read(it) } + }.onFailure { LiveAiSubsLog.w("could not load VAD weights: ${it.message}") }.getOrNull() + }, "LiveAiSubsInit").apply { + isDaemon = true + start() + } + } + + /** True once the VAD model is loaded. */ + val ready: Boolean get() = weights != null + + /** How far ahead of playback the newest copied audio is, or null when unknown. */ + fun tapAheadUs(): Long? { + val tap = lastTapUs + val position = estimatedPositionUs() + return if (tap == UNKNOWN || position == UNKNOWN) null else tap - position + } + + /** The audio and caption sinks for one prepared channel source. */ + fun newSinks(): LiveTapSinks { + val sinkGeneration = ++generation + reset() + return LiveTapSinks(ChannelSink(sinkGeneration), CaptionSink(sinkGeneration)) + } + + /** Forget everything in flight (channel change, seek, resume). */ + fun reset() { + analysedUs = UNKNOWN + lastTapUs = UNKNOWN + decoder?.discontinuity() + } + + /** From the UI thread, about every 100–250 ms: the player's period position. */ + fun onPosition(periodPositionUs: Long, playing: Boolean) { + positionUs = periodPositionUs + positionAtMs = SystemClock.elapsedRealtime() + positionPlaying = playing + } + + fun estimatedPositionUs(): Long { + val position = positionUs + if (position == UNKNOWN) return UNKNOWN + return if (positionPlaying) position + (SystemClock.elapsedRealtime() - positionAtMs) * 1_000L else position + } + + fun release() { + generation++ + active = false + decoder?.release() + decoder = null + } + + private fun ensureDecoder(): AudioSyncDecoder? { + decoder?.let { return it } + val vadWeights = weights ?: return null + synchronized(this) { + decoder?.let { return it } + audioGeneration = generation + val analyzer = SpeechAnalyzer(SileroVad(vadWeights), SpeechTimeline()) + val segmenter = SpeechSegmenter( + maxSegmentFrames = MAX_SEGMENT_FRAMES, + softMaxSegmentFrames = SOFT_MAX_SEGMENT_FRAMES, + ) { startFrame, samples -> + val startUs = startFrame.toLong() * SpeechTimeline.FRAME_DURATION_US + val endUs = startUs + samples.size * 1_000_000L / SileroVad.SAMPLE_RATE + segmentsEmitted++ + onSegment(SpeechSegment(startUs, endUs, samples), audioGeneration) + } + analyzer.chunkListener = object : SpeechAnalyzer.ChunkListener { + override fun onChunk(frame: Int, chunk: FloatArray, probability: Float) { + val chunkUs = frame.toLong() * SpeechTimeline.FRAME_DURATION_US + analysedUs = chunkUs + if (isBehindPlayback(chunkUs)) { + // Once per episode: the chunks still in the decoder are behind too. + if (!skipping) { + skipping = true + skipBacklog() + } + return + } + skipping = false + segmenter.onChunk(frame, chunk, probability) + } + + override fun onReset() { + audioGeneration = generation + segmenter.onReset() + } + } + return AudioSyncDecoder( + analyzer = analyzer, + liveAnalyzer = analyzer, + onUnsupportedFormat = { mime -> + LiveAiSubsLog.w("no look-ahead decoder for $mime") + onUnsupportedAudio(mime) + }, + threadPriority = Process.THREAD_PRIORITY_LESS_FAVORABLE, + ).also { decoder = it } + } + } + + /** + * Audio that will be heard within [MIN_USEFUL_AHEAD_US] cannot be subtitled in time. On a + * busy box (measured: a 1080p channel on a Mi Box 4) analysis can fall 10–20 s behind; going + * on would only make every later subtitle late too, so the backlog is dropped and analysis + * restarts on the newest downloaded audio. + */ + private fun isBehindPlayback(chunkUs: Long): Boolean { + val position = estimatedPositionUs() + return position != UNKNOWN && positionPlaying && chunkUs < position + MIN_USEFUL_AHEAD_US + } + + private fun skipBacklog() { + backlogSkips++ + analysedUs = UNKNOWN + LiveAiSubsLog.i("analysis fell behind playback; skipping to the newest audio (skip #$backlogSkips)") + decoder?.discontinuity() + } + + private inner class ChannelSink(private val sinkGeneration: Int) : AudioSampleSink { + /** A channel can carry several audio tracks; the first one seen is the one analysed. */ + @Volatile private var lockedTrackId: String? = null + + override fun wantsSamples(format: Format): Boolean { + if (sinkGeneration != generation || !active) return false + val locked = lockedTrackId + if (locked != null && locked != (format.id ?: NO_ID)) return false + val current = ensureDecoder() ?: return false + return current.accepts(format) + } + + override fun onSample(format: Format, timeUs: Long, data: ByteArray, offset: Int, size: Int) { + if (sinkGeneration != generation) return + if (lockedTrackId == null) { + lockedTrackId = format.id ?: NO_ID + LiveAiSubsLog.i( + "tapping ${format.sampleMimeType} ${format.channelCount}ch ${format.sampleRate}Hz lang=${format.language}", + ) + } + // Audio already about to play is useless; don't spend the decoder on it. + val position = estimatedPositionUs() + if (position != UNKNOWN && positionPlaying && timeUs < position + MIN_USEFUL_AHEAD_US) return + lastTapUs = timeUs + decoder?.offer(format, timeUs, data, offset, size) + } + + override fun onDiscontinuity() { + if (sinkGeneration != generation) return + decoder?.discontinuity() + } + } + + /** + * Reads the channel's own text subtitles into sentences: CEA-608 closed captions carried in + * the video, and text renditions (WebVTT and the like) the player parsed during extraction. + * Bitmap subtitles (DVB) carry no text; those channels are subtitled from their audio. + */ + private inner class CaptionSink(private val sinkGeneration: Int) : AudioSampleSink { + private val assembler = Cea608TextAssembler { sentence -> + if (sinkGeneration == generation) onCaption(sentence, sinkGeneration) + } + private val cueReader = CaptionCueReader { sentence -> + if (sinkGeneration == generation) onCaption(sentence, sinkGeneration) + } + private val loggedMimes = HashSet() + + override fun wantsSamples(format: Format): Boolean { + if (sinkGeneration != generation || !active) return false + val mime = format.sampleMimeType ?: return false + if (synchronized(loggedMimes) { loggedMimes.add("$mime ${format.codecs} ${format.language}") }) { + LiveAiSubsLog.i("text track offered: $mime codecs=${format.codecs} lang=${format.language}") + } + if (isBitmapSubtitle(format)) return false + return mime == MimeTypes.APPLICATION_CEA608 || mime == MimeTypes.APPLICATION_MEDIA3_CUES + } + + override fun onSample(format: Format, timeUs: Long, data: ByteArray, offset: Int, size: Int) { + if (sinkGeneration != generation) return + synchronized(assembler) { + if (format.sampleMimeType == MimeTypes.APPLICATION_MEDIA3_CUES) { + cueReader.feed(timeUs, data, offset, size) + } else { + assembler.feed(timeUs, data, offset, size) + } + } + } + + override fun onDiscontinuity() { + synchronized(assembler) { + assembler.reset() + cueReader.reset() + } + } + } + + companion object { + private const val UNKNOWN = Long.MIN_VALUE + private const val NO_ID = "(no id)" + /** 5 s hard limit: a line can only be sent once its segment has ended. */ + const val MAX_SEGMENT_FRAMES = 156 + /** From 3.5 s on, a pause between words ends the segment. */ + const val SOFT_MAX_SEGMENT_FRAMES = 110 + /** Audio closer than this to the playhead cannot be subtitled before it is heard. */ + const val MIN_USEFUL_AHEAD_US = 2_500_000L + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt new file mode 100644 index 0000000000..af398008ea --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveSubtitleSession.kt @@ -0,0 +1,280 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.content.Context +import android.os.SystemClock +import kotlinx.coroutines.CoroutineScope +import kotlinx.coroutines.Deferred +import kotlinx.coroutines.Dispatchers +import kotlinx.coroutines.Job +import kotlinx.coroutines.SupervisorJob +import kotlinx.coroutines.async +import kotlinx.coroutines.cancel +import kotlinx.coroutines.channels.BufferOverflow +import kotlinx.coroutines.channels.Channel +import kotlinx.coroutines.flow.MutableStateFlow +import kotlinx.coroutines.flow.StateFlow +import kotlinx.coroutines.flow.asStateFlow +import kotlinx.coroutines.launch +import kotlinx.coroutines.sync.Semaphore +import kotlinx.coroutines.sync.withPermit + +/** What the viewer should be told about Live TV AI subtitles. */ +internal sealed interface LiveAiSubsStatus { + data object Off : LiveAiSubsStatus + data object Running : LiveAiSubsStatus + data object NoKey : LiveAiSubsStatus + data object InvalidKey : LiveAiSubsStatus + /** HTTP 429: sending pauses until [untilMs] (elapsed realtime). */ + data class QuotaPaused(val untilMs: Long) : LiveAiSubsStatus + data object Offline : LiveAiSubsStatus + data class Unsupported(val mimeType: String) : LiveAiSubsStatus +} + +/** + * One piece of work for the AI: a stretch of speech, or a sentence of the channel's captions. + * [generation] is the channel source it was captured from (see [LiveSpeechPipeline.generation]). + */ +internal sealed interface LiveWork { + val generation: Int + class Audio(val segment: SpeechSegment, override val generation: Int) : LiveWork + class Caption(val sentence: CaptionSentence, override val generation: Int) : LiveWork +} + +/** + * Live TV AI subtitles for the one Live TV player: [pipeline] cuts speech segments from the + * channel's audio ahead of playback, [engine] turns each into timed lines, and [scheduler] shows + * them when playback reaches them. The UI calls [tick] with the player position about every + * 100 ms and observes [cue] and [status]. + * + * Requests: at most [MAX_IN_FLIGHT] at once, results applied in segment order. A segment that + * could no longer be shown in time is not sent (cost). Main-thread confined except for segment + * hand-off from the decoder thread. + */ +internal class LiveSubtitleSession( + context: Context, + private val engine: LiveSubtitleEngine, + private val captionTranslator: LiveCaptionTranslator, +) { + private val scope = CoroutineScope(SupervisorJob() + Dispatchers.Main.immediate) + private val segments = Channel(capacity = 8, onBufferOverflow = BufferOverflow.DROP_OLDEST) + + /** + * Captions first: while the channel's own captions keep coming (up to [CAPTION_HOLDOVER_US] + * after the last one), its audio is not sent. When they stop, the audio takes over again. + */ + @Volatile private var captionsUntilUs = Long.MIN_VALUE + private val inFlight = Semaphore(MAX_IN_FLIGHT) + + val pipeline = LiveSpeechPipeline( + context = context, + onSegment = { segment, generation -> segments.trySend(LiveWork.Audio(segment, generation)) }, + onCaption = { sentence, generation -> + captionsUntilUs = sentence.endUs + CAPTION_HOLDOVER_US + segments.trySend(LiveWork.Caption(sentence, generation)) + }, + onUnsupportedAudio = { mime -> scope.launch { setStatus(LiveAiSubsStatus.Unsupported(mime)) } }, + ) + private val scheduler = LiveCueScheduler() + + private val _cue = MutableStateFlow(null) + val cue: StateFlow = _cue.asStateFlow() + + private val _status = MutableStateFlow(LiveAiSubsStatus.Off) + val status: StateFlow = _status.asStateFlow() + + private var languageName = "English" + private var rtl = false + private var running = false + private var epoch = 0 + private var positionUs = 0L + private var quotaBackoffMs = QUOTA_BACKOFF_START_MS + private var pausedUntilMs = 0L + private val recentLines = ArrayDeque() + private val latencies = ArrayDeque() + private var worker: Job? = null + + /** Measured p95 of the AI's answer time, for the Auto delay. */ + val latencyP95Ms: Long? get() = percentile95(latencies.toList()) + + fun setTargetLanguage(name: String, isRtl: Boolean) { + if (name != languageName) recentLines.clear() + languageName = name + rtl = isRtl + } + + /** + * Starts or stops subtitling. [hasKey] false reports [LiveAiSubsStatus.NoKey]. Capture only + * runs while [capture] (fullscreen and playing), to keep cost down. + */ + fun update(enabled: Boolean, hasKey: Boolean, capture: Boolean) { + if (!enabled) { + stop() + return + } + if (!hasKey) { + stop() + setStatus(LiveAiSubsStatus.NoKey) + return + } + if (!running) { + running = true + if (_status.value is LiveAiSubsStatus.Off || _status.value is LiveAiSubsStatus.NoKey) { + setStatus(LiveAiSubsStatus.Running) + } + startWorker() + } + pipeline.active = capture && _status.value !is LiveAiSubsStatus.InvalidKey && + _status.value !is LiveAiSubsStatus.Unsupported + } + + /** A new channel (or a re-prepared source): nothing from before may show. */ + fun onNewSource() { + epoch++ + captionsUntilUs = Long.MIN_VALUE + scheduler.clear() + recentLines.clear() + _cue.value = null + // A channel that could not be decoded says nothing about the next one. + if (_status.value is LiveAiSubsStatus.Unsupported) setStatus(if (running) LiveAiSubsStatus.Running else LiveAiSubsStatus.Off) + } + + /** The key was changed in settings: try again. */ + fun onKeyChanged() { + if (_status.value is LiveAiSubsStatus.InvalidKey || _status.value is LiveAiSubsStatus.NoKey) { + setStatus(if (running) LiveAiSubsStatus.Running else LiveAiSubsStatus.Off) + } + } + + /** UI thread, about every 100 ms. */ + fun tick(periodPositionUs: Long, playing: Boolean) { + positionUs = periodPositionUs + pipeline.onPosition(periodPositionUs, playing) + val now = SystemClock.elapsedRealtime() + (_status.value as? LiveAiSubsStatus.QuotaPaused)?.let { if (now >= it.untilMs) setStatus(LiveAiSubsStatus.Running) } + _cue.value = if (running) scheduler.cueAt(periodPositionUs) else null + } + + fun release() { + stop() + pipeline.release() + scope.cancel() + } + + private fun stop() { + running = false + pipeline.active = false + worker?.cancel() + worker = null + epoch++ + scheduler.clear() + _cue.value = null + if (_status.value !is LiveAiSubsStatus.Off) setStatus(LiveAiSubsStatus.Off) + } + + private fun setStatus(status: LiveAiSubsStatus) { + if (_status.value != status) LiveAiSubsLog.i("status: $status") + _status.value = status + } + + private fun startWorker() { + worker?.cancel() + // Drop anything queued while stopped. + while (segments.tryReceive().isSuccess) Unit + worker = scope.launch { + val ordered = Channel>>(capacity = MAX_IN_FLIGHT) + launch { + for ((resultEpoch, deferred) in ordered) { + val result = deferred.await() ?: continue + if (resultEpoch == epoch) apply(result) + } + } + for (work in segments) { + // Waits for a free request slot first: the channel can change while waiting. + inFlight.acquire() + // Captured from an earlier channel (or before a restart): never sent, never shown. + if (work.generation != pipeline.generation || !shouldSend(work)) { + inFlight.release() + continue + } + val sentEpoch = epoch + val context = recentLines.toList() + val name = languageName + val isRtl = rtl + val deferred = async { + try { + val started = SystemClock.elapsedRealtime() + val result = when (work) { + is LiveWork.Audio -> engine.process(work.segment, context, name, isRtl) + is LiveWork.Caption -> captionTranslator.translate(work.sentence, name, isRtl) + } + if (result is LiveEngineResult.Cues) recordLatency(SystemClock.elapsedRealtime() - started) + result + } finally { + inFlight.release() + } + } + ordered.send(sentEpoch to deferred) + } + } + } + + private fun shouldSend(work: LiveWork): Boolean { + val (startUs, endUs) = when (work) { + is LiveWork.Audio -> work.segment.startUs to work.segment.endUs + is LiveWork.Caption -> work.sentence.startUs to work.sentence.endUs + } + // The channel's captions cover this stretch: no need to listen to its audio. + if (work is LiveWork.Audio && startUs <= captionsUntilUs) return false + val status = _status.value + if (status is LiveAiSubsStatus.InvalidKey || status is LiveAiSubsStatus.Unsupported) return false + val now = SystemClock.elapsedRealtime() + if (status is LiveAiSubsStatus.QuotaPaused && now < status.untilMs) return false + if (status is LiveAiSubsStatus.Offline && now < pausedUntilMs) return false + // Its window would already be over when an answer could arrive. + return endUs + LiveCueScheduler.MIN_DISPLAY_US > pipeline.estimatedPositionUs() + MIN_ANSWER_US + } + + private fun apply(result: LiveEngineResult) { + when (result) { + is LiveEngineResult.Cues -> { + quotaBackoffMs = QUOTA_BACKOFF_START_MS + if (_status.value is LiveAiSubsStatus.Offline || _status.value is LiveAiSubsStatus.QuotaPaused) { + setStatus(LiveAiSubsStatus.Running) + } + scheduler.add(result.cues, positionUs) + result.cues.forEach { + recentLines.addLast(it.text) + while (recentLines.size > CONTEXT_LINES) recentLines.removeFirst() + } + } + LiveEngineResult.InvalidKey -> setStatus(LiveAiSubsStatus.InvalidKey) + LiveEngineResult.QuotaExceeded -> { + setStatus(LiveAiSubsStatus.QuotaPaused(SystemClock.elapsedRealtime() + quotaBackoffMs)) + quotaBackoffMs = (quotaBackoffMs * 2).coerceAtMost(QUOTA_BACKOFF_MAX_MS) + } + LiveEngineResult.Offline -> { + pausedUntilMs = SystemClock.elapsedRealtime() + OFFLINE_RETRY_MS + setStatus(LiveAiSubsStatus.Offline) + } + is LiveEngineResult.Failed -> LiveAiSubsLog.w("segment failed: ${result.reason}") + } + } + + private fun recordLatency(ms: Long) { + latencies.addLast(ms) + while (latencies.size > LATENCY_SAMPLES) latencies.removeFirst() + } + + private companion object { + const val MAX_IN_FLIGHT = 2 + const val CONTEXT_LINES = 3 + const val LATENCY_SAMPLES = 30 + /** The fastest a segment can realistically come back. */ + const val MIN_ANSWER_US = 800_000L + const val QUOTA_BACKOFF_START_MS = 30_000L + const val QUOTA_BACKOFF_MAX_MS = 240_000L + const val OFFLINE_RETRY_MS = 10_000L + /** How long after the last caption the audio stays unused (captions pause between lines). */ + const val CAPTION_HOLDOVER_US = 20_000_000L + } +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveTapExtractors.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveTapExtractors.kt new file mode 100644 index 0000000000..0e927bbcb8 --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveTapExtractors.kt @@ -0,0 +1,81 @@ +@file:androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) + +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.net.Uri +import androidx.media3.common.C +import androidx.media3.extractor.Extractor +import androidx.media3.extractor.ExtractorInput +import androidx.media3.extractor.ExtractorOutput +import androidx.media3.extractor.ExtractorsFactory +import androidx.media3.extractor.PositionHolder +import androidx.media3.extractor.SeekMap +import androidx.media3.extractor.SniffFailure +import androidx.media3.extractor.TrackOutput +import com.arflix.tv.ui.screens.player.audiosync.AudioSampleSink +import com.arflix.tv.ui.screens.player.audiosync.AudioTapTrackOutput + +/** The two copies Live TV AI subtitles take of a channel as it downloads. */ +internal class LiveTapSinks(val audio: AudioSampleSink, val captions: AudioSampleSink) + +/** + * Copies audio samples to [sinks].audio and text samples (closed captions carried in the video, + * subtitle streams) to [sinks].captions, while forwarding everything unchanged to the player. + * The copy happens at load time, so both arrive as far ahead of playback as the player buffers. + */ +internal class LiveTapExtractorOutput( + private val delegate: ExtractorOutput, + private val sinks: LiveTapSinks, +) : ExtractorOutput { + private val outputs = HashMap() + + override fun track(id: Int, type: Int): TrackOutput { + val output = delegate.track(id, type) + return when (type) { + C.TRACK_TYPE_AUDIO -> outputs.getOrPut(id) { AudioTapTrackOutput(output, sinks.audio) } + C.TRACK_TYPE_TEXT -> outputs.getOrPut(id) { AudioTapTrackOutput(output, sinks.captions) } + else -> output + } + } + + override fun endTracks() = delegate.endTracks() + + override fun seekMap(seekMap: SeekMap) = delegate.seekMap(seekMap) +} + +/** Progressive (raw MPEG-TS) streams: every extractor's output goes through [LiveTapExtractorOutput]. */ +internal class LiveTapExtractorsFactory( + private val delegate: ExtractorsFactory, + private val sinks: LiveTapSinks, +) : ExtractorsFactory { + override fun createExtractors(): Array = wrap(delegate.createExtractors()) + + override fun createExtractors(uri: Uri, responseHeaders: Map>): Array = + wrap(delegate.createExtractors(uri, responseHeaders)) + + private fun wrap(extractors: Array): Array = + Array(extractors.size) { LiveTapExtractor(extractors[it], sinks) } +} + +private class LiveTapExtractor( + private val delegate: Extractor, + private val sinks: LiveTapSinks, +) : Extractor { + override fun sniff(input: ExtractorInput): Boolean = delegate.sniff(input) + + override fun getSniffFailureDetails(): List = delegate.sniffFailureDetails + + override fun init(output: ExtractorOutput) = delegate.init(LiveTapExtractorOutput(output, sinks)) + + override fun read(input: ExtractorInput, seekPosition: PositionHolder): Int = delegate.read(input, seekPosition) + + override fun seek(position: Long, timeUs: Long) { + sinks.audio.onDiscontinuity() + sinks.captions.onDiscontinuity() + delegate.seek(position, timeUs) + } + + override fun release() = delegate.release() + + override fun getUnderlyingImplementation(): Extractor = delegate.underlyingImplementation +} diff --git a/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/SubtitleLookaheadLoadControl.kt b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/SubtitleLookaheadLoadControl.kt new file mode 100644 index 0000000000..438cc7daea --- /dev/null +++ b/app/src/main/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/SubtitleLookaheadLoadControl.kt @@ -0,0 +1,109 @@ +@file:androidx.annotation.OptIn(androidx.media3.common.util.UnstableApi::class) + +package com.arflix.tv.ui.screens.tv.live.aisubs + +import android.os.SystemClock +import androidx.media3.common.Timeline +import androidx.media3.exoplayer.LoadControl +import androidx.media3.exoplayer.analytics.PlayerId +import androidx.media3.exoplayer.source.MediaSource +import androidx.media3.exoplayer.source.TrackGroupArray +import androidx.media3.exoplayer.trackselection.ExoTrackSelection +import androidx.media3.exoplayer.upstream.Allocator + +/** + * Holds live playback back until [requiredLookaheadUs] of media is buffered, at channel start and + * after every rebuffer, so audio reaches the subtitle pipeline that much before it is heard. A + * line can only be translated once it has been spoken in full, so without this hold every + * subtitle would be late. Everything else is [delegate]'s decision. + * + * The hold only applies while [requiredLookaheadUs] is above zero (set from the UI thread), so a + * channel without AI subtitles starts as fast as before. Waiting always converges: while playback + * waits, the live edge keeps moving, so the buffered duration keeps growing. + */ +internal class SubtitleLookaheadLoadControl( + private val delegate: LoadControl, +) : LoadControl { + @Volatile + var requiredLookaheadUs: Long = 0L + + /** True while playback is being held for the look-ahead (read by the UI). */ + @Volatile + var holding: Boolean = false + private set + + /** Buffered media when the hold was last checked, for "Preparing subtitles… N s". */ + @Volatile + var lastBufferedUs: Long = 0L + private set + + /** + * The buffer's memory cap was reached before the look-ahead (a high-bitrate channel on a + * low-memory box): playback started with less, so some subtitles may be late. + */ + @Volatile + var cappedByMemory: Boolean = false + + private var lastHoldLogMs = 0L + + override fun onPrepared(playerId: PlayerId) = delegate.onPrepared(playerId) + + override fun onTracksSelected( + parameters: LoadControl.Parameters, + trackGroups: TrackGroupArray, + trackSelections: Array, + ) = delegate.onTracksSelected(parameters, trackGroups, trackSelections) + + override fun onStopped(playerId: PlayerId) = delegate.onStopped(playerId) + + override fun onReleased(playerId: PlayerId) = delegate.onReleased(playerId) + + override fun getAllocator(playerId: PlayerId): Allocator = delegate.getAllocator(playerId) + + override fun getBackBufferDurationUs(playerId: PlayerId): Long = delegate.getBackBufferDurationUs(playerId) + + override fun retainBackBufferFromKeyframe(playerId: PlayerId): Boolean = + delegate.retainBackBufferFromKeyframe(playerId) + + override fun shouldContinueLoading(parameters: LoadControl.Parameters): Boolean = + delegate.shouldContinueLoading(parameters) + + override fun shouldContinuePreloading( + playerId: PlayerId, + timeline: Timeline, + mediaPeriodId: MediaSource.MediaPeriodId, + bufferedDurationUs: Long, + ): Boolean = delegate.shouldContinuePreloading(playerId, timeline, mediaPeriodId, bufferedDurationUs) + + override fun shouldStartPlayback(parameters: LoadControl.Parameters): Boolean { + if (!delegate.shouldStartPlayback(parameters)) return false + val required = requiredLookaheadUs + lastBufferedUs = parameters.bufferedDurationUs + if (required <= 0L || parameters.bufferedDurationUs >= required) { + holding = false + return true + } + // Loading has stopped (memory target reached): waiting longer would never get there and + // would end in the stuck-buffering error, so play with what there is. + if (!delegate.shouldContinueLoading(parameters)) { + if (!cappedByMemory) LiveAiSubsLog.w("look-ahead capped by the buffer memory target at ${parameters.bufferedDurationUs / 1_000}ms") + cappedByMemory = true + holding = false + return true + } + holding = true + val now = SystemClock.elapsedRealtime() + if (now - lastHoldLogMs >= HOLD_LOG_INTERVAL_MS) { + lastHoldLogMs = now + LiveAiSubsLog.i( + "holding playback: buffered=${parameters.bufferedDurationUs / 1_000}ms " + + "required=${required / 1_000}ms rebuffering=${parameters.rebuffering}", + ) + } + return false + } + + private companion object { + const val HOLD_LOG_INTERVAL_MS = 1_000L + } +} diff --git a/app/src/main/res/values-de/strings.xml b/app/src/main/res/values-de/strings.xml index 29aa1b2066..0f69f8c245 100644 --- a/app/src/main/res/values-de/strings.xml +++ b/app/src/main/res/values-de/strings.xml @@ -1919,4 +1919,22 @@ 1. QR-Code mit dem Telefon scannen\n2. In Discord auf \'Autorisieren\' tippen\n3. Verbindet sich automatisch In dieser Version nicht enthalten Zeige deinen Wiedergabestatus in deinem Discord-Profil. + + + KI-Untertitel + Untertitel werden vorbereitet… %1$d s + KI-Untertitel brauchen einen Google-Gemini-API-Schlüssel (Einstellungen → KI-Untertitel) + KI-Untertitel: Der Gemini-API-Schlüssel wurde abgelehnt + KI-Untertitel pausiert: Gemini-Kontingent erreicht + KI-Untertitel: keine Internetverbindung + KI-Untertitel werden auf diesem Sender nicht unterstützt + Dieser Sender kann nicht genug verzögert werden; manche Untertitel kommen evtl. zu spät + KI-Untertitel für Live-TV + Fügt KI-Untertitel zum Untertitelmenü von Live-Sendern hinzu: Gemini transkribiert und übersetzt in die Untertitelsprache des Profils. Live-TV wird einige Sekunden verzögert, damit die Zeilen rechtzeitig erscheinen. + Gemini-API-Schlüssel funktioniert + Gemini hat diesen API-Schlüssel abgelehnt + Schlüssel konnte nicht geprüft werden (offline?) + Füge zuerst einen Google-Gemini-API-Schlüssel hinzu + Automatisch übersetzen, wenn ein Sender startet + Jeder Sender startet mit KI-Untertiteln, ohne sie im Untertitelmenü auszuwählen diff --git a/app/src/main/res/values-es/strings.xml b/app/src/main/res/values-es/strings.xml index f758d8e75a..f50bedea6b 100644 --- a/app/src/main/res/values-es/strings.xml +++ b/app/src/main/res/values-es/strings.xml @@ -1639,4 +1639,22 @@ Muestra tu estado de visionado en tu perfil de Discord. No se encontraron otras fuentes para este canal. No se pudieron cargar las fuentes. Por favor, inténtalo de nuevo. + + + Subtítulos IA + Preparando subtítulos… %1$d s + Los subtítulos IA necesitan una clave de API de Google Gemini (Ajustes → Subtítulos IA) + Subtítulos IA: se rechazó la clave de API de Gemini + Subtítulos IA en pausa: se alcanzó la cuota de Gemini + Subtítulos IA: sin conexión a internet + Subtítulos IA no disponibles en este canal + Este canal no se puede retrasar lo suficiente; algunos subtítulos pueden llegar tarde + Subtítulos IA en TV en directo + Añade subtítulos IA al menú de subtítulos de los canales en directo: Gemini transcribe y traduce al idioma de subtítulos del perfil. La TV en directo se retrasa unos segundos para que las líneas lleguen a tiempo. + La clave de API de Gemini funciona + Gemini rechazó esta clave de API + No se pudo comprobar la clave (¿sin conexión?) + Primero añade una clave de API de Google Gemini + Traducir automáticamente al iniciar un canal + Cada canal empieza con subtítulos IA, sin elegirlos en el menú de subtítulos diff --git a/app/src/main/res/values-iw/strings.xml b/app/src/main/res/values-iw/strings.xml index a97d45d7e3..9026a29dfb 100644 --- a/app/src/main/res/values-iw/strings.xml +++ b/app/src/main/res/values-iw/strings.xml @@ -376,4 +376,22 @@ אין חיבור ל-MDBList, ולכן הסנכרון מושהה. ARVIO ממשיך לנסות; אם זה נמשך, חבר את MDBList מחדש. אין חיבור ל-MDBList כרגע. נסה שוב בעוד כמה דקות. לא ניתן לטעון את הרשימה הזו מ-MDBList. נסה שוב. + + + כתוביות AI + מכין כתוביות… %1$d שנ׳ + כתוביות AI דורשות מפתח Google Gemini API (הגדרות ← כתוביות AI) + כתוביות AI: מפתח ה-Gemini API נדחה + כתוביות AI הושהו: הגעת למכסת Gemini + כתוביות AI: אין חיבור לאינטרנט + כתוביות AI אינן נתמכות בערוץ הזה + אי אפשר לעכב את הערוץ הזה מספיק; חלק מהכתוביות עלולות לאחר + כתוביות AI לטלוויזיה חיה + מוסיף כתוביות AI לתפריט הכתוביות בערוצים חיים: Gemini מתמלל ומתרגם לשפת הכתוביות של הפרופיל. השידור מתעכב כמה שניות כדי שהכתוביות יופיעו בזמן. + מפתח ה-Gemini API תקין + Gemini דחה את המפתח הזה + לא ניתן לבדוק את המפתח (אין חיבור?) + יש להוסיף קודם מפתח Google Gemini API + תרגום אוטומטי כשערוץ מתחיל + כל ערוץ מתחיל עם כתוביות AI, בלי לבחור אותן בתפריט הכתוביות diff --git a/app/src/main/res/values-pt-rBR/strings.xml b/app/src/main/res/values-pt-rBR/strings.xml index a94d5ea804..ce8b9a7f66 100644 --- a/app/src/main/res/values-pt-rBR/strings.xml +++ b/app/src/main/res/values-pt-rBR/strings.xml @@ -1551,4 +1551,22 @@ Portal Ministra / Stalker com autenticação por endereço MAC. Conteúdo a importar URL da lista M3U + + + Legendas IA + Preparando legendas… %1$d s + As legendas IA precisam de uma chave de API do Google Gemini (Configurações → Legendas IA) + Legendas IA: a chave de API do Gemini foi rejeitada + Legendas IA em pausa: quota do Gemini atingida + Legendas IA: sem conexão com a internet + Legendas IA não suportadas neste canal + Não dá para atrasar este canal o suficiente; algumas legendas podem chegar atrasadas + Legendas IA na TV ao vivo + Adiciona legendas IA ao menu de legendas dos canais ao vivo: o Gemini transcreve e traduz para o idioma de legendas do perfil. A TV ao vivo é atrasada alguns segundos para as linhas aparecerem na hora certa. + A chave de API do Gemini funciona + O Gemini rejeitou esta chave de API + Não foi possível verificar a chave (sem conexão?) + Adicione primeiro uma chave de API do Google Gemini + Traduzir automaticamente ao iniciar um canal + Cada canal começa com legendas IA, sem escolher no menu de legendas diff --git a/app/src/main/res/values-pt/strings.xml b/app/src/main/res/values-pt/strings.xml index 96c16dc7e7..64c0cdcc79 100644 --- a/app/src/main/res/values-pt/strings.xml +++ b/app/src/main/res/values-pt/strings.xml @@ -1551,4 +1551,22 @@ Portal Ministra / Stalker com autenticação por endereço MAC. Conteúdo a importar URL da lista M3U + + + Legendas IA + A preparar legendas… %1$d s + As legendas IA precisam de uma chave de API do Google Gemini (Definições → Legendas IA) + Legendas IA: a chave de API do Gemini foi rejeitada + Legendas IA em pausa: quota do Gemini atingida + Legendas IA: sem ligação à internet + Legendas IA não suportadas neste canal + Não é possível atrasar este canal o suficiente; algumas legendas podem chegar tarde + Legendas IA na TV em direto + Adiciona legendas IA ao menu de legendas dos canais em direto: o Gemini transcreve e traduz para o idioma de legendas do perfil. A TV em direto é atrasada alguns segundos para as linhas aparecerem a tempo. + A chave de API do Gemini funciona + O Gemini rejeitou esta chave de API + Não foi possível verificar a chave (sem ligação?) + Adicione primeiro uma chave de API do Google Gemini + Traduzir automaticamente ao iniciar um canal + Cada canal começa com legendas IA, sem as escolher no menu de legendas diff --git a/app/src/main/res/values/strings.xml b/app/src/main/res/values/strings.xml index a2ea370cc8..ffe6915935 100644 --- a/app/src/main/res/values/strings.xml +++ b/app/src/main/res/values/strings.xml @@ -2083,5 +2083,22 @@ Connect WeTrakr Copy code and open WeTrakr Maximum of 10 IPTV playlists reached - + + AI subtitles + Preparing subtitles… %1$d s + AI subtitles need a Google Gemini API key (Settings → AI subtitles) + AI subtitles: the Gemini API key was rejected + AI subtitles paused: Gemini quota reached + AI subtitles: no internet connection + AI subtitles not supported on this channel + This channel can\'t be delayed enough; some subtitles may be late + Live TV AI subtitles + Adds AI subtitles to the subtitle menu of live channels: Gemini transcribes and translates into your profile\'s subtitle language. Live TV is delayed a few seconds so lines appear on time. + Gemini API key works + Gemini rejected this API key + Couldn\'t check the API key (offline?) + Add a Google Gemini API key first + Auto-translate when a channel starts + Every channel starts with AI subtitles on, without picking them in the subtitle menu + diff --git a/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/LiveAiSubtitlesOptionTest.kt b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/LiveAiSubtitlesOptionTest.kt new file mode 100644 index 0000000000..7b8dad8470 --- /dev/null +++ b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/LiveAiSubtitlesOptionTest.kt @@ -0,0 +1,34 @@ +package com.arflix.tv.ui.screens.tv.live + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertSame +import org.junit.Test + +class LiveAiSubtitlesOptionTest { + private val streamOptions = listOf( + LiveTrackOption(LIVE_SUBTITLES_OFF, "Off", selected = false), + LiveTrackOption("en", "English", selected = true), + ) + + @Test fun featureOffLeavesTheStreamChoicesAlone() { + val options = withAiSubtitlesOption(streamOptions, "Off", "AI subtitles", aiAvailable = false, aiSelected = false) + assertSame(streamOptions, options) + } + + @Test fun aiEntryComesAfterTheStreamTracks() { + val options = withAiSubtitlesOption(streamOptions, "Off", "AI subtitles", aiAvailable = true, aiSelected = false) + assertEquals(listOf(LIVE_SUBTITLES_OFF, "en", LIVE_SUBTITLES_AI), options.map { it.key }) + assertEquals(listOf(false, true, false), options.map { it.selected }) + } + + @Test fun selectedAiEntryIsTheOnlySelection() { + val options = withAiSubtitlesOption(streamOptions, "Off", "AI subtitles", aiAvailable = true, aiSelected = true) + assertEquals(listOf(false, false, true), options.map { it.selected }) + } + + @Test fun channelWithoutSubtitleTracksGetsOffAndAi() { + val options = withAiSubtitlesOption(emptyList(), "Off", "AI subtitles", aiAvailable = true, aiSelected = false) + assertEquals(listOf(LIVE_SUBTITLES_OFF, LIVE_SUBTITLES_AI), options.map { it.key }) + assertEquals(listOf(true, false), options.map { it.selected }) + } +} diff --git a/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/Cea608TextAssemblerTest.kt b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/Cea608TextAssemblerTest.kt new file mode 100644 index 0000000000..b836efa143 --- /dev/null +++ b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/Cea608TextAssemblerTest.kt @@ -0,0 +1,96 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test + +class Cea608TextAssemblerTest { + private val sentences = ArrayList() + private val assembler = Cea608TextAssembler { sentences += it } + + /** cc_data triplets on field 1 (header 0xFC = valid, NTSC field 1). */ + private fun cc(vararg pairs: Pair, header: Int = 0xFC): ByteArray = + pairs.flatMap { (a, b) -> listOf(header.toByte(), a.toByte(), b.toByte()) }.toByteArray() + + private fun text(s: String): ByteArray { + val padded = if (s.length % 2 == 0) s else "$s\u0000" + return cc(*padded.chunked(2).map { it[0].code to it[1].code }.toTypedArray()) + } + + private fun feed(ms: Long, data: ByteArray) = assembler.feed(ms * 1_000, data, 0, data.size) + + private val carriageReturn = cc(0x14 to 0x2D) + + @Test fun charactersBecomeASentenceEndingAtAFullStop() { + feed(1_000, text("HELLO ")) + feed(1_500, text("WORLD.")) + assertEquals(listOf(CaptionSentence(1_000_000, 1_500_000, "HELLO WORLD.")), sentences) + } + + @Test fun repeatedControlCodesActOnce() { + feed(1_000, text("ONE")) + // Control codes come twice in a row; the pair must give one word boundary, not two. + feed(1_100, cc(0x14 to 0x2D, 0x14 to 0x2D)) + feed(1_200, text("TWO.")) + assertEquals("ONE TWO.", sentences.single().text) + } + + @Test fun carriageReturnSeparatesWords() { + feed(1_000, text("ROLL")) + feed(1_100, carriageReturn) + feed(1_200, text("UP.")) + assertEquals("ROLL UP.", sentences.single().text) + } + + @Test fun backspaceRemovesTheLastCharacter() { + feed(1_000, text("CATS")) + feed(1_100, cc(0x14 to 0x21)) + feed(1_200, text("!")) + assertEquals("CAT!", sentences.single().text) + } + + @Test fun paddingAndSecondFieldAreIgnored() { + feed(1_000, cc(0x80 to 0x80)) + feed(1_100, text("XY").also { it[0] = 0xFD.toByte() }) + feed(1_200, text("OK.")) + assertEquals("OK.", sentences.single().text) + } + + @Test fun secondCaptionChannelIsIgnored() { + feed(1_000, cc(0x1C to 0x2D)) + feed(1_100, text("A.")) + assertEquals("A.", sentences.single().text) + } + + @Test fun aPauseEndsTheSentence() { + feed(1_000, text("NO STOP HERE")) + assertTrue(sentences.isEmpty()) + feed(1_000 + Cea608TextAssembler.GAP_US / 1_000 + 100, cc(0x80 to 0x80)) + assertEquals("NO STOP HERE", sentences.single().text) + } + + @Test fun longRunOnCaptionsAreCutAtAWordAfterFiveSeconds() { + // Roll-up captions arrive a word at a time, without pauses. + listOf("THIS ", "GOES ", "ON ", "AND ", "ON ", "AND ").forEachIndexed { second, word -> + feed(second * 1_000L, text(word)) + } + assertEquals("THIS GOES ON AND ON AND", sentences.single().text) + assertEquals(0L, sentences.single().startUs) + assertEquals(5_000_000L, sentences.single().endUs) + } + + @Test fun resetDropsWhatWasCollected() { + feed(1_000, text("HALF")) + assembler.reset() + feed(1_100, text("NEW.")) + assertEquals("NEW.", sentences.single().text) + } + + @Test fun musicNoteIsAWordBoundaryNotText() { + feed(1_000, cc(0x11 to 0x37)) + assertFalse(assembler.sawText) + feed(1_100, text("SONG.")) + assertEquals("SONG.", sentences.single().text) + } +} diff --git a/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiLiveProtocolTest.kt b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiLiveProtocolTest.kt new file mode 100644 index 0000000000..06adc1ce77 --- /dev/null +++ b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/GeminiLiveProtocolTest.kt @@ -0,0 +1,130 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import org.json.JSONArray +import org.json.JSONObject +import org.junit.Assert.assertEquals +import org.junit.Assert.assertFalse +import org.junit.Assert.assertTrue +import org.junit.Test +import org.junit.runner.RunWith +import org.robolectric.RobolectricTestRunner +import org.robolectric.annotation.Config +import org.robolectric.annotation.ConscryptMode + +/** org.json is part of the Android framework, so these run under Robolectric. */ +@RunWith(RobolectricTestRunner::class) +@Config(sdk = [28], application = android.app.Application::class, manifest = Config.NONE) +@ConscryptMode(ConscryptMode.Mode.OFF) +class GeminiLiveProtocolTest { + private val segment = SpeechSegment(startUs = 100_000_000, endUs = 104_000_000, pcm16k = FloatArray(16)) + + private fun cues(result: LiveEngineResult) = (result as LiveEngineResult.Cues).cues + + @Test fun documentedObjectIsTimedFromItsOffsets() { + val result = GeminiLiveProtocol.parseLines( + """{"lines":[{"text":"שלום","startOffsetMs":0},{"text":"מה שלומך?","startOffsetMs":1500}],"sourceLanguage":"en"}""", + segment, rtl = true, + ) + val cues = cues(result) + assertEquals(listOf("שלום", "מה שלומך?"), cues.map { it.text }) + assertEquals(100_000_000L, cues[0].startUs) + assertEquals(101_500_000L, cues[0].endUs) + assertEquals(101_500_000L, cues[1].startUs) + assertEquals(104_000_000L, cues[1].endUs) + assertTrue(cues.all { it.rtl }) + assertEquals("en", (result as LiveEngineResult.Cues).sourceLanguage) + } + + @Test fun linesWithoutOffsetsShareTheSegmentByLength() { + val cues = cues(GeminiLiveProtocol.parseLines("""{"lines":[{"text":"aaaa"},{"text":"bbbb"}]}""", segment, rtl = false)) + assertEquals(102_000_000L, cues[1].startUs) + } + + @Test fun offsetsAreKeptInOrderAndInsideTheSegment() { + val cues = cues( + GeminiLiveProtocol.parseLines( + """{"lines":[{"text":"a","startOffsetMs":3000},{"text":"b","startOffsetMs":1000},{"text":"c","startOffsetMs":99000}]}""", + segment, rtl = false, + ), + ) + assertTrue(cues.zipWithNext().all { (x, y) -> y.startUs >= x.startUs }) + assertTrue(cues.all { it.startUs in segment.startUs until segment.endUs }) + } + + @Test fun emptyLinesMeanNoSpeech() { + assertTrue(cues(GeminiLiveProtocol.parseLines("""{"lines":[]}""", segment, rtl = false)).isEmpty()) + } + + @Test fun bareArrayOfStringsIsAccepted() { + val cues = cues(GeminiLiveProtocol.parseLines("""["Hello there", "General Kenobi"]""", segment, rtl = false)) + assertEquals(listOf("Hello there", "General Kenobi"), cues.map { it.text }) + } + + @Test fun fencedReplyIsAccepted() { + val text = "```json\n[{\"text\":\"One\"},{\"text\":\"Two\"}]\n```" + assertEquals(2, cues(GeminiLiveProtocol.parseLines(text, segment, rtl = false)).size) + } + + @Test fun newlinesAndExtraSpacesAreCollapsed() { + val cues = cues(GeminiLiveProtocol.parseLines("""{"lines":[{"text":" two\nlines here "}]}""", segment, rtl = false)) + assertEquals("two lines here", cues.single().text) + } + + @Test fun malformedReplyFails() { + assertTrue(GeminiLiveProtocol.parseLines("not json at all", segment, rtl = false) is LiveEngineResult.Failed) + } + + @Test fun responseSkipsThoughtPartsAndReadsTheText() { + val body = JSONObject().put( + "candidates", + JSONArray().put( + JSONObject().put( + "content", + JSONObject().put( + "parts", + JSONArray() + .put(JSONObject().put("thought", true).put("text", "thinking…")) + .put(JSONObject().put("text", """{"lines":[{"text":"Hi"}]}""")), + ), + ).put("finishReason", "STOP"), + ), + ).toString() + assertEquals("Hi", cues(GeminiLiveProtocol.parseResponse(body, segment, rtl = false)).single().text) + } + + @Test fun responseWithoutCandidatesFails() { + val blocked = """{"promptFeedback":{"blockReason":"SAFETY"}}""" + assertEquals(LiveEngineResult.Failed("blocked"), GeminiLiveProtocol.parseResponse(blocked, segment, rtl = false)) + assertTrue(GeminiLiveProtocol.parseResponse("", segment, rtl = false) is LiveEngineResult.Failed) + } + + @Test fun invalidKeyIsRecognised() { + assertTrue(GeminiLiveProtocol.isInvalidKey(400, """{"error":{"status":"INVALID_ARGUMENT","details":[{"reason":"API_KEY_INVALID"}]}}""")) + assertTrue(GeminiLiveProtocol.isInvalidKey(403, "")) + assertFalse(GeminiLiveProtocol.isInvalidKey(400, """{"error":{"message":"Request too large"}}""")) + assertFalse(GeminiLiveProtocol.isInvalidKey(500, "")) + } + + @Test fun requestCarriesAudioContextAndJsonMode() { + val body = JSONObject(GeminiLiveProtocol.requestBody(segment, listOf("previous line"), "Hebrew")) + val parts = body.getJSONArray("contents").getJSONObject(0).getJSONArray("parts") + assertEquals("audio/wav", parts.getJSONObject(0).getJSONObject("inline_data").getString("mime_type")) + assertTrue(parts.getJSONObject(1).getString("text").contains("previous line")) + val config = body.getJSONObject("generationConfig") + assertEquals("application/json", config.getString("responseMimeType")) + assertEquals("minimal", config.getJSONObject("thinkingConfig").getString("thinkingLevel")) + assertTrue(body.getJSONObject("system_instruction").toString().contains("Hebrew")) + } + + @Test fun wavHeaderDescribes16kMonoPcm16() { + val wav = GeminiLiveProtocol.wav16kMono(floatArrayOf(0f, 1f, -1f)) + assertEquals(44 + 6, wav.size) + assertEquals("RIFF", String(wav, 0, 4, Charsets.US_ASCII)) + assertEquals("WAVE", String(wav, 8, 4, Charsets.US_ASCII)) + val rate = (wav[24].toInt() and 0xFF) or ((wav[25].toInt() and 0xFF) shl 8) or ((wav[26].toInt() and 0xFF) shl 16) + assertEquals(16_000, rate) + assertEquals(1, wav[22].toInt()) + val loud = (wav[46].toInt() and 0xFF) or (wav[47].toInt() shl 8) + assertEquals(Short.MAX_VALUE.toInt(), loud) + } +} diff --git a/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueSchedulerTest.kt b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueSchedulerTest.kt new file mode 100644 index 0000000000..716a4f28f8 --- /dev/null +++ b/app/src/test/kotlin/com/arflix/tv/ui/screens/tv/live/aisubs/LiveCueSchedulerTest.kt @@ -0,0 +1,121 @@ +package com.arflix.tv.ui.screens.tv.live.aisubs + +import org.junit.Assert.assertEquals +import org.junit.Assert.assertNull +import org.junit.Assert.assertTrue +import org.junit.Test + +class LiveCueSchedulerTest { + private fun cue(startMs: Long, endMs: Long, text: String = "line") = LiveCue(startMs * 1_000, endMs * 1_000, text) + private fun LiveCueScheduler.at(ms: Long) = cueAt(ms * 1_000)?.text + + @Test fun cueShowsFromItsStartUntilItsEnd() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 12_500, "hello")), positionUs = 0) + assertNull(scheduler.at(9_999)) + assertEquals("hello", scheduler.at(10_000)) + assertEquals("hello", scheduler.at(12_499)) + assertNull(scheduler.at(12_500)) + } + + @Test fun shortCueStaysUpForTheMinimum() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 10_300, "hi")), positionUs = 0) + assertEquals("hi", scheduler.at(11_100)) + assertNull(scheduler.at(11_200)) + } + + @Test fun longCueIsCappedAtTheMaximum() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 30_000, "long")), positionUs = 0) + assertEquals("long", scheduler.at(15_900)) + assertNull(scheduler.at(16_000)) + } + + @Test fun nextCueCutsThePreviousOne() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 13_000, "one"), cue(11_000, 13_000, "two")), positionUs = 0) + assertEquals("one", scheduler.at(10_500)) + assertEquals("two", scheduler.at(11_000)) + } + + @Test fun cuesFromSegmentsArrivingOutOfOrderAreSorted() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(20_000, 22_000, "later")), positionUs = 0) + scheduler.add(listOf(cue(10_000, 12_000, "earlier")), positionUs = 0) + assertEquals("earlier", scheduler.at(11_000)) + assertEquals("later", scheduler.at(21_000)) + } + + @Test fun lateCueStillInsideItsWindowShowsImmediately() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 13_000, "late")), positionUs = 11_000_000) + assertEquals("late", scheduler.at(11_000)) + assertEquals(0, scheduler.lateDropped) + } + + @Test fun cueArrivingAfterItsWindowIsDroppedAndCounted() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 11_000, "gone")), positionUs = 12_000_000) + assertNull(scheduler.at(12_000)) + assertEquals(1, scheduler.lateDropped) + assertEquals(0, scheduler.pendingCount) + } + + @Test fun clearForgetsEverything() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 12_000)), positionUs = 0) + scheduler.clear() + assertNull(scheduler.at(11_000)) + assertEquals(0, scheduler.pendingCount) + } + + @Test fun blankLinesAreIgnored() { + val scheduler = LiveCueScheduler() + scheduler.add(listOf(cue(10_000, 12_000, " ")), positionUs = 0) + assertEquals(0, scheduler.pendingCount) + } + + @Test fun shortTextIsNotSplit() { + val parts = LiveCueScheduler.split(cue(0, 4_000, "A short line")) + assertEquals(1, parts.size) + } + + @Test fun longTextSplitsAtAWordNearTheMiddleWithProportionalTime() { + val text = "This is a rather long subtitle line that keeps going " + + "and going until it no longer fits on the screen at once" + val parts = LiveCueScheduler.split(cue(0, 6_000, text)) + assertEquals(2, parts.size) + assertEquals(text, parts[0].text + " " + parts[1].text) + assertEquals(parts[0].endUs, parts[1].startUs) + val expectedSplit = 6_000_000L * parts[0].text.length / (parts[0].text.length + parts[1].text.length) + assertEquals(expectedSplit, parts[0].endUs) + assertTrue(parts.all { it.text.length <= LiveCueScheduler.SPLIT_CHARS }) + } + + @Test fun longTextWithoutSpacesStaysWhole() { + val parts = LiveCueScheduler.split(cue(0, 6_000, "x".repeat(120))) + assertEquals(1, parts.size) + } +} + +class LiveAiSubsDelayTest { + @Test fun autoAssumesFiveSecondsBeforeAnyMeasurement() { + assertEquals(11_000L, autoLiveAiSubsDelayMs(null)) + } + + @Test fun autoAddsMeasuredLatencyToSegmentLengthAndMargin() { + assertEquals(8_200L, autoLiveAiSubsDelayMs(2_200L)) + } + + @Test fun autoIsClampedToTheOfferedRange() { + assertEquals(6_000L, autoLiveAiSubsDelayMs(0L)) + assertEquals(15_000L, autoLiveAiSubsDelayMs(30_000L)) + } + + @Test fun percentileNeedsAFewSamples() { + assertNull(percentile95(listOf(1L, 2L))) + assertEquals(96L, percentile95((1L..100L).toList())) + assertEquals(1_900L, percentile95(listOf(1_000L, 1_200L, 1_900L, 1_100L))) + } +}