diff --git a/.github/workflows/speech.yml b/.github/workflows/speech.yml index fed4311..cc91c40 100644 --- a/.github/workflows/speech.yml +++ b/.github/workflows/speech.yml @@ -1,7 +1,10 @@ name: Native speech on: + # Only on main: a branch pushed to this repo would otherwise also run every + # job a second time through its pull request. push: + branches: [main] paths: - crates/speech/** - Cargo.toml diff --git a/crates/speech/java/dev/robius/speech/NativeSpeech.java b/crates/speech/java/dev/robius/speech/NativeSpeech.java index 906fd35..876492c 100644 --- a/crates/speech/java/dev/robius/speech/NativeSpeech.java +++ b/crates/speech/java/dev/robius/speech/NativeSpeech.java @@ -15,6 +15,8 @@ import android.speech.SpeechRecognizer; import java.util.ArrayList; import java.util.HashMap; +import java.util.List; +import java.util.Locale; /** * One dictation session, driving the system SpeechRecognizer. @@ -213,6 +215,22 @@ private String transcript(Bundle results) { return values == null || values.isEmpty() || values.get(0) == null ? "" : values.get(0); } + // Whether `text` is `previous` with words cut off the end, comparing words loosely. + private static boolean isShortenedRevision(String previous, String text) { + List old = looseWords(previous), cut = looseWords(text); + return !cut.isEmpty() && cut.size() < old.size() && old.subList(0, cut.size()).equals(cut); + } + + // Words ignoring case and surrounding punctuation, as the Rust side compares them. + private static List looseWords(String text) { + List words = new ArrayList<>(); + for (String word : text.trim().split("\\s+")) { + String loose = word.replaceAll("^[^\\p{L}\\p{N}]+|[^\\p{L}\\p{N}]+$", "").toLowerCase(Locale.ROOT); + if (!loose.isEmpty()) words.add(loose); + } + return words; + } + private void commitPending() { if (!pending.isEmpty()) { event(id, 2, pending, 0); @@ -261,6 +279,8 @@ private void fail(String message) { private void fail(String message, int kind) { if (disposed) return; + // Keep the words already shown before reporting the error. + commitPending(); dispose(); event(id, kind, message, 0); } @@ -312,7 +332,8 @@ private void dispose() { awaitingResult = false; transientFailures = 0; String text = results == null ? "" : transcript(results); - if (!text.isEmpty()) pending = text; + // A final that only cuts words off the end of the last partial keeps them. + if (!text.isEmpty() && !isShortenedRevision(pending, text)) pending = text; commitPending(); nextUtterance(); } diff --git a/crates/speech/src/dictation.rs b/crates/speech/src/dictation.rs index 5f41ce6..c01d294 100644 --- a/crates/speech/src/dictation.rs +++ b/crates/speech/src/dictation.rs @@ -73,10 +73,21 @@ impl Dictation { /// a final commits it. Returns the edit that brings the field up to date, if /// there is one; apply it and call [`applied`](Self::applied). pub fn transcript(&mut self, text: &str, is_final: bool) -> Option { - self.pending = text.trim().to_owned(); + // Never lose words the recognizer already gave us: an empty transcript changes nothing, + // one that starts over commits what it replaces, and one that stops short keeps the rest. + let text = text.trim(); + if !text.is_empty() { + let previous = std::mem::take(&mut self.pending); + self.pending = text.to_owned(); + if starts_over(&previous, text) { + append_words(&mut self.committed, &previous); + } else { + append_words(&mut self.pending, &words_beyond(text, &previous)); + } + } if is_final { - append_words(&mut self.committed, &self.pending); - self.pending.clear(); + let pending = std::mem::take(&mut self.pending); + append_words(&mut self.committed, &pending); } if let Some(awaiting_final) = self.interrupted.as_mut() { if is_final { @@ -193,6 +204,23 @@ fn floor_char_boundary(text: &str, index: usize) -> usize { index } +/// Whether `new` starts over rather than revising `old`, like a recognizer that restarted after +/// a pause: it drops most words of a long `old`, rather than reformatting a few ("300 and" to "340"). +fn starts_over(old: &str, new: &str) -> bool { + let (old, new) = (loose_words(old), loose_words(new)); + let kept = old.iter().filter(|word| new.contains(word)).count(); + old.len() >= 6 && !new.is_empty() && new.len() * 2 < old.len() && kept * 4 < old.len() +} + +fn loose_words(text: &str) -> Vec { + text.split_whitespace().map(loose).filter(|word| !word.is_empty()).collect() +} + +/// A word compared loosely, because a recognizer routinely re-cases or re-punctuates. +fn loose(word: &str) -> String { + word.trim_matches(|c: char| !c.is_alphanumeric()).to_lowercase() +} + /// The part of `spoken` that goes beyond what the user can already see in /// `shown`. Words are compared loosely, because a recognizer routinely re-cases /// or re-punctuates what it already sent. Returns nothing when the two share no @@ -201,7 +229,6 @@ fn words_beyond(shown: &str, spoken: &str) -> String { if shown.trim().is_empty() { return spoken.trim().to_owned(); } - let loose = |word: &str| word.trim_matches(|c: char| !c.is_alphanumeric()).to_lowercase(); let shown_words: Vec<_> = shown.split_whitespace().map(loose).collect(); let mut matched = 0; let mut rest = spoken.trim(); @@ -556,6 +583,38 @@ mod tests { assert_eq!(field.text, "éX bc"); } + #[test] + fn words_already_shown_are_never_removed() { + // An empty transcript, partial or final, keeps what is shown. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "keep me", false); + field.say(&mut dictation, "", false); + field.say(&mut dictation, "", true); + assert_eq!(field.text, "keep me"); + // A recognizer that starts over after a pause keeps the phrase before it. + field.say(&mut dictation, "a long phrase that took thirty seconds", false); + field.say(&mut dictation, "Short phrase.", false); + field.say(&mut dictation, "Short phrase.", true); + assert_eq!(field.text, "keep me a long phrase that took thirty seconds Short phrase."); + + // A final that stops short keeps the words it left out. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + field.say(&mut dictation, "hello world and more", false); + field.say(&mut dictation, "Hello world.", true); + assert_eq!(field.text, "Hello world. and more"); + + // Reformatting a number as it's spoken is a revision, not a restart. + let mut dictation = Dictation::new("", 0..0); + let mut field = Field::new("", 0); + for partial in ["Three", "300", "300 and", "340", "347 apples"] { + field.say(&mut dictation, partial, false); + } + field.say(&mut dictation, "347 apples.", true); + assert_eq!(field.text, "347 apples."); + } + #[test] fn words_beyond_only_returns_what_was_not_already_shown() { assert_eq!(words_beyond("hello world", "hello world and more"), "and more"); diff --git a/crates/speech/swift/NativeSpeech.swift b/crates/speech/swift/NativeSpeech.swift index 377d210..0d22237 100644 --- a/crates/speech/swift/NativeSpeech.swift +++ b/crates/speech/swift/NativeSpeech.swift @@ -54,6 +54,109 @@ struct SpeechRetryPolicy { } } +/// Turns one recognition task's results into transcripts that never drop words already sent. +/// +/// After a pause, on-device recognition closes a segment with one result that has metadata, +/// then restarts `bestTranscription` from the new words only, without ever setting `isFinal`. +/// Its final result after `endAudio` can hold only the last segment, or nothing at all. +struct SegmentTracker { + /// Words sent as a partial but not yet committed. + private(set) var open = "" + /// The last text committed, and the whole transcription at that point. + private var committed = "" + private var closedTranscription = "" + /// Whether this task restarts its transcription after each segment, rather than growing it. + private var restarts = false + + /// Returns the transcripts to send (kind 1 partial, kind 2 final) for one result. + mutating func result(_ transcription: String, isFinal: Bool, closesSegment: Bool) -> [(Int32, String)] { + var text = transcription.trimmingCharacters(in: .whitespacesAndNewlines) + // A transcription that grows instead of restarting still holds what was committed. + if !closedTranscription.isEmpty && !restarts { + if let rest = looseRemainder(of: text, after: closedTranscription) { + text = rest + } else if !text.isEmpty { + restarts = true + } + } + var out: [(Int32, String)] = [] + if isFinal { + // An empty or cut-short final keeps what was shown, and a repeat of the + // segment just committed isn't committed again. + if text.isEmpty || (open.isEmpty && looselyEqual(text, committed)) || isShortenedRevision(of: open, text) { + return flush() + } + if startsOver(from: open, to: text) { out += flush() } + open = "" + return out + commit(text) + } + guard !text.isEmpty, !(closesSegment && open.isEmpty && looselyEqual(text, committed)) else { return [] } + if startsOver(from: open, to: text) { out += flush() } + if closesSegment { + open = "" + closedTranscription = transcription + return out + commit(text) + } + open = text + out.append((1, text)) + return out + } + + /// Commits the words shown but not yet committed, e.g. when a task ends without a final. + mutating func flush() -> [(Int32, String)] { + defer { open = "" } + return commit(open) + } + + private mutating func commit(_ text: String) -> [(Int32, String)] { + guard !text.isEmpty else { return [] } + committed = text + return [(2, text)] + } +} + +/// Words compared loosely, ignoring case and punctuation, as `dictation.rs` does. +func looseWords(_ text: String) -> [String] { + text.split(whereSeparator: { $0.isWhitespace }) + .map { $0.trimmingCharacters(in: CharacterSet.alphanumerics.inverted).lowercased() } + .filter { !$0.isEmpty } +} + +func looselyEqual(_ a: String, _ b: String) -> Bool { + looseWords(a) == looseWords(b) +} + +/// Whether `new` starts over rather than revising `old`, as after a pause without metadata: +/// it drops most words of a long `old`, rather than reformatting a few (e.g. "300 and" to "340"). +func startsOver(from old: String, to new: String) -> Bool { + let (old, new) = (looseWords(old), looseWords(new)) + let kept = old.filter(new.contains).count + return old.count >= 6 && !new.isEmpty && new.count * 2 < old.count && kept * 4 < old.count +} + +/// Whether `new` is `old` with words cut off the end. +func isShortenedRevision(of old: String, _ new: String) -> Bool { + let (old, new) = (looseWords(old), looseWords(new)) + return !new.isEmpty && new.count < old.count && Array(old.prefix(new.count)) == new +} + +/// The rest of `text` after its first words, if they loosely match all of `prefix`'s words. +func looseRemainder(of text: String, after prefix: String) -> String? { + var expected = looseWords(prefix)[...] + var rest = Substring(text) + while let word = expected.first { + rest = rest.drop(while: { $0.isWhitespace }) + guard !rest.isEmpty else { return nil } + let token = rest.prefix(while: { !$0.isWhitespace }) + rest = rest.dropFirst(token.count) + let loose = looseWords(String(token)) + if loose.isEmpty { continue } + guard loose == [word] else { return nil } + expected = expected.dropFirst() + } + return rest.trimmingCharacters(in: .whitespacesAndNewlines) +} + // The engine keeps recording while a recognizer finalizes an utterance. Only // that short interval needs copies; otherwise buffers go straight to the request. // No engine operations or UI callbacks run while this lock is held. @@ -165,7 +268,7 @@ final class NativeSpeech { var finished = false var tapInstalled = false var started = false - var lastPartial = "" + var segments = SegmentTracker() var generation: UInt64 = 0 var retryPolicy = SpeechRetryPolicy() var endingUtterance = false @@ -301,7 +404,7 @@ final class NativeSpeech { } if #available(macOS 13.0, iOS 16.0, *) { request.addsPunctuation = true } self.request = request - lastPartial = "" + segments = SegmentTracker() let capture = self.capture capture.prepareRequest(utterance) task = recognizer.recognitionTask(with: request) { [weak self] result, error in @@ -313,11 +416,16 @@ final class NativeSpeech { DispatchQueue.main.async { guard let self = self, !self.finished, self.generation == utterance else { return } if let result = result { - self.lastPartial = result.bestTranscription.formattedString - if !self.lastPartial.isEmpty { self.retryPolicy.madeProgress() } - self.send(result.isFinal ? 2 : 1, self.lastPartial) + let text = result.bestTranscription.formattedString + if !text.isEmpty { self.retryPolicy.madeProgress() } + var closesSegment = false + if #available(iOS 14.0, *) { + closesSegment = !result.isFinal && result.speechRecognitionMetadata != nil + } + for (kind, text) in self.segments.result(text, isFinal: result.isFinal, closesSegment: closesSegment) { + self.send(kind, text) + } if result.isFinal { - self.lastPartial = "" if self.stopping { self.finishStoppedUtterance() } else { self.restartUtterance(after: 0.15) } return @@ -502,7 +610,7 @@ final class NativeSpeech { } func commitPartial() { - if !lastPartial.isEmpty { send(2, lastPartial); lastPartial = "" } + for (kind, text) in segments.flush() { send(kind, text) } } func finish() { diff --git a/crates/speech/swift/tests/SpeechTests.swift b/crates/speech/swift/tests/SpeechTests.swift index 8afbeca..5c184cd 100644 --- a/crates/speech/swift/tests/SpeechTests.swift +++ b/crates/speech/swift/tests/SpeechTests.swift @@ -28,6 +28,11 @@ enum SpeechTests { ("utterance rollover commits the draft", rolloverCommitsTheDraft), ("cancelling during rollover emits nothing", cancellingDuringRolloverIsSilent), ("a graceful stop replays buffered speech", gracefulStopReplaysBufferedSpeech), + ("a segment closed before a restart is kept", segmentsSurviveRestarts), + ("a growing transcription commits each segment once", growingTranscriptionIsNotRepeated), + ("a restart without metadata keeps the earlier words", restartWithoutMetadataKeepsWords), + ("an empty, cut-short or repeated final keeps what was shown", finalsNeverRemoveShownWords), + ("a revision of the current utterance stays a revision", revisionsStayRevisions), ] static func main() { @@ -203,7 +208,7 @@ enum SpeechTests { precondition(rollover.engine == nil, "Creating a session must not initialize microphone hardware") rollover.started = true rollover.generation = 7 - rollover.lastPartial = "Keep this draft" + _ = rollover.segments.result("Keep this draft", isFinal: false, closesSegment: false) rollover.endUtteranceBeforeLimit(6, finalizationTimeout: 0) precondition(!rollover.endingUtterance, "An old utterance timer must not affect its successor") rollover.endUtteranceBeforeLimit(7, finalizationTimeout: 0) @@ -219,7 +224,7 @@ enum SpeechTests { let before = nativeEvents.count let cancelled = NativeSpeech(id: 44, locale: "en-US", preferOnDevice: true) cancelled.generation = 1 - cancelled.lastPartial = "Cancelled partial" + _ = cancelled.segments.result("Cancelled partial", isFinal: false, closesSegment: false) cancelled.endUtteranceBeforeLimit(1, finalizationTimeout: 0) cancelled.stop(cancel: true) var drained = false @@ -232,7 +237,7 @@ enum SpeechTests { nativeEvents.removeAll() let stopping = NativeSpeech(id: 45, locale: "", preferOnDevice: true) stopping.stopping = true - stopping.lastPartial = "Previous utterance" + _ = stopping.segments.result("Previous utterance", isFinal: false, closesSegment: false) stopping.capture.beginRequest { _ in } stopping.capture.endRequest() precondition(stopping.capture.append(audioBuffer(sample: 6))) @@ -243,14 +248,65 @@ enum SpeechTests { var replayed: [Float] = [] stopping.capture.beginRequest { replayed.append($0.floatChannelData![0][0]) } precondition(replayed == [6]) - stopping.lastPartial = "Buffered final words" + _ = stopping.segments.result("Buffered final words", isFinal: false, closesSegment: false) stopping.finishStoppedUtterance() precondition(nativeEvents.map(\.kind) == [2, 2, 4]) precondition(nativeEvents[1].text == "Buffered final words" && stopping.finished) } + // MARK: - Segments + + static func segmentsSurviveRestarts() { + // Apple's on-device trace: the closing result repeats the last partial, then the text restarts. + var tracker = SegmentTracker() + let sent = feed(&tracker, [("Can", false, false), ("Can you", false, false), ("Can you", false, true), + ("Hear", false, false), ("Hear this", false, false), ("Hear this", false, true), ("", true, false)]) + precondition(sent == ["1:Can", "1:Can you", "2:Can you", "1:Hear", "1:Hear this", "2:Hear this"], "\(sent)") + } + + static func growingTranscriptionIsNotRepeated() { + var tracker = SegmentTracker() + let sent = feed(&tracker, [("Can you", false, false), ("Can you", false, true), ("Can you hear", false, false), + ("Can you hear this", false, true), ("Can you hear this.", true, false)]) + precondition(sent == ["1:Can you", "2:Can you", "1:hear", "2:hear this"], "\(sent)") + } + + static func restartWithoutMetadataKeepsWords() { + var tracker = SegmentTracker() + let sent = feed(&tracker, [("one two three four five six", false, false), ("seven", false, false), ("seven eight", true, false)]) + precondition(sent == ["1:one two three four five six", "2:one two three four five six", "1:seven", "2:seven eight"], "\(sent)") + } + + static func finalsNeverRemoveShownWords() { + var tracker = SegmentTracker() + var sent = feed(&tracker, [("keep these words", false, false), ("", true, false)]) + precondition(sent == ["1:keep these words", "2:keep these words"], "\(sent)") + tracker = SegmentTracker() + sent = feed(&tracker, [("keep these words", false, false), ("Keep these.", true, false)]) + precondition(sent == ["1:keep these words", "2:keep these words"], "\(sent)") + tracker = SegmentTracker() + sent = feed(&tracker, [("said once", false, true), ("said once", false, true), ("Said once.", true, false)]) + precondition(sent == ["2:said once"], "\(sent)") + } + + static func revisionsStayRevisions() { + var tracker = SegmentTracker() + let sent = feed(&tracker, [("I scream", false, false), ("Ice cream", false, false), ("Ice cream please.", true, false)]) + precondition(sent == ["1:I scream", "1:Ice cream", "2:Ice cream please."], "\(sent)") + // Apple reformats numbers as they're spoken, e.g. "Three hundred and forty seven apples". + var numbers = SegmentTracker() + let reformatted = feed(&numbers, [("Three", false, false), ("300", false, false), ("300 and", false, false), + ("340", false, false), ("347 apples", false, false), ("347 apples.", true, false)]) + precondition(reformatted.last == "2:347 apples." && !reformatted.contains { $0.hasPrefix("2:300") }, "\(reformatted)") + } + // MARK: - Helpers + /// Feeds (text, isFinal, closesSegment) results and returns what would be sent, as "kind:text". + private static func feed(_ tracker: inout SegmentTracker, _ results: [(String, Bool, Bool)]) -> [String] { + results.flatMap { tracker.result($0.0, isFinal: $0.1, closesSegment: $0.2) }.map { "\($0.0):\($0.1)" } + } + private static func audioBuffer(sample: Float, frames: AVAudioFrameCount = 1) -> AVAudioPCMBuffer { let format = AVAudioFormat(standardFormatWithSampleRate: 8_000, channels: 1)! let buffer = AVAudioPCMBuffer(pcmFormat: format, frameCapacity: frames)! diff --git a/crates/speech/tests/android_retry_test.py b/crates/speech/tests/android_retry_test.py index 71d965a..aa43a97 100644 --- a/crates/speech/tests/android_retry_test.py +++ b/crates/speech/tests/android_retry_test.py @@ -656,6 +656,26 @@ NativeSpeech.stop(29, true); Handler.drain(); } + private static void anErrorKeepsTheWordsShown() { + start(30); + SpeechRecognizer.last.listener.onPartialResults(words("keep these words")); + SpeechRecognizer.last.listener.onError(SpeechRecognizer.ERROR_NETWORK); + check(events.indexOf("2:keep these words") >= 0 && events.get(events.size() - 1).startsWith("5:"), + "an error commits the words shown before reporting itself"); + Handler.drain(); + } + private static void aCutShortFinalKeepsTheWordsShown() { + start(31); + SpeechRecognizer.last.listener.onPartialResults(words("hello world and more")); + SpeechRecognizer.last.listener.onResults(words("Hello world.")); + check(events.contains("2:hello world and more"), "a final that cuts words off the end keeps them"); + Handler.runNext(); + SpeechRecognizer.last.listener.onPartialResults(words("I scream for")); + SpeechRecognizer.last.listener.onResults(words("Ice cream for you.")); + check(events.contains("2:Ice cream for you."), "a final that revises words still replaces the partial"); + NativeSpeech.stop(31, true); + Handler.drain(); + } private interface Case { void run(); } public static void main(String[] args) { System.load(args[0]); @@ -672,6 +692,8 @@ {"queued old removal cannot remove a new permission request", (Case) NativeSpeechRetryTest::restartWhileOldParentRemovalIsQueued}, {"configuration teardown stops the old pending session", (Case) NativeSpeechRetryTest::configurationTeardownStopsPendingSession}, {"saved permission state restores through the host classloader", (Case) NativeSpeechRetryTest::processRestoreContainsOnlyHostVisibleClasses}, + {"an error keeps the words already shown", (Case) NativeSpeechRetryTest::anErrorKeepsTheWordsShown}, + {"a cut-short final keeps the words shown", (Case) NativeSpeechRetryTest::aCutShortFinalKeepsTheWordsShown}, }; for (Object[] entry : cases) { System.out.println(" " + entry[0]);