feat: reconcile speaker identities through OpenRouter audio
Short chunk overlaps often contain only one participant, so returning speakers receive new transcript IDs. MAI can also split one voice into several IDs within a single request, which overlap stitching cannot fix. Add an opt-in audio comparison pass: --speakers N selects Gemini 3.8 Flash with a known count, and --speaker-model allows another compatible model. Keep request-local identities separate, select up to three clean excerpts per ID, and compare all excerpts in one OpenRouter request. Gemini uses Google AI Studio because the Vertex backend limits prompts to one audio file. No local model or extra Python dependency is required. Only reconcile identities whose sample assignments agree. Preserve uncertain identities separately, reject incomplete or malformed replies, and retain the original words and timestamps. Bound sample count and request size, clip sampling to the real audio duration, and extend worker cleanup and buffered output to the comparison step. Document the sampled method's limitations and additional provider request. Test Plan: - python3 -B -m unittest -v: all 82 tests passed, including real MP3 encoding with fake transcription/comparison responses and cancellation. - bash -n transcribe: passed. - git diff --cached --check: passed. - Live API and voice accuracy unverified: this workspace could not access the configured 1Password credential. Mocked responses prove pipeline behavior, not model accuracy on the supplied recording.
This commit is contained in:
1 parent
1d03dcd587
commit
138c796399
9 files changed
+988
-19
No files matched your search
@@ -48,6 +48,10 @@ class TranscribeTests(unittest.TestCase):
|
||||
def reset_logs(self):
|
||||
self.log.unlink(missing_ok=True)
|
||||
self.log.with_suffix(".events.jsonl").unlink(missing_ok=True)
|
||||
self.log.with_suffix(".speakers.json").unlink(missing_ok=True)
|
||||
|
||||
def speaker_request(self):
|
||||
return json.loads(self.log.with_suffix(".speakers.json").read_text())
|
||||
|
||||
def assert_processes_stopped(self):
|
||||
for event in self.events():
|
||||
@@ -342,6 +346,144 @@ class TranscribeTests(unittest.TestCase):
|
||||
self.assertIn("integer between 1 and 30", result.stderr)
|
||||
self.assertFalse(self.log.exists())
|
||||
|
||||
def test_voice_comparison_merges_fragments_inside_and_across_requests(self):
|
||||
result = self.run_cli("--speakers", "2", "--chunk-minutes", "1", mode="reconcile")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout, (
|
||||
"[Speaker 0]\nAlpha.\n\n[Speaker 0]\nAgain.\n\n[Speaker 1]\nBeta.\n\n"
|
||||
"[Speaker 0]\nAlphaLate.\n\n[Speaker 1]\nBetaLate.\n"
|
||||
))
|
||||
self.assertNotIn("speaker continuity is uncertain", result.stderr)
|
||||
comparison = self.speaker_request()
|
||||
self.assertEqual(comparison["request"]["model"], "google/gemini-3.8-flash")
|
||||
self.assertEqual(len(comparison["durations"]), 5)
|
||||
self.assertTrue(all(1.5 <= duration <= 8.1 for duration in comparison["durations"]))
|
||||
self.assertEqual(len(self.requests()), 2)
|
||||
self.assert_diarization(True)
|
||||
self.assert_cleaned()
|
||||
|
||||
def test_voice_comparison_model_override_without_count_and_single_request(self):
|
||||
for model, count in (("test/audio-model", ()), ("~audio-model", ("--speakers", "2"))):
|
||||
with self.subTest(model=model):
|
||||
self.reset_logs()
|
||||
result = self.run_cli(*count, "--speaker-model", model, mode="reconcile", duration=30)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout, (
|
||||
"[Speaker 0]\nAlpha.\n\n[Speaker 0]\nAgain.\n\n[Speaker 1]\nBeta.\n"
|
||||
))
|
||||
self.assertEqual(self.speaker_request()["request"]["model"], model)
|
||||
self.assertEqual(len(self.requests()), 1)
|
||||
self.assert_diarization(True)
|
||||
self.assert_cleaned()
|
||||
|
||||
def test_uncertain_voice_assignment_retains_a_separate_id(self):
|
||||
result = self.run_cli("--speakers", "2", "--chunk-minutes", "1", mode="reconcile-null")
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout, (
|
||||
"[Speaker 0]\nAlpha.\n\n[Speaker 1]\nAgain.\n\n[Speaker 2]\nBeta.\n\n"
|
||||
"[Speaker 1]\nAlphaLate.\n\n[Speaker 2]\nBetaLate.\n"
|
||||
))
|
||||
self.assertIn("Warning:", result.stderr)
|
||||
self.assert_cleaned()
|
||||
|
||||
def test_voice_comparison_srt_preserves_words_and_original_timestamps(self):
|
||||
serial = self.run_cli("--jobs", "1", "--speakers", "2", "--srt", "--chunk-minutes", "1",
|
||||
mode="reconcile")
|
||||
self.assertEqual(serial.returncode, 0, serial.stderr)
|
||||
self.assertEqual(serial.stdout, (
|
||||
"1\n00:00:01,000 --> 00:00:03,000\nAlpha.\n\n"
|
||||
"2\n00:00:06,000 --> 00:00:08,000\nAgain.\n\n"
|
||||
"3\n00:00:12,000 --> 00:00:14,000\nBeta.\n\n"
|
||||
"4\n00:01:00,000 --> 00:01:02,000\nAlphaLate.\n\n"
|
||||
"5\n00:01:05,000 --> 00:01:07,000\nBetaLate.\n\n"
|
||||
))
|
||||
self.assert_cleaned()
|
||||
self.reset_logs()
|
||||
parallel = self.run_cli("--jobs", "2", "--speakers", "2", "--srt", "--chunk-minutes", "1",
|
||||
mode="reconcile")
|
||||
self.assertEqual(parallel.returncode, 0, parallel.stderr)
|
||||
self.assertEqual(parallel.stdout, serial.stdout)
|
||||
self.assert_cleaned()
|
||||
|
||||
def test_failed_voice_comparison_emits_no_partial_transcript_and_cleans_up(self):
|
||||
for mode in ("reconcile-http-error", "reconcile-bad-json", "reconcile-missing",
|
||||
"reconcile-duplicate", "reconcile-unknown-sample", "reconcile-invalid-speaker",
|
||||
"reconcile-truncated"):
|
||||
with self.subTest(mode=mode):
|
||||
self.reset_logs()
|
||||
result = self.run_cli("--speakers", "2", "--chunk-minutes", "1", mode=mode)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout, "")
|
||||
self.assertTrue(self.log.with_suffix(".speakers.json").exists())
|
||||
self.assertTrue(result.stderr)
|
||||
self.assert_cleaned()
|
||||
|
||||
def test_missing_or_too_short_voice_samples_skip_comparison(self):
|
||||
for mode, expected in (("ok", "Hello.\n"),
|
||||
("speakers", "[Speaker 0]\nHello\n\n[Speaker 1]\nthere\n")):
|
||||
with self.subTest(mode=mode):
|
||||
self.reset_logs()
|
||||
result = self.run_cli("--speakers", "2", mode=mode, duration=5)
|
||||
self.assertEqual(result.returncode, 0, result.stderr)
|
||||
self.assertEqual(result.stdout, expected)
|
||||
self.assertIn("Warning:", result.stderr)
|
||||
self.assertFalse(self.log.with_suffix(".speakers.json").exists())
|
||||
self.assert_cleaned()
|
||||
|
||||
def test_invalid_speaker_options_fail_before_credentials_or_provider_access(self):
|
||||
credential_access = self.path / "credential-access"
|
||||
(self.bin / "op").write_text(f"#!/bin/sh\ntouch '{credential_access}'\nexit 1\n")
|
||||
options = [("--speakers", value) for value in
|
||||
("0", "65", "-1", "1.5", "abc", "99999999999999999999")]
|
||||
options += [("--speaker-model", value) for value in ("", "-bad", "bad model", "a\nb")]
|
||||
options += [("--no-diarization", "--speakers", "2"),
|
||||
("--speakers", "2", "--no-diarization"),
|
||||
("--speaker-model", "test/audio", "--no-diarization")]
|
||||
for args in options:
|
||||
with self.subTest(options=args):
|
||||
result = self.run_cli(*args, duration=1)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertEqual(result.stdout, "")
|
||||
self.assertIn("--speaker", result.stderr)
|
||||
for option in ("--speakers", "--speaker-model"):
|
||||
result = subprocess.run([str(ROOT / "transcribe"), option], env=self.env,
|
||||
capture_output=True, text=True, timeout=5)
|
||||
self.assertNotEqual(result.returncode, 0)
|
||||
self.assertIn(option, result.stderr)
|
||||
self.assertFalse(credential_access.exists())
|
||||
self.assertFalse(self.log.exists())
|
||||
|
||||
def test_interrupting_voice_comparison_stops_descendants_and_cleans_up(self):
|
||||
make_audio(self.audio, 30)
|
||||
process = subprocess.Popen(
|
||||
[str(ROOT / "transcribe"), "--speakers", "2", str(self.audio)],
|
||||
env=dict(self.env, TRANSCRIBE_TEST_MODE="reconcile-blocked"),
|
||||
stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True,
|
||||
)
|
||||
try:
|
||||
deadline = time.monotonic() + 8
|
||||
while time.monotonic() < deadline:
|
||||
if any(e["event"] == "child" and e["index"] == -1 for e in self.events()):
|
||||
break
|
||||
time.sleep(0.01)
|
||||
else:
|
||||
self.fail("Voice comparison did not start")
|
||||
process.send_signal(signal.SIGTERM)
|
||||
stdout, stderr = process.communicate(timeout=5)
|
||||
self.assertEqual(process.returncode, 128 + signal.SIGTERM, stderr)
|
||||
self.assertEqual(stdout, "")
|
||||
self.assert_processes_stopped()
|
||||
self.assert_cleaned()
|
||||
finally:
|
||||
if process.poll() is None:
|
||||
process.kill()
|
||||
for event in self.events():
|
||||
try:
|
||||
os.killpg(event["group"], signal.SIGKILL)
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
process.communicate(timeout=5)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in new issue
Block a user