import json import os from pathlib import Path import shutil import signal import subprocess import tempfile import time import unittest from test_chunks import make_audio ROOT = Path(__file__).resolve().parent TOOLS = ("bash", "ffmpeg", "ffprobe", "jq", "base64", "mktemp", "realpath") @unittest.skipUnless(all(shutil.which(tool) for tool in TOOLS), "CLI media dependencies missing") class TranscribeTests(unittest.TestCase): def setUp(self): self.directory = tempfile.TemporaryDirectory() self.addCleanup(self.directory.cleanup) self.path = Path(self.directory.name) self.bin = self.path / "bin" self.bin.mkdir() for name in ("op", "curl"): fake = self.bin / name shutil.copyfile(ROOT / "tests" / "fake_provider.py", fake) fake.chmod(0o755) self.log = self.path / "requests.jsonl" self.env = dict(os.environ, PATH=f"{self.bin}{os.pathsep}{os.environ['PATH']}", TRANSCRIBE_TEST_LOG=str(self.log), TRANSCRIBE_TEST_MODE="ok") self.audio = self.path / "input with spaces.wav" def run_cli(self, *args, mode="ok", duration=70, env_overrides=None): make_audio(self.audio, duration) return subprocess.run([str(ROOT / "transcribe"), *args, str(self.audio)], env=dict(self.env, TRANSCRIBE_TEST_MODE=mode, **(env_overrides or {})), capture_output=True, text=True, timeout=30) def requests(self): return sorted((json.loads(line) for line in self.log.read_text().splitlines()), key=lambda request: request["index"]) def events(self): path = self.log.with_suffix(".events.jsonl") return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] def reset_logs(self): self.log.unlink(missing_ok=True) self.log.with_suffix(".events.jsonl").unlink(missing_ok=True) self.log.with_suffix(".speakers.json").unlink(missing_ok=True) def speaker_request(self): return json.loads(self.log.with_suffix(".speakers.json").read_text()) def assert_automatic_speaker_comparison(self): request = self.speaker_request()["request"] self.assertEqual(request["model"], "google/gemini-3.8-flash") schema = request["response_format"]["json_schema"]["schema"] speaker_schema = schema["properties"]["assignments"]["items"]["properties"]["speaker"] self.assertNotIn("maximum", speaker_schema) self.assertNotIn("complete recording has exactly", request["messages"][0]["content"]) def assert_processes_stopped(self): for event in self.events(): for pid in (event["pid"], event.get("child_pid")): if pid is None: continue try: # A killed descendant can await reaping by init; it must # not still be running or holding our temporary files. state = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[0] self.assertEqual(state, "Z", f"Process {pid} survived worker cleanup") except FileNotFoundError: pass def assert_cleaned(self): self.assertTrue(all(not Path(r["staging"]).exists() for r in self.requests())) def test_parallel_default_and_job_limit_with_ordered_output(self): for options, limit, duration in (((), 8, 500), (("--jobs", "2"), 2, 180), (("--jobs", "1"), 1, 130)): with self.subTest(options=options): self.reset_logs() result = self.run_cli( *options, "--chunk-minutes", "1", mode="parallel", duration=duration, env_overrides={"TRANSCRIBE_TEST_BARRIER": str(limit)}, ) self.assertEqual(result.returncode, 0, result.stderr) requests = self.requests() self.assertEqual(result.stdout.split(), [f"Part{i}." for i in range(len(requests))]) active = maximum = 0 events = sorted(self.events(), key=lambda event: event["time"]) for event in events: active += 1 if event["event"] == "start" else -1 maximum = max(maximum, active) self.assertEqual(active, 0) self.assertEqual(maximum, limit) self.assertEqual(len({r["work"] for r in requests}), len(requests)) if limit > 1: first_finished = next(e for e in events if e["event"] == "finish") self.assertNotEqual(first_finished["index"], 0) refilled = next(e["time"] for e in events if e["event"] == "start" and e["index"] == limit) slow_finished = next(e["time"] for e in events if e["event"] == "finish" and e["index"] == 0) self.assertLess(refilled, slow_finished) self.assert_processes_stopped() self.assert_cleaned() def test_parallel_srt_matches_sequential_output(self): serial = self.run_cli("--jobs", "1", "--chunk-minutes", "1", "--srt", mode="parallel", duration=130) self.assertEqual(serial.returncode, 0, serial.stderr) self.reset_logs() parallel = self.run_cli( "--jobs", "3", "--chunk-minutes", "1", "--srt", mode="parallel", duration=130, env_overrides={"TRANSCRIBE_TEST_BARRIER": "3"}, ) self.assertEqual(parallel.returncode, 0, parallel.stderr) self.assertEqual(parallel.stdout, serial.stdout) self.assertEqual([block.splitlines()[0] for block in parallel.stdout.strip().split("\n\n")], ["1", "2", "3"]) self.assert_cleaned() def test_parallel_failure_cancels_workers_and_their_children(self): result = self.run_cli( "--jobs", "2", "--chunk-minutes", "1", mode="parallel-error", duration=180, env_overrides={"TRANSCRIBE_TEST_BARRIER": "2"}, ) self.assertNotEqual(result.returncode, 0) self.assertEqual(result.stdout, "") self.assertIn("simulated concurrent failure", result.stderr) self.assertIn("stopping remaining workers", result.stderr) self.assertEqual({e["index"] for e in self.events() if e["event"] == "start"}, {0, 1}) self.assertTrue(any(e["event"] == "child" for e in self.events())) self.assert_processes_stopped() self.assert_cleaned() def test_interruptions_stop_workers_before_removing_staging(self): for signum in (signal.SIGINT, signal.SIGTERM): with self.subTest(signal=signum): self.reset_logs() make_audio(self.audio, 180) process = subprocess.Popen( [str(ROOT / "transcribe"), "--jobs", "2", "--chunk-minutes", "1", str(self.audio)], env=dict(self.env, TRANSCRIBE_TEST_MODE="blocked"), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True, ) try: deadline = time.monotonic() + 5 while time.monotonic() < deadline: events = self.events() if (sum(e["event"] == "start" for e in events) >= 2 and any(e["event"] == "child" for e in events)): break time.sleep(0.01) else: self.fail("Workers did not start") process.send_signal(signum) stdout, stderr = process.communicate(timeout=5) self.assertEqual(process.returncode, 128 + signum, stderr) self.assertEqual(stdout, "") self.assert_processes_stopped() self.assertTrue(all(not Path(e["staging"]).exists() for e in self.events())) finally: if process.poll() is None: process.kill() for event in self.events(): try: os.killpg(event["group"], signal.SIGKILL) except ProcessLookupError: pass process.communicate(timeout=5) def test_immediate_encoder_failure_is_not_lost_by_worker_waiting(self): encoder = self.bin / "ffmpeg" encoder.write_text( "#!/usr/bin/env python3\nimport os, sys\n" "if '-ss' in sys.argv:\n sys.exit(42)\n" f"os.execv({shutil.which('ffmpeg')!r}, ['ffmpeg', *sys.argv[1:]])\n" ) encoder.chmod(0o755) result = self.run_cli("--jobs", "64", "--chunk-minutes", "1", duration=500) self.assertEqual(result.returncode, 42, result.stderr) self.assertEqual(result.stdout, "") self.assertIn("stopping remaining workers", result.stderr) self.assertFalse(self.log.exists()) def test_invalid_jobs_fail_before_provider_access(self): for value in ("0", "65", "-1", "1.5", "abc", "9999999999999999999999"): with self.subTest(value=value): result = self.run_cli("--jobs", value, duration=1) self.assertNotEqual(result.returncode, 0) self.assertIn("integer between 1 and 64", result.stderr) result = subprocess.run([str(ROOT / "transcribe"), "--jobs"], env=self.env, capture_output=True, text=True, timeout=5) self.assertNotEqual(result.returncode, 0) self.assertIn("integer between 1 and 64", result.stderr) self.assertFalse(self.log.exists()) def assert_diarization(self, enabled): for request in self.requests(): self.assertEqual(request["provider"], { "options": {"azure": {"diarization": {"enabled": enabled}}}, }) self.assertIs( request["provider"]["options"]["azure"]["diarization"]["enabled"], enabled, ) def test_long_text_uploads_bounded_mp3_and_stitches_once(self): result = self.run_cli("--chunk-minutes", "1") self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, "Hello.\n\nboundary continues.\n\nLast.\n") self.assertIn("chunk 2/2", result.stderr) requests = self.requests() self.assertEqual(len(requests), 2) self.assertTrue(all(r["upload_duration"] <= 60 for r in requests)) self.assertAlmostEqual(requests[0]["chunk"]["end"] - requests[1]["chunk"]["start"], 6) self.assert_diarization(True) self.assert_cleaned() def test_long_srt_uses_original_timeline_and_continuous_numbers(self): result = self.run_cli("--srt", "--chunk-minutes", "1") self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, ( "1\n00:00:01,000 --> 00:00:01,400\nHello.\n\n" "2\n00:00:56,700 --> 00:00:57,800\nboundary continues.\n\n" "3\n00:01:05,000 --> 00:01:05,400\nLast.\n\n" )) self.assert_diarization(True) self.assert_cleaned() def test_short_text_without_diarization_retains_provider_line_breaks(self): result = self.run_cli("--no-diarization", duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, "Provider's original formatting.\nNext line.\n") self.assertEqual(len(self.requests()), 1) self.assertIsNone(self.requests()[0]["response_format"]) self.assert_diarization(False) self.assert_cleaned() def test_short_text_requests_and_prints_speaker_labels(self): result = self.run_cli(mode="speakers", duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, "[Speaker 0]\nHello\n\n[Speaker 1]\nthere\n") self.assertEqual(self.requests()[0]["response_format"], "verbose_json") self.assertEqual(self.requests()[0]["timestamp_granularities"], ["segment", "word"]) self.assert_diarization(True) self.assertIn("no clean speaker samples", result.stderr) self.assertFalse(self.log.with_suffix(".speakers.json").exists()) self.assert_cleaned() def test_chunked_text_has_speaker_markers_without_chunk_details(self): result = self.run_cli("--chunk-minutes", "1", mode="speakers") self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, ( "[Speaker 0]\nHello.\n\n" "[Speaker 0]\nboundary\n\n" "[Speaker 1]\ncontinues.\n\n" "[Speaker 1]\nLast.\n" )) self.assertIn("no clean speaker samples", result.stderr) self.assertFalse(self.log.with_suffix(".speakers.json").exists()) self.assertNotIn("Chunk", result.stdout) self.assert_cleaned() def test_missing_speaker_labels_warn_instead_of_implying_detection_worked(self): for output in ((), ("--srt",)): with self.subTest(output=output): result = self.run_cli(*output, duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("returned no speaker labels", result.stderr) self.assertNotIn("[Speaker", result.stdout) result = self.run_cli(mode="partial-speakers", duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("some words without speaker labels", result.stderr) self.assertEqual(result.stdout, "[Speaker 0]\nHello\n\n[Speaker unknown]\nthere\n") def test_short_srt_still_requests_word_timestamps(self): result = self.run_cli("--srt", duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, "1\n00:00:01,000 --> 00:00:01,400\nHello.\n\n") self.assertEqual(self.requests()[0]["response_format"], "verbose_json") self.assertEqual(self.requests()[0]["timestamp_granularities"], ["segment", "word"]) self.assert_diarization(True) def test_no_diarization_applies_to_text_srt_and_every_chunk(self): for output in ((), ("--srt",)): for chunked in (False, True): with self.subTest(output=output, chunked=chunked): self.reset_logs() options = ("--chunk-minutes", "1") if chunked else () result = self.run_cli( *output, *options, "--no-diarization", duration=70 if chunked else 5, ) self.assertEqual(result.returncode, 0, result.stderr) self.assertTrue(result.stdout) self.assertEqual(len(self.requests()), 2 if chunked else 1) self.assert_diarization(False) self.assertFalse(self.log.with_suffix(".speakers.json").exists()) for request in self.requests(): if output or chunked: self.assertEqual(request["response_format"], "verbose_json") self.assertEqual(request["timestamp_granularities"], ["segment", "word"]) else: self.assertIsNone(request["response_format"]) self.assertIsNone(request["timestamp_granularities"]) self.assertLessEqual(request["upload_duration"], 60) self.assert_cleaned() # The opt-out must also preserve a limit specified after it. self.log.unlink() result = self.run_cli("--no-diarization", "--chunk-minutes", "1") self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(len(self.requests()), 2) self.assert_diarization(False) self.assert_cleaned() def test_no_diarization_skips_voice_comparison_with_usable_samples(self): # Do not compare voices even if a provider returns labels after opt-out. for output in ((), ("--srt",)): for options in (("--no-diarization", "--chunk-minutes", "1", "--jobs", "1"), ("--chunk-minutes", "1", "--jobs", "2", "--no-diarization")): with self.subTest(output=output, options=options): self.reset_logs() result = self.run_cli(*output, *options, mode="reconcile") self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("AlphaLate.", result.stdout) self.assertEqual(len(self.requests()), 2) self.assert_diarization(False) self.assertFalse(self.log.with_suffix(".speakers.json").exists()) self.assertNotIn("Acoustic speaker matching", result.stderr) self.assert_cleaned() def test_default_upload_limits_depend_on_diarization(self): for options, limit in (((), 600), (("--no-diarization",), 1800)): with self.subTest(options=options): self.log.unlink(missing_ok=True) result = self.run_cli(*options, duration=limit + 10) self.assertEqual(result.returncode, 0, result.stderr) requests = self.requests() self.assertEqual(len(requests), 2) self.assertGreaterEqual(requests[0]["upload_duration"], limit - 0.1) self.assertTrue(all(r["upload_duration"] <= limit for r in requests)) self.assert_diarization(not options) self.assert_cleaned() def test_diarization_breaks_srt_at_speaker_changes(self): result = self.run_cli("--srt", mode="speakers", duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, ( "1\n00:00:01,000 --> 00:00:01,400\nHello\n\n" "2\n00:00:01,500 --> 00:00:01,900\nthere\n\n" )) self.assert_diarization(True) self.log.unlink() result = self.run_cli("--no-diarization", "--srt", mode="speakers", duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, "1\n00:00:01,000 --> 00:00:01,900\nHello there\n\n") self.assert_diarization(False) def test_failed_later_chunk_has_no_partial_stdout_and_cleans_up(self): for mode in ("http-error", "bad-json", "api-error", "missing-words"): with self.subTest(mode=mode): result = self.run_cli("--chunk-minutes", "1", mode=mode) self.assertNotEqual(result.returncode, 0) self.assertEqual(result.stdout, "") self.assertTrue(result.stderr) self.assert_cleaned() def test_invalid_limits_fail_before_provider_access(self): for value in ("0", "31", "abc", "1.5", "-1"): with self.subTest(value=value): result = self.run_cli("--chunk-minutes", value, duration=1) self.assertNotEqual(result.returncode, 0) self.assertIn("integer between 1 and 30", result.stderr) self.assertFalse(self.log.exists()) def test_voice_comparison_merges_fragments_inside_and_across_requests(self): result = self.run_cli("--speakers", "2", "--chunk-minutes", "1", mode="reconcile") self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, ( "[Speaker 0]\nAlpha.\n\n[Speaker 0]\nAgain.\n\n[Speaker 1]\nBeta.\n\n" "[Speaker 0]\nAlphaLate.\n\n[Speaker 1]\nBetaLate.\n" )) self.assertNotIn("speaker continuity is uncertain", result.stderr) comparison = self.speaker_request() self.assertEqual(comparison["request"]["model"], "google/gemini-3.8-flash") schema = comparison["request"]["response_format"]["json_schema"]["schema"] self.assertEqual(schema["properties"]["assignments"]["items"]["properties"]["speaker"]["maximum"], 1) self.assertIn("exactly 2 speakers", comparison["request"]["messages"][0]["content"]) self.assertEqual(len(comparison["durations"]), 5) self.assertTrue(all(1.5 <= duration <= 8.1 for duration in comparison["durations"])) self.assertEqual(len(self.requests()), 2) self.assert_diarization(True) self.assert_cleaned() def test_bare_command_compares_voices_without_a_known_speaker_count(self): for duration in (30, 611): with self.subTest(duration=duration): self.reset_logs() result = self.run_cli(mode="reconcile", duration=duration) self.assertEqual(result.returncode, 0, result.stderr) expected = "[Speaker 0]\nAlpha.\n\n[Speaker 0]\nAgain.\n\n[Speaker 1]\nBeta.\n" if duration > 600: expected += "\n[Speaker 0]\nAlphaLate.\n\n[Speaker 1]\nBetaLate.\n" self.assertEqual(result.stdout, expected) self.assert_automatic_speaker_comparison() self.assertEqual(len(self.requests()), 2 if duration > 600 else 1) self.assertEqual(len(self.speaker_request()["samples"]), 5 if duration > 600 else 3) self.assert_diarization(True) self.assert_cleaned() def test_default_srt_compares_voices_without_speaker_options(self): result = self.run_cli("--srt", mode="reconcile", duration=30) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, ( "1\n00:00:01,000 --> 00:00:03,000\nAlpha.\n\n" "2\n00:00:06,000 --> 00:00:08,000\nAgain.\n\n" "3\n00:00:12,000 --> 00:00:14,000\nBeta.\n\n" )) self.assert_automatic_speaker_comparison() self.assertEqual(len(self.requests()), 1) self.assert_diarization(True) self.assert_cleaned() def test_voice_comparison_model_override_without_count_and_single_request(self): for model, count in (("test/audio-model", ()), ("~audio-model", ("--speakers", "2"))): with self.subTest(model=model): self.reset_logs() result = self.run_cli(*count, "--speaker-model", model, mode="reconcile", duration=30) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, ( "[Speaker 0]\nAlpha.\n\n[Speaker 0]\nAgain.\n\n[Speaker 1]\nBeta.\n" )) self.assertEqual(self.speaker_request()["request"]["model"], model) self.assertEqual(len(self.requests()), 1) self.assert_diarization(True) self.assert_cleaned() def test_uncertain_voice_assignment_retains_a_separate_id(self): result = self.run_cli("--speakers", "2", "--chunk-minutes", "1", mode="reconcile-null") self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, ( "[Speaker 0]\nAlpha.\n\n[Speaker 1]\nAgain.\n\n[Speaker 2]\nBeta.\n\n" "[Speaker 1]\nAlphaLate.\n\n[Speaker 2]\nBetaLate.\n" )) self.assertIn("Warning:", result.stderr) self.assert_cleaned() def test_voice_comparison_srt_preserves_words_and_original_timestamps(self): serial = self.run_cli("--jobs", "1", "--speakers", "2", "--srt", "--chunk-minutes", "1", mode="reconcile") self.assertEqual(serial.returncode, 0, serial.stderr) self.assertEqual(serial.stdout, ( "1\n00:00:01,000 --> 00:00:03,000\nAlpha.\n\n" "2\n00:00:06,000 --> 00:00:08,000\nAgain.\n\n" "3\n00:00:12,000 --> 00:00:14,000\nBeta.\n\n" "4\n00:01:00,000 --> 00:01:02,000\nAlphaLate.\n\n" "5\n00:01:05,000 --> 00:01:07,000\nBetaLate.\n\n" )) self.assert_cleaned() self.reset_logs() parallel = self.run_cli("--jobs", "2", "--speakers", "2", "--srt", "--chunk-minutes", "1", mode="reconcile") self.assertEqual(parallel.returncode, 0, parallel.stderr) self.assertEqual(parallel.stdout, serial.stdout) self.assert_cleaned() def test_failed_voice_comparison_emits_no_partial_transcript_and_cleans_up(self): for mode in ("reconcile-http-error", "reconcile-bad-json", "reconcile-missing", "reconcile-duplicate", "reconcile-unknown-sample", "reconcile-invalid-speaker", "reconcile-truncated"): with self.subTest(mode=mode): self.reset_logs() result = self.run_cli("--chunk-minutes", "1", mode=mode) self.assertNotEqual(result.returncode, 0) self.assertEqual(result.stdout, "") self.assertTrue(self.log.with_suffix(".speakers.json").exists()) self.assertTrue(result.stderr) self.assert_cleaned() def test_missing_or_too_short_voice_samples_skip_comparison(self): for mode, expected in (("ok", "Hello.\n"), ("speakers", "[Speaker 0]\nHello\n\n[Speaker 1]\nthere\n")): with self.subTest(mode=mode): self.reset_logs() result = self.run_cli("--speakers", "2", mode=mode, duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, expected) self.assertIn("Warning:", result.stderr) self.assertFalse(self.log.with_suffix(".speakers.json").exists()) self.assert_cleaned() def test_invalid_speaker_options_fail_before_credentials_or_provider_access(self): credential_access = self.path / "credential-access" (self.bin / "op").write_text(f"#!/bin/sh\ntouch '{credential_access}'\nexit 1\n") options = [("--speakers", value) for value in ("0", "65", "-1", "1.5", "abc", "99999999999999999999")] options += [("--speaker-model", value) for value in ("", "-bad", "bad model", "a\nb")] options += [("--no-diarization", "--speakers", "2"), ("--speakers", "2", "--no-diarization"), ("--no-diarization", "--speaker-model", "test/audio"), ("--speaker-model", "test/audio", "--no-diarization")] for args in options: with self.subTest(options=args): result = self.run_cli(*args, duration=1) self.assertNotEqual(result.returncode, 0) self.assertEqual(result.stdout, "") self.assertIn("--speaker", result.stderr) for option in ("--speakers", "--speaker-model"): result = subprocess.run([str(ROOT / "transcribe"), option], env=self.env, capture_output=True, text=True, timeout=5) self.assertNotEqual(result.returncode, 0) self.assertIn(option, result.stderr) self.assertFalse(credential_access.exists()) self.assertFalse(self.log.exists()) def test_interrupting_voice_comparison_stops_descendants_and_cleans_up(self): make_audio(self.audio, 30) process = subprocess.Popen( [str(ROOT / "transcribe"), "--speakers", "2", str(self.audio)], env=dict(self.env, TRANSCRIBE_TEST_MODE="reconcile-blocked"), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True, ) try: deadline = time.monotonic() + 8 while time.monotonic() < deadline: if any(e["event"] == "child" and e["index"] == -1 for e in self.events()): break time.sleep(0.01) else: self.fail("Voice comparison did not start") process.send_signal(signal.SIGTERM) stdout, stderr = process.communicate(timeout=5) self.assertEqual(process.returncode, 128 + signal.SIGTERM, stderr) self.assertEqual(stdout, "") self.assert_processes_stopped() self.assert_cleaned() finally: if process.poll() is None: process.kill() for event in self.events(): try: os.killpg(event["group"], signal.SIGKILL) except ProcessLookupError: pass process.communicate(timeout=5) if __name__ == "__main__": unittest.main()