import json import os from pathlib import Path import shutil import signal import subprocess import tempfile import time import unittest from test_chunks import make_audio ROOT = Path(__file__).resolve().parent TOOLS = ("bash", "ffmpeg", "ffprobe", "jq", "base64", "mktemp", "realpath") @unittest.skipUnless(all(shutil.which(tool) for tool in TOOLS), "CLI media dependencies missing") class TranscribeTests(unittest.TestCase): def setUp(self): self.directory = tempfile.TemporaryDirectory() self.addCleanup(self.directory.cleanup) self.path = Path(self.directory.name) self.bin = self.path / "bin" self.bin.mkdir() for name in ("op", "curl"): fake = self.bin / name shutil.copyfile(ROOT / "tests" / "fake_provider.py", fake) fake.chmod(0o755) self.log = self.path / "requests.jsonl" self.env = dict(os.environ, PATH=f"{self.bin}{os.pathsep}{os.environ['PATH']}", TRANSCRIBE_TEST_LOG=str(self.log), TRANSCRIBE_TEST_MODE="ok") self.audio = self.path / "input with spaces.wav" def run_cli(self, *args, mode="ok", duration=70, env_overrides=None): make_audio(self.audio, duration) return subprocess.run([str(ROOT / "transcribe"), *args, str(self.audio)], env=dict(self.env, TRANSCRIBE_TEST_MODE=mode, **(env_overrides or {})), capture_output=True, text=True, timeout=30) def requests(self): return sorted((json.loads(line) for line in self.log.read_text().splitlines()), key=lambda request: request["index"]) def events(self): path = self.log.with_suffix(".events.jsonl") return [json.loads(line) for line in path.read_text().splitlines()] if path.exists() else [] def reset_logs(self): self.log.unlink(missing_ok=True) self.log.with_suffix(".events.jsonl").unlink(missing_ok=True) def assert_processes_stopped(self): for event in self.events(): for pid in (event["pid"], event.get("child_pid")): if pid is None: continue try: # A killed descendant can await reaping by init; it must # not still be running or holding our temporary files. state = Path(f"/proc/{pid}/stat").read_text().rsplit(")", 1)[1].split()[0] self.assertEqual(state, "Z", f"Process {pid} survived worker cleanup") except FileNotFoundError: pass def assert_cleaned(self): self.assertTrue(all(not Path(r["staging"]).exists() for r in self.requests())) def test_parallel_default_and_job_limit_with_ordered_output(self): for options, limit, duration in (((), 8, 500), (("--jobs", "2"), 2, 180), (("--jobs", "1"), 1, 130)): with self.subTest(options=options): self.reset_logs() result = self.run_cli( *options, "--chunk-minutes", "1", mode="parallel", duration=duration, env_overrides={"TRANSCRIBE_TEST_BARRIER": str(limit)}, ) self.assertEqual(result.returncode, 0, result.stderr) requests = self.requests() self.assertEqual(result.stdout.split(), [f"Part{i}." for i in range(len(requests))]) active = maximum = 0 events = sorted(self.events(), key=lambda event: event["time"]) for event in events: active += 1 if event["event"] == "start" else -1 maximum = max(maximum, active) self.assertEqual(active, 0) self.assertEqual(maximum, limit) self.assertEqual(len({r["work"] for r in requests}), len(requests)) if limit > 1: first_finished = next(e for e in events if e["event"] == "finish") self.assertNotEqual(first_finished["index"], 0) refilled = next(e["time"] for e in events if e["event"] == "start" and e["index"] == limit) slow_finished = next(e["time"] for e in events if e["event"] == "finish" and e["index"] == 0) self.assertLess(refilled, slow_finished) self.assert_processes_stopped() self.assert_cleaned() def test_parallel_srt_matches_sequential_output(self): serial = self.run_cli("--jobs", "1", "--chunk-minutes", "1", "--srt", mode="parallel", duration=130) self.assertEqual(serial.returncode, 0, serial.stderr) self.reset_logs() parallel = self.run_cli( "--jobs", "3", "--chunk-minutes", "1", "--srt", mode="parallel", duration=130, env_overrides={"TRANSCRIBE_TEST_BARRIER": "3"}, ) self.assertEqual(parallel.returncode, 0, parallel.stderr) self.assertEqual(parallel.stdout, serial.stdout) self.assertEqual([block.splitlines()[0] for block in parallel.stdout.strip().split("\n\n")], ["1", "2", "3"]) self.assert_cleaned() def test_parallel_failure_cancels_workers_and_their_children(self): result = self.run_cli( "--jobs", "2", "--chunk-minutes", "1", mode="parallel-error", duration=180, env_overrides={"TRANSCRIBE_TEST_BARRIER": "2"}, ) self.assertNotEqual(result.returncode, 0) self.assertEqual(result.stdout, "") self.assertIn("simulated concurrent failure", result.stderr) self.assertIn("stopping remaining workers", result.stderr) self.assertEqual({e["index"] for e in self.events() if e["event"] == "start"}, {0, 1}) self.assertTrue(any(e["event"] == "child" for e in self.events())) self.assert_processes_stopped() self.assert_cleaned() def test_interruptions_stop_workers_before_removing_staging(self): for signum in (signal.SIGINT, signal.SIGTERM): with self.subTest(signal=signum): self.reset_logs() make_audio(self.audio, 180) process = subprocess.Popen( [str(ROOT / "transcribe"), "--jobs", "2", "--chunk-minutes", "1", str(self.audio)], env=dict(self.env, TRANSCRIBE_TEST_MODE="blocked"), stdout=subprocess.PIPE, stderr=subprocess.PIPE, text=True, start_new_session=True, ) try: deadline = time.monotonic() + 5 while time.monotonic() < deadline: events = self.events() if (sum(e["event"] == "start" for e in events) >= 2 and any(e["event"] == "child" for e in events)): break time.sleep(0.01) else: self.fail("Workers did not start") process.send_signal(signum) stdout, stderr = process.communicate(timeout=5) self.assertEqual(process.returncode, 128 + signum, stderr) self.assertEqual(stdout, "") self.assert_processes_stopped() self.assertTrue(all(not Path(e["staging"]).exists() for e in self.events())) finally: if process.poll() is None: process.kill() for event in self.events(): try: os.killpg(event["group"], signal.SIGKILL) except ProcessLookupError: pass process.communicate(timeout=5) def test_immediate_encoder_failure_is_not_lost_by_worker_waiting(self): encoder = self.bin / "ffmpeg" encoder.write_text( "#!/usr/bin/env python3\nimport os, sys\n" "if '-ss' in sys.argv:\n sys.exit(42)\n" f"os.execv({shutil.which('ffmpeg')!r}, ['ffmpeg', *sys.argv[1:]])\n" ) encoder.chmod(0o755) result = self.run_cli("--jobs", "64", "--chunk-minutes", "1", duration=500) self.assertEqual(result.returncode, 42, result.stderr) self.assertEqual(result.stdout, "") self.assertIn("stopping remaining workers", result.stderr) self.assertFalse(self.log.exists()) def test_invalid_jobs_fail_before_provider_access(self): for value in ("0", "65", "-1", "1.5", "abc", "9999999999999999999999"): with self.subTest(value=value): result = self.run_cli("--jobs", value, duration=1) self.assertNotEqual(result.returncode, 0) self.assertIn("integer between 1 and 64", result.stderr) result = subprocess.run([str(ROOT / "transcribe"), "--jobs"], env=self.env, capture_output=True, text=True, timeout=5) self.assertNotEqual(result.returncode, 0) self.assertIn("integer between 1 and 64", result.stderr) self.assertFalse(self.log.exists()) def assert_diarization(self, enabled): for request in self.requests(): self.assertEqual(request["provider"], { "options": {"azure": {"diarization": {"enabled": enabled}}}, }) self.assertIs( request["provider"]["options"]["azure"]["diarization"]["enabled"], enabled, ) def test_long_text_uploads_bounded_mp3_and_stitches_once(self): result = self.run_cli("--chunk-minutes", "1") self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, "Hello.\n\nboundary continues.\n\nLast.\n") self.assertIn("chunk 2/2", result.stderr) requests = self.requests() self.assertEqual(len(requests), 2) self.assertTrue(all(r["upload_duration"] <= 60 for r in requests)) self.assertAlmostEqual(requests[0]["chunk"]["end"] - requests[1]["chunk"]["start"], 6) self.assert_diarization(True) self.assert_cleaned() def test_long_srt_uses_original_timeline_and_continuous_numbers(self): result = self.run_cli("--srt", "--chunk-minutes", "1") self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, ( "1\n00:00:01,000 --> 00:00:01,400\nHello.\n\n" "2\n00:00:56,700 --> 00:00:57,800\nboundary continues.\n\n" "3\n00:01:05,000 --> 00:01:05,400\nLast.\n\n" )) self.assert_diarization(True) self.assert_cleaned() def test_short_text_without_diarization_retains_provider_line_breaks(self): result = self.run_cli("--no-diarization", duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, "Provider's original formatting.\nNext line.\n") self.assertEqual(len(self.requests()), 1) self.assertIsNone(self.requests()[0]["response_format"]) self.assert_diarization(False) self.assert_cleaned() def test_short_text_requests_and_prints_speaker_labels(self): result = self.run_cli(mode="speakers", duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, "[Speaker 0]\nHello\n\n[Speaker 1]\nthere\n") self.assertEqual(self.requests()[0]["response_format"], "verbose_json") self.assertEqual(self.requests()[0]["timestamp_granularities"], ["segment", "word"]) self.assert_diarization(True) self.assertEqual(result.stderr, "") self.assert_cleaned() def test_chunked_text_has_speaker_markers_without_chunk_details(self): result = self.run_cli("--chunk-minutes", "1", mode="speakers") self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, ( "[Speaker 0]\nHello.\n\n" "[Speaker 0]\nboundary\n\n" "[Speaker 1]\ncontinues.\n\n" "[Speaker 1]\nLast.\n" )) self.assertIn("speaker continuity is uncertain", result.stderr) self.assertNotIn("Chunk", result.stdout) self.assert_cleaned() def test_missing_speaker_labels_warn_instead_of_implying_detection_worked(self): for output in ((), ("--srt",)): with self.subTest(output=output): result = self.run_cli(*output, duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("returned no speaker labels", result.stderr) self.assertNotIn("[Speaker", result.stdout) result = self.run_cli(mode="partial-speakers", duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertIn("some words without speaker labels", result.stderr) self.assertEqual(result.stdout, "[Speaker 0]\nHello\n\n[Speaker unknown]\nthere\n") def test_short_srt_still_requests_word_timestamps(self): result = self.run_cli("--srt", duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, "1\n00:00:01,000 --> 00:00:01,400\nHello.\n\n") self.assertEqual(self.requests()[0]["response_format"], "verbose_json") self.assertEqual(self.requests()[0]["timestamp_granularities"], ["segment", "word"]) self.assert_diarization(True) def test_no_diarization_applies_to_text_srt_and_every_chunk(self): for output in ((), ("--srt",)): for chunked in (False, True): with self.subTest(output=output, chunked=chunked): self.log.unlink(missing_ok=True) options = ("--chunk-minutes", "1") if chunked else () result = self.run_cli( *output, *options, "--no-diarization", duration=70 if chunked else 5, ) self.assertEqual(result.returncode, 0, result.stderr) self.assertTrue(result.stdout) self.assertEqual(len(self.requests()), 2 if chunked else 1) self.assert_diarization(False) for request in self.requests(): if output or chunked: self.assertEqual(request["response_format"], "verbose_json") self.assertEqual(request["timestamp_granularities"], ["segment", "word"]) else: self.assertIsNone(request["response_format"]) self.assertIsNone(request["timestamp_granularities"]) self.assertLessEqual(request["upload_duration"], 60) self.assert_cleaned() # The opt-out must also preserve a limit specified after it. self.log.unlink() result = self.run_cli("--no-diarization", "--chunk-minutes", "1") self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(len(self.requests()), 2) self.assert_diarization(False) self.assert_cleaned() def test_default_upload_limits_depend_on_diarization(self): for options, limit in (((), 600), (("--no-diarization",), 1800)): with self.subTest(options=options): self.log.unlink(missing_ok=True) result = self.run_cli(*options, duration=limit + 10) self.assertEqual(result.returncode, 0, result.stderr) requests = self.requests() self.assertEqual(len(requests), 2) self.assertGreaterEqual(requests[0]["upload_duration"], limit - 0.1) self.assertTrue(all(r["upload_duration"] <= limit for r in requests)) self.assert_diarization(not options) self.assert_cleaned() def test_diarization_breaks_srt_at_speaker_changes(self): result = self.run_cli("--srt", mode="speakers", duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, ( "1\n00:00:01,000 --> 00:00:01,400\nHello\n\n" "2\n00:00:01,500 --> 00:00:01,900\nthere\n\n" )) self.assert_diarization(True) self.log.unlink() result = self.run_cli("--no-diarization", "--srt", mode="speakers", duration=5) self.assertEqual(result.returncode, 0, result.stderr) self.assertEqual(result.stdout, "1\n00:00:01,000 --> 00:00:01,900\nHello there\n\n") self.assert_diarization(False) def test_failed_later_chunk_has_no_partial_stdout_and_cleans_up(self): for mode in ("http-error", "bad-json", "api-error", "missing-words"): with self.subTest(mode=mode): result = self.run_cli("--chunk-minutes", "1", mode=mode) self.assertNotEqual(result.returncode, 0) self.assertEqual(result.stdout, "") self.assertTrue(result.stderr) self.assert_cleaned() def test_invalid_limits_fail_before_provider_access(self): for value in ("0", "31", "abc", "1.5", "-1"): with self.subTest(value=value): result = self.run_cli("--chunk-minutes", value, duration=1) self.assertNotEqual(result.returncode, 0) self.assertIn("integer between 1 and 30", result.stderr) self.assertFalse(self.log.exists()) if __name__ == "__main__": unittest.main()