ddidderr/transcribe
1
0 Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat!: use whole-recording Pyannote speaker diarization

Browse Source 
Sampled Gemini voice matching could leave one person under separate IDs
across transcription chunks. Use hosted Precision-2 on the complete
recording and align its exclusive speaker timeline with MAI word timing.
Preserve transcription text, order, and timestamps; leave ambiguous word
assignments unknown instead of merging identities to satisfy a count.

Upload lossless FLAC, bound polling, retry transient status reads against
the same job, and validate results before emitting stdout. Keep optional
speaker-count and hosted model overrides. Disable MAI diarization and use
30-minute transcription uploads in both modes. Respect environment keys,
keep authentication out of process arguments, and save optional diagnostics
without audio, credentials, or signed upload URLs.

Test Plan:
- python3 -B -m unittest -v: all 101 regression tests passed, including
  mocked provider upload, polling, alignment, failure, and cleanup paths.
- bash -n transcribe and shellcheck transcribe: passed.
- git diff --cached --check: passed.
- Prepared the 74-minute input locally and verified identical decoded PCM
  hashes for the normalized WAV and full-recording lossless FLAC.
- Live API testing was explicitly deferred by the user; mocked results do
  not establish speaker accuracy on the recording.

BREAKING CHANGE: Default diarization requires PYANNOTE_API_KEY or its
1Password reference. --speaker-model accepts hosted precision-2 or
community-1 instead of OpenRouter audio-chat model IDs.
This commit is contained in:
ddidderr committed 2026-09-12 11:18:30 +02:00
1 parent d7827d39f5
commit 3042b7743b
7 files changed
+1336 -864
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

tests/fake_provider.py
+139 -67
View File
@@ -9,20 +9,37 @@ import subprocess
import sys
import time
log_path = Path(os.environ["TRANSCRIBE_TEST_LOG"])
if Path(sys.argv[0]).name == "op":
print("test-api-key")
with log_path.with_suffix(".credentials.jsonl").open("a") as log:
log.write(json.dumps({"args": sys.argv[1:]}) + "\n")
if os.environ.get("TRANSCRIBE_TEST_OP_FAIL"):
sys.exit(1)
print("test-op-api-key")
sys.exit(0)
args = sys.argv[1:]
request_path = Path(args[args.index("--data-binary") + 1][1:])
request = json.loads(request_path.read_text())
output = Path(args[args.index("-o") + 1])
url = next(value for value in args if value.startswith("https://"))
headers = {}
for position, value in enumerate(args):
if value in ("-H", "--header"):
value = args[position + 1]
header_lines = Path(value[1:]).read_text().splitlines() if value.startswith("@") else [value]
for header_line in header_lines:
key, _, header_value = header_line.partition(":")
headers[key.lower()] = header_value.strip()
request = None
request_path = None
if "--data-binary" in args:
request_path = Path(args[args.index("--data-binary") + 1][1:])
request = json.loads(request_path.read_text())
mode = os.environ.get("TRANSCRIBE_TEST_MODE", "ok")
events_path = Path(os.environ["TRANSCRIBE_TEST_LOG"]).with_suffix(".events.jsonl")
is_comparison = "https://openrouter.ai/api/v1/chat/completions" in args
index = -1 if is_comparison else int(output.stem.split("-")[1])
staging = output.parent.parent if is_comparison else output.parent
events_path = log_path.with_suffix(".events.jsonl")
is_pyannote = "openrouter.ai" not in url
index = -1 if is_pyannote else int(output.stem.split("-")[1])
staging = next(parent for parent in (output.parent, *output.parents)
if (parent / "chunks.json").is_file())
def event(kind, **extra):
@@ -32,68 +49,121 @@ def event(kind, **extra):
log.write(json.dumps(record) + "\n")
def reply(value, status=0, http_status=200):
if "--write-out" in args:
print(str(http_status), end="")
output.write_text(json.dumps(value))
event("finish")
sys.exit(status)
event("start")
if is_comparison:
samples = json.loads((output.parent / "samples.json").read_text())
manifest = json.loads((staging / "chunks.json").read_text())
late_start = round(manifest[0]["end"]) if len(manifest) > 1 else 60
content = request["messages"][-1]["content"]
audio_parts = [part["input_audio"] for part in content if part["type"] == "input_audio"]
assert len(content) == 2 * len(samples) == 2 * len(audio_parts)
for number, sample in enumerate(samples):
assert content[number * 2] == {"type": "text", "text": f"Sample ID {sample['id']}"}
assert content[number * 2 + 1]["type"] == "input_audio"
assert request["response_format"]["type"] == "json_schema"
assert request["response_format"]["json_schema"]["strict"] is True
prompt = "\n".join(part["text"] for message in request["messages"]
for part in (message["content"] if isinstance(message["content"], list)
else [{"type": "text", "text": message["content"]}])
if part["type"] == "text")
assert not any(word in prompt for word in ("Alpha.", "Again.", "Beta.", "AlphaLate.", "BetaLate."))
durations = []
for number, part in enumerate(audio_parts):
assert part["format"] == "mp3"
audio = output.parent / f"checked-{number}.mp3"
audio.write_bytes(base64.b64decode(part["data"], validate=True))
if is_pyannote:
pyannote_log = log_path.with_suffix(".pyannote.jsonl")
previous = [json.loads(line) for line in pyannote_log.read_text().splitlines()] if pyannote_log.exists() else []
record = {"url": url, "request": request, "headers": headers, "staging": str(staging)}
if url == "https://api.pyannote.ai/v1/media/input":
stage = "media"
assert request["url"].startswith("media://") and request["url"].endswith(".flac")
response = {"url": "https://storage.test.invalid/upload?signature=fixture-signed-url"}
elif url.startswith("https://storage.test.invalid/"):
stage = "upload"
assert not headers.get("authorization"), "API bearer leaked to signed storage URL"
upload = Path(args[args.index("--upload-file") + 1])
info = json.loads(subprocess.check_output([
"ffprobe", "-v", "error", "-show_entries",
"format=duration:stream=codec_name,sample_rate,channels", "-of", "json", str(audio),
"format=duration:stream=codec_name,sample_rate,channels", "-of", "json", str(upload),
], text=True))
assert info["streams"][0] == {"codec_name": "mp3", "sample_rate": "16000", "channels": 1}
durations.append(float(info["format"]["duration"]))
assert 1.5 <= durations[-1] <= 8.1
Path(os.environ["TRANSCRIBE_TEST_LOG"]).with_suffix(".speakers.json").write_text(json.dumps({
"request": request, "samples": samples, "durations": durations, "staging": str(staging),
}))
if mode == "reconcile-blocked":
assert info["streams"][0] == {"codec_name": "flac", "sample_rate": "16000", "channels": 1}
record["upload_duration"] = float(info["format"]["duration"])
record["stream"] = info["streams"][0]
response = {}
elif url == "https://api.pyannote.ai/v1/diarize":
stage = "submit"
assert request["model"] in ("precision-2", "community-1")
assert request["url"] == next(item["request"]["url"] for item in previous if item["stage"] == "media")
assert request["exclusive"] is True
assert request["transcription"] is False
response = {"jobId": "test-job-42"}
elif url == "https://api.pyannote.ai/v1/jobs/test-job-42":
stage = "poll"
duration = next(item["upload_duration"] for item in previous if item["stage"] == "upload")
manifest = json.loads((staging / "chunks.json").read_text())
if mode == "silence":
turns = []
elif mode.startswith("reconcile"):
late_start = round(manifest[0]["end"]) if len(manifest) > 1 else 60
turns = [(0, min(duration, 10), "voice_a"), (10, min(duration, 16), "voice_b"),
(late_start, min(duration, late_start + 3), "voice_a"),
(late_start + 4, duration, "voice_b")]
elif mode in ("speakers", "partial-speakers"):
boundary = 1.45 if len(manifest) == 1 else 57.3
turns = [(0, boundary, "voice_a"), (boundary, duration, "voice_b")]
else:
turns = [(0, duration, "voice_a")]
timeline = [{"start": start, "end": end, "speaker": speaker}
for start, end, speaker in turns if start < end]
# Standard diarization intentionally disagrees: clients must select the
# exclusive timeline requested for word-level speaker attribution.
response = {"jobId": "test-job-42", "status": "succeeded", "output": {
"exclusiveDiarization": timeline,
"diarization": [{"start": 0, "end": duration, "speaker": "wrong_timeline"}],
}}
if mode == "reconcile-diagnostics-metadata":
response["request"] = {"url": "media://fixture-recording.flac", "key": "test-pyannote-key"}
response["output"]["debug"] = "https://storage.test.invalid/?signature=fixture-signed-url"
response["output"]["exclusiveDiarization"][0]["request"] = "test-openrouter-key"
response["output"]["exclusiveDiarization"][0]["confidence"] = {"voice_a": 99.0}
response["output"]["diarization"][0]["debug"] = "fixture-signed-url"
poll_count = sum(item["stage"] == "poll" for item in previous)
if mode == "pyannote-lifecycle" and poll_count < 2:
response = {"jobId": "test-job-42", "status": ("created", "running")[poll_count]}
elif mode in ("pyannote-failed", "pyannote-canceled"):
response = {"jobId": "test-job-42", "status": mode.removeprefix("pyannote-"),
"error": "simulated diarization failure"}
elif mode == "pyannote-missing-timeline":
response["output"].pop("exclusiveDiarization")
elif mode == "pyannote-malformed-timeline":
response["output"]["exclusiveDiarization"][0]["end"] = "not a timestamp"
elif mode == "pyannote-empty-timeline":
response["output"]["exclusiveDiarization"] = []
elif mode == "pyannote-wrong-job":
response["jobId"] = "different-job"
elif mode == "pyannote-output-error":
response["output"]["error"] = "simulated incomplete diarization output"
elif mode == "pyannote-partial-timeline":
response["output"]["exclusiveDiarization"] = [{"start": 0, "end": 2, "speaker": "voice_a"}]
else:
raise AssertionError(f"Unexpected curl URL: {url}")
assert args[args.index("--request") + 1] == {
"media": "POST", "upload": "PUT", "submit": "POST", "poll": "GET",
}[stage]
if stage != "upload":
assert headers.get("authorization") in ("Bearer test-pyannote-key", "Bearer test-op-api-key")
record["stage"] = stage
record["response"] = response
with pyannote_log.open("a") as log:
log.write(json.dumps(record) + "\n")
if mode == f"pyannote-{stage}-http-error":
reply({"error": f"simulated {stage} failure"}, http_status=401)
if stage == "poll" and mode == "pyannote-transient-poll" and poll_count == 0:
reply({"error": "simulated transient poll failure"}, status=22)
if stage == "upload" and mode == "pyannote-transport-error":
reply({"error": "simulated transport failure"}, status=22)
if stage == "poll" and mode == "pyannote-bad-json":
if "--write-out" in args:
print("200", end="")
output.write_text("not json")
event("finish")
sys.exit(0)
if stage == "poll" and mode == "pyannote-blocked":
child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"])
event("child", child_pid=child.pid)
time.sleep(30)
if mode == "reconcile-http-error":
output.write_text('{"error":{"message":"simulated comparison failure"}}')
sys.exit(22)
if mode == "reconcile-bad-json":
output.write_text("not json")
sys.exit(0)
assignments = [{"sample": sample["id"], "speaker": 0 if round(sample["start"]) in (1, 6, late_start) else 1}
for sample in samples]
if mode == "reconcile-missing":
assignments.pop()
elif mode == "reconcile-duplicate":
assignments.append(assignments[0])
elif mode == "reconcile-unknown-sample":
assignments[0]["sample"] = "not-a-sample"
elif mode == "reconcile-invalid-speaker":
assignments[0]["speaker"] = -1
elif mode == "reconcile-null":
assignments[0]["speaker"] = None
output.write_text(json.dumps({"choices": [{
"finish_reason": "length" if mode == "reconcile-truncated" else "stop",
"message": {"content": json.dumps({"assignments": assignments})},
}]}))
event("finish")
sys.exit(0)
reply(response)
assert url == "https://openrouter.ai/api/v1/audio/transcriptions"
assert headers.get("authorization") in ("Bearer test-openrouter-key", "Bearer test-op-api-key")
manifest = json.loads((output.parent / "chunks.json").read_text())
chunk = manifest[index]
if mode in ("parallel-error", "blocked") and index == 0:
@@ -121,6 +191,7 @@ with open(os.environ["TRANSCRIBE_TEST_LOG"], "a") as log:
"provider": request.get("provider"),
"response_format": request.get("response_format"),
"timestamp_granularities": request.get("timestamp_granularities"),
"authorization": headers.get("authorization"),
}) + "\n")
if mode in ("parallel", "parallel-error"):
@@ -173,11 +244,12 @@ else:
{"word": "Hello", "start": 1, "end": 1.4},
{"word": "there", "start": 1.5, "end": 1.9},
]
if request["provider"]["options"]["azure"]["diarization"]["enabled"]:
for speaker, word in enumerate(words):
if mode == "speakers" or speaker == 0:
word["speaker"] = (speaker if len(manifest) == 1 else
int(word["start"] + chunk["start"] >= 57.3))
for speaker, word in enumerate(words):
if mode == "speakers" or speaker == 0:
word["speaker"] = (speaker if len(manifest) == 1 else
int(word["start"] + chunk["start"] >= 57.3))
if mode == "silence":
words = []
if mode == "parallel":
middle = (chunk["end"] - chunk["start"]) / 2
words = [{"word": f"Part{index}.", "start": middle, "end": middle + 0.4}]