ddidderr
1d03dcd587
Process up to eight chunks concurrently by default and add --jobs 1..64, with --jobs 1 retaining sequential operation. Give each worker separate encoding and request files, release payloads after completion, and merge responses by original chunk index regardless of completion order. Use a bounded worker scheduler that handles jobs finishing before wait is called. Stop scheduling on an observed worker failure and terminate worker process groups on failure or interruption before removing temporary files. Include jobs interrupted between launch and scheduler registration in cleanup. Preserve all-or-nothing stdout for both text and SRT output. Document the Bash 5.1 requirement, concurrency option, resource usage, and limits of cancelling requests already accepted by a provider. Test Plan: - python3 -B -m unittest -v: all 52 tests passed. - Mocked provider tests verified eight concurrent requests by default, configurable limits, rolling worker replacement, isolated payloads, ordered text, and identical sequential/parallel SRT output. - Verified immediate encoder failure, concurrent API failure, SIGINT and SIGTERM, child-process cancellation, cleanup, and no partial stdout. - bash -n transcribe, ./transcribe --help, and git diff --cached --check: passed. - No live provider request; credentials are unavailable in this environment.
124 lines
5.0 KiB
Python
124 lines
5.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Test-only replacements for op/curl; no network or real credentials."""
|
|
|
|
import base64
|
|
import json
|
|
import os
|
|
from pathlib import Path
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
|
|
if Path(sys.argv[0]).name == "op":
|
|
print("test-api-key")
|
|
sys.exit(0)
|
|
|
|
args = sys.argv[1:]
|
|
request_path = Path(args[args.index("--data-binary") + 1][1:])
|
|
request = json.loads(request_path.read_text())
|
|
output = Path(args[args.index("-o") + 1])
|
|
index = int(output.stem.split("-")[1])
|
|
manifest = json.loads((output.parent / "chunks.json").read_text())
|
|
chunk = manifest[index]
|
|
mode = os.environ.get("TRANSCRIBE_TEST_MODE", "ok")
|
|
events_path = Path(os.environ["TRANSCRIBE_TEST_LOG"]).with_suffix(".events.jsonl")
|
|
|
|
|
|
def event(kind, **extra):
|
|
record = dict(event=kind, index=index, pid=os.getpid(), group=os.getpgrp(),
|
|
time=time.monotonic(), staging=str(output.parent), **extra)
|
|
with events_path.open("a") as log:
|
|
log.write(json.dumps(record) + "\n")
|
|
|
|
|
|
event("start")
|
|
if mode in ("parallel-error", "blocked") and index == 0:
|
|
child = subprocess.Popen([sys.executable, "-c", "import time; time.sleep(30)"])
|
|
event("child", child_pid=child.pid)
|
|
assert request["model"] == "microsoft/mai-transcribe-2"
|
|
assert request["input_audio"]["format"] == "mp3"
|
|
if len(manifest) > 1:
|
|
assert request["response_format"] == "verbose_json"
|
|
assert request["timestamp_granularities"] == ["segment", "word"]
|
|
|
|
audio = output.parent / f"uploaded-{index}.mp3"
|
|
audio.write_bytes(base64.b64decode(request["input_audio"]["data"], validate=True))
|
|
info = json.loads(subprocess.check_output([
|
|
"ffprobe", "-v", "error", "-show_entries",
|
|
"format=duration:stream=codec_name,sample_rate,channels", "-of", "json", str(audio),
|
|
], text=True))
|
|
assert info["streams"][0] == {"codec_name": "mp3", "sample_rate": "16000", "channels": 1}
|
|
assert abs(float(info["format"]["duration"]) - (chunk["end"] - chunk["start"])) < 0.1
|
|
with open(os.environ["TRANSCRIBE_TEST_LOG"], "a") as log:
|
|
log.write(json.dumps({
|
|
"index": index, "chunk": chunk, "staging": str(output.parent),
|
|
"work": str(request_path.parent),
|
|
"upload_duration": float(info["format"]["duration"]),
|
|
"provider": request.get("provider"),
|
|
"response_format": request.get("response_format"),
|
|
"timestamp_granularities": request.get("timestamp_granularities"),
|
|
}) + "\n")
|
|
|
|
if mode in ("parallel", "parallel-error"):
|
|
deadline = time.monotonic() + 5
|
|
barrier = int(os.environ.get("TRANSCRIBE_TEST_BARRIER", "1"))
|
|
while True:
|
|
events = [json.loads(line) for line in events_path.read_text().splitlines()]
|
|
enough = sum(e["event"] == "start" for e in events) >= barrier
|
|
child_ready = mode != "parallel-error" or any(e["event"] == "child" for e in events)
|
|
if enough and child_ready:
|
|
break
|
|
if time.monotonic() >= deadline:
|
|
raise RuntimeError("Worker concurrency barrier timed out")
|
|
time.sleep(0.01)
|
|
if mode == "parallel-error" and index == 1:
|
|
output.write_text('{"error":{"message":"simulated concurrent failure"}}')
|
|
event("finish")
|
|
sys.exit(22)
|
|
time.sleep(30 if mode == "parallel-error" else (0.8 if index == 0 else 0.05))
|
|
elif mode == "blocked":
|
|
time.sleep(30)
|
|
|
|
if index == 1:
|
|
if mode == "http-error":
|
|
output.write_text('{"error":{"message":"simulated timeout"}}')
|
|
sys.exit(22)
|
|
if mode == "bad-json":
|
|
output.write_text("not json")
|
|
sys.exit(0)
|
|
if mode == "api-error":
|
|
output.write_text('{"error":{"message":"simulated API error"},"text":""}')
|
|
sys.exit(0)
|
|
if mode == "missing-words":
|
|
output.write_text('{"text":"speech without timestamps"}')
|
|
sys.exit(0)
|
|
|
|
if "response_format" not in request:
|
|
response = {"text": "Provider's original formatting.\nNext line."}
|
|
else:
|
|
words = [
|
|
{"word": text, "start": start - chunk["start"], "end": end - chunk["start"]}
|
|
for text, start, end in [
|
|
("Hello.", 1, 1.4), ("boundary", 56.7, 57.1),
|
|
("continues.", 57.4, 57.8), ("Last.", 65, 65.4),
|
|
] if start >= chunk["start"] and end <= chunk["end"]
|
|
]
|
|
if mode in ("speakers", "partial-speakers"):
|
|
if len(manifest) == 1:
|
|
words = [
|
|
{"word": "Hello", "start": 1, "end": 1.4},
|
|
{"word": "there", "start": 1.5, "end": 1.9},
|
|
]
|
|
if request["provider"]["options"]["azure"]["diarization"]["enabled"]:
|
|
for speaker, word in enumerate(words):
|
|
if mode == "speakers" or speaker == 0:
|
|
word["speaker"] = (speaker if len(manifest) == 1 else
|
|
int(word["start"] + chunk["start"] >= 57.3))
|
|
if mode == "parallel":
|
|
middle = (chunk["end"] - chunk["start"]) / 2
|
|
words = [{"word": f"Part{index}.", "start": middle, "end": middle + 0.4}]
|
|
response = {"text": " ".join(w["word"] for w in words), "words": words}
|
|
output.write_text(json.dumps(response))
|
|
event("finish")
|