Transcribe Anything

# ffmpeg — audio extraction, preprocessing, silence detection
brew install ffmpeg

# yt-dlp — downloading video/audio from URLs (optional but recommended)
brew install yt-dlp

# Default ASR backend — OpenAI's whisper CLI
pip3 install --break-system-packages openai-whisper

# Apple Silicon accelerated ASR, preferred when available
uv tool install mlx-whisper

# curl_cffi — prevents OAuth errors when downloading private videos
pip3 install --break-system-packages curl_cffi

# faster-whisper — 4x faster than whisper, built-in VAD silence skipping, lower memory
# Best local backend for long files (1hr+)
pip3 install --break-system-packages faster-whisper

# pyannote.audio — preferred local diarization when HF_TOKEN is available
uv venv --python 3.11 .venv-pyannote
uv pip install --python .venv-pyannote/bin/python "pyannote.audio" soundfile

# whisperX — adds precise word-level timestamps; diarization requires HF_TOKEN
# Bundles faster-whisper + pyannote alignment/diarization
pip3 install --break-system-packages whisperx

# Prefer Python 3.11 for NeMo audio dependencies.
uv venv --python 3.11 .venv-nemo
source .venv-nemo/bin/activate
uv pip install "nemo_toolkit[asr]"
git clone --depth 1 https://github.com/NVIDIA/NeMo.git /tmp/NeMo

# insanely-fast-whisper — batched GPU inference, 10-20x faster on NVIDIA GPUs
pip3 install --break-system-packages insanely-fast-whisper

# whisper.cpp — C++ native with Metal acceleration on Apple Silicon
# Best option if you want to avoid Python entirely
brew install whisper-cpp

# OpenAI — best accuracy with gpt-4o-transcribe ($0.006/min)
export OPENAI_API_KEY=sk-...

# Groq — cheapest and fastest cloud option ($0.00004/min with turbo)
export GROQ_API_KEY=gsk_...

# Deepgram — best cloud diarization ($0.0043/min)
export DEEPGRAM_API_KEY=...

# AssemblyAI — cloud diarization + auto-chapters ($0.0062/min)
export ASSEMBLYAI_API_KEY=...

# Gemini — handles 9.5hr files natively, flexible prompting
export GEMINI_API_KEY=...

echo "=== Required ==="
which ffmpeg && echo "ffmpeg: OK" || echo "ffmpeg: MISSING (brew install ffmpeg)"
which whisper && echo "whisper: OK" || echo "whisper: MISSING (pip3 install --break-system-packages openai-whisper)"

echo ""
echo "=== Local Backends ==="
which whisperx && echo "whisperx: OK" || echo "whisperx: not installed"
python3 -c "import faster_whisper" 2>/dev/null && echo "faster-whisper: OK" || echo "faster-whisper: not installed"
python3 -c "import pyannote.audio" 2>/dev/null && echo "pyannote.audio: OK" || echo "pyannote.audio: not installed in system Python"
python3 -c "import nemo.collections.asr" 2>/dev/null && echo "NeMo ASR: OK" || echo "NeMo ASR: not installed"
which mlx_whisper && echo "mlx_whisper: OK" || echo "mlx_whisper: not installed"
which insanely-fast-whisper 2>/dev/null && echo "insanely-fast-whisper: OK" || echo "insanely-fast-whisper: not installed"
which whisper-cpp 2>/dev/null && echo "whisper.cpp: OK" || echo "whisper.cpp: not installed"

echo ""
echo "=== Cloud APIs ==="
[ -n "$OPENAI_API_KEY" ] && echo "OpenAI: configured" || echo "OpenAI: not set"
[ -n "$GROQ_API_KEY" ] && echo "Groq: configured" || echo "Groq: not set"
[ -n "$DEEPGRAM_API_KEY" ] && echo "Deepgram: configured" || echo "Deepgram: not set"
[ -n "$ASSEMBLYAI_API_KEY" ] && echo "AssemblyAI: configured" || echo "AssemblyAI: not set"
[ -n "$GEMINI_API_KEY" ] && echo "Gemini: configured" || echo "Gemini: not set"

echo ""
echo "=== Optional ==="
which yt-dlp && echo "yt-dlp: OK" || echo "yt-dlp: not installed (brew install yt-dlp)"
python3 -c "import curl_cffi" 2>/dev/null && echo "curl_cffi: OK" || echo "curl_cffi: not installed"
[ -n "$HF_TOKEN" ] && echo "HF token: configured (whisperX/pyannote diarization ready if model terms accepted)" || echo "HF token: not set (use NeMo or cloud API for diarization)"

yt-dlp -x --audio-format wav -o "%(title)s.%(ext)s" "{url}"

# Extract audio from video (or re-encode audio) to ASR-optimal format
ffmpeg -i "{input}" \
  -vn \
  -ac 1 \
  -ar 16000 \
  -acodec pcm_s16le \
  -af "highpass=f=80,lowpass=f=8000,loudnorm=I=-16:TP=-1.5:LRA=11" \
  "{output_stem}_preprocessed.wav"

DURATION=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "{preprocessed_file}" | cut -d. -f1)
echo "Duration: ${DURATION}s ($((DURATION / 3600))h $(((DURATION % 3600) / 60))m)"

# Detect silent regions (informational — see what we're working with)
ffmpeg -i "{preprocessed_file}" \
  -af silencedetect=noise=-30dB:d=2.0 \
  -f null - 2>&1 | grep -c "silence_end"
# Shows number of silence gaps >= 2 seconds

from faster_whisper import WhisperModel

model = WhisperModel("large-v3", device="cpu", compute_type="int8")
segments, info = model.transcribe(
    "preprocessed.wav",
    language="en",
    word_timestamps=True,
    vad_filter=True,
    vad_parameters=dict(
        min_silence_duration_ms=1000,
        speech_pad_ms=400,
        threshold=0.5,
    ),
    condition_on_previous_text=False,  # prevents hallucination cascades on long files
)

# Remove silences longer than 2s, keeping 0.3s padding
ffmpeg -i "{preprocessed_file}" \
  -af "silenceremove=start_periods=1:start_threshold=-30dB:stop_periods=-1:stop_duration=2.0:stop_threshold=-30dB" \
  "{output_stem}_trimmed.wav"

# Then transcribe the trimmed file
whisper "{output_stem}_trimmed.wav" --model turbo --language en \
  --condition_on_previous_text False \
  --word_timestamps True \
  --output_format json \
  --output_dir ./

# Compress to opus (smallest format for speech) — 1 hour ≈ 14MB
ffmpeg -i "{preprocessed_file}" -ac 1 -ar 16000 -c:a libopus -b:a 32k "{output_stem}.ogg"

# Check file size
SIZE_MB=$(du -m "{output_stem}.ogg" | cut -f1)
echo "File size: ${SIZE_MB}MB"

# Split into ~20-minute chunks on silence boundaries
# (under 25MB each at opus 32kbps)
ffmpeg -i "{output_stem}.ogg" \
  -f segment \
  -segment_time 1200 \
  -c copy \
  "{output_stem}_chunk_%03d.ogg"

# Get duration of each chunk for timestamp reassembly
for f in {output_stem}_chunk_*.ogg; do
  dur=$(ffprobe -v error -show_entries format=duration -of csv=p=0 "$f")
  echo "$f: ${dur}s"
done

whisper "{input_file}" \
  --model turbo \
  --language en \
  --output_format json \
  --output_dir "{output_dir}" \
  --word_timestamps True \
  --condition_on_previous_text False \
  --fp16 False

whisperx "{input_file}" \
  --model large-v3 \
  --language en \
  --diarize \
  --min_speakers 2 \
  --max_speakers 6 \
  --hf_token "{HF_TOKEN}" \
  --compute_type int8 \
  --output_dir "{output_dir}" \
  --output_format json

whisperx "{input_file}" \
  --model large-v3 \
  --language en \
  --compute_type int8 \
  --output_dir "{output_dir}" \
  --output_format json

import os
import torch
from pyannote.audio import Pipeline

audio = "{preprocessed_wav}"  # mono 16 kHz WAV
num_speakers = 5              # set when known; improves clustering
token = os.environ["HF_TOKEN"]

pipeline = Pipeline.from_pretrained(
    "pyannote/speaker-diarization-community-1",
    token=token,
)

if not torch.cuda.is_available():
    raise RuntimeError(
        "No CUDA GPU available. Do not run long diarization on CPU unless the user explicitly opts in."
    )
device = "cuda"
pipeline.to(torch.device(device))

output = pipeline(audio, num_speakers=num_speakers)
diarization = output.speaker_diarization
exclusive = output.exclusive_speaker_diarization

with open("diarization.rttm", "w") as f:
    diarization.write_rttm(f)

with open("diarization-exclusive.rttm", "w") as f:
    exclusive.write_rttm(f)

AUDIO="{input_file}"
WORK="{output_dir}/nemo-diarization"
NEMO_REPO="${NEMO_REPO:-/tmp/NeMo}"
mkdir -p "$WORK/input" "$WORK/output"

ffmpeg -y -i "$AUDIO" -ac 1 -ar 16000 "$WORK/input/audio.16k.wav"
DURATION=$(ffprobe -v error -show_entries format=duration -of default=nw=1:nk=1 "$WORK/input/audio.16k.wav")

python - <<PY
import json
manifest = {
    "audio_filepath": "$WORK/input/audio.16k.wav",
    "offset": 0,
    "duration": float("$DURATION"),
    "label": "infer",
    "text": "-",
    "num_speakers": 2,
    "rttm_filepath": None,
    "uem_filepath": None,
}
open("$WORK/input/manifest.json", "w").write(json.dumps(manifest) + "\\n")
PY

test -d "$NEMO_REPO" || git clone --depth 1 https://github.com/NVIDIA/NeMo.git "$NEMO_REPO"

uv venv --python 3.11 "$WORK/.venv"
source "$WORK/.venv/bin/activate"
uv pip install "nemo_toolkit[asr]"

python "$NEMO_REPO/examples/speaker_tasks/diarization/clustering_diarizer/offline_diar_infer.py" \
  diarizer.manifest_filepath="$WORK/input/manifest.json" \
  diarizer.out_dir="$WORK/output" \
  diarizer.speaker_embeddings.model_path=titanet_large \
  diarizer.vad.model_path=vad_multilingual_marblenet \
  diarizer.speaker_embeddings.parameters.save_embeddings=False \
  diarizer.clustering.parameters.oracle_num_speakers=True

find "$WORK/output" -name '*.rttm' -print