Compare commits
22 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
 ZGY
|
616d3d3e63 | ||
|
ZGY
|
07c937b295 | ||
|
Dennis Huang
|
b9fbaecf87 | ||
|
Dennis Huang
|
53115a200b | ||
|
Labmem-Zhouyx
|
30118a3352 | ||
|
ZGY
|
e1e9a6a788 | ||
|
ZGY
|
2b26f1ff8d | ||
|
Rafael Figuereo
|
09fce8e9db | ||
|
Eliseu Silva
|
5e293dee2f | ||
|
Eliseu Silva
|
5ef0b3db4c | ||
|
Eliseu Silva
|
b567707deb | ||
|
Labmem-Zhouyx
|
856d2fc2a8 | ||
|
Labmem-Zhouyx
|
8e3f039615 | ||
|
Labmem-Zhouyx
|
43acad2ac8 | ||
|
cccpro12
|
87af7d0854 | ||
|
zhuxiaoxuhit
|
ae8fa7e0b4 | ||
|
Eliseu Silva
|
9f1548b631 | ||
|
ZGY
|
f3b65758c6 | ||
|
muyleanging
|
e08754bcc0 | ||
|
muyleanging
|
36a1378e53 | ||
|
muyleanging
|
4d94dd3f54 | ||
|
muyleanging
|
05fe0cb08d |
+2
-1
@@ -1,6 +1,7 @@
|
||||
launch.json
|
||||
.venv/
|
||||
__pycache__
|
||||
voxcpm.egg-info
|
||||
.DS_Store
|
||||
./pretrained_models/
|
||||
app_local.py
|
||||
app_local.py
|
||||
|
||||
@@ -10,8 +10,8 @@
|
||||
<a href="https://voxcpm.readthedocs.io/en/latest/"><img src="https://img.shields.io/badge/Docs-ReadTheDocs-8CA1AF" alt="Documentation"></a>
|
||||
<a href="https://huggingface.co/openbmb/VoxCPM2"><img src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-VoxCPM2-yellow" alt="Hugging Face"></a>
|
||||
<a href="https://modelscope.cn/models/OpenBMB/VoxCPM2"><img src="https://img.shields.io/badge/ModelScope-VoxCPM2-purple" alt="ModelScope"></a>
|
||||
<a href="https://openbmb.github.io/voxcpm2-demopage/"><img src="https://img.shields.io/badge/DemoPage-Audio Samples-red"></a>
|
||||
|
||||
<a href="https://openbmb.github.io/voxcpm2-demopage/"><img src="https://img.shields.io/badge/DemoPage-Audio Samples-red" alt="DemoPage"></a>
|
||||
<a href="https://arxiv.org/abs/2606.06928"><img src="https://img.shields.io/badge/arXiv-VoxCPM2%20Technical%20Report-red" alt="VoxCPM2 Technical Report"></a>
|
||||
</p>
|
||||
|
||||
<div align="center">
|
||||
@@ -49,20 +49,17 @@ VoxCPM is a **tokenizer-free** Text-to-Speech system that directly generates con
|
||||
- ⚡ **Real-Time Streaming** — RTF as low as ~0.3 on NVIDIA RTX 4090, and ~0.13 accelerated by [Nano-vLLM](https://github.com/a710128/nanovllm-voxcpm) or [vLLM-Omni](https://github.com/vllm-project/vllm-omni) — official vLLM omni-modal serving for VoxCPM2 with PagedAttention and an OpenAI-compatible API
|
||||
- 📜 **Fully Open-Source & Commercial-Ready** — Weights and code released under the [Apache-2.0](LICENSE) license, free for commercial use
|
||||
|
||||
|
||||
<summary><b>🌍 Supported Languages (30)</b></summary>
|
||||
<br>
|
||||
**🌍 Supported Languages (30)**
|
||||
Arabic, Burmese, Chinese, Danish, Dutch, English, Finnish, French, German, Greek, Hebrew, Hindi, Indonesian, Italian, Japanese, Khmer, Korean, Lao, Malay, Norwegian, Polish, Portuguese, Russian, Spanish, Swahili, Swedish, Tagalog, Thai, Turkish, Vietnamese
|
||||
|
||||
Chinese Dialect: 四川话, 粤语, 吴语, 东北话, 河南话, 陕西话, 山东话, 天津话, 闽南话
|
||||
|
||||
|
||||
### News
|
||||
|
||||
* **[2026.04]** 🔥 We release **VoxCPM2** — 2B, 30 languages, Voice Design & Controllable Voice Cloning, 48kHz audio output! [Weights](https://huggingface.co/openbmb/VoxCPM2) | [Docs](https://voxcpm.readthedocs.io/en/latest/) | [Playground](https://huggingface.co/spaces/OpenBMB/VoxCPM-Demo)
|
||||
* **[2025.12]** 🎉 Open-source **VoxCPM1.5** [weights](https://huggingface.co/openbmb/VoxCPM1.5) with SFT & LoRA fine-tuning. (**🏆 #1 GitHub Trending**)
|
||||
* **[2025.09]** 🔥 Release VoxCPM [Technical Report](https://arxiv.org/abs/2509.24650).
|
||||
* **[2025.09]** 🎉 Open-source **VoxCPM-0.5B** [weights](https://huggingface.co/openbmb/VoxCPM-0.5B) (**🏆 #1 HuggingFace Trending**)
|
||||
- **[2026.04]** 🔥 We release **VoxCPM2** — 2B, 30 languages, Voice Design & Controllable Voice Cloning, 48kHz audio output! [Weights](https://huggingface.co/openbmb/VoxCPM2) | [Docs](https://voxcpm.readthedocs.io/en/latest/) | [Playground](https://huggingface.co/spaces/OpenBMB/VoxCPM-Demo) | [Technical Report](https://arxiv.org/abs/2606.06928)
|
||||
- **[2025.12]** 🎉 Open-source **VoxCPM1.5** [weights](https://huggingface.co/openbmb/VoxCPM1.5) with SFT & LoRA fine-tuning. (**🏆 #1 GitHub Trending**)
|
||||
- **[2025.09]** 🔥 Release VoxCPM [Technical Report](https://arxiv.org/abs/2509.24650).
|
||||
- **[2025.09]** 🎉 Open-source **VoxCPM-0.5B** [weights](https://huggingface.co/openbmb/VoxCPM-0.5B) (**🏆 #1 HuggingFace Trending**)
|
||||
|
||||
---
|
||||
|
||||
@@ -74,6 +71,7 @@ Chinese Dialect: 四川话, 粤语, 吴语, 东北话, 河南话, 陕西话, 山
|
||||
- [CLI Usage](#cli-usage)
|
||||
- [Web Demo](#web-demo)
|
||||
- [Production Deployment](#-production-deployment-nano-vllm)
|
||||
- [On-Device Inference (llama.cpp-omni)](#-on-device-inference-llamacpp-omni)
|
||||
- [Models & Versions](#-models--versions)
|
||||
- [Performance](#-performance)
|
||||
- [Fine-tuning](#%EF%B8%8F-fine-tuning)
|
||||
@@ -111,6 +109,7 @@ wav = model.generate(
|
||||
text="VoxCPM2 is the current recommended release for realistic multilingual speech synthesis.",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
||||
print("saved: demo.wav")
|
||||
@@ -134,6 +133,7 @@ wav = model.generate(
|
||||
text="VoxCPM2 is the current recommended release for realistic multilingual speech synthesis.",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
||||
```
|
||||
@@ -147,6 +147,7 @@ wav = model.generate(
|
||||
text="(A young woman, gentle and sweet voice)Hello, welcome to VoxCPM2!",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("voice_design.wav", wav, model.tts_model.sample_rate)
|
||||
```
|
||||
@@ -167,6 +168,7 @@ wav = model.generate(
|
||||
reference_wav_path="path/to/voice.wav",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("controllable_clone.wav", wav, model.tts_model.sample_rate)
|
||||
```
|
||||
@@ -185,8 +187,7 @@ wav = model.generate(
|
||||
sf.write("hifi_clone.wav", wav, model.tts_model.sample_rate)
|
||||
```
|
||||
|
||||
<details>
|
||||
<summary><b>🔄 Streaming API</b></summary>
|
||||
**🔄 Streaming API**
|
||||
|
||||
```python
|
||||
import numpy as np
|
||||
@@ -199,7 +200,8 @@ for chunk in model.generate_streaming(
|
||||
wav = np.concatenate(chunks)
|
||||
sf.write("streaming.wav", wav, model.tts_model.sample_rate)
|
||||
```
|
||||
</details>
|
||||
|
||||
|
||||
|
||||
### CLI Usage
|
||||
|
||||
@@ -213,6 +215,7 @@ voxcpm design \
|
||||
voxcpm design \
|
||||
--text "VoxCPM2 brings studio-quality multilingual speech synthesis." \
|
||||
--control "Young female voice, warm and gentle, slightly smiling" \
|
||||
--seed 42 \
|
||||
--output out.wav
|
||||
|
||||
# Voice cloning (reference audio)
|
||||
@@ -232,6 +235,23 @@ voxcpm clone \
|
||||
# Batch processing
|
||||
voxcpm batch --input examples/input.txt --output-dir outs
|
||||
|
||||
# Optional post-generation timestamps with stable-ts
|
||||
pip install "voxcpm[timestamps]"
|
||||
voxcpm design \
|
||||
--text "VoxCPM2 brings studio-quality multilingual speech synthesis." \
|
||||
--output out.wav \
|
||||
--timestamps \
|
||||
--timestamp-level word \
|
||||
--timestamp-language en
|
||||
|
||||
# Character timestamps are best-effort and are derived from word alignment
|
||||
voxcpm design \
|
||||
--text "欢迎使用 VoxCPM2。" \
|
||||
--output out.wav \
|
||||
--timestamps \
|
||||
--timestamp-level char \
|
||||
--timestamp-language zh
|
||||
|
||||
# Help
|
||||
voxcpm --help
|
||||
```
|
||||
@@ -242,9 +262,17 @@ voxcpm --help
|
||||
python app.py --port 8808 # then open in browser: http://localhost:8808
|
||||
```
|
||||
|
||||
Use `--device` to choose the runtime device:
|
||||
|
||||
```bash
|
||||
python app.py --device auto
|
||||
```
|
||||
|
||||
Supported values are `auto`, `cpu`, `mps`, `cuda`, and `cuda:N`. On Apple Silicon Macs, `auto` uses MPS when available.
|
||||
|
||||
### 🚢 Production Deployment (Nano-vLLM)
|
||||
|
||||
For high-throughput serving, use [**Nano-vLLM-VoxCPM**](https://github.com/a710128/nanovllm-voxcpm) — a dedicated inference engine built on Nano-vLLM with concurrent request support and an async API.
|
||||
For high-throughput serving, use **[Nano-vLLM-VoxCPM](https://github.com/a710128/nanovllm-voxcpm)** — a dedicated inference engine built on Nano-vLLM with concurrent request support and an async API.
|
||||
|
||||
```bash
|
||||
pip install nano-vllm-voxcpm
|
||||
@@ -264,7 +292,7 @@ server.stop()
|
||||
|
||||
### 🏭 Production Serving (vLLM-Omni)
|
||||
|
||||
For production multi-tenant deployments, use [**vLLM-Omni**](https://github.com/vllm-project/vllm-omni) — the official vLLM project's omni-modal extension with native **VoxCPM2** support. PagedAttention KV cache, continuous batching, and a drop-in **OpenAI-compatible** `/v1/audio/speech` endpoint.
|
||||
For production multi-tenant deployments, use **[vLLM-Omni](https://github.com/vllm-project/vllm-omni)** — the official vLLM project's omni-modal extension with native **VoxCPM2** support. PagedAttention KV cache, continuous batching, and a drop-in **OpenAI-compatible** `/v1/audio/speech` endpoint.
|
||||
|
||||
```bash
|
||||
# Install from source (latest main — vllm-omni is rapidly evolving)
|
||||
@@ -288,34 +316,73 @@ curl http://localhost:8000/v1/audio/speech \
|
||||
|
||||
> Built on the upstream vLLM scheduler, with batched concurrent requests, streaming chunk delivery, and multi-GPU deployment out of the box. See the [VoxCPM2 example](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/voxcpm2) for full deployment recipes.
|
||||
|
||||
### 📱 On-Device Inference (llama.cpp-omni)
|
||||
|
||||
For on-device / edge deployment without Python, use **[llama.cpp-omni](https://github.com/tc-mb/llama.cpp-omni)** — a high-performance C++ inference engine built on llama.cpp, with native VoxCPM2 GGUF support on **CPU / Metal / CUDA / Vulkan**.
|
||||
|
||||
**1. Download GGUF weights** from [HuggingFace](https://huggingface.co/DennisHuang648/VoxCPM2-GGUF) | [ModelScope](https://modelscope.cn/models/DennisHuang/VoxCPM2-GGUF) — you need one **BaseLM** (F16 or Q8_0) + the **Acoustic** file. Q8_0 halves the download with negligible quality loss.
|
||||
|
||||
**2. Build**
|
||||
|
||||
```bash
|
||||
git clone https://github.com/tc-mb/llama.cpp-omni.git && cd llama.cpp-omni
|
||||
cmake -B build -DCMAKE_BUILD_TYPE=Release
|
||||
cmake --build build --target voxcpm2-cli -j
|
||||
```
|
||||
|
||||
> CMake auto-detects Metal (macOS) or CUDA (Linux with NVIDIA GPU).
|
||||
|
||||
**3. Run**
|
||||
|
||||
```bash
|
||||
# Basic TTS
|
||||
./build/bin/voxcpm2-cli \
|
||||
-t "Hello, this is VoxCPM2 running through llama.cpp-omni." \
|
||||
-o output.wav VoxCPM2-BaseLM-Q8_0.gguf VoxCPM2-Acoustic-F16.gguf
|
||||
|
||||
# Voice cloning (reference audio)
|
||||
./build/bin/voxcpm2-cli \
|
||||
-t "Cloned voice." -r speaker.wav -o clone.wav \
|
||||
VoxCPM2-BaseLM-Q8_0.gguf VoxCPM2-Acoustic-F16.gguf
|
||||
|
||||
# Ultimate cloning (reference audio + transcript)
|
||||
./build/bin/voxcpm2-cli \
|
||||
-t "Target text." --prompt-wav speaker.wav --prompt-text "transcript of speaker.wav" \
|
||||
-o clone.wav VoxCPM2-BaseLM-Q8_0.gguf VoxCPM2-Acoustic-F16.gguf
|
||||
```
|
||||
|
||||
> **RTF ~1.76 (Q8_0) on Apple M4 Pro / Metal.** Key flags: `--cfg` (guidance scale), `--timesteps` (CFM steps), `--seed`, `--temperature`, `--stream`. See the [llama.cpp-omni repo](https://github.com/tc-mb/llama.cpp-omni) and [GGUF weights page](https://huggingface.co/DennisHuang648/VoxCPM2-GGUF) for full details.
|
||||
|
||||
> **Full parameter reference, multi-scenario examples, and voice cloning tips →** [Quick Start Guide](https://voxcpm.readthedocs.io/en/latest/quickstart.html) | [Usage Guide](https://voxcpm.readthedocs.io/en/latest/usage_guide.html) | [Cookbook](https://voxcpm.readthedocs.io/en/latest/cookbook.html)
|
||||
|
||||
---
|
||||
|
||||
## 📦 Models & Versions
|
||||
|
||||
| | **VoxCPM2** | **VoxCPM1.5** | **VoxCPM-0.5B** |
|
||||
|---|:---:|:---:|:---:|
|
||||
| **Status** | 🟢 Latest | Stable | Legacy |
|
||||
| **Backbone Parameters** | 2B | 0.6B | 0.5B |
|
||||
| **Audio Sample Rate** | 48kHz | 44.1kHz | 16kHz |
|
||||
| **LM Token Rate** | 6.25Hz | 6.25Hz | 12.5Hz |
|
||||
| **Languages** | 30 | 2 (zh, en) | 2 (zh, en) |
|
||||
| **Cloning Mode** | Isolated Reference & Continuation | Continuation only | Continuation only |
|
||||
| **Voice Design** | ✅ | — | — |
|
||||
| **Controllable Voice Cloning** | ✅ | — | — |
|
||||
| **SFT / LoRA** | ✅ | ✅ | ✅ |
|
||||
| **RTF (RTX 4090)** | ~0.30 | ~0.15 | ~0.17 |
|
||||
| **RTF in Nano-VLLM (RTX 4090)** | ~0.13 | ~0.08 | ~0.10 |
|
||||
| **VRAM** | ~8 GB | ~6 GB | ~5 GB |
|
||||
| **Weights** | [🤗 HF](https://huggingface.co/openbmb/VoxCPM2) / [MS](https://modelscope.cn/models/OpenBMB/VoxCPM2) | [🤗 HF](https://huggingface.co/openbmb/VoxCPM1.5) / [MS](https://modelscope.cn/models/OpenBMB/VoxCPM1.5) | [🤗 HF](https://huggingface.co/openbmb/VoxCPM-0.5B) / [MS](https://modelscope.cn/models/OpenBMB/VoxCPM-0.5B) |
|
||||
| **Technical Report** | Coming soon | — | [arXiv](https://arxiv.org/abs/2509.24650) [ICLR 2026](https://openreview.net/forum?id=h5KLpGoqzC) |
|
||||
| **Demo Page** | [Audio Samples](https://openbmb.github.io/voxcpm2-demopage) | — | [Audio Samples](https://openbmb.github.io/VoxCPM-demopage) |
|
||||
|
||||
| | **VoxCPM2** | **VoxCPM1.5** | **VoxCPM-0.5B** |
|
||||
| ------------------------------- | ---------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------ |
|
||||
| **Status** | 🟢 Latest | Stable | Legacy |
|
||||
| **Backbone Parameters** | 2B | 0.6B | 0.5B |
|
||||
| **Audio Sample Rate** | 48kHz | 44.1kHz | 16kHz |
|
||||
| **LM Token Rate** | 6.25Hz | 6.25Hz | 12.5Hz |
|
||||
| **Languages** | 30 | 2 (zh, en) | 2 (zh, en) |
|
||||
| **Cloning Mode** | Isolated Reference & Continuation | Continuation only | Continuation only |
|
||||
| **Voice Design** | ✅ | — | — |
|
||||
| **Controllable Voice Cloning** | ✅ | — | — |
|
||||
| **SFT / LoRA** | ✅ | ✅ | ✅ |
|
||||
| **RTF (RTX 4090)** | ~0.30 | ~0.15 | ~0.17 |
|
||||
| **RTF in Nano-VLLM (RTX 4090)** | ~0.13 | ~0.08 | ~0.10 |
|
||||
| **VRAM** | ~8 GB | ~6 GB | ~5 GB |
|
||||
| **Weights** | [🤗 HF](https://huggingface.co/openbmb/VoxCPM2) / [MS](https://modelscope.cn/models/OpenBMB/VoxCPM2) | [🤗 HF](https://huggingface.co/openbmb/VoxCPM1.5) / [MS](https://modelscope.cn/models/OpenBMB/VoxCPM1.5) | [🤗 HF](https://huggingface.co/openbmb/VoxCPM-0.5B) / [MS](https://modelscope.cn/models/OpenBMB/VoxCPM-0.5B) |
|
||||
| **Technical Report** | [arXiv](https://arxiv.org/abs/2606.06928) | — | [arXiv](https://arxiv.org/abs/2509.24650) [ICLR 2026](https://openreview.net/forum?id=h5KLpGoqzC) |
|
||||
| **Demo Page** | [Audio Samples](https://openbmb.github.io/voxcpm2-demopage) | — | [Audio Samples](https://openbmb.github.io/VoxCPM-demopage) |
|
||||
|
||||
|
||||
VoxCPM2 is built on a **tokenizer-free, diffusion autoregressive** paradigm. The model operates entirely in the latent space of **AudioVAE V2**, following a four-stage pipeline: **LocEnc → TSLM → RALM → LocDiT**, enabling rich expressiveness and 48kHz native audio output.
|
||||
|
||||
<div align="center">
|
||||
<img src="assets/voxcpm_model.png" alt="VoxCPM2 Model Architecture" width="90%">
|
||||
<img src="assets/voxcpm2_model.png" alt="VoxCPM2 Model Architecture" width="90%">
|
||||
</div>
|
||||
|
||||
> For full architectural details, VoxCPM2-specific upgrades, and a model comparison table, see the [Architecture Design](https://voxcpm.readthedocs.io/en/latest/models/architecture.html).
|
||||
@@ -328,186 +395,187 @@ VoxCPM2 achieves state-of-the-art or comparable results on public zero-shot and
|
||||
|
||||
### Seed-TTS-eval
|
||||
|
||||
<details>
|
||||
<summary><b>Seed-TTS-eval WER(⬇)&SIM(⬆) Results (click to expand)</b></summary>
|
||||
|
||||
| Model | Parameters | Open-Source | test-EN | | test-ZH | | test-Hard | |
|
||||
|------|------|------|:------------:|:--:|:------------:|:--:|:-------------:|:--:|
|
||||
| | | | WER/%⬇ | SIM/%⬆| CER/%⬇| SIM/%⬆ | CER/%⬇ | SIM/%⬆ |
|
||||
| MegaTTS3 | 0.5B | ❌ | 2.79 | 77.1 | 1.52 | 79.0 | - | - |
|
||||
| DiTAR | 0.6B | ❌ | 1.69 | 73.5 | 1.02 | 75.3 | - | - |
|
||||
| CosyVoice3 | 0.5B | ❌ | 2.02 | 71.8 | 1.16 | 78.0 | 6.08 | 75.8 |
|
||||
| CosyVoice3 | 1.5B | ❌ | 2.22 | 72.0 | 1.12 | 78.1 | 5.83 | 75.8 |
|
||||
| Seed-TTS | - | ❌ | 2.25 | 76.2 | 1.12 | 79.6 | 7.59 | 77.6 |
|
||||
| MiniMax-Speech | - | ❌ | 1.65 | 69.2 | 0.83 | 78.3 | - | - |
|
||||
| F5-TTS | 0.3B | ✅ | 2.00 | 67.0 | 1.53 | 76.0 | 8.67 | 71.3 |
|
||||
| MaskGCT | 1B | ✅ | 2.62 | 71.7 | 2.27 | 77.4 | - | - |
|
||||
| CosyVoice | 0.3B | ✅ | 4.29 | 60.9 | 3.63 | 72.3 | 11.75 | 70.9 |
|
||||
| CosyVoice2 | 0.5B | ✅ | 3.09 | 65.9 | 1.38 | 75.7 | 6.83 | 72.4 |
|
||||
| SparkTTS | 0.5B | ✅ | 3.14 | 57.3 | 1.54 | 66.0 | - | - |
|
||||
| FireRedTTS | 0.5B | ✅ | 3.82 | 46.0 | 1.51 | 63.5 | 17.45 | 62.1 |
|
||||
| FireRedTTS-2 | 1.5B | ✅ | 1.95 | 66.5 | 1.14 | 73.6 | - | - |
|
||||
| Qwen2.5-Omni | 7B | ✅ | 2.72 | 63.2 | 1.70 | 75.2 | 7.97 | 74.7 |
|
||||
| Qwen3-Omni | 30B-A3B | ✅ | 1.39 | - | 1.07 | - | - | - |
|
||||
| OpenAudio-s1-mini | 0.5B | ✅ | 1.94 | 55.0 | 1.18 | 68.5 | 23.37 | 64.3 |
|
||||
| IndexTTS2 | 1.5B | ✅ | 2.23 | 70.6 | 1.03 | 76.5 | 7.12 | 75.5 |
|
||||
| VibeVoice | 1.5B | ✅ | 3.04 | 68.9 | 1.16 | 74.4 | - | - |
|
||||
| HiggsAudio-v2 | 3B | ✅ | 2.44 | 67.7 | 1.50 | 74.0 | 55.07 | 65.6 |
|
||||
| VoxCPM-0.5B | 0.6B | ✅ | 1.85 | 72.9 | 0.93 | 77.2 | 8.87 | 73.0 |
|
||||
| VoxCPM1.5 | 0.8B | ✅ | 2.12 | 71.4 | 1.18 | 77.0 | 7.74 | 73.1 |
|
||||
| MOSS-TTS | | ✅ | 1.85 | 73.4 | 1.20 | 78.8 | - | - |
|
||||
| Qwen3-TTS | 1.7B | ✅ | 1.23 | 71.7 | 1.22 | 77.0 | 6.76 | 74.8 |
|
||||
| FishAudio S2 | 4B | ✅ | 0.99 | - | 0.54 | - | 5.99 | - |
|
||||
| LongCat-Audio-DiT | 3.5B | ✅ | 1.50 | 78.6 | 1.09 | 81.8 | 6.04 | 79.7 |
|
||||
| **VoxCPM2** | 2B | ✅ | 1.84 | 75.3 | 0.97| 79.5| 8.13 | 75.3 |
|
||||
</details>
|
||||
**Seed-TTS-eval WER(⬇)&SIM(⬆) Results (click to expand)**
|
||||
|
||||
|
||||
| Model | Parameters | Open-Source | test-EN | | test-ZH | | test-Hard | |
|
||||
| ----------------- | ---------- | ----------- | ------- | ------ | ------- | ------ | --------- | ------ |
|
||||
| | | | WER/%⬇ | SIM/%⬆ | CER/%⬇ | SIM/%⬆ | CER/%⬇ | SIM/%⬆ |
|
||||
| MegaTTS3 | 0.5B | ❌ | 2.79 | 77.1 | 1.52 | 79.0 | - | - |
|
||||
| DiTAR | 0.6B | ❌ | 1.69 | 73.5 | 1.02 | 75.3 | - | - |
|
||||
| CosyVoice3 | 0.5B | ❌ | 2.02 | 71.8 | 1.16 | 78.0 | 6.08 | 75.8 |
|
||||
| CosyVoice3 | 1.5B | ❌ | 2.22 | 72.0 | 1.12 | 78.1 | 5.83 | 75.8 |
|
||||
| Seed-TTS | - | ❌ | 2.25 | 76.2 | 1.12 | 79.6 | 7.59 | 77.6 |
|
||||
| MiniMax-Speech | - | ❌ | 1.65 | 69.2 | 0.83 | 78.3 | - | - |
|
||||
| F5-TTS | 0.3B | ✅ | 2.00 | 67.0 | 1.53 | 76.0 | 8.67 | 71.3 |
|
||||
| MaskGCT | 1B | ✅ | 2.62 | 71.7 | 2.27 | 77.4 | - | - |
|
||||
| CosyVoice | 0.3B | ✅ | 4.29 | 60.9 | 3.63 | 72.3 | 11.75 | 70.9 |
|
||||
| CosyVoice2 | 0.5B | ✅ | 3.09 | 65.9 | 1.38 | 75.7 | 6.83 | 72.4 |
|
||||
| SparkTTS | 0.5B | ✅ | 3.14 | 57.3 | 1.54 | 66.0 | - | - |
|
||||
| FireRedTTS | 0.5B | ✅ | 3.82 | 46.0 | 1.51 | 63.5 | 17.45 | 62.1 |
|
||||
| FireRedTTS-2 | 1.5B | ✅ | 1.95 | 66.5 | 1.14 | 73.6 | - | - |
|
||||
| Qwen2.5-Omni | 7B | ✅ | 2.72 | 63.2 | 1.70 | 75.2 | 7.97 | 74.7 |
|
||||
| Qwen3-Omni | 30B-A3B | ✅ | 1.39 | - | 1.07 | - | - | - |
|
||||
| OpenAudio-s1-mini | 0.5B | ✅ | 1.94 | 55.0 | 1.18 | 68.5 | 23.37 | 64.3 |
|
||||
| IndexTTS2 | 1.5B | ✅ | 2.23 | 70.6 | 1.03 | 76.5 | 7.12 | 75.5 |
|
||||
| VibeVoice | 1.5B | ✅ | 3.04 | 68.9 | 1.16 | 74.4 | - | - |
|
||||
| HiggsAudio-v2 | 3B | ✅ | 2.44 | 67.7 | 1.50 | 74.0 | 55.07 | 65.6 |
|
||||
| VoxCPM-0.5B | 0.6B | ✅ | 1.85 | 72.9 | 0.93 | 77.2 | 8.87 | 73.0 |
|
||||
| VoxCPM1.5 | 0.8B | ✅ | 2.12 | 71.4 | 1.18 | 77.0 | 7.74 | 73.1 |
|
||||
| MOSS-TTS | | ✅ | 1.85 | 73.4 | 1.20 | 78.8 | - | - |
|
||||
| Qwen3-TTS | 1.7B | ✅ | 1.23 | 71.7 | 1.22 | 77.0 | 6.76 | 74.8 |
|
||||
| FishAudio S2 | 4B | ✅ | 0.99 | - | 0.54 | - | 5.99 | - |
|
||||
| LongCat-Audio-DiT | 3.5B | ✅ | 1.50 | 78.6 | 1.09 | 81.8 | 6.04 | 79.7 |
|
||||
| **VoxCPM2** | 2B | ✅ | 1.84 | 75.3 | 0.97 | 79.5 | 8.13 | 75.3 |
|
||||
|
||||
|
||||
|
||||
|
||||
### CV3-eval
|
||||
|
||||
**CV3-eval Multilingual WER/CER(⬇) Results (click to expand)**
|
||||
|
||||
|
||||
| Model | zh | en | hard-zh | hard-en | ja | ko | de | es | fr | it | ru |
|
||||
| --------------- | ---- | ---- | ------- | ------- | ---- | ---- | ---- | ---- | ---- | ---- | ---- |
|
||||
| CosyVoice2 | 4.08 | 6.32 | 12.58 | 11.96 | 9.13 | 19.7 | - | - | - | - | - |
|
||||
| CosyVoice3-1.5B | 3.91 | 4.99 | 9.77 | 10.55 | 7.57 | 5.69 | 6.43 | 4.47 | 11.8 | 10.5 | 6.64 |
|
||||
| Fish Audio S2 | 2.65 | 2.43 | 9.10 | 4.40 | 3.96 | 2.76 | 2.22 | 2.00 | 6.26 | 2.04 | 2.78 |
|
||||
| **VoxCPM2** | 3.65 | 5.00 | 8.55 | 8.48 | 5.96 | 5.69 | 4.77 | 3.80 | 9.85 | 4.25 | 5.21 |
|
||||
|
||||
|
||||
### CV3-eval
|
||||
<details>
|
||||
<summary><b>CV3-eval Multilingual WER/CER(⬇) Results (click to expand)</b></summary>
|
||||
|
||||
| Model | zh | en | hard-zh | hard-en | ja | ko | de | es | fr | it | ru |
|
||||
|-------|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|:--:|
|
||||
| CosyVoice2 | 4.08 | 6.32 | 12.58| 11.96| 9.13 | 19.7 |- | - | - | - | - |
|
||||
| CosyVoice3-1.5B | 3.91 | 4.99 | 9.77 | 10.55 | 7.57 | 5.69 | 6.43 | 4.47 | 11.8 | 10.5 | 6.64 |
|
||||
| Fish Audio S2 | 2.65 | 2.43 | 9.10 | 4.40 | 3.96 | 2.76 | 2.22 | 2.00 | 6.26 | 2.04 | 2.78 |
|
||||
| **VoxCPM2** | 3.65 | 5.00 | 8.55 | 8.48 | 5.96 | 5.69 | 4.77 | 3.80 | 9.85 | 4.25 | 5.21 |
|
||||
</details>
|
||||
|
||||
### MiniMax-Multilingual-Test
|
||||
|
||||
<details>
|
||||
<summary><b>Minimax-MLS-test WER(⬇) Results (click to expand)</b></summary>
|
||||
**Minimax-MLS-test WER(⬇) Results (click to expand)**
|
||||
|
||||
| Language | Minimax | ElevenLabs | Qwen3-TTS | FishAudio S2 | **VoxCPM2** |
|
||||
|----------|:-------:|:----------:|:--------------------:|:------------:|:-----------:|
|
||||
| Arabic | **1.665** | 1.666 | – | 3.500 | 13.046 |
|
||||
| Cantonese | 34.111 | 51.513 | – | **30.670** | 38.584 |
|
||||
| Chinese | 2.252 | 16.026 | 0.928 | **0.730** | 1.136 |
|
||||
| Czech | 3.875 | **2.108** | – | 2.840 | 24.132 |
|
||||
| Dutch | 1.143 | **0.803** | – | 0.990 | 0.913 |
|
||||
| English | 2.164 | 2.339 | **0.934** | 1.620 | 2.289 |
|
||||
| Finnish | 4.666 | 2.964 | – | 3.330 | **2.632** |
|
||||
| French | 4.099 | 5.216 | **2.858** | 3.050 | 4.534 |
|
||||
| German | 1.906 | 0.572 | 1.235 | **0.550** | 0.679 |
|
||||
| Greek | 2.016 | **0.991** | – | 5.740 | 2.844 |
|
||||
| Hindi | 6.962 | **5.827** | – | 14.640 | 19.699 |
|
||||
| Indonesian | 1.237 | **1.059** | – | 1.460 | 1.084 |
|
||||
| Italian | 1.543 | 1.743 | **0.948** | 1.270 | 1.563 |
|
||||
| Japanese | 3.519 | 10.646 | 3.823 | **2.760** | 4.628 |
|
||||
| Korean | 1.747 | 1.865 | 1.755 | **1.180** | 1.962 |
|
||||
| Polish | 1.415 | **0.766** | – | 1.260 | 1.141 |
|
||||
| Portuguese | 1.877 | 1.331 | 1.526 | **1.140** | 1.938 |
|
||||
| Romanian | 2.878 | **1.347** | – | 10.740 | 21.577 |
|
||||
| Russian | 4.281 | 3.878 | 3.212 | **2.400** | 3.634 |
|
||||
| Spanish | 1.029 | 1.084 | 1.126 | **0.910** | 1.438 |
|
||||
| Thai | 2.701 | 73.936 | – | 4.230 | 2.961 |
|
||||
| Turkish | 1.52 | 0.699 | – | 0.870 | 0.817 |
|
||||
| Ukrainian | 1.082 | **0.997** | – | 2.300 | 6.316 |
|
||||
| Vietnamese | **0.88** | 73.415 | – | 7.410 | 3.307 |
|
||||
|
||||
</details>
|
||||
| Language | Minimax | ElevenLabs | Qwen3-TTS | FishAudio S2 | **VoxCPM2** |
|
||||
| ---------- | --------- | ---------- | --------- | ------------ | ----------- |
|
||||
| Arabic | **1.665** | 1.666 | – | 3.500 | 13.046 |
|
||||
| Cantonese | 34.111 | 51.513 | – | **30.670** | 38.584 |
|
||||
| Chinese | 2.252 | 16.026 | 0.928 | **0.730** | 1.136 |
|
||||
| Czech | 3.875 | **2.108** | – | 2.840 | 24.132 |
|
||||
| Dutch | 1.143 | **0.803** | – | 0.990 | 0.913 |
|
||||
| English | 2.164 | 2.339 | **0.934** | 1.620 | 2.289 |
|
||||
| Finnish | 4.666 | 2.964 | – | 3.330 | **2.632** |
|
||||
| French | 4.099 | 5.216 | **2.858** | 3.050 | 4.534 |
|
||||
| German | 1.906 | 0.572 | 1.235 | **0.550** | 0.679 |
|
||||
| Greek | 2.016 | **0.991** | – | 5.740 | 2.844 |
|
||||
| Hindi | 6.962 | **5.827** | – | 14.640 | 19.699 |
|
||||
| Indonesian | 1.237 | **1.059** | – | 1.460 | 1.084 |
|
||||
| Italian | 1.543 | 1.743 | **0.948** | 1.270 | 1.563 |
|
||||
| Japanese | 3.519 | 10.646 | 3.823 | **2.760** | 4.628 |
|
||||
| Korean | 1.747 | 1.865 | 1.755 | **1.180** | 1.962 |
|
||||
| Polish | 1.415 | **0.766** | – | 1.260 | 1.141 |
|
||||
| Portuguese | 1.877 | 1.331 | 1.526 | **1.140** | 1.938 |
|
||||
| Romanian | 2.878 | **1.347** | – | 10.740 | 21.577 |
|
||||
| Russian | 4.281 | 3.878 | 3.212 | **2.400** | 3.634 |
|
||||
| Spanish | 1.029 | 1.084 | 1.126 | **0.910** | 1.438 |
|
||||
| Thai | 2.701 | 73.936 | – | 4.230 | 2.961 |
|
||||
| Turkish | 1.52 | 0.699 | – | 0.870 | 0.817 |
|
||||
| Ukrainian | 1.082 | **0.997** | – | 2.300 | 6.316 |
|
||||
| Vietnamese | **0.88** | 73.415 | – | 7.410 | 3.307 |
|
||||
|
||||
<details>
|
||||
<summary><b>Minimax-MLS-test SIM(⬆) Results (click to expand)</b></summary>
|
||||
|
||||
| Language | Minimax | ElevenLabs | Qwen3-TTS | FishAudio S2 | **VoxCPM2** |
|
||||
|----------|:-------:|:----------:|:--------------------:|:------------:|:-----------:|
|
||||
| Arabic | 73.6 | 70.6 | – | 75.0 | **79.1** |
|
||||
| Cantonese | 77.8 | 67.0 | – | 80.5 | **83.5** |
|
||||
| Chinese | 78.0 | 67.7 | 79.9 | 81.6 | **82.5** |
|
||||
| Czech | 79.6 | 68.5 | – | **79.8** | 78.3 |
|
||||
| Dutch | 73.8 | 68.0 | – | 73.0 | **80.8** |
|
||||
| English | 75.6 | 61.3 | 77.5 | 79.7 | **85.4** |
|
||||
| Finnish | 83.5 | 75.9 | – | 81.9 | **89.0** |
|
||||
| French | 62.8 | 53.5 | 62.8 | 69.8 | **73.5** |
|
||||
| German | 73.3 | 61.4 | 77.5 | 76.7 | **80.3** |
|
||||
| Greek | 82.6 | 73.3 | – | 79.5 | **86.0** |
|
||||
| Hindi | 81.8 | 73.0 | – | 82.1 | **85.6** |
|
||||
| Indonesian | 72.9 | 66.0 | – | 76.3 | **80.0** |
|
||||
| Italian | 69.9 | 57.9 | 81.7 | 74.7 | **78.0** |
|
||||
| Japanese | 77.6 | 73.8 | 78.8 | 79.6 | **82.8** |
|
||||
| Korean | 77.6 | 70.0 | 79.9 | 81.7 | **83.3** |
|
||||
| Polish | 80.2 | 72.9 | – | 81.9 | **88.4** |
|
||||
| Portuguese | 80.5 | 71.1 | 81.7 | 78.1 | **83.7** |
|
||||
| Romanian | **80.9** | 69.9 | – | 73.3 | 79.7 |
|
||||
| Russian | 76.1 | 67.6 | 79.2 | 79.0 | **81.1** |
|
||||
| Spanish | 76.2 | 61.5 | 81.4 | 77.6 | **83.1** |
|
||||
| Thai | 80.0 | 58.8 | – | 78.6 | **84.0** |
|
||||
| Turkish | 77.9 | 59.6 | – | 83.5 | **87.1** |
|
||||
| Ukrainian | 73.0 | 64.7 | – | 74.7 | **79.8** |
|
||||
| Vietnamese | 74.3 | 36.9 | – | 74.0 | **80.6** |
|
||||
|
||||
</details>
|
||||
|
||||
**Minimax-MLS-test SIM(⬆) Results (click to expand)**
|
||||
|
||||
|
||||
| Language | Minimax | ElevenLabs | Qwen3-TTS | FishAudio S2 | **VoxCPM2** |
|
||||
| ---------- | -------- | ---------- | --------- | ------------ | ----------- |
|
||||
| Arabic | 73.6 | 70.6 | – | 75.0 | **79.1** |
|
||||
| Cantonese | 77.8 | 67.0 | – | 80.5 | **83.5** |
|
||||
| Chinese | 78.0 | 67.7 | 79.9 | 81.6 | **82.5** |
|
||||
| Czech | 79.6 | 68.5 | – | **79.8** | 78.3 |
|
||||
| Dutch | 73.8 | 68.0 | – | 73.0 | **80.8** |
|
||||
| English | 75.6 | 61.3 | 77.5 | 79.7 | **85.4** |
|
||||
| Finnish | 83.5 | 75.9 | – | 81.9 | **89.0** |
|
||||
| French | 62.8 | 53.5 | 62.8 | 69.8 | **73.5** |
|
||||
| German | 73.3 | 61.4 | 77.5 | 76.7 | **80.3** |
|
||||
| Greek | 82.6 | 73.3 | – | 79.5 | **86.0** |
|
||||
| Hindi | 81.8 | 73.0 | – | 82.1 | **85.6** |
|
||||
| Indonesian | 72.9 | 66.0 | – | 76.3 | **80.0** |
|
||||
| Italian | 69.9 | 57.9 | 81.7 | 74.7 | **78.0** |
|
||||
| Japanese | 77.6 | 73.8 | 78.8 | 79.6 | **82.8** |
|
||||
| Korean | 77.6 | 70.0 | 79.9 | 81.7 | **83.3** |
|
||||
| Polish | 80.2 | 72.9 | – | 81.9 | **88.4** |
|
||||
| Portuguese | 80.5 | 71.1 | 81.7 | 78.1 | **83.7** |
|
||||
| Romanian | **80.9** | 69.9 | – | 73.3 | 79.7 |
|
||||
| Russian | 76.1 | 67.6 | 79.2 | 79.0 | **81.1** |
|
||||
| Spanish | 76.2 | 61.5 | 81.4 | 77.6 | **83.1** |
|
||||
| Thai | 80.0 | 58.8 | – | 78.6 | **84.0** |
|
||||
| Turkish | 77.9 | 59.6 | – | 83.5 | **87.1** |
|
||||
| Ukrainian | 73.0 | 64.7 | – | 74.7 | **79.8** |
|
||||
| Vietnamese | 74.3 | 36.9 | – | 74.0 | **80.6** |
|
||||
|
||||
|
||||
|
||||
|
||||
### Internal 30-Language ASR Benchmark
|
||||
|
||||
We additionally run an internal multilingual intelligibility benchmark with **30 languages × 500 samples**. ASR transcription is evaluated via **Gemini 3.1 Flash Lite API**.
|
||||
|
||||
<details>
|
||||
<summary><b>Internal 30-Language ASR Benchmark (click to expand)</b></summary>
|
||||
**Internal 30-Language ASR Benchmark (click to expand)**
|
||||
|
||||
|
||||
| Language | Metric | VoxCPM2 | Fish S2-Pro |
|
||||
| ---------------------- | ------ | --------- | ----------- |
|
||||
| ar (Arabic) | CER | 1.23% | 0.30% |
|
||||
| da (Danish) | WER | 2.70% | 3.52% |
|
||||
| de (German) | WER | 0.96% | 0.64% |
|
||||
| el (Greek) | WER | 3.17% | 4.61% |
|
||||
| en (English) | WER | 0.42% | 1.03% |
|
||||
| es (Spanish) | WER | 1.33% | 0.64% |
|
||||
| fi (Finnish) | WER | 2.24% | 2.80% |
|
||||
| fr (French) | WER | 2.16% | 2.34% |
|
||||
| he (Hebrew) | CER | 2.98% | 15.27% |
|
||||
| hi (Hindi) | CER | 0.79% | 0.91% |
|
||||
| id (Indonesian) | WER | 1.36% | 1.68% |
|
||||
| it (Italian) | WER | 1.65% | 1.08% |
|
||||
| ja (Japanese) | CER | 2.40% | 1.82% |
|
||||
| km (Khmer) | CER | 2.05% | 75.15% |
|
||||
| ko (Korean) | CER | 0.95% | 0.29% |
|
||||
| lo (Lao) | CER | 1.90% | 87.40% |
|
||||
| ms (Malay) | WER | 1.75% | 1.41% |
|
||||
| my (Burmese) | CER | 1.42% | 85.27% |
|
||||
| nl (Dutch) | WER | 1.25% | 1.68% |
|
||||
| no (Norwegian) | WER | 2.49% | 3.76% |
|
||||
| pl (Polish) | WER | 1.90% | 1.65% |
|
||||
| pt (Portuguese) | WER | 1.48% | 1.49% |
|
||||
| ru (Russian) | WER | 0.90% | 0.86% |
|
||||
| sv (Swedish) | WER | 2.22% | 2.63% |
|
||||
| sw (Swahili) | CER | 1.07% | 2.02% |
|
||||
| th (Thai) | CER | 0.94% | 1.92% |
|
||||
| tl (Tagalog) | WER | 2.63% | 4.00% |
|
||||
| tr (Turkish) | WER | 1.65% | 1.65% |
|
||||
| vi (Vietnamese) | WER | 1.56% | 5.56% |
|
||||
| zh (Chinese) | CER | 0.92% | 1.02% |
|
||||
| Average (30 languages) | | **1.68%** | - |
|
||||
|
||||
|
||||
| Language | Metric | VoxCPM2 | Fish S2-Pro |
|
||||
|---|---:|---:|---:|
|
||||
| ar (Arabic) | CER | 1.23% | 0.30% |
|
||||
| da (Danish) | WER | 2.70% | 3.52% |
|
||||
| de (German) | WER | 0.96% | 0.64% |
|
||||
| el (Greek) | WER | 3.17% | 4.61% |
|
||||
| en (English) | WER | 0.42% | 1.03% |
|
||||
| es (Spanish) | WER | 1.33% | 0.64% |
|
||||
| fi (Finnish) | WER | 2.24% | 2.80% |
|
||||
| fr (French) | WER | 2.16% | 2.34% |
|
||||
| he (Hebrew) | CER | 2.98% | 15.27% |
|
||||
| hi (Hindi) | CER | 0.79% | 0.91% |
|
||||
| id (Indonesian) | WER | 1.36% | 1.68% |
|
||||
| it (Italian) | WER | 1.65% | 1.08% |
|
||||
| ja (Japanese) | CER | 2.40% | 1.82% |
|
||||
| km (Khmer) | CER | 2.05% | 75.15% |
|
||||
| ko (Korean) | CER | 0.95% | 0.29% |
|
||||
| lo (Lao) | CER | 1.90% | 87.40% |
|
||||
| ms (Malay) | WER | 1.75% | 1.41% |
|
||||
| my (Burmese) | CER | 1.42% | 85.27% |
|
||||
| nl (Dutch) | WER | 1.25% | 1.68% |
|
||||
| no (Norwegian) | WER | 2.49% | 3.76% |
|
||||
| pl (Polish) | WER | 1.90% | 1.65% |
|
||||
| pt (Portuguese) | WER | 1.48% | 1.49% |
|
||||
| ru (Russian) | WER | 0.90% | 0.86% |
|
||||
| sv (Swedish) | WER | 2.22% | 2.63% |
|
||||
| sw (Swahili) | CER | 1.07% | 2.02% |
|
||||
| th (Thai) | CER | 0.94% | 1.92% |
|
||||
| tl (Tagalog) | WER | 2.63% | 4.00% |
|
||||
| tr (Turkish) | WER | 1.65% | 1.65% |
|
||||
| vi (Vietnamese) | WER | 1.56% | 5.56% |
|
||||
| zh (Chinese) | CER | 0.92% | 1.02% |
|
||||
| Average (30 languages) | | **1.68%** | - |
|
||||
|
||||
</details>
|
||||
|
||||
### InstructTTSEval
|
||||
|
||||
<details>
|
||||
<summary><b>Instruction-Guided Voice Design Results (click to expand)</b></summary>
|
||||
|
||||
| Model | InstructTTSEval-ZH | | | InstructTTSEval-EN | | |
|
||||
|-------|:---:|:----:|:----:|:----:|:----:|:----:|
|
||||
| | APS⬆| DSD⬆ | RP⬆| APS⬆ | DSD⬆ | RP⬆ |
|
||||
| Hume | – | – | – | 83.0 | 75.3 | 54.3 |
|
||||
| VoxInstruct | 47.5 | 52.3 | 42.6 | 54.9 | 57.0 | 39.3 |
|
||||
| Parler-tts-mini | – | – | – | 63.4 | 48.7 | 28.6 |
|
||||
| Parler-tts-large | – | – | – | 60.0 | 45.9 | 31.2 |
|
||||
| PromptTTS | – | – | – | 64.3 | 47.2 | 31.4 |
|
||||
| PromptStyle | – | – | – | 57.4 | 46.4 | 30.9 |
|
||||
| VoiceSculptor | 75.7 | 64.7 | 61.5 | – | – | – |
|
||||
| Mimo-Audio-7B-Instruct | 75.7 | 74.3 | 61.5 | 80.6 | 77.6 | 59.5 |
|
||||
| Qwen3TTS-12Hz-1.7B-VD | **85.2** | **81.1** | **65.1** | 82.9 | 82.4 | 68.4 |
|
||||
| **VoxCPM2** | **85.2** | 71.5 | 60.8 | **84.2** | **83.2** | **71.4** |
|
||||
|
||||
</details>
|
||||
|
||||
**Instruction-Guided Voice Design Results (click to expand)**
|
||||
|
||||
|
||||
| Model | InstructTTSEval-ZH | | | InstructTTSEval-EN | | |
|
||||
| ---------------------- | ------------------ | -------- | -------- | ------------------ | -------- | -------- |
|
||||
| | APS⬆ | DSD⬆ | RP⬆ | APS⬆ | DSD⬆ | RP⬆ |
|
||||
| Hume | – | – | – | 83.0 | 75.3 | 54.3 |
|
||||
| VoxInstruct | 47.5 | 52.3 | 42.6 | 54.9 | 57.0 | 39.3 |
|
||||
| Parler-tts-mini | – | – | – | 63.4 | 48.7 | 28.6 |
|
||||
| Parler-tts-large | – | – | – | 60.0 | 45.9 | 31.2 |
|
||||
| PromptTTS | – | – | – | 64.3 | 47.2 | 31.4 |
|
||||
| PromptStyle | – | – | – | 57.4 | 46.4 | 30.9 |
|
||||
| VoiceSculptor | 75.7 | 64.7 | 61.5 | – | – | – |
|
||||
| Mimo-Audio-7B-Instruct | 75.7 | 74.3 | 61.5 | 80.6 | 77.6 | 59.5 |
|
||||
| Qwen3TTS-12Hz-1.7B-VD | **85.2** | **81.1** | **65.1** | 82.9 | 82.4 | 68.4 |
|
||||
| **VoxCPM2** | **85.2** | 71.5 | 60.8 | **84.2** | **83.2** | **71.4** |
|
||||
|
||||
|
||||
|
||||
@@ -539,30 +607,36 @@ python lora_ft_webui.py # then open http://localhost:7860
|
||||
|
||||
Full documentation: **[voxcpm.readthedocs.io](https://voxcpm.readthedocs.io/en/latest/)**
|
||||
|
||||
| Topic | Link |
|
||||
|---|---|
|
||||
| Quick Start & Installation | [Quick Start](https://voxcpm.readthedocs.io/en/latest/quickstart.html) |
|
||||
| Usage Guide & Cookbook | [User Guide](https://voxcpm.readthedocs.io/en/latest/usage_guide.html) |
|
||||
| VoxCPM Series | [Models](https://voxcpm.readthedocs.io/en/latest/models/version_history.html) |
|
||||
| Fine-tuning (SFT & LoRA) | [Fine-tuning Guide](https://voxcpm.readthedocs.io/en/latest/finetuning/finetune.html) |
|
||||
| FAQ & Troubleshooting | [FAQ](https://voxcpm.readthedocs.io/en/latest/faq.html) |
|
||||
|
||||
| Topic | Link |
|
||||
| -------------------------- | ------------------------------------------------------------------------------------- |
|
||||
| Quick Start & Installation | [Quick Start](https://voxcpm.readthedocs.io/en/latest/quickstart.html) |
|
||||
| Usage Guide & Cookbook | [User Guide](https://voxcpm.readthedocs.io/en/latest/usage_guide.html) |
|
||||
| VoxCPM Series | [Models](https://voxcpm.readthedocs.io/en/latest/models/version_history.html) |
|
||||
| Fine-tuning (SFT & LoRA) | [Fine-tuning Guide](https://voxcpm.readthedocs.io/en/latest/finetuning/finetune.html) |
|
||||
| FAQ & Troubleshooting | [FAQ](https://voxcpm.readthedocs.io/en/latest/faq.html) |
|
||||
|
||||
|
||||
---
|
||||
|
||||
## 🌟 Ecosystem & Community
|
||||
|
||||
| Project | Description |
|
||||
|---|---|
|
||||
| [**Nano-vLLM**](https://github.com/a710128/nanovllm-voxcpm) | High-throughput and Fast GPU serving |
|
||||
| [**vLLM-Omni**](https://github.com/vllm-project/vllm-omni) | Official vLLM omni-modal serving for VoxCPM2 — PagedAttention, OpenAI-compatible API |
|
||||
| [**VoxCPM.cpp**](https://github.com/bluryar/VoxCPM.cpp) | GGML/GGUF: CPU, CUDA, Vulkan inference |
|
||||
| [**VoxCPM-ONNX**](https://github.com/bluryar/VoxCPM-ONNX) | ONNX export for CPU inference |
|
||||
| [**VoxCPMANE**](https://github.com/0seba/VoxCPMANE) | Apple Neural Engine backend |
|
||||
| [**voxcpm_rs**](https://github.com/madushan1000/voxcpm_rs) | Rust re-implementation |
|
||||
| [**ComfyUI-VoxCPM**](https://github.com/wildminder/ComfyUI-VoxCPM) | ComfyUI node-based workflows |
|
||||
| [**ComfyUI_RH_VoxCPM**](https://github.com/HM-RunningHub/ComfyUI_RH_VoxCPM) | Feature-complete ComfyUI workflow for VoxCPM 2 with multi-speaker generation, LoRA, and auto-ASR |
|
||||
| [**ComfyUI-VoxCPMTTS**](https://github.com/1038lab/ComfyUI-VoxCPMTTS) | ComfyUI TTS extension |
|
||||
| [**TTS WebUI**](https://github.com/rsxdalv/tts_webui_extension.vox_cpm) | Browser-based TTS extension |
|
||||
|
||||
| Project | Description |
|
||||
| --------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------ |
|
||||
| **[Nano-vLLM](https://github.com/a710128/nanovllm-voxcpm)** | High-throughput and Fast GPU serving |
|
||||
| **[vLLM-Omni](https://github.com/vllm-project/vllm-omni)** | Official vLLM omni-modal serving for VoxCPM2 — PagedAttention, OpenAI-compatible API |
|
||||
| **[llama.cpp-omni](https://github.com/tc-mb/llama.cpp-omni)** | Full-duplex omni inference engine — VoxCPM2 GGUF on CPU / Metal / CUDA / Vulkan |
|
||||
| **[VoxCPM.cpp](https://github.com/bluryar/VoxCPM.cpp)** | GGML/GGUF: CPU, CUDA, Vulkan inference |
|
||||
| **[audio.cpp](https://github.com/0xShug0/audio.cpp)** | ggml-based unified C++ inference framework — CPU/CUDA/Vulkan/Metal, CLI & server, no Python |
|
||||
| **[VoxCPM-ONNX](https://github.com/bluryar/VoxCPM-ONNX)** | ONNX export for CPU inference |
|
||||
| **[VoxCPMANE](https://github.com/0seba/VoxCPMANE)** | Apple Neural Engine backend |
|
||||
| **[voxcpm_rs](https://github.com/madushan1000/voxcpm_rs)** | Rust re-implementation |
|
||||
| **[ComfyUI-VoxCPM](https://github.com/wildminder/ComfyUI-VoxCPM)** | ComfyUI node-based workflows |
|
||||
| **[ComfyUI_RH_VoxCPM](https://github.com/HM-RunningHub/ComfyUI_RH_VoxCPM)** | Feature-complete ComfyUI workflow for VoxCPM 2 with multi-speaker generation, LoRA, and auto-ASR |
|
||||
| **[ComfyUI-VoxCPMTTS](https://github.com/1038lab/ComfyUI-VoxCPMTTS)** | ComfyUI TTS extension |
|
||||
| **[TTS WebUI](https://github.com/rsxdalv/tts_webui_extension.vox_cpm)** | Browser-based TTS extension |
|
||||
|
||||
|
||||
> See the full [Ecosystem](https://voxcpm.readthedocs.io/en/latest/) in the docs. Community projects are not officially maintained by OpenBMB. Built something cool? [Open an issue or PR](https://github.com/OpenBMB/VoxCPM/issues) to add it!
|
||||
|
||||
@@ -582,21 +656,18 @@ Full documentation: **[voxcpm.readthedocs.io](https://voxcpm.readthedocs.io/en/l
|
||||
If you find VoxCPM helpful, please consider citing our work and starring ⭐ the repository!
|
||||
|
||||
```bib
|
||||
@article{voxcpm2_2026,
|
||||
title = {VoxCPM2: Tokenizer-Free TTS for Multilingual Speech Generation, Creative Voice Design, and True-to-Life Cloning},
|
||||
author = {VoxCPM Team},
|
||||
journal = {GitHub},
|
||||
@article{zhou2026voxcpm2,
|
||||
title = {VoxCPM2 Technical Report},
|
||||
author = {Zhou, Yixuan and Zeng, Guoyang and Liu, Xin and Li, Xiang and Yu, Renjie and Gui, Jiancheng and Wu, Jiaheng and Wang, Ziyang and Shen, Xudong and Ye, Runchuan and Zhang, Zhisheng and Zhou, Jiuyang and Bai, Bingsong and Sun, Weiyue and Deng, Mengyuan and Shi, Qundong and Wu, Zhiyong and Liu, Zhiyuan},
|
||||
journal = {arXiv preprint arXiv:2606.06928},
|
||||
year = {2026},
|
||||
}
|
||||
|
||||
@article{voxcpm2025,
|
||||
title = {VoxCPM: Tokenizer-Free TTS for Context-Aware Speech Generation
|
||||
and True-to-Life Voice Cloning},
|
||||
author = {Zhou, Yixuan and Zeng, Guoyang and Liu, Xin and Li, Xiang and
|
||||
Yu, Renjie and Wang, Ziyang and Ye, Runchuan and Sun, Weiyue and
|
||||
Gui, Jiancheng and Li, Kehan and Wu, Zhiyong and Liu, Zhiyuan},
|
||||
@article{zhou2025voxcpm,
|
||||
title = {Voxcpm: Tokenizer-free TTS for context-aware speech generation and true-to-life voice cloning},
|
||||
author = {Zhou, Yixuan and Zeng, Guoyang and Liu, Xin and Li, Xiang and Yu, Renjie and Wang, Ziyang and Ye, Runchuan and Sun, Weiyue and Gui, Jiancheng and Li, Kehan and Wu, Zhiyong and Liu, Zhiyuan},
|
||||
journal = {arXiv preprint arXiv:2509.24650},
|
||||
year = {2025},
|
||||
year = {2025}
|
||||
}
|
||||
```
|
||||
|
||||
@@ -614,12 +685,8 @@ VoxCPM model weights and code are open-sourced under the [Apache-2.0](LICENSE) l
|
||||
|
||||
## Institutions
|
||||
|
||||
<p>
|
||||
<a href="https://modelbest.cn/"><img src="assets/modelbest_logo.png" width="28px"> ModelBest</a>
|
||||
|
||||
<a href="https://github.com/thuhcsi"><img src="assets/thuhcsi_logo.png" width="28px"> THUHCSI</a>
|
||||
</p>
|
||||
[ModelBest](https://modelbest.cn/) [THUHCSI](https://github.com/thuhcsi)
|
||||
|
||||
## ⭐ Star History
|
||||
|
||||
[](https://star-history.com/#OpenBMB/VoxCPM&Date)
|
||||
[Star History Chart](https://star-history.com/#OpenBMB/VoxCPM&Date)
|
||||
+82
-15
@@ -10,7 +10,7 @@
|
||||
<a href="https://voxcpm.readthedocs.io/zh-cn/latest/"><img src="https://img.shields.io/badge/Docs-ReadTheDocs-8CA1AF" alt="Documentation"></a>
|
||||
<a href="https://huggingface.co/openbmb/VoxCPM2"><img src="https://img.shields.io/badge/%F0%9F%A4%97%20Hugging%20Face-VoxCPM2-yellow" alt="Hugging Face"></a>
|
||||
<a href="https://modelscope.cn/models/OpenBMB/VoxCPM2"><img src="https://img.shields.io/badge/ModelScope-VoxCPM2-purple" alt="ModelScope"></a>
|
||||
<a href="https://openbmb.github.io/voxcpm2-demopage/"><img src="https://img.shields.io/badge/DemoPage-Audio Samples-red"></a>
|
||||
<a href="https://openbmb.github.io/voxcpm2-demopage/"><img src="https://img.shields.io/badge/DemoPage-Audio Samples-red" alt="DemoPage"></a>
|
||||
|
||||
</p>
|
||||
|
||||
@@ -58,7 +58,7 @@ VoxCPM 是一个**无离散音频分词器**（Tokenizer-Free）的语音合成
|
||||
|
||||
### 最新动态
|
||||
|
||||
* **[2026.04]** 🔥 发布 **VoxCPM2** — 20亿参数，30种语言，音色设计与可控声音克隆，48kHz 音频输出！[模型权重](https://huggingface.co/openbmb/VoxCPM2) | [使用文档](https://voxcpm.readthedocs.io/zh-cn/latest/) | [在线体验](https://huggingface.co/spaces/OpenBMB/VoxCPM-Demo) | [官网体验](https://voxcpm.modelbest.cn/) (适用国内访问)
|
||||
* **[2026.04]** 🔥 发布 **VoxCPM2** — 20亿参数，30种语言，音色设计与可控声音克隆，48kHz 音频输出！[模型权重](https://huggingface.co/openbmb/VoxCPM2) | [使用文档](https://voxcpm.readthedocs.io/zh-cn/latest/) | [在线体验](https://huggingface.co/spaces/OpenBMB/VoxCPM-Demo) | [官网体验](https://voxcpm.modelbest.cn/) (适用国内访问) | [技术报告](https://arxiv.org/abs/2606.06928)
|
||||
* **[2025.12]** 🎉 开源 **VoxCPM1.5** [模型权重](https://huggingface.co/openbmb/VoxCPM1.5)，支持 SFT 和 LoRA 微调。(**🏆 GitHub Trending #1**)
|
||||
* **[2025.09]** 🔥 发布 VoxCPM [技术报告](https://arxiv.org/abs/2509.24650)。
|
||||
* **[2025.09]** 🎉 开源 **VoxCPM-0.5B** [模型权重](https://huggingface.co/openbmb/VoxCPM-0.5B) (**🏆 HuggingFace Trending #1**)
|
||||
@@ -73,6 +73,7 @@ VoxCPM 是一个**无离散音频分词器**（Tokenizer-Free）的语音合成
|
||||
- [命令行使用](#命令行使用)
|
||||
- [Web Demo](#web-demo)
|
||||
- [生产部署](#-生产部署nano-vllm)
|
||||
- [端侧推理（llama.cpp-omni）](#-端侧推理llamacpp-omni)
|
||||
- [模型与版本](#-模型与版本)
|
||||
- [性能评测](#-性能评测)
|
||||
- [微调](#%EF%B8%8F-微调)
|
||||
@@ -110,6 +111,7 @@ wav = model.generate(
|
||||
text="VoxCPM2 是目前推荐使用的多语言语音合成版本。",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
||||
print("已保存: demo.wav")
|
||||
@@ -133,6 +135,7 @@ wav = model.generate(
|
||||
text="VoxCPM2 是目前推荐使用的多语言语音合成版本。",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("demo.wav", wav, model.tts_model.sample_rate)
|
||||
```
|
||||
@@ -146,6 +149,7 @@ wav = model.generate(
|
||||
text="(年轻女性，声音温柔甜美)你好，欢迎使用VoxCPM2！",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("voice_design.wav", wav, model.tts_model.sample_rate)
|
||||
```
|
||||
@@ -166,6 +170,7 @@ wav = model.generate(
|
||||
reference_wav_path="path/to/voice.wav",
|
||||
cfg_value=2.0,
|
||||
inference_timesteps=10,
|
||||
seed=42,
|
||||
)
|
||||
sf.write("controllable_clone.wav", wav, model.tts_model.sample_rate)
|
||||
```
|
||||
@@ -212,6 +217,7 @@ voxcpm design \
|
||||
voxcpm design \
|
||||
--text "VoxCPM2带来全新语音合成体验。" \
|
||||
--control "年轻女声，温暖温柔，略带微笑" \
|
||||
--seed 42 \
|
||||
--output out.wav
|
||||
|
||||
# 声音克隆（参考音频）
|
||||
@@ -231,6 +237,23 @@ voxcpm clone \
|
||||
# 批量处理
|
||||
voxcpm batch --input examples/input.txt --output-dir outs
|
||||
|
||||
# 可选的生成后时间戳对齐（基于 stable-ts）
|
||||
pip install "voxcpm[timestamps]"
|
||||
voxcpm design \
|
||||
--text "VoxCPM2带来全新语音合成体验。" \
|
||||
--output out.wav \
|
||||
--timestamps \
|
||||
--timestamp-level word \
|
||||
--timestamp-language zh
|
||||
|
||||
# 字级时间戳是 best-effort，会基于词级对齐结果拆分
|
||||
voxcpm design \
|
||||
--text "欢迎使用 VoxCPM2。" \
|
||||
--output out.wav \
|
||||
--timestamps \
|
||||
--timestamp-level char \
|
||||
--timestamp-language zh
|
||||
|
||||
# 帮助
|
||||
voxcpm --help
|
||||
```
|
||||
@@ -241,6 +264,14 @@ voxcpm --help
|
||||
python app.py --port 8808 # 然后在浏览器打开 http://localhost:8808
|
||||
```
|
||||
|
||||
使用 `--device` 选择运行设备：
|
||||
|
||||
```bash
|
||||
python app.py --device auto
|
||||
```
|
||||
|
||||
支持的取值包括 `auto`、`cpu`、`mps`、`cuda` 和 `cuda:N`。在 Apple Silicon Mac 上，`auto` 会在可用时使用 MPS。
|
||||
|
||||
### 🚢 生产部署（Nano-vLLM）
|
||||
|
||||
如需高吞吐量部署，使用 [**Nano-vLLM-VoxCPM**](https://github.com/a710128/nanovllm-voxcpm) — 基于 Nano-vLLM 构建的专用推理引擎，支持并发请求和异步 API。
|
||||
@@ -287,6 +318,43 @@ curl http://localhost:8000/v1/audio/speech \
|
||||
|
||||
> 基于上游 vLLM 调度器构建，开箱即用支持批量并发、流式分块输出和多 GPU 部署。完整示例见 [VoxCPM2 部署样例](https://github.com/vllm-project/vllm-omni/tree/main/examples/online_serving/voxcpm2)。
|
||||
|
||||
### 📱 端侧推理（llama.cpp-omni）
|
||||
|
||||
如需在端侧/消费级硬件上无 Python 运行，使用 **[llama.cpp-omni](https://github.com/tc-mb/llama.cpp-omni)** — 基于 llama.cpp 的高性能 C++ 推理引擎，原生支持 VoxCPM2 GGUF，可在 **CPU / Metal / CUDA / Vulkan** 上运行。
|
||||
|
||||
**1. 下载 GGUF 权重**：从 [HF下载](https://huggingface.co/DennisHuang648/VoxCPM2-GGUF) | [ModelScope](https://modelscope.cn/models/DennisHuang/VoxCPM2-GGUF)，需要一个 **BaseLM**（F16 或 Q8_0）+ **Acoustic** 文件。Q8_0 体积减半，质量损失可忽略。
|
||||
|
||||
**2. 编译**
|
||||
|
||||
```bash
|
||||
git clone https://github.com/tc-mb/llama.cpp-omni.git && cd llama.cpp-omni
|
||||
cmake -B build -DCMAKE_BUILD_TYPE=Release
|
||||
cmake --build build --target voxcpm2-cli -j
|
||||
```
|
||||
|
||||
> CMake 会自动检测并启用 Metal（macOS）或 CUDA（Linux + NVIDIA GPU）。
|
||||
|
||||
**3. 运行**
|
||||
|
||||
```bash
|
||||
# 基础 TTS
|
||||
./build/bin/voxcpm2-cli \
|
||||
-t "你好，我是通过 llama.cpp-omni 运行的 VoxCPM2。" \
|
||||
-o output.wav VoxCPM2-BaseLM-Q8_0.gguf VoxCPM2-Acoustic-F16.gguf
|
||||
|
||||
# 声音克隆（参考音频）
|
||||
./build/bin/voxcpm2-cli \
|
||||
-t "克隆的声音。" -r speaker.wav -o clone.wav \
|
||||
VoxCPM2-BaseLM-Q8_0.gguf VoxCPM2-Acoustic-F16.gguf
|
||||
|
||||
# 精准克隆（参考音频 + 转写文本）
|
||||
./build/bin/voxcpm2-cli \
|
||||
-t "目标文本。" --prompt-wav speaker.wav --prompt-text "参考音频的转写文本" \
|
||||
-o clone.wav VoxCPM2-BaseLM-Q8_0.gguf VoxCPM2-Acoustic-F16.gguf
|
||||
```
|
||||
|
||||
> **在 Apple M4 Pro / Metal 上 RTF ~1.76（Q8_0）。** 主要参数：`--cfg`（引导尺度）、`--timesteps`（CFM 步数）、`--seed`、`--temperature`、`--stream`。详见 [llama.cpp-omni 仓库](https://github.com/tc-mb/llama.cpp-omni) 和 [GGUF 权重页面](https://huggingface.co/DennisHuang648/VoxCPM2-GGUF)。
|
||||
|
||||
> **完整参数说明、多场景示例与声音克隆技巧 →** [快速开始指南](https://voxcpm.readthedocs.io/zh-cn/latest/quickstart.html) | [使用指南](https://voxcpm.readthedocs.io/zh-cn/latest/usage_guide.html) | [Cookbook](https://voxcpm.readthedocs.io/zh-cn/latest/cookbook.html)
|
||||
|
||||
---
|
||||
@@ -308,13 +376,13 @@ curl http://localhost:8000/v1/audio/speech \
|
||||
| **RTF Nano-VLLM (RTX 4090)** | ~0.13 | ~0.08 | ~0.10 |
|
||||
| **显存占用** | ~8 GB | ~6 GB | ~5 GB |
|
||||
| **模型权重** | [🤗 HF](https://huggingface.co/openbmb/VoxCPM2) / [MS](https://modelscope.cn/models/OpenBMB/VoxCPM2) | [🤗 HF](https://huggingface.co/openbmb/VoxCPM1.5) / [MS](https://modelscope.cn/models/OpenBMB/VoxCPM1.5) | [🤗 HF](https://huggingface.co/openbmb/VoxCPM-0.5B) / [MS](https://modelscope.cn/models/OpenBMB/VoxCPM-0.5B) |
|
||||
| **技术报告** | 即将发布 | — | [arXiv](https://arxiv.org/abs/2509.24650) [ICLR 2026](https://openreview.net/forum?id=h5KLpGoqzC) |
|
||||
| **技术报告** | [arXiv](https://arxiv.org/abs/2606.06928) | — | [arXiv](https://arxiv.org/abs/2509.24650) [ICLR 2026](https://openreview.net/forum?id=h5KLpGoqzC) |
|
||||
| **Demo 页面** | [音频示例](https://openbmb.github.io/voxcpm2-demopage) | — | [音频示例](https://openbmb.github.io/VoxCPM-demopage) |
|
||||
|
||||
VoxCPM2 采用**连续音频表征、扩散自回归**范式，模型在 **AudioVAE** 的连续隐空间中通过四阶段处理：**LocEnc → TSLM → RALM → LocDiT**，实现丰富的表现力语音合成和 48kHz 原生音频输出。
|
||||
|
||||
<div align="center">
|
||||
<img src="assets/voxcpm_model.png" alt="VoxCPM2 模型架构" width="90%">
|
||||
<img src="assets/voxcpm2_model.png" alt="VoxCPM2 模型架构" width="90%">
|
||||
</div>
|
||||
|
||||
> 完整架构细节、VoxCPM2 升级内容和模型对比表见 [架构设计文档](https://voxcpm.readthedocs.io/zh-cn/latest/models/architecture.html)。
|
||||
@@ -548,7 +616,9 @@ python lora_ft_webui.py # 然后打开 http://localhost:7860
|
||||
|---|---|
|
||||
| [**Nano-vLLM**](https://github.com/a710128/nanovllm-voxcpm) | 高吞吐快速 GPU 推理引擎 |
|
||||
| [**vLLM-Omni**](https://github.com/vllm-project/vllm-omni) | 官方 vLLM 全模态服务（原生支持 VoxCPM2）— PagedAttention、OpenAI 兼容 API |
|
||||
| [**llama.cpp-omni**](https://github.com/tc-mb/llama.cpp-omni) | 全双工全模态推理引擎 — VoxCPM2 GGUF，支持 CPU / Metal / CUDA / Vulkan |
|
||||
| [**VoxCPM.cpp**](https://github.com/bluryar/VoxCPM.cpp) | GGML/GGUF：CPU、CUDA、Vulkan 推理 |
|
||||
| [**audio.cpp**](https://github.com/0xShug0/audio.cpp) | 基于 ggml 的统一 C++ 推理框架 — CPU/CUDA/Vulkan/Metal，CLI 与服务端，无需 Python |
|
||||
| [**VoxCPM-ONNX**](https://github.com/bluryar/VoxCPM-ONNX) | ONNX 导出，支持 CPU 推理 |
|
||||
| [**VoxCPMANE**](https://github.com/0seba/VoxCPMANE) | Apple Neural Engine 后端 |
|
||||
| [**voxcpm_rs**](https://github.com/madushan1000/voxcpm_rs) | Rust 重新实现 |
|
||||
@@ -575,21 +645,18 @@ python lora_ft_webui.py # 然后打开 http://localhost:7860
|
||||
如果 VoxCPM 对您有帮助，请考虑引用我们的工作并为仓库加星 ⭐！
|
||||
|
||||
```bib
|
||||
@article{voxcpm2_2026,
|
||||
title = {VoxCPM2: Tokenizer-Free TTS for Multilingual Speech Generation, Creative Voice Design, and True-to-Life Cloning},
|
||||
author = {VoxCPM Team},
|
||||
journal = {GitHub},
|
||||
@article{zhou2026voxcpm2,
|
||||
title = {VoxCPM2 Technical Report},
|
||||
author = {Zhou, Yixuan and Zeng, Guoyang and Liu, Xin and Li, Xiang and Yu, Renjie and Gui, Jiancheng and Wu, Jiaheng and Wang, Ziyang and Shen, Xudong and Ye, Runchuan and Zhang, Zhisheng and Zhou, Jiuyang and Bai, Bingsong and Sun, Weiyue and Deng, Mengyuan and Shi, Qundong and Wu, Zhiyong and Liu, Zhiyuan},
|
||||
journal = {arXiv preprint arXiv:2606.06928},
|
||||
year = {2026},
|
||||
}
|
||||
|
||||
@article{voxcpm2025,
|
||||
title = {VoxCPM: Tokenizer-Free TTS for Context-Aware Speech Generation
|
||||
and True-to-Life Voice Cloning},
|
||||
author = {Zhou, Yixuan and Zeng, Guoyang and Liu, Xin and Li, Xiang and
|
||||
Yu, Renjie and Wang, Ziyang and Ye, Runchuan and Sun, Weiyue and
|
||||
Gui, Jiancheng and Li, Kehan and Wu, Zhiyong and Liu, Zhiyuan},
|
||||
@article{zhou2025voxcpm,
|
||||
title = {Voxcpm: Tokenizer-free TTS for context-aware speech generation and true-to-life voice cloning},
|
||||
author = {Zhou, Yixuan and Zeng, Guoyang and Liu, Xin and Li, Xiang and Yu, Renjie and Wang, Ziyang and Ye, Runchuan and Sun, Weiyue and Gui, Jiancheng and Li, Kehan and Wu, Zhiyong and Liu, Zhiyuan},
|
||||
journal = {arXiv preprint arXiv:2509.24650},
|
||||
year = {2025},
|
||||
year = {2025}
|
||||
}
|
||||
```
|
||||
|
||||
|
||||
@@ -2,8 +2,8 @@ import os
|
||||
import re
|
||||
import sys
|
||||
import logging
|
||||
import random
|
||||
import numpy as np
|
||||
import torch
|
||||
import gradio as gr
|
||||
from typing import Optional, Tuple
|
||||
from funasr import AutoModel
|
||||
@@ -12,6 +12,7 @@ from pathlib import Path
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
||||
|
||||
import voxcpm
|
||||
from voxcpm.model.utils import resolve_runtime_device
|
||||
|
||||
logging.basicConfig(
|
||||
level=logging.INFO,
|
||||
@@ -44,8 +45,8 @@ _EXAMPLES_FOOTER_EN = (
|
||||
"**Example 1 — Gentle & Melancholic Girl** \n"
|
||||
'`Control Instruction`: *"A young girl with a soft, sweet voice. '
|
||||
'Speaks slowly with a melancholic, slightly tsundere tone."* \n'
|
||||
'`Target Text`: *"I never asked you to stay… It\'s not like I care or anything. '
|
||||
'But… why does it still hurt so much now that you\'re gone?"* \n\n'
|
||||
"`Target Text`: *\"I never asked you to stay… It's not like I care or anything. "
|
||||
"But… why does it still hurt so much now that you're gone?\"* \n\n"
|
||||
"**Example 2 — Laid-Back Surfer Dude** \n"
|
||||
'`Control Instruction`: *"Relaxed young male voice, slightly nasal, '
|
||||
'lazy drawl, very casual and chill."* \n'
|
||||
@@ -115,6 +116,10 @@ _I18N_TRANSLATIONS = {
|
||||
"cfg_info": "Higher → closer to the prompt / reference; lower → more creative variation",
|
||||
"dit_steps_label": "LocDiT flow-matching steps",
|
||||
"dit_steps_info": "LocDiT flow-matching steps — more steps → maybe better audio quality, but slower",
|
||||
"seed_label": "Seed",
|
||||
"seed_info": "Seed used for reproducible generation. Updated with the actual successful seed after generation.",
|
||||
"random_seed_label": "Random Seed",
|
||||
"random_seed_info": "Generate a new seed before each inference run.",
|
||||
"usage_instructions": _USAGE_INSTRUCTIONS_EN,
|
||||
"examples_footer": _EXAMPLES_FOOTER_EN,
|
||||
},
|
||||
@@ -142,7 +147,7 @@ _I18N_TRANSLATIONS = {
|
||||
"examples_footer": _EXAMPLES_FOOTER_ZH,
|
||||
},
|
||||
"zh-Hans": None, # alias, filled below
|
||||
"zh": None, # alias, filled below
|
||||
"zh": None, # alias, filled below
|
||||
}
|
||||
_I18N_TRANSLATIONS["zh-Hans"] = _I18N_TRANSLATIONS["zh-CN"]
|
||||
_I18N_TRANSLATIONS["zh"] = _I18N_TRANSLATIONS["zh-CN"]
|
||||
@@ -155,8 +160,7 @@ for _d in _I18N_TRANSLATIONS.values():
|
||||
I18N = gr.I18n(**_I18N_TRANSLATIONS)
|
||||
|
||||
DEFAULT_TARGET_TEXT = (
|
||||
"VoxCPM2 is a creative multilingual TTS model from ModelBest, "
|
||||
"designed to generate highly realistic speech."
|
||||
"VoxCPM2 is a creative multilingual TTS model from ModelBest, " "designed to generate highly realistic speech."
|
||||
)
|
||||
|
||||
_CUSTOM_CSS = """
|
||||
@@ -219,18 +223,16 @@ _APP_THEME = gr.themes.Soft(
|
||||
|
||||
# ---------- Model ----------
|
||||
|
||||
|
||||
class VoxCPMDemo:
|
||||
def __init__(self, model_id: str = "openbmb/VoxCPM2") -> None:
|
||||
self.device = "cuda" if torch.cuda.is_available() else "cpu"
|
||||
logger.info(f"Running on device: {self.device}")
|
||||
def __init__(self, model_id: str = "openbmb/VoxCPM2", device: str = "auto") -> None:
|
||||
self.device = resolve_runtime_device(device, "cuda")
|
||||
logger.info(f"Running VoxCPM on device: {self.device}")
|
||||
self.optimize = self.device.startswith("cuda")
|
||||
|
||||
self.asr_model_id = "iic/SenseVoiceSmall"
|
||||
self.asr_model: Optional[AutoModel] = AutoModel(
|
||||
model=self.asr_model_id,
|
||||
disable_update=True,
|
||||
log_level="DEBUG",
|
||||
device="cuda:0" if self.device == "cuda" else "cpu",
|
||||
)
|
||||
self.asr_device = "cuda:0" if self.device.startswith("cuda") else "cpu"
|
||||
self.asr_model: Optional[AutoModel] = None
|
||||
|
||||
self.voxcpm_model: Optional[voxcpm.VoxCPM] = None
|
||||
self._model_id = model_id
|
||||
@@ -239,14 +241,35 @@ class VoxCPMDemo:
|
||||
if self.voxcpm_model is not None:
|
||||
return self.voxcpm_model
|
||||
logger.info(f"Loading model: {self._model_id}")
|
||||
self.voxcpm_model = voxcpm.VoxCPM.from_pretrained(self._model_id, optimize=True)
|
||||
self.voxcpm_model = voxcpm.VoxCPM.from_pretrained(
|
||||
self._model_id,
|
||||
optimize=self.optimize,
|
||||
device=self.device,
|
||||
)
|
||||
logger.info("Model loaded successfully.")
|
||||
return self.voxcpm_model
|
||||
|
||||
def get_or_load_asr_model(self) -> AutoModel:
|
||||
if self.asr_model is not None:
|
||||
return self.asr_model
|
||||
logger.info(f"Loading ASR model: {self.asr_model_id} on device: {self.asr_device}")
|
||||
self.asr_model = AutoModel(
|
||||
model=self.asr_model_id,
|
||||
disable_update=True,
|
||||
log_level="DEBUG",
|
||||
device=self.asr_device,
|
||||
)
|
||||
logger.info("ASR model loaded successfully.")
|
||||
return self.asr_model
|
||||
|
||||
def prompt_wav_recognition(self, prompt_wav: Optional[str]) -> str:
|
||||
if prompt_wav is None:
|
||||
return ""
|
||||
res = self.asr_model.generate(input=prompt_wav, language="auto", use_itn=True)
|
||||
res = self.get_or_load_asr_model().generate(
|
||||
input=prompt_wav,
|
||||
language="auto",
|
||||
use_itn=True,
|
||||
)
|
||||
return res[0]["text"].split("|>")[-1]
|
||||
|
||||
def _build_generate_kwargs(
|
||||
@@ -259,6 +282,7 @@ class VoxCPMDemo:
|
||||
do_normalize: bool,
|
||||
denoise: bool,
|
||||
inference_timesteps: int = 10,
|
||||
seed: Optional[int] = None,
|
||||
) -> dict:
|
||||
generate_kwargs = dict(
|
||||
text=final_text,
|
||||
@@ -267,6 +291,7 @@ class VoxCPMDemo:
|
||||
inference_timesteps=inference_timesteps,
|
||||
normalize=do_normalize,
|
||||
denoise=denoise,
|
||||
seed=seed,
|
||||
)
|
||||
if prompt_text_clean and audio_path:
|
||||
generate_kwargs["prompt_wav_path"] = audio_path
|
||||
@@ -283,7 +308,8 @@ class VoxCPMDemo:
|
||||
do_normalize: bool = True,
|
||||
denoise: bool = True,
|
||||
inference_timesteps: int = 10,
|
||||
) -> Tuple[int, np.ndarray]:
|
||||
seed: Optional[int] = None,
|
||||
) -> Tuple[int, np.ndarray, Optional[int]]:
|
||||
current_model = self.get_or_load_voxcpm()
|
||||
|
||||
text = (text_input or "").strip()
|
||||
@@ -315,16 +341,32 @@ class VoxCPMDemo:
|
||||
do_normalize=do_normalize,
|
||||
denoise=denoise,
|
||||
inference_timesteps=inference_timesteps,
|
||||
seed=seed,
|
||||
)
|
||||
wav = current_model.generate(**generate_kwargs)
|
||||
return (current_model.tts_model.sample_rate, wav)
|
||||
last_successful_seed = getattr(current_model.tts_model, "last_successful_seed", seed)
|
||||
return (current_model.tts_model.sample_rate, wav, last_successful_seed)
|
||||
|
||||
|
||||
# ---------- UI ----------
|
||||
|
||||
|
||||
def create_demo_interface(demo: VoxCPMDemo):
|
||||
gr.set_static_paths(paths=[Path.cwd().absolute() / "assets"])
|
||||
|
||||
def _coerce_seed(seed_value) -> Optional[int]:
|
||||
if seed_value is None or seed_value == "":
|
||||
return None
|
||||
return int(seed_value)
|
||||
|
||||
def _prepare_seed(use_random_seed: bool, seed_value):
|
||||
if use_random_seed:
|
||||
return random.randint(0, 2**32 - 1)
|
||||
return _coerce_seed(seed_value)
|
||||
|
||||
def _on_random_seed_toggle(checked):
|
||||
return gr.update(interactive=not checked)
|
||||
|
||||
def _generate(
|
||||
text: str,
|
||||
control_instruction: str,
|
||||
@@ -335,10 +377,12 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
do_normalize: bool,
|
||||
denoise: bool,
|
||||
dit_steps: int,
|
||||
seed_value,
|
||||
):
|
||||
actual_prompt_text = prompt_text_value.strip() if use_prompt_text else ""
|
||||
actual_control = "" if use_prompt_text else control_instruction
|
||||
sr, wav_np = demo.generate_tts_audio(
|
||||
seed = _coerce_seed(seed_value)
|
||||
sr, wav_np, last_successful_seed = demo.generate_tts_audio(
|
||||
text_input=text,
|
||||
control_instruction=actual_control,
|
||||
reference_wav_path_input=ref_wav,
|
||||
@@ -347,8 +391,9 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
do_normalize=do_normalize,
|
||||
denoise=denoise,
|
||||
inference_timesteps=int(dit_steps),
|
||||
seed=seed,
|
||||
)
|
||||
return (sr, wav_np)
|
||||
return (sr, wav_np), last_successful_seed
|
||||
|
||||
def _on_toggle_instant(checked):
|
||||
"""Instant UI toggle — no ASR, no blocking."""
|
||||
@@ -445,6 +490,20 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
label=I18N("dit_steps_label"),
|
||||
info=I18N("dit_steps_info"),
|
||||
)
|
||||
with gr.Row():
|
||||
seed_value = gr.Number(
|
||||
value=random.randint(0, 2**32 - 1),
|
||||
precision=0,
|
||||
label=I18N("seed_label"),
|
||||
info=I18N("seed_info"),
|
||||
interactive=False,
|
||||
)
|
||||
random_seed = gr.Checkbox(
|
||||
value=True,
|
||||
label=I18N("random_seed_label"),
|
||||
elem_classes=["switch-toggle"],
|
||||
info=I18N("random_seed_info"),
|
||||
)
|
||||
|
||||
run_btn = gr.Button(I18N("generate_btn"), variant="primary", size="lg")
|
||||
|
||||
@@ -462,7 +521,18 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
outputs=[prompt_text],
|
||||
)
|
||||
|
||||
random_seed.change(
|
||||
fn=_on_random_seed_toggle,
|
||||
inputs=[random_seed],
|
||||
outputs=[seed_value],
|
||||
)
|
||||
|
||||
run_btn.click(
|
||||
fn=_prepare_seed,
|
||||
inputs=[random_seed, seed_value],
|
||||
outputs=[seed_value],
|
||||
show_progress=False,
|
||||
).then(
|
||||
fn=_generate,
|
||||
inputs=[
|
||||
text,
|
||||
@@ -474,21 +544,24 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
DoNormalizeText,
|
||||
DoDenoisePromptAudio,
|
||||
dit_steps,
|
||||
seed_value,
|
||||
],
|
||||
outputs=[audio_output],
|
||||
outputs=[audio_output, seed_value],
|
||||
show_progress=True,
|
||||
api_name="generate",
|
||||
)
|
||||
|
||||
return interface
|
||||
|
||||
|
||||
def run_demo(
|
||||
server_name: str = "0.0.0.0",
|
||||
server_port: int = 8808,
|
||||
show_error: bool = True,
|
||||
model_id: str = "openbmb/VoxCPM2",
|
||||
device: str = "auto",
|
||||
):
|
||||
demo = VoxCPMDemo(model_id=model_id)
|
||||
demo = VoxCPMDemo(model_id=model_id, device=device)
|
||||
interface = create_demo_interface(demo)
|
||||
interface.queue(max_size=10, default_concurrency_limit=1).launch(
|
||||
server_name=server_name,
|
||||
@@ -502,11 +575,32 @@ def run_demo(
|
||||
|
||||
if __name__ == "__main__":
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser()
|
||||
parser.add_argument(
|
||||
"--model-id", type=str, default="openbmb/VoxCPM2",
|
||||
"--model-id",
|
||||
type=str,
|
||||
default="openbmb/VoxCPM2",
|
||||
help="Local path or HuggingFace repo ID (default: openbmb/VoxCPM2)",
|
||||
)
|
||||
parser.add_argument("--port", type=int, default=8808, help="Server port")
|
||||
parser.add_argument(
|
||||
"--host",
|
||||
type=str,
|
||||
default="0.0.0.0",
|
||||
help="Bind address. Use 127.0.0.1 to restrict access to the local machine; "
|
||||
"the default 0.0.0.0 exposes the unauthenticated UI/API to the network (default: 0.0.0.0)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--device",
|
||||
type=str,
|
||||
default="auto",
|
||||
help="Runtime device: auto, cpu, mps, cuda, or cuda:N (default: auto)",
|
||||
)
|
||||
args = parser.parse_args()
|
||||
run_demo(model_id=args.model_id, server_port=args.port)
|
||||
run_demo(
|
||||
model_id=args.model_id,
|
||||
server_name=args.host,
|
||||
server_port=args.port,
|
||||
device=args.device,
|
||||
)
|
||||
|
||||
+24
-12
@@ -6,6 +6,7 @@ import gradio as gr
|
||||
from typing import Optional, Tuple
|
||||
from funasr import AutoModel
|
||||
from pathlib import Path
|
||||
|
||||
os.environ["TOKENIZERS_PARALLELISM"] = "false"
|
||||
if os.environ.get("HF_REPO_ID", "").strip() == "":
|
||||
os.environ["HF_REPO_ID"] = "openbmb/VoxCPM1.5"
|
||||
@@ -23,7 +24,7 @@ class VoxCPMDemo:
|
||||
self.asr_model: Optional[AutoModel] = AutoModel(
|
||||
model=self.asr_model_id,
|
||||
disable_update=True,
|
||||
log_level='DEBUG',
|
||||
log_level="DEBUG",
|
||||
device="cuda:0" if self.device == "cuda" else "cpu",
|
||||
)
|
||||
|
||||
@@ -48,6 +49,7 @@ class VoxCPMDemo:
|
||||
if not os.path.isdir(target_dir):
|
||||
try:
|
||||
from huggingface_hub import snapshot_download # type: ignore
|
||||
|
||||
os.makedirs(target_dir, exist_ok=True)
|
||||
print(f"Downloading model from HF repo '{repo_id}' to '{target_dir}' ...", file=sys.stderr)
|
||||
snapshot_download(repo_id=repo_id, local_dir=target_dir, local_dir_use_symlinks=False)
|
||||
@@ -72,7 +74,7 @@ class VoxCPMDemo:
|
||||
if prompt_wav is None:
|
||||
return ""
|
||||
res = self.asr_model.generate(input=prompt_wav, language="auto", use_itn=True)
|
||||
text = res[0]["text"].split('|>')[-1]
|
||||
text = res[0]["text"].split("|>")[-1]
|
||||
return text
|
||||
|
||||
def generate_tts_audio(
|
||||
@@ -149,11 +151,13 @@ _CUSTOM_CSS = """
|
||||
|
||||
def create_demo_interface(demo: VoxCPMDemo):
|
||||
"""Build the Gradio UI for VoxCPM demo."""
|
||||
gr.set_static_paths(paths=[Path.cwd().absolute()/"assets"])
|
||||
gr.set_static_paths(paths=[Path.cwd().absolute() / "assets"])
|
||||
|
||||
with gr.Blocks() as interface:
|
||||
# Header logo
|
||||
gr.HTML('<div class="logo-container"><img src="/gradio_api/file=assets/voxcpm_logo.png" alt="VoxCPM Logo"></div>')
|
||||
gr.HTML(
|
||||
'<div class="logo-container"><img src="/gradio_api/file=assets/voxcpm_logo.png" alt="VoxCPM Logo"></div>'
|
||||
)
|
||||
|
||||
# Quick Start
|
||||
with gr.Accordion("📋 Quick Start Guide ｜快速入门", open=False, elem_id="acc_quick"):
|
||||
@@ -201,7 +205,7 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
with gr.Row():
|
||||
with gr.Column():
|
||||
prompt_wav = gr.Audio(
|
||||
sources=["upload", 'microphone'],
|
||||
sources=["upload", "microphone"],
|
||||
type="filepath",
|
||||
label="Prompt Speech (Optional, or let VoxCPM improvise)",
|
||||
value="./examples/example.wav",
|
||||
@@ -210,13 +214,13 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
value=False,
|
||||
label="Prompt Speech Enhancement",
|
||||
elem_id="chk_denoise",
|
||||
info="We use ZipEnhancer model to denoise the prompt audio."
|
||||
info="We use ZipEnhancer model to denoise the prompt audio.",
|
||||
)
|
||||
with gr.Row():
|
||||
prompt_text = gr.Textbox(
|
||||
value="Just by listening a few minutes a day, you'll be able to eliminate negative thoughts by conditioning your mind to be more positive.",
|
||||
label="Prompt Text",
|
||||
placeholder="Please enter the prompt text. Automatic recognition is supported, and you can correct the results yourself..."
|
||||
placeholder="Please enter the prompt text. Automatic recognition is supported, and you can correct the results yourself...",
|
||||
)
|
||||
run_btn = gr.Button("Generate Speech", variant="primary")
|
||||
|
||||
@@ -227,7 +231,7 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
value=2.0,
|
||||
step=0.1,
|
||||
label="CFG Value (Guidance Scale)",
|
||||
info="Higher values increase adherence to prompt, lower values allow more creativity"
|
||||
info="Higher values increase adherence to prompt, lower values allow more creativity",
|
||||
)
|
||||
inference_timesteps = gr.Slider(
|
||||
minimum=4,
|
||||
@@ -235,7 +239,7 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
value=10,
|
||||
step=1,
|
||||
label="Inference Timesteps",
|
||||
info="Number of inference timesteps for generation (higher values may improve quality but slower)"
|
||||
info="Number of inference timesteps for generation (higher values may improve quality but slower)",
|
||||
)
|
||||
with gr.Row():
|
||||
text = gr.Textbox(
|
||||
@@ -247,14 +251,22 @@ def create_demo_interface(demo: VoxCPMDemo):
|
||||
value=False,
|
||||
label="Text Normalization",
|
||||
elem_id="chk_normalize",
|
||||
info="We use wetext library to normalize the input text."
|
||||
info="We use wetext library to normalize the input text.",
|
||||
)
|
||||
audio_output = gr.Audio(label="Output Audio")
|
||||
|
||||
# Wiring
|
||||
run_btn.click(
|
||||
fn=demo.generate_tts_audio,
|
||||
inputs=[text, prompt_wav, prompt_text, cfg_value, inference_timesteps, DoNormalizeText, DoDenoisePromptAudio],
|
||||
inputs=[
|
||||
text,
|
||||
prompt_wav,
|
||||
prompt_text,
|
||||
cfg_value,
|
||||
inference_timesteps,
|
||||
DoNormalizeText,
|
||||
DoDenoisePromptAudio,
|
||||
],
|
||||
outputs=[audio_output],
|
||||
show_progress=True,
|
||||
api_name="generate",
|
||||
@@ -277,4 +289,4 @@ def run_demo(server_name: str = "localhost", server_port: int = 7860, show_error
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
run_demo()
|
||||
run_demo()
|
||||
|
||||
Binary file not shown.
|
After Width: | Height: | Size: 1.1 MiB |
+3
-4
@@ -308,7 +308,8 @@ def run_inference(text, prompt_wav, prompt_text, lora_selection, cfg_scale, step
|
||||
if new_r is not None and current_r is not None and new_r != current_r:
|
||||
print(f"LoRA rank mismatch (model r={current_r}, checkpoint r={new_r}), reloading...", file=sys.stderr)
|
||||
reload_base = (
|
||||
new_base_model if new_base_model and os.path.exists(new_base_model)
|
||||
new_base_model
|
||||
if new_base_model and os.path.exists(new_base_model)
|
||||
else (pretrained_path if pretrained_path and pretrained_path.strip() else default_pretrained_path)
|
||||
)
|
||||
try:
|
||||
@@ -982,9 +983,7 @@ with gr.Blocks(title="VoxCPM LoRA WebUI", theme=gr.themes.Soft(), css=custom_css
|
||||
|
||||
gr.Markdown("#### 分发选项 (Distribution)")
|
||||
with gr.Row():
|
||||
hf_model_id = gr.Textbox(
|
||||
label="HuggingFace Model ID (e.g., openbmb/VoxCPM2)", value=""
|
||||
)
|
||||
hf_model_id = gr.Textbox(label="HuggingFace Model ID (e.g., openbmb/VoxCPM2)", value="")
|
||||
distribute = gr.Checkbox(label="分发模式 (distribute)", value=False)
|
||||
|
||||
with gr.Column(scale=2, elem_classes="form-section"):
|
||||
|
||||
@@ -59,6 +59,9 @@ dev = [
|
||||
"flake8>=3.8",
|
||||
"pre-commit>=2.0",
|
||||
]
|
||||
timestamps = [
|
||||
"stable-ts>=2.19.1",
|
||||
]
|
||||
|
||||
[project.scripts]
|
||||
voxcpm = "voxcpm.cli:main"
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
Loads src/voxcpm/model/utils.py directly to avoid the heavy voxcpm package
|
||||
init. Run with: `python scripts/test_pick_runtime_dtype.py`.
|
||||
"""
|
||||
|
||||
import importlib.util
|
||||
import os
|
||||
import pathlib
|
||||
|
||||
@@ -19,6 +19,7 @@ With voice cloning:
|
||||
--text "Hello, this is voice cloning result." \
|
||||
--prompt_audio path/to/ref.wav \
|
||||
--prompt_text "Reference audio transcript" \
|
||||
--seed 42 \
|
||||
--output ft_clone.wav
|
||||
"""
|
||||
|
||||
@@ -86,6 +87,12 @@ def parse_args():
|
||||
action="store_true",
|
||||
help="Enable text normalization",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Random seed for generation (default: None)",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -109,6 +116,9 @@ def main():
|
||||
print(f"[FT Inference] Using reference audio: {prompt_wav_path}", file=sys.stderr)
|
||||
print(f"[FT Inference] Reference text: {prompt_text}", file=sys.stderr)
|
||||
|
||||
if args.seed is not None:
|
||||
print(f"[FT Inference] Using seed: {args.seed}", file=sys.stderr)
|
||||
|
||||
audio_np = model.generate(
|
||||
text=args.text,
|
||||
prompt_wav_path=prompt_wav_path,
|
||||
@@ -118,6 +128,7 @@ def main():
|
||||
max_len=args.max_len,
|
||||
normalize=args.normalize,
|
||||
denoise=False,
|
||||
seed=args.seed,
|
||||
)
|
||||
|
||||
# Save audio
|
||||
|
||||
@@ -16,6 +16,7 @@ With voice cloning:
|
||||
--text "This is voice cloning result." \
|
||||
--prompt_audio path/to/ref.wav \
|
||||
--prompt_text "Reference audio transcript" \
|
||||
--seed 42 \
|
||||
--output lora_clone.wav
|
||||
|
||||
Note: The script reads base_model path and lora_config from lora_config.json
|
||||
@@ -94,6 +95,12 @@ def parse_args():
|
||||
action="store_true",
|
||||
help="Enable text normalization",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--seed",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Random seed for generation (default: None)",
|
||||
)
|
||||
return parser.parse_args()
|
||||
|
||||
|
||||
@@ -130,6 +137,8 @@ def main():
|
||||
print(
|
||||
f" LoRA config: r={lora_cfg.r}, alpha={lora_cfg.alpha}" if lora_cfg else " LoRA config: None", file=sys.stderr
|
||||
)
|
||||
if args.seed is not None:
|
||||
print(f" Seed: {args.seed}", file=sys.stderr)
|
||||
|
||||
# 3. Load model with LoRA (no denoiser)
|
||||
print(f"\n[1/2] Loading model with LoRA: {pretrained_path}", file=sys.stderr)
|
||||
@@ -161,6 +170,7 @@ def main():
|
||||
max_len=args.max_len,
|
||||
normalize=args.normalize,
|
||||
denoise=False,
|
||||
seed=args.seed,
|
||||
)
|
||||
lora_output = out_path.with_stem(out_path.stem + "_with_lora")
|
||||
sf.write(str(lora_output), audio_np, model.tts_model.sample_rate)
|
||||
@@ -181,6 +191,7 @@ def main():
|
||||
max_len=args.max_len,
|
||||
normalize=args.normalize,
|
||||
denoise=False,
|
||||
seed=args.seed,
|
||||
)
|
||||
disabled_output = out_path.with_stem(out_path.stem + "_lora_disabled")
|
||||
sf.write(str(disabled_output), audio_np, model.tts_model.sample_rate)
|
||||
@@ -201,6 +212,7 @@ def main():
|
||||
max_len=args.max_len,
|
||||
normalize=args.normalize,
|
||||
denoise=False,
|
||||
seed=args.seed,
|
||||
)
|
||||
reenabled_output = out_path.with_stem(out_path.stem + "_lora_reenabled")
|
||||
sf.write(str(reenabled_output), audio_np, model.tts_model.sample_rate)
|
||||
@@ -221,6 +233,7 @@ def main():
|
||||
max_len=args.max_len,
|
||||
normalize=args.normalize,
|
||||
denoise=False,
|
||||
seed=args.seed,
|
||||
)
|
||||
reset_output = out_path.with_stem(out_path.stem + "_lora_reset")
|
||||
sf.write(str(reset_output), audio_np, model.tts_model.sample_rate)
|
||||
@@ -242,6 +255,7 @@ def main():
|
||||
max_len=args.max_len,
|
||||
normalize=args.normalize,
|
||||
denoise=False,
|
||||
seed=args.seed,
|
||||
)
|
||||
reload_output = out_path.with_stem(out_path.stem + "_lora_reloaded")
|
||||
sf.write(str(reload_output), audio_np, model.tts_model.sample_rate)
|
||||
|
||||
@@ -599,7 +599,9 @@ def generate_sample_audio(
|
||||
)
|
||||
with torch.no_grad():
|
||||
with autocast_ctx:
|
||||
generated = unwrapped_model.generate(target_text=text, inference_timesteps=10, cfg_value=2.0)
|
||||
generated = unwrapped_model.generate(
|
||||
target_text=text, inference_timesteps=10, cfg_value=2.0, seed=42
|
||||
)
|
||||
|
||||
# Restore training setup
|
||||
# unwrapped_model.to(torch.float32)
|
||||
@@ -682,7 +684,7 @@ def load_checkpoint(model, optimizer, scheduler, save_dir: Path, rank: int = 0):
|
||||
|
||||
state_dict = load_file(str(lora_weights_path))
|
||||
else:
|
||||
ckpt = torch.load(lora_weights_path, map_location="cpu")
|
||||
ckpt = torch.load(lora_weights_path, map_location="cpu", weights_only=True)
|
||||
state_dict = ckpt.get("state_dict", ckpt)
|
||||
|
||||
unwrapped.load_state_dict(state_dict, strict=False)
|
||||
@@ -700,7 +702,7 @@ def load_checkpoint(model, optimizer, scheduler, save_dir: Path, rank: int = 0):
|
||||
|
||||
state_dict = load_file(str(model_path))
|
||||
else:
|
||||
ckpt = torch.load(model_path, map_location="cpu")
|
||||
ckpt = torch.load(model_path, map_location="cpu", weights_only=True)
|
||||
state_dict = ckpt.get("state_dict", ckpt)
|
||||
|
||||
unwrapped.load_state_dict(state_dict, strict=False)
|
||||
@@ -710,14 +712,14 @@ def load_checkpoint(model, optimizer, scheduler, save_dir: Path, rank: int = 0):
|
||||
# Load optimizer state
|
||||
optimizer_path = latest_folder / "optimizer.pth"
|
||||
if optimizer_path.exists():
|
||||
optimizer.load_state_dict(torch.load(optimizer_path, map_location="cpu"))
|
||||
optimizer.load_state_dict(torch.load(optimizer_path, map_location="cpu", weights_only=True))
|
||||
if rank == 0:
|
||||
print(f"Loaded optimizer state from {optimizer_path}", file=sys.stderr)
|
||||
|
||||
# Load scheduler state
|
||||
scheduler_path = latest_folder / "scheduler.pth"
|
||||
if scheduler_path.exists():
|
||||
scheduler.load_state_dict(torch.load(scheduler_path, map_location="cpu"))
|
||||
scheduler.load_state_dict(torch.load(scheduler_path, map_location="cpu", weights_only=True))
|
||||
if rank == 0:
|
||||
print(f"Loaded scheduler state from {scheduler_path}", file=sys.stderr)
|
||||
|
||||
|
||||
+146
-106
@@ -11,6 +11,8 @@ import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
from voxcpm.timestamps import align_audio_file
|
||||
|
||||
DEFAULT_HF_MODEL_ID = "openbmb/VoxCPM2"
|
||||
|
||||
# -----------------------------
|
||||
@@ -86,9 +88,7 @@ def resolve_prompt_text(args, parser) -> str | None:
|
||||
|
||||
|
||||
def detect_model_architecture(args) -> str | None:
|
||||
model_location = getattr(args, "model_path", None) or getattr(
|
||||
args, "hf_model_id", None
|
||||
)
|
||||
model_location = getattr(args, "model_path", None) or getattr(args, "hf_model_id", None)
|
||||
if not model_location:
|
||||
return None
|
||||
|
||||
@@ -103,11 +103,7 @@ def detect_model_architecture(args) -> str | None:
|
||||
model_hint = str(model_location).lower()
|
||||
if "voxcpm2" in model_hint:
|
||||
return "voxcpm2"
|
||||
if (
|
||||
"voxcpm1.5" in model_hint
|
||||
or "voxcpm-1.5" in model_hint
|
||||
or "voxcpm_1.5" in model_hint
|
||||
):
|
||||
if "voxcpm1.5" in model_hint or "voxcpm-1.5" in model_hint or "voxcpm_1.5" in model_hint:
|
||||
return "voxcpm"
|
||||
|
||||
return None
|
||||
@@ -121,9 +117,7 @@ def validate_prompt_related_args(args, parser, prompt_text: str | None):
|
||||
parser.error("--prompt-audio requires --prompt-text or --prompt-file.")
|
||||
|
||||
if args.control and prompt_text:
|
||||
parser.error(
|
||||
"--control cannot be used together with --prompt-text or --prompt-file."
|
||||
)
|
||||
parser.error("--control cannot be used together with --prompt-text or --prompt-file.")
|
||||
|
||||
|
||||
def validate_reference_support(args, parser):
|
||||
@@ -138,9 +132,7 @@ def validate_reference_support(args, parser):
|
||||
def validate_design_args(args, parser):
|
||||
prompt_text = resolve_prompt_text(args, parser)
|
||||
if args.prompt_audio or args.reference_audio or prompt_text:
|
||||
parser.error(
|
||||
"`design` does not accept prompt/reference audio. Use `clone` instead."
|
||||
)
|
||||
parser.error("`design` does not accept prompt/reference audio. Use `clone` instead.")
|
||||
|
||||
|
||||
def validate_clone_args(args, parser):
|
||||
@@ -149,9 +141,7 @@ def validate_clone_args(args, parser):
|
||||
validate_reference_support(args, parser)
|
||||
|
||||
if not args.prompt_audio and not args.reference_audio:
|
||||
parser.error(
|
||||
"`clone` requires --reference-audio, or --prompt-audio with --prompt-text/--prompt-file."
|
||||
)
|
||||
parser.error("`clone` requires --reference-audio, or --prompt-audio with --prompt-text/--prompt-file.")
|
||||
|
||||
return prompt_text
|
||||
|
||||
@@ -173,9 +163,7 @@ def load_model(args):
|
||||
|
||||
print("Loading VoxCPM model...", file=sys.stderr)
|
||||
|
||||
zipenhancer_path = getattr(args, "zipenhancer_path", None) or os.environ.get(
|
||||
"ZIPENHANCER_MODEL_PATH", None
|
||||
)
|
||||
zipenhancer_path = getattr(args, "zipenhancer_path", None) or os.environ.get("ZIPENHANCER_MODEL_PATH", None)
|
||||
|
||||
# Build LoRA config if provided
|
||||
lora_config = None
|
||||
@@ -259,8 +247,8 @@ def _run_single(args, parser, *, text: str, output: str, prompt_text: str | None
|
||||
cfg_value=args.cfg_value,
|
||||
inference_timesteps=args.inference_timesteps,
|
||||
normalize=args.normalize,
|
||||
denoise=args.denoise
|
||||
and (args.prompt_audio is not None or args.reference_audio is not None),
|
||||
denoise=args.denoise and (args.prompt_audio is not None or args.reference_audio is not None),
|
||||
seed=args.seed,
|
||||
)
|
||||
|
||||
import soundfile as sf
|
||||
@@ -269,22 +257,24 @@ def _run_single(args, parser, *, text: str, output: str, prompt_text: str | None
|
||||
|
||||
duration = len(audio_array) / model.tts_model.sample_rate
|
||||
print(f"Saved audio to: {output_path} ({duration:.2f}s)", file=sys.stderr)
|
||||
maybe_write_timestamps(
|
||||
args,
|
||||
text=text,
|
||||
audio_path=output_path,
|
||||
sample_rate=model.tts_model.sample_rate,
|
||||
)
|
||||
|
||||
|
||||
def cmd_design(args, parser):
|
||||
validate_design_args(args, parser)
|
||||
final_text = build_final_text(args.text, args.control)
|
||||
return _run_single(
|
||||
args, parser, text=final_text, output=args.output, prompt_text=None
|
||||
)
|
||||
return _run_single(args, parser, text=final_text, output=args.output, prompt_text=None)
|
||||
|
||||
|
||||
def cmd_clone(args, parser):
|
||||
prompt_text = validate_clone_args(args, parser)
|
||||
final_text = build_final_text(args.text, args.control)
|
||||
return _run_single(
|
||||
args, parser, text=final_text, output=args.output, prompt_text=prompt_text
|
||||
)
|
||||
return _run_single(args, parser, text=final_text, output=args.output, prompt_text=prompt_text)
|
||||
|
||||
|
||||
def cmd_validate(args, parser):
|
||||
@@ -306,8 +296,6 @@ def cmd_validate(args, parser):
|
||||
|
||||
|
||||
def cmd_batch(args, parser):
|
||||
import soundfile as sf
|
||||
|
||||
input_file = require_file_exists(args.input, parser, "input file")
|
||||
output_dir = Path(args.output_dir)
|
||||
output_dir.mkdir(parents=True, exist_ok=True)
|
||||
@@ -321,17 +309,15 @@ def cmd_batch(args, parser):
|
||||
prompt_text = validate_batch_args(args, parser)
|
||||
model = load_model(args)
|
||||
|
||||
import soundfile as sf
|
||||
|
||||
prompt_audio_path = None
|
||||
if args.prompt_audio:
|
||||
prompt_audio_path = str(
|
||||
require_file_exists(args.prompt_audio, parser, "prompt audio file")
|
||||
)
|
||||
prompt_audio_path = str(require_file_exists(args.prompt_audio, parser, "prompt audio file"))
|
||||
|
||||
reference_audio_path = None
|
||||
if args.reference_audio:
|
||||
reference_audio_path = str(
|
||||
require_file_exists(args.reference_audio, parser, "reference audio file")
|
||||
)
|
||||
reference_audio_path = str(require_file_exists(args.reference_audio, parser, "reference audio file"))
|
||||
|
||||
success_count = 0
|
||||
|
||||
@@ -346,8 +332,8 @@ def cmd_batch(args, parser):
|
||||
cfg_value=args.cfg_value,
|
||||
inference_timesteps=args.inference_timesteps,
|
||||
normalize=args.normalize,
|
||||
denoise=args.denoise
|
||||
and (prompt_audio_path is not None or reference_audio_path is not None),
|
||||
denoise=args.denoise and (prompt_audio_path is not None or reference_audio_path is not None),
|
||||
seed=args.seed,
|
||||
)
|
||||
|
||||
output_file = output_dir / f"output_{i:03d}.wav"
|
||||
@@ -355,6 +341,12 @@ def cmd_batch(args, parser):
|
||||
|
||||
duration = len(audio_array) / model.tts_model.sample_rate
|
||||
print(f"Saved: {output_file} ({duration:.2f}s)", file=sys.stderr)
|
||||
maybe_write_timestamps(
|
||||
args,
|
||||
text=final_text,
|
||||
audio_path=output_file,
|
||||
sample_rate=model.tts_model.sample_rate,
|
||||
)
|
||||
success_count += 1
|
||||
|
||||
except Exception as e:
|
||||
@@ -363,6 +355,41 @@ def cmd_batch(args, parser):
|
||||
print(f"\nBatch finished: {success_count}/{len(texts)} succeeded", file=sys.stderr)
|
||||
|
||||
|
||||
def default_timestamp_path(audio_path: Path) -> Path:
|
||||
return audio_path.with_suffix(".timestamps.json")
|
||||
|
||||
|
||||
def maybe_write_timestamps(args, *, text: str, audio_path: Path, sample_rate: int) -> None:
|
||||
if not getattr(args, "timestamps", False):
|
||||
return
|
||||
|
||||
timestamp_output = getattr(args, "timestamp_output", None)
|
||||
output_path = Path(timestamp_output) if timestamp_output else default_timestamp_path(audio_path)
|
||||
output_path.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
try:
|
||||
result = align_audio_file(
|
||||
audio_path=str(audio_path),
|
||||
text=text,
|
||||
sample_rate=sample_rate,
|
||||
backend=args.timestamp_backend,
|
||||
level=args.timestamp_level,
|
||||
model_name=args.timestamp_model,
|
||||
device=args.timestamp_device,
|
||||
language=args.timestamp_language,
|
||||
)
|
||||
except Exception as exc:
|
||||
if getattr(args, "timestamp_strict", False):
|
||||
raise SystemExit(f"Timestamp alignment failed: {exc}") from exc
|
||||
print(f"Warning: Timestamp alignment failed: {exc}", file=sys.stderr)
|
||||
return
|
||||
|
||||
with open(output_path, "w", encoding="utf-8") as f:
|
||||
json.dump(result, f, ensure_ascii=False, indent=2)
|
||||
f.write("\n")
|
||||
print(f"Saved timestamps to: {output_path}", file=sys.stderr)
|
||||
|
||||
|
||||
# -----------------------------
|
||||
# Parser
|
||||
# -----------------------------
|
||||
@@ -387,8 +414,12 @@ def _add_common_generation_args(parser):
|
||||
default=10,
|
||||
help="Inference steps (int, recommended 4–30, default: 10)",
|
||||
)
|
||||
parser.add_argument("--normalize", action="store_true", help="Enable text normalization")
|
||||
parser.add_argument(
|
||||
"--normalize", action="store_true", help="Enable text normalization"
|
||||
"--seed",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Random seed for generation (default: None)",
|
||||
)
|
||||
|
||||
|
||||
@@ -398,12 +429,8 @@ def _add_prompt_reference_args(parser):
|
||||
"-pa",
|
||||
help="Prompt audio file path (continuation mode, requires --prompt-text or --prompt-file)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt-text", "-pt", help="Text corresponding to the prompt audio"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--prompt-file", type=str, help="Text file corresponding to the prompt audio"
|
||||
)
|
||||
parser.add_argument("--prompt-text", "-pt", help="Text corresponding to the prompt audio")
|
||||
parser.add_argument("--prompt-file", type=str, help="Text file corresponding to the prompt audio")
|
||||
parser.add_argument(
|
||||
"--reference-audio",
|
||||
"-ra",
|
||||
@@ -430,15 +457,9 @@ def _add_model_args(parser):
|
||||
default="auto",
|
||||
help="Runtime device: auto, cpu, mps, cuda, or cuda:N (default: auto)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--cache-dir", type=str, help="Cache directory for Hub downloads"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--local-files-only", action="store_true", help="Disable network access"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--no-denoiser", action="store_true", help="Disable denoiser model loading"
|
||||
)
|
||||
parser.add_argument("--cache-dir", type=str, help="Cache directory for Hub downloads")
|
||||
parser.add_argument("--local-files-only", action="store_true", help="Disable network access")
|
||||
parser.add_argument("--no-denoiser", action="store_true", help="Disable denoiser model loading")
|
||||
parser.add_argument(
|
||||
"--no-optimize",
|
||||
action="store_true",
|
||||
@@ -453,9 +474,7 @@ def _add_model_args(parser):
|
||||
|
||||
def _add_lora_args(parser):
|
||||
parser.add_argument("--lora-path", type=str, help="Path to LoRA weights")
|
||||
parser.add_argument(
|
||||
"--lora-r", type=int, default=32, help="LoRA rank (positive int, default: 32)"
|
||||
)
|
||||
parser.add_argument("--lora-r", type=int, default=32, help="LoRA rank (positive int, default: 32)")
|
||||
parser.add_argument(
|
||||
"--lora-alpha",
|
||||
type=int,
|
||||
@@ -468,12 +487,8 @@ def _add_lora_args(parser):
|
||||
default=0.0,
|
||||
help="LoRA dropout rate (0.0–1.0, default: 0.0)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lora-disable-lm", action="store_true", help="Disable LoRA on LM layers"
|
||||
)
|
||||
parser.add_argument(
|
||||
"--lora-disable-dit", action="store_true", help="Disable LoRA on DiT layers"
|
||||
)
|
||||
parser.add_argument("--lora-disable-lm", action="store_true", help="Disable LoRA on LM layers")
|
||||
parser.add_argument("--lora-disable-dit", action="store_true", help="Disable LoRA on DiT layers")
|
||||
parser.add_argument(
|
||||
"--lora-enable-proj",
|
||||
action="store_true",
|
||||
@@ -481,6 +496,52 @@ def _add_lora_args(parser):
|
||||
)
|
||||
|
||||
|
||||
def _add_timestamp_args(parser, *, include_output: bool = True):
|
||||
parser.add_argument(
|
||||
"--timestamps",
|
||||
action="store_true",
|
||||
help="Run post-generation timestamp alignment and write a JSON sidecar file",
|
||||
)
|
||||
if include_output:
|
||||
parser.add_argument(
|
||||
"--timestamp-output",
|
||||
type=str,
|
||||
help="Output timestamp JSON path (default: output audio path with .timestamps.json suffix)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timestamp-level",
|
||||
choices=["segment", "word", "char"],
|
||||
default="word",
|
||||
help="Timestamp granularity (default: word; char is best-effort)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timestamp-backend",
|
||||
choices=["stable-ts"],
|
||||
default="stable-ts",
|
||||
help="Timestamp alignment backend (default: stable-ts)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timestamp-model",
|
||||
default="base",
|
||||
help="stable-ts Whisper model name (default: base)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timestamp-language",
|
||||
default=None,
|
||||
help="Language hint for timestamp alignment, e.g. zh or en",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timestamp-device",
|
||||
default=None,
|
||||
help="Device for timestamp alignment, e.g. cuda or cpu",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--timestamp-strict",
|
||||
action="store_true",
|
||||
help="Fail the command if timestamp alignment fails",
|
||||
)
|
||||
|
||||
|
||||
def _build_parser():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="VoxCPM CLI - VoxCPM2-first voice design, cloning, and batch processing",
|
||||
@@ -496,37 +557,25 @@ Examples:
|
||||
|
||||
subparsers = parser.add_subparsers(dest="command")
|
||||
|
||||
design_parser = subparsers.add_parser(
|
||||
"design", help="Generate speech with VoxCPM2-first voice design"
|
||||
)
|
||||
design_parser = subparsers.add_parser("design", help="Generate speech with VoxCPM2-first voice design")
|
||||
_add_common_generation_args(design_parser)
|
||||
_add_prompt_reference_args(design_parser)
|
||||
_add_model_args(design_parser)
|
||||
_add_lora_args(design_parser)
|
||||
design_parser.add_argument(
|
||||
"--output", "-o", required=True, help="Output audio file path"
|
||||
)
|
||||
_add_timestamp_args(design_parser)
|
||||
design_parser.add_argument("--output", "-o", required=True, help="Output audio file path")
|
||||
|
||||
clone_parser = subparsers.add_parser(
|
||||
"clone", help="Clone a voice with reference/prompt audio"
|
||||
)
|
||||
clone_parser = subparsers.add_parser("clone", help="Clone a voice with reference/prompt audio")
|
||||
_add_common_generation_args(clone_parser)
|
||||
_add_prompt_reference_args(clone_parser)
|
||||
_add_model_args(clone_parser)
|
||||
_add_lora_args(clone_parser)
|
||||
clone_parser.add_argument(
|
||||
"--output", "-o", required=True, help="Output audio file path"
|
||||
)
|
||||
_add_timestamp_args(clone_parser)
|
||||
clone_parser.add_argument("--output", "-o", required=True, help="Output audio file path")
|
||||
|
||||
batch_parser = subparsers.add_parser(
|
||||
"batch", help="Batch-generate one line per output file"
|
||||
)
|
||||
batch_parser.add_argument(
|
||||
"--input", "-i", required=True, help="Input text file (one text per line)"
|
||||
)
|
||||
batch_parser.add_argument(
|
||||
"--output-dir", "-od", required=True, help="Output directory"
|
||||
)
|
||||
batch_parser = subparsers.add_parser("batch", help="Batch-generate one line per output file")
|
||||
batch_parser.add_argument("--input", "-i", required=True, help="Input text file (one text per line)")
|
||||
batch_parser.add_argument("--output-dir", "-od", required=True, help="Output directory")
|
||||
batch_parser.add_argument(
|
||||
"--control",
|
||||
type=str,
|
||||
@@ -545,20 +594,23 @@ Examples:
|
||||
default=10,
|
||||
help="Inference steps (int, recommended 4–30, default: 10)",
|
||||
)
|
||||
batch_parser.add_argument("--normalize", action="store_true", help="Enable text normalization")
|
||||
batch_parser.add_argument(
|
||||
"--normalize", action="store_true", help="Enable text normalization"
|
||||
"--seed",
|
||||
type=int,
|
||||
default=None,
|
||||
help="Random seed for generation (default: None)",
|
||||
)
|
||||
_add_model_args(batch_parser)
|
||||
_add_lora_args(batch_parser)
|
||||
_add_timestamp_args(batch_parser, include_output=False)
|
||||
|
||||
# Validate subcommand
|
||||
validate_parser = subparsers.add_parser(
|
||||
"validate",
|
||||
help="Validate a training data manifest (JSONL) before fine-tuning",
|
||||
)
|
||||
validate_parser.add_argument(
|
||||
"--manifest", "-m", required=True, help="Path to JSONL training manifest"
|
||||
)
|
||||
validate_parser.add_argument("--manifest", "-m", required=True, help="Path to JSONL training manifest")
|
||||
validate_parser.add_argument(
|
||||
"--sample-rate",
|
||||
type=int,
|
||||
@@ -571,22 +623,17 @@ Examples:
|
||||
default=0,
|
||||
help="Maximum number of samples to validate (0 = all, default: 0)",
|
||||
)
|
||||
validate_parser.add_argument(
|
||||
"--verbose", "-v", action="store_true", help="Print per-sample progress"
|
||||
)
|
||||
validate_parser.add_argument("--verbose", "-v", action="store_true", help="Print per-sample progress")
|
||||
|
||||
# Legacy root arguments
|
||||
parser.add_argument("--input", "-i", help="Input text file (batch mode only)")
|
||||
parser.add_argument(
|
||||
"--output-dir", "-od", help="Output directory (batch mode only)"
|
||||
)
|
||||
parser.add_argument("--output-dir", "-od", help="Output directory (batch mode only)")
|
||||
_add_common_generation_args(parser)
|
||||
parser.add_argument(
|
||||
"--output", "-o", help="Output audio file path (single or clone mode)"
|
||||
)
|
||||
parser.add_argument("--output", "-o", help="Output audio file path (single or clone mode)")
|
||||
_add_prompt_reference_args(parser)
|
||||
_add_model_args(parser)
|
||||
_add_lora_args(parser)
|
||||
_add_timestamp_args(parser)
|
||||
|
||||
return parser
|
||||
|
||||
@@ -595,9 +642,7 @@ def _dispatch_legacy(args, parser):
|
||||
warn_legacy_mode()
|
||||
|
||||
if args.input and args.text:
|
||||
parser.error(
|
||||
"Use either batch mode (--input) or single mode (--text), not both."
|
||||
)
|
||||
parser.error("Use either batch mode (--input) or single mode (--text), not both.")
|
||||
|
||||
if args.input:
|
||||
if not args.output_dir:
|
||||
@@ -607,12 +652,7 @@ def _dispatch_legacy(args, parser):
|
||||
if not args.text or not args.output:
|
||||
parser.error("Single-sample legacy mode requires --text and --output")
|
||||
|
||||
if (
|
||||
args.prompt_audio
|
||||
or args.prompt_text
|
||||
or args.prompt_file
|
||||
or args.reference_audio
|
||||
):
|
||||
if args.prompt_audio or args.prompt_text or args.prompt_file or args.reference_audio:
|
||||
return cmd_clone(args, parser)
|
||||
|
||||
return cmd_design(args, parser)
|
||||
|
||||
+13
-7
@@ -45,14 +45,17 @@ class VoxCPM:
|
||||
file=sys.stderr,
|
||||
)
|
||||
|
||||
# If lora_weights_path is provided but no lora_config, create a default one
|
||||
# If lora_weights_path is provided but no lora_config, load the saved
|
||||
# lora_config.json (so r/alpha match the checkpoint); else use a default.
|
||||
if lora_weights_path is not None and lora_config is None:
|
||||
lora_config = LoRAConfig(
|
||||
enable_lm=True,
|
||||
enable_dit=True,
|
||||
enable_proj=False,
|
||||
)
|
||||
print(f"Auto-created default LoRAConfig for loading weights from: {lora_weights_path}", file=sys.stderr)
|
||||
cfg_path = os.path.join(lora_weights_path, "lora_config.json")
|
||||
if os.path.isdir(lora_weights_path) and os.path.isfile(cfg_path):
|
||||
with open(cfg_path, "r", encoding="utf-8") as f:
|
||||
lora_config = LoRAConfig(**json.load(f)["lora_config"])
|
||||
print(f"Loaded LoRAConfig from: {cfg_path}", file=sys.stderr)
|
||||
else:
|
||||
lora_config = LoRAConfig(enable_lm=True, enable_dit=True, enable_proj=False)
|
||||
print(f"Auto-created default LoRAConfig for loading weights from: {lora_weights_path}", file=sys.stderr)
|
||||
|
||||
# Determine model type from config.json architecture field
|
||||
config_path = os.path.join(voxcpm_model_path, "config.json")
|
||||
@@ -193,6 +196,7 @@ class VoxCPM:
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
streaming: bool = False,
|
||||
seed: Optional[int] = None,
|
||||
) -> Generator[np.ndarray, None, None]:
|
||||
"""Synthesize speech for the given text and return a single waveform.
|
||||
|
||||
@@ -215,6 +219,7 @@ class VoxCPM:
|
||||
retry_badcase_max_times: Maximum number of times to retry badcase.
|
||||
retry_badcase_ratio_threshold: Threshold for audio-to-text ratio.
|
||||
streaming: Whether to return a generator of audio chunks.
|
||||
seed: Optional random seed for reproducibility.
|
||||
Returns:
|
||||
Generator of numpy.ndarray: 1D waveform array (float32) on CPU.
|
||||
Yields audio chunks for each generation step if ``streaming=True``,
|
||||
@@ -291,6 +296,7 @@ class VoxCPM:
|
||||
retry_badcase_max_times=retry_badcase_max_times,
|
||||
retry_badcase_ratio_threshold=retry_badcase_ratio_threshold,
|
||||
streaming=streaming,
|
||||
seed=seed,
|
||||
)
|
||||
|
||||
if streaming:
|
||||
|
||||
+22
-11
@@ -5,9 +5,12 @@ from transformers import PreTrainedTokenizer
|
||||
|
||||
_LOW_PRECISION_DTYPES = {"bfloat16", "bf16", "float16", "fp16"}
|
||||
_VALID_DTYPE_OVERRIDES = {
|
||||
"bfloat16", "bf16",
|
||||
"float16", "fp16",
|
||||
"float32", "fp32",
|
||||
"bfloat16",
|
||||
"bf16",
|
||||
"float16",
|
||||
"fp16",
|
||||
"float32",
|
||||
"fp32",
|
||||
}
|
||||
|
||||
|
||||
@@ -21,6 +24,19 @@ def next_and_close(gen):
|
||||
gen.close()
|
||||
|
||||
|
||||
def materialize_generation_seed(seed: Optional[int]) -> int:
|
||||
"""Return a concrete seed for a generation request."""
|
||||
if seed is not None:
|
||||
return int(seed)
|
||||
return int(torch.seed() & 0xFFFFFFFF)
|
||||
|
||||
|
||||
def apply_generation_seed(seed: int) -> None:
|
||||
torch.manual_seed(seed)
|
||||
if torch.cuda.is_available():
|
||||
torch.cuda.manual_seed_all(seed)
|
||||
|
||||
|
||||
def mask_multichar_chinese_tokens(tokenizer: PreTrainedTokenizer):
|
||||
"""Create a tokenizer wrapper that converts multi-character Chinese tokens to single characters.
|
||||
|
||||
@@ -160,10 +176,7 @@ def pick_runtime_dtype(device: str, configured_dtype: str) -> str:
|
||||
override = os.environ.get("VOXCPM_MPS_DTYPE", "").strip().lower()
|
||||
if override:
|
||||
if override not in _VALID_DTYPE_OVERRIDES:
|
||||
raise ValueError(
|
||||
f"VOXCPM_MPS_DTYPE='{override}' is not one of "
|
||||
f"{sorted(_VALID_DTYPE_OVERRIDES)}"
|
||||
)
|
||||
raise ValueError(f"VOXCPM_MPS_DTYPE='{override}' is not one of " f"{sorted(_VALID_DTYPE_OVERRIDES)}")
|
||||
return override
|
||||
|
||||
if (configured_dtype or "").lower() in _LOW_PRECISION_DTYPES:
|
||||
@@ -211,15 +224,13 @@ def resolve_runtime_device(device: Optional[str], configured_device: str = "cuda
|
||||
if explicit.startswith("cuda"):
|
||||
if not torch.cuda.is_available():
|
||||
raise ValueError(
|
||||
f"Requested device '{device}', but CUDA is not available. "
|
||||
"Use device='auto' for automatic fallback."
|
||||
f"Requested device '{device}', but CUDA is not available. " "Use device='auto' for automatic fallback."
|
||||
)
|
||||
return explicit
|
||||
if explicit == "mps":
|
||||
if not _has_mps():
|
||||
raise ValueError(
|
||||
"Requested device 'mps', but MPS is not available. "
|
||||
"Use device='auto' for automatic fallback."
|
||||
"Requested device 'mps', but MPS is not available. " "Use device='auto' for automatic fallback."
|
||||
)
|
||||
return "mps"
|
||||
if explicit == "cpu":
|
||||
|
||||
@@ -45,7 +45,9 @@ from ..modules.locdit import CfmConfig, UnifiedCFM, VoxCPMLocDiT
|
||||
from ..modules.locenc import VoxCPMLocEnc
|
||||
from ..modules.minicpm4 import MiniCPM4Config, MiniCPMModel
|
||||
from .utils import (
|
||||
apply_generation_seed,
|
||||
get_dtype,
|
||||
materialize_generation_seed,
|
||||
mask_multichar_chinese_tokens,
|
||||
next_and_close,
|
||||
pick_runtime_dtype,
|
||||
@@ -140,6 +142,7 @@ class VoxCPMModel(nn.Module):
|
||||
self.text_tokenizer = mask_multichar_chinese_tokens(tokenizer)
|
||||
self.audio_start_token = 101
|
||||
self.audio_end_token = 102
|
||||
self.last_successful_seed = None
|
||||
|
||||
# Residual Acoustic LM
|
||||
residual_lm_config = config.lm_config.model_copy(deep=True)
|
||||
@@ -367,6 +370,7 @@ class VoxCPMModel(nn.Module):
|
||||
retry_badcase_max_times: int = 3,
|
||||
retry_badcase_ratio_threshold: float = 6.0, # setting acceptable ratio of audio length to text length (for badcase detection)
|
||||
streaming: bool = False,
|
||||
seed: Optional[int] = None,
|
||||
) -> Generator[torch.Tensor, None, None]:
|
||||
if retry_badcase and streaming:
|
||||
warnings.warn("Retry on bad cases is not supported in streaming mode, setting retry_badcase=False.")
|
||||
@@ -452,7 +456,12 @@ class VoxCPMModel(nn.Module):
|
||||
target_text_length = len(self.text_tokenizer(target_text))
|
||||
|
||||
retry_badcase_times = 0
|
||||
current_seed = materialize_generation_seed(seed)
|
||||
last_attempt_seed = current_seed
|
||||
while retry_badcase_times < retry_badcase_max_times:
|
||||
last_attempt_seed = current_seed
|
||||
apply_generation_seed(last_attempt_seed)
|
||||
|
||||
inference_result = self._inference(
|
||||
text_token,
|
||||
text_mask,
|
||||
@@ -471,6 +480,7 @@ class VoxCPMModel(nn.Module):
|
||||
for latent_pred, _ in inference_result:
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||
decode_audio = decode_audio[..., -patch_len:].squeeze(1).cpu()
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
yield decode_audio
|
||||
break
|
||||
else:
|
||||
@@ -482,6 +492,7 @@ class VoxCPMModel(nn.Module):
|
||||
file=sys.stderr,
|
||||
)
|
||||
retry_badcase_times += 1
|
||||
current_seed += 1
|
||||
continue
|
||||
else:
|
||||
break
|
||||
@@ -489,6 +500,7 @@ class VoxCPMModel(nn.Module):
|
||||
break
|
||||
|
||||
if not streaming:
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32)).squeeze(1).cpu()
|
||||
yield decode_audio
|
||||
|
||||
@@ -603,6 +615,7 @@ class VoxCPMModel(nn.Module):
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
streaming: bool = False,
|
||||
streaming_prefix_len: int = 3,
|
||||
seed: Optional[int] = None,
|
||||
) -> Generator[Tuple[torch.Tensor, torch.Tensor, Union[torch.Tensor, List[torch.Tensor]]], None, None]:
|
||||
"""
|
||||
Generate audio using pre-built prompt cache.
|
||||
@@ -678,7 +691,12 @@ class VoxCPMModel(nn.Module):
|
||||
# run inference
|
||||
target_text_length = len(self.text_tokenizer(target_text))
|
||||
retry_badcase_times = 0
|
||||
current_seed = materialize_generation_seed(seed)
|
||||
last_attempt_seed = current_seed
|
||||
while retry_badcase_times < retry_badcase_max_times:
|
||||
last_attempt_seed = current_seed
|
||||
apply_generation_seed(last_attempt_seed)
|
||||
|
||||
inference_result = self._inference(
|
||||
text_token,
|
||||
text_mask,
|
||||
@@ -698,6 +716,7 @@ class VoxCPMModel(nn.Module):
|
||||
for latent_pred, pred_audio_feat in inference_result:
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||
decode_audio = decode_audio[..., -patch_len:].squeeze(1).cpu()
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
yield (decode_audio, target_text_token, pred_audio_feat)
|
||||
break
|
||||
else:
|
||||
@@ -709,12 +728,14 @@ class VoxCPMModel(nn.Module):
|
||||
file=sys.stderr,
|
||||
)
|
||||
retry_badcase_times += 1
|
||||
current_seed += 1
|
||||
continue
|
||||
else:
|
||||
break
|
||||
else:
|
||||
break
|
||||
if not streaming:
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||
patch_len = self.patch_size * self.chunk_size
|
||||
if audio_mask.sum().item() > 0:
|
||||
|
||||
@@ -46,7 +46,9 @@ from ..modules.locdit import CfmConfig, UnifiedCFM, VoxCPMLocDiTV2
|
||||
from ..modules.locenc import VoxCPMLocEnc
|
||||
from ..modules.minicpm4 import MiniCPM4Config, MiniCPMModel
|
||||
from .utils import (
|
||||
apply_generation_seed,
|
||||
get_dtype,
|
||||
materialize_generation_seed,
|
||||
mask_multichar_chinese_tokens,
|
||||
next_and_close,
|
||||
pick_runtime_dtype,
|
||||
@@ -55,7 +57,9 @@ from .utils import (
|
||||
|
||||
|
||||
# A simple function to trim audio silence using VAD, not used default
|
||||
def _trim_audio_silence_vad(audio: torch.Tensor, sample_rate: int, max_silence_ms: float = 200.0, top_db: float = 35.0) -> torch.Tensor:
|
||||
def _trim_audio_silence_vad(
|
||||
audio: torch.Tensor, sample_rate: int, max_silence_ms: float = 200.0, top_db: float = 35.0
|
||||
) -> torch.Tensor:
|
||||
if audio.numel() == 0:
|
||||
return audio
|
||||
y = audio.squeeze(0).numpy()
|
||||
@@ -184,6 +188,7 @@ class VoxCPM2Model(nn.Module):
|
||||
self.audio_end_token = 102
|
||||
self.ref_audio_start_token = 103
|
||||
self.ref_audio_end_token = 104
|
||||
self.last_successful_seed = None
|
||||
|
||||
# Residual Acoustic LM
|
||||
residual_lm_config = config.lm_config.model_copy(deep=True)
|
||||
@@ -476,6 +481,7 @@ class VoxCPM2Model(nn.Module):
|
||||
trim_silence_vad: bool = False,
|
||||
streaming: bool = False,
|
||||
streaming_prefix_len: int = 4,
|
||||
seed: Optional[int] = None,
|
||||
) -> Generator[torch.Tensor, None, None]:
|
||||
if retry_badcase and streaming:
|
||||
warnings.warn("Retry on bad cases is not supported in streaming mode, setting retry_badcase=False.")
|
||||
@@ -633,7 +639,12 @@ class VoxCPM2Model(nn.Module):
|
||||
target_text_length = len(self.text_tokenizer(target_text))
|
||||
|
||||
retry_badcase_times = 0
|
||||
current_seed = materialize_generation_seed(seed)
|
||||
last_attempt_seed = current_seed
|
||||
while retry_badcase_times < retry_badcase_max_times:
|
||||
last_attempt_seed = current_seed
|
||||
apply_generation_seed(last_attempt_seed)
|
||||
|
||||
inference_result = self._inference(
|
||||
text_token,
|
||||
text_mask,
|
||||
@@ -651,6 +662,7 @@ class VoxCPM2Model(nn.Module):
|
||||
for latent_pred, _, _ctx in inference_result:
|
||||
decode_audio = vae_dec.decode_chunk(latent_pred.to(torch.float32))
|
||||
decode_audio = decode_audio.squeeze(1).cpu()
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
yield decode_audio
|
||||
break
|
||||
else:
|
||||
@@ -662,6 +674,7 @@ class VoxCPM2Model(nn.Module):
|
||||
file=sys.stderr,
|
||||
)
|
||||
retry_badcase_times += 1
|
||||
current_seed += 1
|
||||
continue
|
||||
else:
|
||||
break
|
||||
@@ -669,10 +682,11 @@ class VoxCPM2Model(nn.Module):
|
||||
break
|
||||
|
||||
if not streaming:
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||
decode_patch_len = self.patch_size * self._decode_chunk_size
|
||||
if context_len > 0:
|
||||
decode_audio = decode_audio[..., decode_patch_len * context_len:].squeeze(1).cpu()
|
||||
decode_audio = decode_audio[..., decode_patch_len * context_len :].squeeze(1).cpu()
|
||||
else:
|
||||
decode_audio = decode_audio.squeeze(1).cpu()
|
||||
yield decode_audio
|
||||
@@ -793,6 +807,7 @@ class VoxCPM2Model(nn.Module):
|
||||
retry_badcase_ratio_threshold: float = 6.0,
|
||||
streaming: bool = False,
|
||||
streaming_prefix_len: int = 4,
|
||||
seed: Optional[int] = None,
|
||||
) -> Generator[Tuple[torch.Tensor, torch.Tensor, Union[torch.Tensor, List[torch.Tensor]]], None, None]:
|
||||
"""
|
||||
Generate audio using pre-built prompt cache.
|
||||
@@ -920,7 +935,12 @@ class VoxCPM2Model(nn.Module):
|
||||
# run inference
|
||||
target_text_length = len(self.text_tokenizer(target_text))
|
||||
retry_badcase_times = 0
|
||||
current_seed = materialize_generation_seed(seed)
|
||||
last_attempt_seed = current_seed
|
||||
while retry_badcase_times < retry_badcase_max_times:
|
||||
last_attempt_seed = current_seed
|
||||
apply_generation_seed(last_attempt_seed)
|
||||
|
||||
inference_result = self._inference(
|
||||
text_token,
|
||||
text_mask,
|
||||
@@ -938,6 +958,7 @@ class VoxCPM2Model(nn.Module):
|
||||
for latent_pred, pred_audio_feat, _ctx in inference_result:
|
||||
decode_audio = vae_dec.decode_chunk(latent_pred.to(torch.float32))
|
||||
decode_audio = decode_audio.squeeze(1).cpu()
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
yield (decode_audio, target_text_token, pred_audio_feat)
|
||||
break
|
||||
else:
|
||||
@@ -949,16 +970,18 @@ class VoxCPM2Model(nn.Module):
|
||||
file=sys.stderr,
|
||||
)
|
||||
retry_badcase_times += 1
|
||||
current_seed += 1
|
||||
continue
|
||||
else:
|
||||
break
|
||||
else:
|
||||
break
|
||||
if not streaming:
|
||||
self.last_successful_seed = last_attempt_seed
|
||||
decode_audio = self.audio_vae.decode(latent_pred.to(torch.float32))
|
||||
decode_patch_len = self.patch_size * self._decode_chunk_size
|
||||
if context_len > 0:
|
||||
decode_audio = decode_audio[..., decode_patch_len * context_len:].squeeze(1).cpu()
|
||||
decode_audio = decode_audio[..., decode_patch_len * context_len :].squeeze(1).cpu()
|
||||
else:
|
||||
decode_audio = decode_audio.squeeze(1).cpu()
|
||||
yield (decode_audio, target_text_token, pred_audio_feat)
|
||||
|
||||
@@ -551,8 +551,7 @@ class StreamingVAEDecoder:
|
||||
if x.shape[-1] >= _p:
|
||||
states[_k] = x[:, :, -_p:].detach()
|
||||
else:
|
||||
prev = states.get(_k, torch.zeros(x.shape[0], x.shape[1], _p,
|
||||
device=x.device, dtype=x.dtype))
|
||||
prev = states.get(_k, torch.zeros(x.shape[0], x.shape[1], _p, device=x.device, dtype=x.dtype))
|
||||
states[_k] = torch.cat([prev, x], dim=-1)[:, :, -_p:].detach()
|
||||
return nn.Conv1d.forward(_m, x_pad)
|
||||
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
from .base import TimestampItem, TimestampResult
|
||||
from .postprocess import align_audio_file
|
||||
|
||||
__all__ = [
|
||||
"TimestampItem",
|
||||
"TimestampResult",
|
||||
"align_audio_file",
|
||||
]
|
||||
@@ -0,0 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import asdict, dataclass
|
||||
from typing import Literal
|
||||
|
||||
TimestampLevel = Literal["segment", "word", "char"]
|
||||
|
||||
|
||||
@dataclass
|
||||
class TimestampItem:
|
||||
text: str
|
||||
start: float
|
||||
end: float
|
||||
level: str
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
return asdict(self)
|
||||
|
||||
|
||||
@dataclass
|
||||
class TimestampResult:
|
||||
audio_path: str
|
||||
sample_rate: int | None
|
||||
backend: str
|
||||
level: str
|
||||
text: str
|
||||
items: list[TimestampItem]
|
||||
warning: str | None = None
|
||||
|
||||
def to_dict(self) -> dict:
|
||||
payload = asdict(self)
|
||||
payload["items"] = [item.to_dict() for item in self.items]
|
||||
return payload
|
||||
@@ -0,0 +1,33 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from .base import TimestampLevel
|
||||
|
||||
|
||||
def align_audio_file(
|
||||
*,
|
||||
audio_path: str,
|
||||
text: str,
|
||||
sample_rate: int | None = None,
|
||||
backend: str = "stable-ts",
|
||||
level: TimestampLevel = "word",
|
||||
model_name: str = "base",
|
||||
device: str | None = None,
|
||||
language: str | None = None,
|
||||
) -> dict:
|
||||
if backend != "stable-ts":
|
||||
raise ValueError(f"Unsupported timestamp backend: {backend}")
|
||||
|
||||
from .stable_ts import StableTSAligner
|
||||
|
||||
aligner = StableTSAligner(
|
||||
model_name=model_name,
|
||||
device=device,
|
||||
language=language,
|
||||
)
|
||||
result = aligner.align(
|
||||
audio_path=audio_path,
|
||||
text=text,
|
||||
sample_rate=sample_rate,
|
||||
level=level,
|
||||
)
|
||||
return result.to_dict()
|
||||
@@ -0,0 +1,105 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from typing import Any
|
||||
|
||||
from .base import TimestampItem, TimestampLevel, TimestampResult
|
||||
|
||||
|
||||
class StableTSAligner:
|
||||
def __init__(
|
||||
self,
|
||||
model_name: str = "base",
|
||||
device: str | None = None,
|
||||
language: str | None = None,
|
||||
) -> None:
|
||||
try:
|
||||
import stable_whisper
|
||||
except ImportError as exc:
|
||||
raise ImportError(
|
||||
"stable-ts is required for timestamp alignment. " 'Install with: pip install "voxcpm[timestamps]"'
|
||||
) from exc
|
||||
|
||||
self.model = stable_whisper.load_model(model_name, device=device)
|
||||
self.model_name = model_name
|
||||
self.device = device
|
||||
self.language = language
|
||||
|
||||
def align(
|
||||
self,
|
||||
*,
|
||||
audio_path: str,
|
||||
text: str,
|
||||
sample_rate: int | None = None,
|
||||
level: TimestampLevel = "word",
|
||||
) -> TimestampResult:
|
||||
result = self.model.align(audio_path, text, language=self.language)
|
||||
items = extract_timestamp_items(result, level)
|
||||
return TimestampResult(
|
||||
audio_path=audio_path,
|
||||
sample_rate=sample_rate,
|
||||
backend="stable-ts",
|
||||
level=level,
|
||||
text=text,
|
||||
items=items,
|
||||
)
|
||||
|
||||
|
||||
def extract_timestamp_items(result: Any, level: TimestampLevel) -> list[TimestampItem]:
|
||||
segments = _get_value(result, "segments", []) or []
|
||||
if level == "segment":
|
||||
return [
|
||||
TimestampItem(
|
||||
text=str(_get_value(segment, "text", "")).strip(),
|
||||
start=float(_get_value(segment, "start", 0.0) or 0.0),
|
||||
end=float(_get_value(segment, "end", 0.0) or 0.0),
|
||||
level="segment",
|
||||
)
|
||||
for segment in segments
|
||||
if str(_get_value(segment, "text", "")).strip()
|
||||
]
|
||||
|
||||
words = []
|
||||
for segment in segments:
|
||||
for word in _get_value(segment, "words", []) or []:
|
||||
text = str(_get_value(word, "word", _get_value(word, "text", ""))).strip()
|
||||
if not text:
|
||||
continue
|
||||
words.append(
|
||||
TimestampItem(
|
||||
text=text,
|
||||
start=float(_get_value(word, "start", 0.0) or 0.0),
|
||||
end=float(_get_value(word, "end", 0.0) or 0.0),
|
||||
level="word",
|
||||
)
|
||||
)
|
||||
|
||||
if level == "char":
|
||||
return split_word_items_to_chars(words)
|
||||
return words
|
||||
|
||||
|
||||
def split_word_items_to_chars(words: list[TimestampItem]) -> list[TimestampItem]:
|
||||
chars = []
|
||||
for word in words:
|
||||
text = word.text.strip()
|
||||
if not text:
|
||||
continue
|
||||
|
||||
duration = max(word.end - word.start, 0.0)
|
||||
step = duration / len(text)
|
||||
for idx, char in enumerate(text):
|
||||
chars.append(
|
||||
TimestampItem(
|
||||
text=char,
|
||||
start=word.start + idx * step,
|
||||
end=word.start + (idx + 1) * step,
|
||||
level="char",
|
||||
)
|
||||
)
|
||||
return chars
|
||||
|
||||
|
||||
def _get_value(obj: Any, key: str, default: Any = None) -> Any:
|
||||
if isinstance(obj, dict):
|
||||
return obj.get(key, default)
|
||||
return getattr(obj, key, default)
|
||||
@@ -350,45 +350,80 @@ class AudioFeatureProcessingPacker:
|
||||
|
||||
# -- text token track --
|
||||
# [103, 0×R, 104, text_ids, 101, 0×A, 102]
|
||||
text_token_info = torch.cat([
|
||||
_tok([self.audio_prompt_start_id]),
|
||||
torch.zeros(ref_len, dtype=torch.int32, device=device),
|
||||
_tok([self.audio_prompt_end_id]),
|
||||
text_token,
|
||||
_tok([self.audio_start_id]),
|
||||
torch.zeros(tgt_len, dtype=torch.int32, device=device),
|
||||
_tok([self.audio_end_id]),
|
||||
])
|
||||
text_token_info = torch.cat(
|
||||
[
|
||||
_tok([self.audio_prompt_start_id]),
|
||||
torch.zeros(ref_len, dtype=torch.int32, device=device),
|
||||
_tok([self.audio_prompt_end_id]),
|
||||
text_token,
|
||||
_tok([self.audio_start_id]),
|
||||
torch.zeros(tgt_len, dtype=torch.int32, device=device),
|
||||
_tok([self.audio_end_id]),
|
||||
]
|
||||
)
|
||||
|
||||
# -- audio feature track --
|
||||
zero_1 = torch.zeros((1,) + feat_shape, dtype=torch.float32, device=device)
|
||||
zero_txt = torch.zeros((txt_len,) + feat_shape, dtype=torch.float32, device=device)
|
||||
audio_feat_info = torch.cat([
|
||||
zero_1, ref_feats, zero_1, # 103, ref, 104
|
||||
zero_txt, # text
|
||||
zero_1, tgt_feats, zero_1, # 101, target, 102
|
||||
], dim=0)
|
||||
audio_feat_info = torch.cat(
|
||||
[
|
||||
zero_1,
|
||||
ref_feats,
|
||||
zero_1, # 103, ref, 104
|
||||
zero_txt, # text
|
||||
zero_1,
|
||||
tgt_feats,
|
||||
zero_1, # 101, target, 102
|
||||
],
|
||||
dim=0,
|
||||
)
|
||||
|
||||
# -- masks --
|
||||
text_mask = torch.cat([
|
||||
torch.ones(1), torch.zeros(ref_len), torch.ones(1),
|
||||
torch.ones(txt_len),
|
||||
torch.ones(1), torch.zeros(tgt_len), torch.ones(1),
|
||||
]).to(torch.int32).to(device)
|
||||
text_mask = (
|
||||
torch.cat(
|
||||
[
|
||||
torch.ones(1),
|
||||
torch.zeros(ref_len),
|
||||
torch.ones(1),
|
||||
torch.ones(txt_len),
|
||||
torch.ones(1),
|
||||
torch.zeros(tgt_len),
|
||||
torch.ones(1),
|
||||
]
|
||||
)
|
||||
.to(torch.int32)
|
||||
.to(device)
|
||||
)
|
||||
|
||||
audio_mask = torch.cat([
|
||||
torch.zeros(1), torch.ones(ref_len), torch.zeros(1),
|
||||
torch.zeros(txt_len),
|
||||
torch.zeros(1), torch.ones(tgt_len), torch.zeros(1),
|
||||
]).to(torch.int32).to(device)
|
||||
audio_mask = (
|
||||
torch.cat(
|
||||
[
|
||||
torch.zeros(1),
|
||||
torch.ones(ref_len),
|
||||
torch.zeros(1),
|
||||
torch.zeros(txt_len),
|
||||
torch.zeros(1),
|
||||
torch.ones(tgt_len),
|
||||
torch.zeros(1),
|
||||
]
|
||||
)
|
||||
.to(torch.int32)
|
||||
.to(device)
|
||||
)
|
||||
|
||||
loss_mask = torch.cat([
|
||||
torch.zeros(1 + ref_len + 1), # ref part: no loss
|
||||
torch.zeros(txt_len), # text: no loss
|
||||
torch.zeros(1), # 101: no loss
|
||||
torch.ones(tgt_len), # target audio: LOSS
|
||||
torch.zeros(1), # 102: no loss
|
||||
]).to(torch.int32).to(device)
|
||||
loss_mask = (
|
||||
torch.cat(
|
||||
[
|
||||
torch.zeros(1 + ref_len + 1), # ref part: no loss
|
||||
torch.zeros(txt_len), # text: no loss
|
||||
torch.zeros(1), # 101: no loss
|
||||
torch.ones(tgt_len), # target audio: LOSS
|
||||
torch.zeros(1), # 102: no loss
|
||||
]
|
||||
)
|
||||
.to(torch.int32)
|
||||
.to(device)
|
||||
)
|
||||
|
||||
total_len = 1 + ref_len + 1 + txt_len + 1 + tgt_len + 1
|
||||
labels = torch.zeros(total_len, dtype=torch.int32, device=device)
|
||||
|
||||
@@ -44,10 +44,7 @@ def _check_audio_file(audio_path: str, sample_rate: int) -> Optional[str]:
|
||||
if info.frames == 0:
|
||||
return f"Audio file is empty: {audio_path}"
|
||||
if info.samplerate != sample_rate:
|
||||
return (
|
||||
f"Sample rate mismatch in {audio_path}: "
|
||||
f"expected {sample_rate} Hz, got {info.samplerate} Hz"
|
||||
)
|
||||
return f"Sample rate mismatch in {audio_path}: " f"expected {sample_rate} Hz, got {info.samplerate} Hz"
|
||||
return None
|
||||
except ImportError:
|
||||
# soundfile not available; just check existence
|
||||
@@ -187,13 +184,9 @@ def validate_manifest(
|
||||
if duration is not None:
|
||||
result.audio_durations.append(duration)
|
||||
if duration < 0.3:
|
||||
result.warnings.append(
|
||||
f"Line {i + 1}: Very short audio ({duration:.2f}s)"
|
||||
)
|
||||
result.warnings.append(f"Line {i + 1}: Very short audio ({duration:.2f}s)")
|
||||
elif duration > 30.0:
|
||||
result.warnings.append(
|
||||
f"Line {i + 1}: Very long audio ({duration:.1f}s), may cause OOM"
|
||||
)
|
||||
result.warnings.append(f"Line {i + 1}: Very long audio ({duration:.1f}s), may cause OOM")
|
||||
else:
|
||||
result.errors.append(f"Line {i + 1}: Invalid audio path")
|
||||
has_error = True
|
||||
@@ -209,9 +202,7 @@ def validate_manifest(
|
||||
if os.path.isfile(ref_path):
|
||||
result.has_ref_audio += 1
|
||||
else:
|
||||
result.warnings.append(
|
||||
f"Line {i + 1}: ref_audio file not found: {ref_path}"
|
||||
)
|
||||
result.warnings.append(f"Line {i + 1}: ref_audio file not found: {ref_path}")
|
||||
|
||||
if not has_error:
|
||||
result.valid_samples += 1
|
||||
@@ -222,14 +213,10 @@ def validate_manifest(
|
||||
# Summarize truncated errors
|
||||
if missing_audio_count > 5:
|
||||
result.errors.append(
|
||||
f"... and {missing_audio_count - 5} more missing audio files "
|
||||
f"({missing_audio_count} total)"
|
||||
f"... and {missing_audio_count - 5} more missing audio files " f"({missing_audio_count} total)"
|
||||
)
|
||||
if empty_text_count > 5:
|
||||
result.warnings.append(
|
||||
f"... and {empty_text_count - 5} more empty text entries "
|
||||
f"({empty_text_count} total)"
|
||||
)
|
||||
result.warnings.append(f"... and {empty_text_count - 5} more empty text entries " f"({empty_text_count} total)")
|
||||
|
||||
return result
|
||||
|
||||
|
||||
+209
-16
@@ -1,6 +1,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib.util
|
||||
import json
|
||||
import sys
|
||||
import types
|
||||
from pathlib import Path
|
||||
@@ -54,6 +55,11 @@ def run_main(monkeypatch, argv):
|
||||
cli.main()
|
||||
|
||||
|
||||
def patch_soundfile_write(monkeypatch):
|
||||
soundfile_stub = types.SimpleNamespace(write=lambda *args, **kwargs: None)
|
||||
monkeypatch.setitem(sys.modules, "soundfile", soundfile_stub)
|
||||
|
||||
|
||||
def test_parser_defaults_to_voxcpm2():
|
||||
parser = cli._build_parser()
|
||||
args = parser.parse_args(["design", "--text", "hello", "--output", "out.wav"])
|
||||
@@ -70,7 +76,7 @@ def test_load_model_respects_no_optimize_for_local_model(monkeypatch):
|
||||
calls["kwargs"] = kwargs
|
||||
self.tts_model = DummyTTSModel()
|
||||
|
||||
monkeypatch.setattr(cli, "VoxCPM", FakeVoxCPM)
|
||||
monkeypatch.setattr(core_stub, "VoxCPM", FakeVoxCPM)
|
||||
args = cli._build_parser().parse_args(
|
||||
[
|
||||
"design",
|
||||
@@ -99,7 +105,7 @@ def test_load_model_defaults_optimize_for_hf(monkeypatch):
|
||||
calls["kwargs"] = kwargs
|
||||
return DummyModel()
|
||||
|
||||
monkeypatch.setattr(cli, "VoxCPM", FakeVoxCPM)
|
||||
monkeypatch.setattr(core_stub, "VoxCPM", FakeVoxCPM)
|
||||
args = cli._build_parser().parse_args(
|
||||
[
|
||||
"design",
|
||||
@@ -125,7 +131,7 @@ def test_load_model_respects_no_optimize_for_hf(monkeypatch):
|
||||
calls["kwargs"] = kwargs
|
||||
return DummyModel()
|
||||
|
||||
monkeypatch.setattr(cli, "VoxCPM", FakeVoxCPM)
|
||||
monkeypatch.setattr(core_stub, "VoxCPM", FakeVoxCPM)
|
||||
args = cli._build_parser().parse_args(
|
||||
[
|
||||
"design",
|
||||
@@ -152,7 +158,7 @@ def test_load_model_passes_explicit_device_to_hf(monkeypatch):
|
||||
calls["kwargs"] = kwargs
|
||||
return DummyModel()
|
||||
|
||||
monkeypatch.setattr(cli, "VoxCPM", FakeVoxCPM)
|
||||
monkeypatch.setattr(core_stub, "VoxCPM", FakeVoxCPM)
|
||||
args = cli._build_parser().parse_args(
|
||||
[
|
||||
"design",
|
||||
@@ -173,7 +179,7 @@ def test_load_model_passes_explicit_device_to_hf(monkeypatch):
|
||||
def test_design_subcommand_applies_control(monkeypatch, tmp_path):
|
||||
dummy_model = DummyModel()
|
||||
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||
monkeypatch.setattr(cli.sf, "write", lambda *args, **kwargs: None)
|
||||
patch_soundfile_write(monkeypatch)
|
||||
|
||||
run_main(
|
||||
monkeypatch,
|
||||
@@ -201,7 +207,7 @@ def test_clone_subcommand_reads_prompt_file(monkeypatch, tmp_path):
|
||||
prompt_file.write_text("prompt transcript\n", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||
monkeypatch.setattr(cli.sf, "write", lambda *args, **kwargs: None)
|
||||
patch_soundfile_write(monkeypatch)
|
||||
|
||||
run_main(
|
||||
monkeypatch,
|
||||
@@ -273,7 +279,7 @@ def test_clone_rejects_reference_audio_for_v1_hf_model_id(monkeypatch, tmp_path)
|
||||
def test_legacy_root_args_still_work_and_warn(monkeypatch, tmp_path, capsys):
|
||||
dummy_model = DummyModel()
|
||||
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||
monkeypatch.setattr(cli.sf, "write", lambda *args, **kwargs: None)
|
||||
patch_soundfile_write(monkeypatch)
|
||||
|
||||
run_main(
|
||||
monkeypatch,
|
||||
@@ -296,7 +302,7 @@ def test_batch_subcommand_applies_control(monkeypatch, tmp_path):
|
||||
input_file.write_text("hello\nworld\n", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||
monkeypatch.setattr(cli.sf, "write", lambda *args, **kwargs: None)
|
||||
patch_soundfile_write(monkeypatch)
|
||||
|
||||
run_main(
|
||||
monkeypatch,
|
||||
@@ -317,6 +323,140 @@ def test_batch_subcommand_applies_control(monkeypatch, tmp_path):
|
||||
]
|
||||
|
||||
|
||||
def test_design_writes_timestamp_json_when_requested(monkeypatch, tmp_path):
|
||||
dummy_model = DummyModel()
|
||||
timestamp_calls = []
|
||||
|
||||
def fake_align_audio_file(**kwargs):
|
||||
timestamp_calls.append(kwargs)
|
||||
return {
|
||||
"audio_path": kwargs["audio_path"],
|
||||
"sample_rate": kwargs["sample_rate"],
|
||||
"backend": kwargs["backend"],
|
||||
"level": kwargs["level"],
|
||||
"text": kwargs["text"],
|
||||
"items": [{"text": "hello", "start": 0.0, "end": 0.5, "level": "word"}],
|
||||
"warning": None,
|
||||
}
|
||||
|
||||
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||
monkeypatch.setattr(cli, "align_audio_file", fake_align_audio_file)
|
||||
patch_soundfile_write(monkeypatch)
|
||||
|
||||
output = tmp_path / "out.wav"
|
||||
run_main(
|
||||
monkeypatch,
|
||||
[
|
||||
"design",
|
||||
"--text",
|
||||
"hello",
|
||||
"--output",
|
||||
str(output),
|
||||
"--timestamps",
|
||||
"--timestamp-language",
|
||||
"en",
|
||||
],
|
||||
)
|
||||
|
||||
timestamp_path = tmp_path / "out.timestamps.json"
|
||||
payload = json.loads(timestamp_path.read_text(encoding="utf-8"))
|
||||
assert payload["audio_path"] == str(output)
|
||||
assert payload["text"] == "hello"
|
||||
assert payload["items"][0]["text"] == "hello"
|
||||
assert timestamp_calls[0]["language"] == "en"
|
||||
assert timestamp_calls[0]["level"] == "word"
|
||||
|
||||
|
||||
def test_timestamp_alignment_failure_warns_by_default(monkeypatch, tmp_path, capsys):
|
||||
dummy_model = DummyModel()
|
||||
|
||||
def fake_align_audio_file(**kwargs):
|
||||
raise RuntimeError("alignment failed")
|
||||
|
||||
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||
monkeypatch.setattr(cli, "align_audio_file", fake_align_audio_file)
|
||||
patch_soundfile_write(monkeypatch)
|
||||
|
||||
run_main(
|
||||
monkeypatch,
|
||||
[
|
||||
"design",
|
||||
"--text",
|
||||
"hello",
|
||||
"--output",
|
||||
str(tmp_path / "out.wav"),
|
||||
"--timestamps",
|
||||
],
|
||||
)
|
||||
|
||||
assert "Timestamp alignment failed" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_timestamp_alignment_failure_exits_in_strict_mode(monkeypatch, tmp_path):
|
||||
dummy_model = DummyModel()
|
||||
|
||||
def fake_align_audio_file(**kwargs):
|
||||
raise RuntimeError("alignment failed")
|
||||
|
||||
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||
monkeypatch.setattr(cli, "align_audio_file", fake_align_audio_file)
|
||||
patch_soundfile_write(monkeypatch)
|
||||
|
||||
with pytest.raises(SystemExit):
|
||||
run_main(
|
||||
monkeypatch,
|
||||
[
|
||||
"design",
|
||||
"--text",
|
||||
"hello",
|
||||
"--output",
|
||||
str(tmp_path / "out.wav"),
|
||||
"--timestamps",
|
||||
"--timestamp-strict",
|
||||
],
|
||||
)
|
||||
|
||||
|
||||
def test_batch_writes_one_timestamp_json_per_output(monkeypatch, tmp_path):
|
||||
dummy_model = DummyModel()
|
||||
|
||||
def fake_align_audio_file(**kwargs):
|
||||
return {
|
||||
"audio_path": kwargs["audio_path"],
|
||||
"sample_rate": kwargs["sample_rate"],
|
||||
"backend": kwargs["backend"],
|
||||
"level": kwargs["level"],
|
||||
"text": kwargs["text"],
|
||||
"items": [{"text": kwargs["text"], "start": 0.0, "end": 0.5, "level": "word"}],
|
||||
"warning": None,
|
||||
}
|
||||
|
||||
input_file = tmp_path / "texts.txt"
|
||||
input_file.write_text("hello\nworld\n", encoding="utf-8")
|
||||
output_dir = tmp_path / "outs"
|
||||
|
||||
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||
monkeypatch.setattr(cli, "align_audio_file", fake_align_audio_file)
|
||||
patch_soundfile_write(monkeypatch)
|
||||
|
||||
run_main(
|
||||
monkeypatch,
|
||||
[
|
||||
"batch",
|
||||
"--input",
|
||||
str(input_file),
|
||||
"--output-dir",
|
||||
str(output_dir),
|
||||
"--timestamps",
|
||||
],
|
||||
)
|
||||
|
||||
first = json.loads((output_dir / "output_001.timestamps.json").read_text(encoding="utf-8"))
|
||||
second = json.loads((output_dir / "output_002.timestamps.json").read_text(encoding="utf-8"))
|
||||
assert first["text"] == "hello"
|
||||
assert second["text"] == "world"
|
||||
|
||||
|
||||
def test_legacy_clone_with_prompt_file_still_works(monkeypatch, tmp_path, capsys):
|
||||
dummy_model = DummyModel()
|
||||
prompt_audio = tmp_path / "prompt.wav"
|
||||
@@ -325,7 +465,7 @@ def test_legacy_clone_with_prompt_file_still_works(monkeypatch, tmp_path, capsys
|
||||
prompt_file.write_text("legacy transcript", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||
monkeypatch.setattr(cli.sf, "write", lambda *args, **kwargs: None)
|
||||
patch_soundfile_write(monkeypatch)
|
||||
|
||||
run_main(
|
||||
monkeypatch,
|
||||
@@ -450,10 +590,7 @@ def test_clone_rejects_prompt_audio_without_transcript(monkeypatch, tmp_path, ca
|
||||
with pytest.raises(SystemExit):
|
||||
cli.main()
|
||||
|
||||
assert (
|
||||
"--prompt-audio requires --prompt-text or --prompt-file"
|
||||
in capsys.readouterr().err
|
||||
)
|
||||
assert "--prompt-audio requires --prompt-text or --prompt-file" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_clone_rejects_transcript_without_prompt_audio(monkeypatch, tmp_path, capsys):
|
||||
@@ -475,9 +612,7 @@ def test_clone_rejects_transcript_without_prompt_audio(monkeypatch, tmp_path, ca
|
||||
with pytest.raises(SystemExit):
|
||||
cli.main()
|
||||
|
||||
assert (
|
||||
"--prompt-text/--prompt-file requires --prompt-audio" in capsys.readouterr().err
|
||||
)
|
||||
assert "--prompt-text/--prompt-file requires --prompt-audio" in capsys.readouterr().err
|
||||
|
||||
|
||||
def test_batch_rejects_control_with_prompt_transcript(monkeypatch, tmp_path, capsys):
|
||||
@@ -541,3 +676,61 @@ def test_detect_model_architecture_uses_local_configs():
|
||||
|
||||
assert cli.detect_model_architecture(v1_args) == "voxcpm"
|
||||
assert cli.detect_model_architecture(v2_args) == "voxcpm2"
|
||||
|
||||
|
||||
def test_parser_accepts_seed():
|
||||
parser = cli._build_parser()
|
||||
# Default seed should be None
|
||||
args = parser.parse_args(["design", "--text", "hello", "--output", "out.wav"])
|
||||
assert args.seed is None
|
||||
|
||||
# Custom seed should be parsed as int
|
||||
args = parser.parse_args(["design", "--text", "hello", "--output", "out.wav", "--seed", "42"])
|
||||
assert args.seed == 42
|
||||
|
||||
|
||||
def test_design_subcommand_passes_seed(monkeypatch, tmp_path):
|
||||
dummy_model = DummyModel()
|
||||
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||
patch_soundfile_write(monkeypatch)
|
||||
|
||||
run_main(
|
||||
monkeypatch,
|
||||
[
|
||||
"design",
|
||||
"--text",
|
||||
"hello",
|
||||
"--seed",
|
||||
"123",
|
||||
"--output",
|
||||
str(tmp_path / "out.wav"),
|
||||
],
|
||||
)
|
||||
|
||||
assert dummy_model.calls[0]["seed"] == 123
|
||||
|
||||
|
||||
def test_batch_subcommand_passes_seed(monkeypatch, tmp_path):
|
||||
dummy_model = DummyModel()
|
||||
input_file = tmp_path / "texts.txt"
|
||||
input_file.write_text("hello\nworld\n", encoding="utf-8")
|
||||
|
||||
monkeypatch.setattr(cli, "load_model", lambda args: dummy_model)
|
||||
patch_soundfile_write(monkeypatch)
|
||||
|
||||
run_main(
|
||||
monkeypatch,
|
||||
[
|
||||
"batch",
|
||||
"--input",
|
||||
str(input_file),
|
||||
"--output-dir",
|
||||
str(tmp_path / "outs"),
|
||||
"--seed",
|
||||
"999",
|
||||
],
|
||||
)
|
||||
|
||||
assert len(dummy_model.calls) == 2
|
||||
assert dummy_model.calls[0]["seed"] == 999
|
||||
assert dummy_model.calls[1]["seed"] == 999
|
||||
|
||||
@@ -47,3 +47,25 @@ def test_resolve_runtime_device_rejects_unavailable_explicit_cuda(monkeypatch):
|
||||
|
||||
with pytest.raises(ValueError, match="CUDA is not available"):
|
||||
utils.resolve_runtime_device("cuda:0", "cuda")
|
||||
|
||||
|
||||
def test_materialize_generation_seed_preserves_explicit_seed():
|
||||
assert utils.materialize_generation_seed(42) == 42
|
||||
|
||||
|
||||
def test_materialize_generation_seed_creates_concrete_seed_for_none(monkeypatch):
|
||||
monkeypatch.setattr(utils.torch, "seed", lambda: 0x123456789)
|
||||
|
||||
assert utils.materialize_generation_seed(None) == 0x23456789
|
||||
|
||||
|
||||
def test_apply_generation_seed_sets_cpu_and_cuda_rng(monkeypatch):
|
||||
calls = []
|
||||
|
||||
monkeypatch.setattr(utils.torch, "manual_seed", lambda seed: calls.append(("cpu", seed)))
|
||||
monkeypatch.setattr(utils.torch.cuda, "is_available", lambda: True)
|
||||
monkeypatch.setattr(utils.torch.cuda, "manual_seed_all", lambda seed: calls.append(("cuda", seed)))
|
||||
|
||||
utils.apply_generation_seed(123)
|
||||
|
||||
assert calls == [("cpu", 123), ("cuda", 123)]
|
||||
|
||||
@@ -0,0 +1,34 @@
|
||||
from pathlib import Path
|
||||
import sys
|
||||
import types
|
||||
|
||||
ROOT = Path(__file__).resolve().parents[1]
|
||||
pkg = types.ModuleType("voxcpm")
|
||||
pkg.__path__ = [str(ROOT / "src" / "voxcpm")]
|
||||
sys.modules.setdefault("voxcpm", pkg)
|
||||
|
||||
from voxcpm.timestamps.base import TimestampItem
|
||||
from voxcpm.timestamps.stable_ts import split_word_items_to_chars
|
||||
|
||||
|
||||
def test_split_word_items_to_chars_evenly_distributes_word_duration():
|
||||
chars = split_word_items_to_chars([TimestampItem(text="欢迎", start=0.5, end=0.9, level="word")])
|
||||
|
||||
assert [item.text for item in chars] == ["欢", "迎"]
|
||||
assert chars[0].start == 0.5
|
||||
assert chars[0].end == 0.7
|
||||
assert chars[1].start == 0.7
|
||||
assert chars[1].end == 0.9
|
||||
assert all(item.level == "char" for item in chars)
|
||||
|
||||
|
||||
def test_split_word_items_to_chars_skips_empty_text():
|
||||
chars = split_word_items_to_chars(
|
||||
[
|
||||
TimestampItem(text=" ", start=0.0, end=0.2, level="word"),
|
||||
TimestampItem(text="你", start=0.2, end=0.4, level="word"),
|
||||
]
|
||||
)
|
||||
|
||||
assert len(chars) == 1
|
||||
assert chars[0].text == "你"
|
||||
@@ -0,0 +1,85 @@
|
||||
"""Regression guard: every ``torch.load`` call must set ``weights_only=True``.
|
||||
|
||||
VoxCPM deliberately loads checkpoints with ``weights_only=True`` so that a
|
||||
crafted ``.ckpt``/``.pth``/``.bin`` file cannot execute arbitrary code via
|
||||
pickle during unpickling (see
|
||||
``tests/test_lora_checkpoint_loading.py::test_load_lora_weights_rejects_malicious_pickle_payloads``).
|
||||
|
||||
The fine-tuning resume path in ``scripts/train_voxcpm_finetune.py`` originally
|
||||
called ``torch.load`` without that flag, leaving an arbitrary-code-execution
|
||||
gap when resuming from an attacker-supplied checkpoint directory. This test
|
||||
statically asserts the flag is present on every ``torch.load`` call across the
|
||||
package and scripts so the gap cannot silently reappear.
|
||||
"""
|
||||
|
||||
import ast
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
REPO_ROOT = Path(__file__).resolve().parents[1]
|
||||
|
||||
# Directories whose Python files load checkpoints at runtime / on resume.
|
||||
SCANNED_DIRS = [REPO_ROOT / "src", REPO_ROOT / "scripts", REPO_ROOT / "app.py", REPO_ROOT / "lora_ft_webui.py"]
|
||||
|
||||
|
||||
def _python_files():
|
||||
for entry in SCANNED_DIRS:
|
||||
if entry.is_file() and entry.suffix == ".py":
|
||||
yield entry
|
||||
elif entry.is_dir():
|
||||
yield from entry.rglob("*.py")
|
||||
|
||||
|
||||
def _is_torch_load(node: ast.Call) -> bool:
|
||||
func = node.func
|
||||
# Matches ``torch.load(...)`` and ``load(...)`` aliased from torch.
|
||||
if isinstance(func, ast.Attribute) and func.attr == "load":
|
||||
return isinstance(func.value, ast.Name) and func.value.id == "torch"
|
||||
return False
|
||||
|
||||
|
||||
def _has_weights_only_true(node: ast.Call) -> bool:
|
||||
for kw in node.keywords:
|
||||
if kw.arg == "weights_only":
|
||||
return isinstance(kw.value, ast.Constant) and kw.value.value is True
|
||||
return False
|
||||
|
||||
|
||||
def test_every_torch_load_sets_weights_only_true():
|
||||
offenders = []
|
||||
checked = 0
|
||||
for path in _python_files():
|
||||
tree = ast.parse(path.read_text(encoding="utf-8"), filename=str(path))
|
||||
for node in ast.walk(tree):
|
||||
if isinstance(node, ast.Call) and _is_torch_load(node):
|
||||
checked += 1
|
||||
if not _has_weights_only_true(node):
|
||||
offenders.append(f"{path.relative_to(REPO_ROOT)}:{node.lineno}")
|
||||
|
||||
assert checked > 0, "expected to find at least one torch.load call to verify"
|
||||
assert not offenders, (
|
||||
"torch.load without weights_only=True (pickle RCE risk):\n "
|
||||
+ "\n ".join(offenders)
|
||||
)
|
||||
|
||||
|
||||
def test_torch_load_weights_only_blocks_malicious_pickle(tmp_path):
|
||||
"""Behavioral check that weights_only=True actually rejects a code-exec payload."""
|
||||
torch = pytest.importorskip("torch")
|
||||
|
||||
marker = tmp_path / "pwned.txt"
|
||||
|
||||
class Exploit:
|
||||
def __reduce__(self):
|
||||
import pathlib
|
||||
|
||||
return (pathlib.Path.write_text, (marker, "executed\n"))
|
||||
|
||||
ckpt = tmp_path / "optimizer.pth"
|
||||
torch.save({"state_dict": {"w": torch.zeros(1)}, "boom": Exploit()}, ckpt)
|
||||
|
||||
with pytest.raises(Exception):
|
||||
torch.load(ckpt, map_location="cpu", weights_only=True)
|
||||
|
||||
assert not marker.exists(), "malicious pickle executed despite weights_only=True"
|
||||
@@ -207,6 +207,7 @@ class TestValidateManifest:
|
||||
|
||||
audio = tmp_dir / "audio_8k.wav"
|
||||
import numpy as np
|
||||
|
||||
samples = np.zeros(8000, dtype=np.float32)
|
||||
sf.write(str(audio), samples, 8000)
|
||||
|
||||
@@ -240,6 +241,7 @@ class TestValidateManifest:
|
||||
def test_cli_validate_exit_code(self, tmp_dir):
|
||||
"""validate subcommand must exit 1 on validation error (missing audio)."""
|
||||
import subprocess
|
||||
|
||||
manifest = tmp_dir / "bad.jsonl"
|
||||
_write_manifest(manifest, [{"text": "hi", "audio": "/nonexistent/x.wav"}])
|
||||
|
||||
|
||||
Reference in New Issue
Block a user