Compare commits

...

28 Commits

Author SHA1 Message Date
mute a80bd8f669 rvsttd: increase fade-out to 25ms 2026-08-17 07:41:08 +00:00
mute c174576e14 rvsttd: increase fade-out to 10ms 2026-08-17 07:00:35 +00:00
mute b8ee014cc3 rvsttd: hold-back last drain chunk for fade, same faded audio to Moonshine + WAV 2026-08-17 06:58:18 +00:00
mute cc9fc95519 rvsttd: single faded audio stream feeds both Moonshine and debug WAV 2026-08-17 06:51:27 +00:00
mute 411f1c0952 rvsttd: fade-out on last audio chunk fed to Moonshine, debug WAV stays raw 2026-08-17 06:49:21 +00:00
mute b40a06b01b rvsttd: 5ms fade-out on debug WAV to eliminate trailing click 2026-08-17 06:47:46 +00:00
mute b6361ec080 Revert "fix click: 5ms fade-out from last sample to zero before silence padding"
This reverts commit 84474731e7.
2026-08-17 06:46:43 +00:00
mute 84474731e7 fix click: 5ms fade-out from last sample to zero before silence padding 2026-08-17 06:45:06 +00:00
mute b3e0fe3b30 fix: drain loop uses fixed 100ms deadline instead of idle timeout 2026-08-17 06:41:32 +00:00
mute 20e5c6c542 rvsttd: 100ms trailing audio drain before final flush, cpal stays alive during drain 2026-08-17 06:06:16 +00:00
mute 56bf55ba42 rvsttd: mpsc channel for audio capture, Fixed(800) buffer, deterministic shutdown 2026-08-17 05:57:01 +00:00
mute 004cd10f78 rvsttd: add --debug flag, save session audio (WAV) + transcript log 2026-08-14 06:39:58 +00:00
mute 8473ab6ba5 remove debug/trace logging and clean up log files 2026-08-13 12:51:41 +00:00
mute e09eb804b6 fix audio cutoff: don't Stop() after playback, let NAudio drain naturally 2026-08-13 12:50:20 +00:00
mute b4f05d6301 remove unused _recvThread field 2026-08-13 12:40:06 +00:00
mute caad0ed50e v0.9: VoicePipeline refactor — queue-based pipeline, lock/release model, thread-based TCP 2026-08-13 12:38:27 +00:00
mute 41a53d35b0 rvsttd: remove leftover session_id_counter references 2026-08-13 11:00:45 +00:00
mute fbbb9968b1 move PROTOCOL.md to repo root 2026-08-13 10:57:56 +00:00
mute b034ed15df rvsttd: enable TTS build (required for linking), fix unused var 2026-08-13 10:52:25 +00:00
mute b842f8b1f1 rvsttd: fix SocketAddr Default trait bound 2026-08-13 10:50:45 +00:00
mute d8bdec7543 rvsttd: drop ureq, use curl for model downloads (rustc 1.85 compat) 2026-08-13 10:49:49 +00:00
mute 23de208e96 rvsttd: fix build - strip -Werror, add TTS include path 2026-08-13 10:43:10 +00:00
mute 8811a15e1c rvsttd: patch all CMakeLists.txt for CMake 3.18 2026-08-13 10:37:43 +00:00
mute 912c380c02 rvsttd: patch CMake minimum version for Debian 11 2026-08-13 10:35:26 +00:00
mute b2b0c0d579 rvsttd: model fetch mode, build from source, ~/.rvsttd convention 2026-08-13 10:31:46 +00:00
mute 4be00bc4f1 add Rust target/ to .gitignore 2026-08-13 10:00:19 +00:00
mute 7765c6967c rename Server/server to rvsttd 2026-08-13 09:59:31 +00:00
mute facbfe6a5c v0.8: TCP transport via gatuna tunnel, session IDs, pre-synth playback, server Rust rewrite 2026-08-13 09:58:33 +00:00
21 changed files with 1824 additions and 1120 deletions
+3
View File
@@ -27,3 +27,6 @@ Robovoice.Tts.LibPiper/piper.h
## Temp files ## Temp files
*.wav *.wav
*.raw *.raw
## Rust build output
target/
+21
View File
@@ -0,0 +1,21 @@
# Robovoice Backlog
## Second-order PTT
When a game is running, it has its own PTT button (e.g. a voip push-to-talk
key). Robovoice should simulate pressing the game's PTT key before TTS audio
output begins, and release it after playback finishes.
This lets the TTS audio be transmitted through the game's voip channel to
other players.
### Considerations
- Needs a configurable "game PTT key" (separate from Robovoice's own PTT key)
- Use `SendInput` or `keybd_event` to synthesize the keypress
- Press the game PTT key right before buffered audio starts playing
- Release it after `AudioOutput` finishes playback (need a playback-complete
signal — currently `Flush()` doesn't provide one)
- Edge cases: what if the user presses Robovoice PTT while game PTT is still
held from a previous utterance? Flush should release game PTT too.
- Should this be a per-output-device setting? (CABLE Output vs speakers)
-106
View File
@@ -1,106 +0,0 @@
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Text;
// Args: [interface_ip] [interval_ms]
// Defaults: auto-detect LAN IP, 50ms
string localIp = args.Length > 0 ? args[0] : GetLanInterfaceIp() ?? "0.0.0.0";
int intervalMs = args.Length > 1 && int.TryParse(args[1], out int iv) ? iv : 50;
using var sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
sock.EnableBroadcast = true;
var localEp = new IPEndPoint(IPAddress.Parse(localIp), 68);
sock.Bind(localEp);
Console.WriteLine($"Bound to {localEp} (SO_REUSEADDR)");
Console.WriteLine("Sending NOPs + listening. Press Ctrl+C to stop.");
Console.WriteLine();
var cts = new CancellationTokenSource();
var recvTask = Task.Run(() => ReceiveLoop(sock, cts.Token));
var destEp = new IPEndPoint(IPAddress.Broadcast, 67);
uint session = 1;
uint nonce = 0;
while (!cts.Token.IsCancellationRequested)
{
nonce++;
string msg = $"HKMSTR {session} {nonce}\n";
byte[] payload = Encoding.UTF8.GetBytes(msg);
try
{
int sent = sock.SendTo(payload, destEp);
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] SENT s={session} n={nonce} {sent} bytes");
}
catch (Exception ex)
{
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] SEND FAILED: {ex.Message}");
}
try { Thread.Sleep(intervalMs); }
catch (OperationCanceledException) { break; }
}
cts.Cancel();
await recvTask;
static string? GetLanInterfaceIp()
{
foreach (var nic in NetworkInterface.GetAllNetworkInterfaces())
{
if (nic.OperationalStatus != OperationalStatus.Up)
continue;
if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback)
continue;
if (nic.Description.Contains("WireGuard", StringComparison.OrdinalIgnoreCase))
continue;
foreach (var addr in nic.GetIPProperties().UnicastAddresses)
{
if (addr.Address.AddressFamily == AddressFamily.InterNetwork)
{
var ip = addr.Address.ToString();
if (ip.StartsWith("192.168.") || ip.StartsWith("10.") || ip.StartsWith("172."))
return ip;
}
}
}
return null;
}
static void ReceiveLoop(Socket sock, CancellationToken ct)
{
byte[] buffer = new byte[4096];
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
while (!ct.IsCancellationRequested)
{
int received;
try
{
if (!sock.Poll(500_000, SelectMode.SelectRead))
continue;
received = sock.ReceiveFrom(buffer, ref fromEp);
}
catch (Exception ex)
{
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV ERROR: {ex.Message}");
continue;
}
string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');
if (!text.StartsWith("HKMSTR"))
{
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV {received} bytes from {fromEp} (no magic)");
continue;
}
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV {received} bytes from {fromEp}: {text}");
}
}
@@ -1,10 +0,0 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>
+106
View File
@@ -0,0 +1,106 @@
# Robovoice STT Protocol
## Overview
Robovoice connects to the STT server over TCP (typically through a gatuna
L2 tunnel). The server captures audio, runs Moonshine STT, and sends
transcript segments back. The client pre-synthesizes TTS on segments and
plays audio on final.
```
[Robovoice client] --TCP--> [STT server 127.0.0.1:6996]
│ │
├── ON <session>\n ────────►│ (abort old, start new session)
├── OFF <session>\n ────────►│ (stop, final STT pass)
│◄── P <session> <text>\n ──┤ (completed segment)
│◄── F <session> <text>\n ──┤ (all done; text may be empty)
```
## Transport
- **Protocol:** TCP (reliable, ordered, connection-oriented)
- **Server:** `127.0.0.1:6996` (hardcoded loopback)
- **Framing:** newline-delimited text (`\n`), UTF-8
- **Auto-reconnect:** client retries every 3s if connection drops
## Wire format
### Client → Server
**ON (PTT pressed):**
```
ON <session>\n
```
Starts a new STT session. The server aborts any active session and starts
recording. `<session>` is an incrementing unsigned integer chosen by the
client. Replies from the server echo this session ID.
**OFF (PTT released):**
```
OFF <session>\n
```
Stops the session. The server does a final STT pass on remaining audio and
sends any new segments followed by `F`.
### Server → Client
**Segment (completed VAD segment):**
```
P <session> <text>\n
```
A completed, VAD-separated utterance segment. The client starts TTS
synthesis immediately and buffers the audio (does not play yet).
**Final (all done):**
```
F <session> <text>\n
```
Signals all segments have been sent. `<text>` may be empty (`F <session>\n`).
Triggers playback of all buffered audio on the client. If text is non-empty,
the client synthesizes it before playing.
## Session IDs
- Client increments session ID on each PTT press
- Server echoes the session ID in all replies for that session
- Client drops any reply with a stale session ID (handles the race where
stale segments from an aborted session are still in the TCP buffer)
- Server aborts old session on receiving `ON` with a new session ID
## Client playback model
1. `P` arrives → start TTS synthesis immediately, buffer audio (don't play)
2. More `P` arrive → keep synthesizing and buffering
3. `F` arrives → play all buffered audio immediately
4. PTT pressed → flush: stop playback, cancel synthesis, clear buffers
The purpose of pre-synthesis is to minimize latency between PTT release
and audio playback. By the time `F` arrives, audio is already buffered.
## Server state machine
```
┌──────────┐ ON <session> ┌──────────────┐
│ IDLE │ ──────────────► │ RECORDING │
└──────────┘ └──────────────┘
│ │
OFF │ │
recv'd │ │
▼ │
┌─────────────┐
│ PROCESSING │
└─────────────┘
send │
P/F │
back to IDLE
```
- **IDLE → RECORDING:** `ON <session>` received, start mic capture
- **RECORDING:** Moonshine streaming produces completed segments → send `P`
- **RECORDING → PROCESSING:** `OFF <session>` received
- **PROCESSING → IDLE:** final STT pass, send remaining `P` + `F`
If `ON` arrives while recording, the current session is aborted (no final
flush) and a new session starts immediately.
+1 -1
View File
@@ -12,7 +12,7 @@ public sealed class AppConfig
public int LengthScale { get; set; } = 100; public int LengthScale { get; set; } = 100;
public int NoiseWScale { get; set; } = 800; public int NoiseWScale { get; set; } = 800;
public bool MinimizeToTray { get; set; } = true; public bool MinimizeToTray { get; set; } = true;
public string InterfaceIp { get; set; } = string.Empty; public string SttEndpoint { get; set; } = "127.0.0.1:6996";
public static string AppDataDir => Path.Combine( public static string AppDataDir => Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
+26 -82
View File
@@ -9,21 +9,15 @@ partial class MainForm
private TextBox txtPttKey = null!; private TextBox txtPttKey = null!;
private Label lblVoice = null!; private Label lblVoice = null!;
private ComboBox cmbVoice = null!; private ComboBox cmbVoice = null!;
private Button btnTestVoice = null!;
private Button btnManageVoices = null!; private Button btnManageVoices = null!;
private Label lblOutput = null!; private Label lblOutput = null!;
private ComboBox cmbOutput = null!; private ComboBox cmbOutput = null!;
private Label lblFile = null!;
private Button btnBrowseFile = null!;
private Label lblFileName = null!;
private Label lblServer = null!; private Label lblServer = null!;
private ComboBox cmbInterface = null!; private TextBox txtSttEndpoint = null!;
private Label lblLineStatus = null!; private Button btnLock = null!;
private RichTextBox txtLog = null!; private RichTextBox txtLog = null!;
private CheckBox chkMinimizeToTray = null!; private CheckBox chkMinimizeToTray = null!;
private Button btnClearLog = null!; private Button btnClearLog = null!;
private Label lblTextInput = null!;
private TextBox txtTextInput = null!;
private Label lblNoise = null!; private Label lblNoise = null!;
private TrackBar trkNoise = null!; private TrackBar trkNoise = null!;
private Label lblNoiseVal = null!; private Label lblNoiseVal = null!;
@@ -49,21 +43,15 @@ partial class MainForm
txtPttKey = new TextBox(); txtPttKey = new TextBox();
lblVoice = new Label(); lblVoice = new Label();
cmbVoice = new ComboBox(); cmbVoice = new ComboBox();
btnTestVoice = new Button();
btnManageVoices = new Button(); btnManageVoices = new Button();
lblOutput = new Label(); lblOutput = new Label();
cmbOutput = new ComboBox(); cmbOutput = new ComboBox();
lblFile = new Label();
btnBrowseFile = new Button();
lblFileName = new Label();
lblServer = new Label(); lblServer = new Label();
cmbInterface = new ComboBox(); txtSttEndpoint = new TextBox();
lblLineStatus = new Label(); btnLock = new Button();
txtLog = new RichTextBox(); txtLog = new RichTextBox();
chkMinimizeToTray = new CheckBox(); chkMinimizeToTray = new CheckBox();
btnClearLog = new Button(); btnClearLog = new Button();
lblTextInput = new Label();
txtTextInput = new TextBox();
lblNoise = new Label(); lblNoise = new Label();
trkNoise = new TrackBar(); trkNoise = new TrackBar();
lblNoiseVal = new Label(); lblNoiseVal = new Label();
@@ -100,65 +88,38 @@ partial class MainForm
cmbVoice.Size = new Size(200, 23); cmbVoice.Size = new Size(200, 23);
cmbVoice.DropDownStyle = ComboBoxStyle.DropDownList; cmbVoice.DropDownStyle = ComboBoxStyle.DropDownList;
// btnTestVoice
btnTestVoice.Text = "Test";
btnTestVoice.Location = new Point(423, 11);
btnTestVoice.Size = new Size(45, 25);
btnTestVoice.UseVisualStyleBackColor = true;
// btnManageVoices // btnManageVoices
btnManageVoices.Text = "Add/Remove..."; btnManageVoices.Text = "Add/Remove...";
btnManageVoices.Location = new Point(474, 11); btnManageVoices.Location = new Point(423, 11);
btnManageVoices.Size = new Size(95, 25); btnManageVoices.Size = new Size(95, 25);
btnManageVoices.UseVisualStyleBackColor = true; btnManageVoices.UseVisualStyleBackColor = true;
// lblOutput // lblOutput
lblOutput.Text = "Output:"; lblOutput.Text = "Output:";
lblOutput.Location = new Point(580, 15); lblOutput.Location = new Point(530, 15);
lblOutput.Size = new Size(50, 23); lblOutput.Size = new Size(50, 23);
lblOutput.TextAlign = ContentAlignment.MiddleLeft; lblOutput.TextAlign = ContentAlignment.MiddleLeft;
// cmbOutput // cmbOutput
cmbOutput.Location = new Point(633, 12); cmbOutput.Location = new Point(583, 12);
cmbOutput.Size = new Size(180, 23); cmbOutput.Size = new Size(150, 23);
cmbOutput.DropDownStyle = ComboBoxStyle.DropDownList; cmbOutput.DropDownStyle = ComboBoxStyle.DropDownList;
// lblFile
lblFile.Text = "Text File:";
lblFile.Location = new Point(12, 48);
lblFile.Size = new Size(60, 23);
lblFile.TextAlign = ContentAlignment.MiddleLeft;
// btnBrowseFile
btnBrowseFile.Text = "Browse...";
btnBrowseFile.Location = new Point(75, 45);
btnBrowseFile.Size = new Size(75, 25);
btnBrowseFile.UseVisualStyleBackColor = true;
// lblFileName
lblFileName.Text = "(none)";
lblFileName.Location = new Point(155, 48);
lblFileName.Size = new Size(280, 23);
lblFileName.TextAlign = ContentAlignment.MiddleLeft;
lblFileName.ForeColor = Color.Gray;
// lblServer // lblServer
lblServer.Text = "Interface:"; lblServer.Text = "STT:";
lblServer.Location = new Point(440, 48); lblServer.Location = new Point(12, 48);
lblServer.Size = new Size(55, 23); lblServer.Size = new Size(35, 23);
lblServer.TextAlign = ContentAlignment.MiddleLeft; lblServer.TextAlign = ContentAlignment.MiddleLeft;
// cmbInterface // txtSttEndpoint
cmbInterface.Location = new Point(498, 45); txtSttEndpoint.Location = new Point(50, 45);
cmbInterface.Size = new Size(165, 23); txtSttEndpoint.Size = new Size(200, 23);
cmbInterface.DropDownStyle = ComboBoxStyle.DropDownList;
// lblLineStatus // btnLock
lblLineStatus.Text = ""; btnLock.Text = "Lock Model";
lblLineStatus.Location = new Point(645, 48); btnLock.Location = new Point(260, 44);
lblLineStatus.Size = new Size(160, 23); btnLock.Size = new Size(90, 25);
lblLineStatus.TextAlign = ContentAlignment.MiddleRight; btnLock.UseVisualStyleBackColor = true;
lblLineStatus.ForeColor = Color.DarkBlue;
// lblNoise // lblNoise
lblNoise.Text = "Noise:"; lblNoise.Text = "Noise:";
@@ -166,7 +127,7 @@ partial class MainForm
lblNoise.Size = new Size(40, 23); lblNoise.Size = new Size(40, 23);
lblNoise.TextAlign = ContentAlignment.MiddleLeft; lblNoise.TextAlign = ContentAlignment.MiddleLeft;
// trkNoise (0-1000 → 0.0-1.0, default 667) // trkNoise
trkNoise.Location = new Point(52, 78); trkNoise.Location = new Point(52, 78);
trkNoise.Size = new Size(120, 45); trkNoise.Size = new Size(120, 45);
trkNoise.Minimum = 0; trkNoise.Minimum = 0;
@@ -187,7 +148,7 @@ partial class MainForm
lblSpeed.Size = new Size(40, 23); lblSpeed.Size = new Size(40, 23);
lblSpeed.TextAlign = ContentAlignment.MiddleLeft; lblSpeed.TextAlign = ContentAlignment.MiddleLeft;
// trkSpeed (50-300 → 0.5-3.0, default 100) // trkSpeed
trkSpeed.Location = new Point(260, 78); trkSpeed.Location = new Point(260, 78);
trkSpeed.Size = new Size(120, 45); trkSpeed.Size = new Size(120, 45);
trkSpeed.Minimum = 50; trkSpeed.Minimum = 50;
@@ -208,7 +169,7 @@ partial class MainForm
lblNoiseW.Size = new Size(45, 23); lblNoiseW.Size = new Size(45, 23);
lblNoiseW.TextAlign = ContentAlignment.MiddleLeft; lblNoiseW.TextAlign = ContentAlignment.MiddleLeft;
// trkNoiseW (0-1000 → 0.0-1.0, default 800) // trkNoiseW
trkNoiseW.Location = new Point(475, 78); trkNoiseW.Location = new Point(475, 78);
trkNoiseW.Size = new Size(120, 45); trkNoiseW.Size = new Size(120, 45);
trkNoiseW.Minimum = 0; trkNoiseW.Minimum = 0;
@@ -224,19 +185,8 @@ partial class MainForm
lblNoiseWVal.TextAlign = ContentAlignment.MiddleLeft; lblNoiseWVal.TextAlign = ContentAlignment.MiddleLeft;
// txtLog // txtLog
txtLog.Location = new Point(12, 149); txtLog.Location = new Point(12, 121);
txtLog.Size = new Size(800, 310); txtLog.Size = new Size(800, 338);
// lblTextInput
lblTextInput.Text = "Text:";
lblTextInput.Location = new Point(12, 121);
lblTextInput.Size = new Size(35, 23);
lblTextInput.TextAlign = ContentAlignment.MiddleLeft;
// txtTextInput
txtTextInput.Location = new Point(50, 118);
txtTextInput.Size = new Size(762, 23);
txtLog.ReadOnly = true; txtLog.ReadOnly = true;
txtLog.Font = new Font("Consolas", 9F); txtLog.Font = new Font("Consolas", 9F);
txtLog.BackColor = Color.FromArgb(30, 30, 30); txtLog.BackColor = Color.FromArgb(30, 30, 30);
@@ -262,16 +212,12 @@ partial class MainForm
Controls.Add(txtPttKey); Controls.Add(txtPttKey);
Controls.Add(lblVoice); Controls.Add(lblVoice);
Controls.Add(cmbVoice); Controls.Add(cmbVoice);
Controls.Add(btnTestVoice);
Controls.Add(btnManageVoices); Controls.Add(btnManageVoices);
Controls.Add(lblOutput); Controls.Add(lblOutput);
Controls.Add(cmbOutput); Controls.Add(cmbOutput);
Controls.Add(lblFile);
Controls.Add(btnBrowseFile);
Controls.Add(lblFileName);
Controls.Add(lblServer); Controls.Add(lblServer);
Controls.Add(cmbInterface); Controls.Add(txtSttEndpoint);
Controls.Add(lblLineStatus); Controls.Add(btnLock);
Controls.Add(lblNoise); Controls.Add(lblNoise);
Controls.Add(trkNoise); Controls.Add(trkNoise);
Controls.Add(lblNoiseVal); Controls.Add(lblNoiseVal);
@@ -281,8 +227,6 @@ partial class MainForm
Controls.Add(lblNoiseW); Controls.Add(lblNoiseW);
Controls.Add(trkNoiseW); Controls.Add(trkNoiseW);
Controls.Add(lblNoiseWVal); Controls.Add(lblNoiseWVal);
Controls.Add(lblTextInput);
Controls.Add(txtTextInput);
Controls.Add(txtLog); Controls.Add(txtLog);
Controls.Add(chkMinimizeToTray); Controls.Add(chkMinimizeToTray);
Controls.Add(btnClearLog); Controls.Add(btnClearLog);
+190 -222
View File
@@ -2,7 +2,7 @@ using NAudio.Wave;
using Robovoice.App; using Robovoice.App;
using Robovoice.Core; using Robovoice.Core;
using Robovoice.Core.Voices; using Robovoice.Core.Voices;
using Robovoice.Stt.Dhcp; using Robovoice.Stt.Tcp;
using Robovoice.Tts.LibPiper; using Robovoice.Tts.LibPiper;
using System.Diagnostics; using System.Diagnostics;
@@ -16,11 +16,12 @@ internal sealed partial class MainForm : Form
private LibPiperTtsEngine? _tts; private LibPiperTtsEngine? _tts;
private AudioOutput? _audioOutput; private AudioOutput? _audioOutput;
private DhcpSttSource? _sttSource; private TcpSttSource? _sttSource;
private Orchestrator? _orchestrator; private VoicePipeline? _pipeline;
private PttHotkey? _pttHotkey; private PttHotkey? _pttHotkey;
private NotifyIcon? _trayIcon; private NotifyIcon? _trayIcon;
private bool _trayInit; private bool _trayInit;
private bool _locked;
public MainForm() public MainForm()
{ {
@@ -36,57 +37,59 @@ internal sealed partial class MainForm : Form
private async void OnLoad(object? sender, EventArgs e) private async void OnLoad(object? sender, EventArgs e)
{ {
ActiveControl = txtLog; try
PopulateVoices(); {
PopulateOutputDevices(); ActiveControl = txtLog;
PopulateVoices();
PopulateOutputDevices();
_pttKey = (Keys)_config.PttKey; _pttKey = (Keys)_config.PttKey;
if (_pttKey == Keys.None) if (_pttKey == Keys.None)
_pttKey = Keys.F8; _pttKey = Keys.F8;
txtPttKey.Text = KeyToDisplayString(_pttKey); txtPttKey.Text = KeyToDisplayString(_pttKey);
if (!string.IsNullOrEmpty(_config.Voice) && cmbVoice.Items.Contains(_config.Voice)) if (!string.IsNullOrEmpty(_config.Voice) && cmbVoice.Items.Contains(_config.Voice))
cmbVoice.SelectedItem = _config.Voice; cmbVoice.SelectedItem = _config.Voice;
else if (cmbVoice.Items.Count > 0) else if (cmbVoice.Items.Count > 0)
cmbVoice.SelectedIndex = 0; cmbVoice.SelectedIndex = 0;
if (!string.IsNullOrEmpty(_config.OutputDevice) && cmbOutput.Items.Contains(_config.OutputDevice)) if (!string.IsNullOrEmpty(_config.OutputDevice) && cmbOutput.Items.Contains(_config.OutputDevice))
cmbOutput.SelectedItem = _config.OutputDevice; cmbOutput.SelectedItem = _config.OutputDevice;
else else
AutoSelectCableOutput(); AutoSelectCableOutput();
trkNoise.Value = _config.NoiseScale; trkNoise.Value = _config.NoiseScale;
trkSpeed.Value = _config.LengthScale; trkSpeed.Value = _config.LengthScale;
trkNoiseW.Value = _config.NoiseWScale; trkNoiseW.Value = _config.NoiseWScale;
OnSliderScroll(null, EventArgs.Empty); OnSliderScroll(null, EventArgs.Empty);
chkMinimizeToTray.Checked = _config.MinimizeToTray; chkMinimizeToTray.Checked = _config.MinimizeToTray;
PopulateInterfaces(); txtSttEndpoint.Text = _config.SttEndpoint;
btnBrowseFile.Click += OnBrowseFile; btnManageVoices.Click += OnManageVoices;
btnTestVoice.Click += OnTestVoice; btnClearLog.Click += (_, _) => txtLog.Clear();
btnManageVoices.Click += OnManageVoices; btnLock.Click += OnLockToggle;
btnClearLog.Click += (_, _) => txtLog.Clear(); txtPttKey.Enter += OnPttKeyFocus;
txtPttKey.Enter += OnPttKeyFocus; txtPttKey.KeyDown += OnPttKeyDown;
txtPttKey.KeyDown += OnPttKeyDown; txtSttEndpoint.Leave += OnSttEndpointChanged;
cmbOutput.SelectedIndexChanged += OnOutputChanged;
cmbVoice.SelectedIndexChanged += OnVoiceChanged;
cmbInterface.SelectedIndexChanged += OnInterfaceChanged;
trkNoise.Scroll += OnSliderScroll; trkNoise.Scroll += OnSliderScroll;
trkSpeed.Scroll += OnSliderScroll; trkSpeed.Scroll += OnSliderScroll;
trkNoiseW.Scroll += OnSliderScroll; trkNoiseW.Scroll += OnSliderScroll;
trkNoise.MouseUp += OnSliderReleased; trkNoise.MouseUp += OnSliderReleased;
trkSpeed.MouseUp += OnSliderReleased; trkSpeed.MouseUp += OnSliderReleased;
trkNoiseW.MouseUp += OnSliderReleased; trkNoiseW.MouseUp += OnSliderReleased;
chkMinimizeToTray.CheckedChanged += (_, _) => SaveConfig(); chkMinimizeToTray.CheckedChanged += (_, _) => SaveConfig();
Resize += OnResize; Resize += OnResize;
SetupTray(); SetupTray();
}
await InitializeEngineAsync(); catch (Exception ex)
{
Debug.WriteLine($"OnLoad failed: {ex}");
}
} }
private Keys _pttKey = Keys.F8; private Keys _pttKey = Keys.F8;
@@ -209,7 +212,21 @@ internal sealed partial class MainForm : Form
cmbOutput.SelectedIndex = 0; cmbOutput.SelectedIndex = 0;
} }
private async Task InitializeEngineAsync() // ─── Lock / Release ────────────────────────────────────────────────────
private async void OnLockToggle(object? sender, EventArgs e)
{
if (_locked)
{
ReleaseModel();
}
else
{
await LockModelAsync();
}
}
private async Task LockModelAsync()
{ {
if (cmbVoice.SelectedItem is not string voiceName) if (cmbVoice.SelectedItem is not string voiceName)
{ {
@@ -230,43 +247,98 @@ internal sealed partial class MainForm : Form
return; return;
} }
btnLock.Enabled = false;
Log($"Loading voice: {voiceName}..."); Log($"Loading voice: {voiceName}...");
_audioOutput?.Dispose();
_tts?.DisposeAsync().AsTask().Wait();
_tts = new LibPiperTtsEngine(
modelPath,
_espeakDataPath,
noiseScale: trkNoise.Value / 1000.0f,
lengthScale: trkSpeed.Value / 100.0f,
noiseWScale: trkNoiseW.Value / 1000.0f);
_audioOutput = new AudioOutput();
if (_sttSource is null)
{
string ifaceIp = cmbInterface.SelectedItem as string ?? "";
_sttSource = new DhcpSttSource { InterfaceIp = ifaceIp, Log = Log };
await _sttSource.StartAsync();
}
_orchestrator?.DisposeAsync().AsTask().Wait();
_orchestrator = new Orchestrator(_tts, _audioOutput, _sttSource, Log)
{
OutputDeviceName = cmbOutput.SelectedItem as string ?? string.Empty,
};
try try
{ {
await _orchestrator.InitializeTtsAsync(); _tts = new LibPiperTtsEngine(
Log("Engine ready. Press PTT to send to STT server."); modelPath,
_espeakDataPath,
noiseScale: trkNoise.Value / 1000.0f,
lengthScale: trkSpeed.Value / 100.0f,
noiseWScale: trkNoiseW.Value / 1000.0f);
await Task.Run(() => _tts.InitializeAsync());
Log($"TTS ready (sample rate: {_tts.SampleRate} Hz)");
_audioOutput = new AudioOutput();
_sttSource = new TcpSttSource { Endpoint = txtSttEndpoint.Text, Log = Log };
_sttSource.TranscriptReceived += OnTranscript;
await _sttSource.StartAsync();
_pipeline = new VoicePipeline(_tts, _audioOutput, Log)
{
OutputDeviceName = cmbOutput.SelectedItem as string ?? string.Empty,
};
_pipeline.Start();
_locked = true;
SetControlsLocked(true);
btnLock.Text = "Release Model";
btnLock.Enabled = true;
SetupHotkey(); SetupHotkey();
Log("Model locked. Press PTT to talk.");
} }
catch (Exception ex) catch (Exception ex)
{ {
Log($"Init failed: {ex.Message}"); Log($"Lock failed: {ex.Message}");
btnLock.Enabled = true;
_tts?.DisposeAsync().AsTask().Wait();
_tts = null;
_audioOutput?.Dispose();
_audioOutput = null;
if (_sttSource is not null)
{
_sttSource.TranscriptReceived -= OnTranscript;
_sttSource.DisposeAsync().AsTask().Wait();
_sttSource = null;
}
} }
} }
private void ReleaseModel()
{
if (!_locked) return;
_pttHotkey?.Dispose();
_pttHotkey = null;
if (_sttSource is not null)
{
_sttSource.TranscriptReceived -= OnTranscript;
_sttSource.DisposeAsync().AsTask().Wait();
_sttSource = null;
}
_pipeline?.Dispose();
_pipeline = null;
// VoicePipeline.Dispose calls _tts.DisposeAsync and _audioOutput.Dispose
_tts = null;
_audioOutput = null;
_locked = false;
SetControlsLocked(false);
btnLock.Text = "Lock Model";
Log("Model released.");
}
private void SetControlsLocked(bool locked)
{
cmbVoice.Enabled = !locked;
cmbOutput.Enabled = !locked;
trkNoise.Enabled = !locked;
trkSpeed.Enabled = !locked;
trkNoiseW.Enabled = !locked;
txtSttEndpoint.Enabled = !locked;
btnManageVoices.Enabled = !locked;
}
// ─── PTT ────────────────────────────────────────────────────────────────
private void SetupHotkey() private void SetupHotkey()
{ {
_pttHotkey?.Dispose(); _pttHotkey?.Dispose();
@@ -279,86 +351,56 @@ internal sealed partial class MainForm : Form
private void OnPttPressed(object? sender, EventArgs e) private void OnPttPressed(object? sender, EventArgs e)
{ {
Log("PTT pressed"); _pipeline?.OnPttPressed();
_orchestrator?.NotifyPttPressed(); _sttSource?.SendOn();
try
{
_sttSource?.SendOn();
}
catch (Exception ex)
{
Log($"SendOn failed: {ex.Message}");
}
} }
private void OnPttReleased(object? sender, EventArgs e) private void OnPttReleased(object? sender, EventArgs e)
{ {
Log("PTT released"); _sttSource?.SendOff();
try _pipeline?.OnPttReleased();
{
_sttSource?.SendOff();
}
catch (Exception ex)
{
Log($"SendOff failed: {ex.Message}");
}
} }
private void OnBrowseFile(object? sender, EventArgs e) // ─── STT transcript → pipeline ──────────────────────────────────────────
private void OnTranscript(object? sender, TranscriptEventArgs e)
{ {
using var dlg = new OpenFileDialog if (_pipeline is null || !_locked) return;
{
Filter = "Text files (*.txt)|*.txt|All files (*.*)|*.*",
Title = "Select a text file to speak",
};
if (dlg.ShowDialog() == DialogResult.OK) var msg = e.Message;
if (msg.Type == TranscriptType.Partial)
{ {
string text = File.ReadAllText(dlg.FileName); if (!string.IsNullOrWhiteSpace(msg.Text))
lblFileName.Text = Path.GetFileName(dlg.FileName); {
lblFileName.ForeColor = Color.Black; Log($"SEGMENT: \"{msg.Text}\"");
Log($"Loaded: {dlg.FileName} ({text.Length} chars)"); _pipeline.EnqueueSegment(msg.Text);
_ = _orchestrator?.SynthesizeAsync(text); }
}
else
{
if (!string.IsNullOrWhiteSpace(msg.Text))
{
Log($"FINAL: \"{msg.Text}\"");
_pipeline.EnqueueFinal(msg.Text);
}
else
{
Log("FINAL: (empty)");
_pipeline.EnqueueFinal("");
}
} }
} }
private void OnTestVoice(object? sender, EventArgs e) // ─── Voice manager ─────────────────────────────────────────────────────
{
if (_orchestrator is null)
{
Log("Engine not initialized.");
return;
}
if (cmbVoice.SelectedItem is not string voiceName)
{
Log("No voice selected.");
return;
}
if (!VoiceCatalogue.IsVoiceInstalled(_voicesDir, voiceName))
{
Log($"Voice '{voiceName}' is not installed. Use Add/Remove to download it.");
return;
}
string text = txtTextInput.Text.Trim();
if (string.IsNullOrEmpty(text))
text = "Hello, this is a voice test.";
btnTestVoice.Enabled = false;
try
{
_ = _orchestrator.SynthesizeAsync(text);
}
finally
{
btnTestVoice.Enabled = true;
}
}
private void OnManageVoices(object? sender, EventArgs e) private void OnManageVoices(object? sender, EventArgs e)
{ {
if (_locked)
{
Log("Release the model before managing voices.");
return;
}
string currentVoice = cmbVoice.SelectedItem as string ?? string.Empty; string currentVoice = cmbVoice.SelectedItem as string ?? string.Empty;
using var dlg = new VoiceManagerForm(_voicesDir, currentVoice); using var dlg = new VoiceManagerForm(_voicesDir, currentVoice);
dlg.ShowDialog(this); dlg.ShowDialog(this);
@@ -376,31 +418,9 @@ internal sealed partial class MainForm : Form
} }
SaveConfig(); SaveConfig();
_ = InitializeEngineAsync();
} }
private void OnVoiceChanged(object? sender, EventArgs e) // ─── Slider / endpoint ─────────────────────────────────────────────────
{
btnTestVoice.Enabled = cmbVoice.SelectedItem is string voiceName
&& VoiceCatalogue.IsVoiceInstalled(_voicesDir, voiceName);
if (cmbVoice.SelectedItem is string name)
{
SaveConfig();
_ = ReinitializeEngineAsync(name);
}
}
private async Task ReinitializeEngineAsync(string voiceName)
{
if (!VoiceCatalogue.IsVoiceInstalled(_voicesDir, voiceName))
{
btnTestVoice.Enabled = false;
return;
}
await InitializeEngineAsync();
}
private void OnSliderScroll(object? sender, EventArgs e) private void OnSliderScroll(object? sender, EventArgs e)
{ {
@@ -412,72 +432,15 @@ internal sealed partial class MainForm : Form
private void OnSliderReleased(object? sender, MouseEventArgs e) private void OnSliderReleased(object? sender, MouseEventArgs e)
{ {
SaveConfig(); SaveConfig();
if (cmbVoice.SelectedItem is string name && VoiceCatalogue.IsVoiceInstalled(_voicesDir, name))
{
_ = InitializeEngineAsync();
}
} }
private void OnOutputChanged(object? sender, EventArgs e) private void OnSttEndpointChanged(object? sender, EventArgs e)
{ {
if (_orchestrator is not null) _config.SttEndpoint = txtSttEndpoint.Text;
_orchestrator.OutputDeviceName = cmbOutput.SelectedItem as string ?? string.Empty;
Log($"Output device: {_orchestrator?.OutputDeviceName}");
SaveConfig(); SaveConfig();
} }
private void PopulateInterfaces() // ─── Tray ──────────────────────────────────────────────────────────────
{
cmbInterface.Items.Clear();
foreach (var (ip, name) in DhcpSttSource.GetAvailableInterfaces())
{
cmbInterface.Items.Add(name);
cmbInterface.Items[^1] = name;
cmbInterface.Items[cmbInterface.Items.Count - 1] = name;
}
if (!string.IsNullOrEmpty(_config.InterfaceIp))
{
for (int i = 0; i < cmbInterface.Items.Count; i++)
{
if (cmbInterface.Items[i] is string s && s.Contains(_config.InterfaceIp))
{
cmbInterface.SelectedIndex = i;
return;
}
}
}
if (cmbInterface.Items.Count > 0)
cmbInterface.SelectedIndex = 0;
}
private void OnInterfaceChanged(object? sender, EventArgs e)
{
if (cmbInterface.SelectedItem is not string selected)
return;
string? ip = ExtractIpFromDisplay(selected);
if (ip is null) return;
_config.InterfaceIp = ip;
SaveConfig();
if (_sttSource is not null)
{
_sttSource.DisposeAsync().AsTask().Wait(2000);
_sttSource = null;
_ = InitializeEngineAsync();
}
}
private static string? ExtractIpFromDisplay(string display)
{
int start = display.IndexOf('(');
int end = display.IndexOf(')');
if (start < 0 || end <= start) return null;
return display[(start + 1)..end];
}
private void SetupTray() private void SetupTray()
{ {
@@ -535,6 +498,8 @@ internal sealed partial class MainForm : Form
Activate(); Activate();
} }
// ─── Config ────────────────────────────────────────────────────────────
private void SaveConfig() private void SaveConfig()
{ {
_config.Voice = cmbVoice.SelectedItem as string ?? string.Empty; _config.Voice = cmbVoice.SelectedItem as string ?? string.Empty;
@@ -544,10 +509,12 @@ internal sealed partial class MainForm : Form
_config.LengthScale = trkSpeed.Value; _config.LengthScale = trkSpeed.Value;
_config.NoiseWScale = trkNoiseW.Value; _config.NoiseWScale = trkNoiseW.Value;
_config.MinimizeToTray = chkMinimizeToTray.Checked; _config.MinimizeToTray = chkMinimizeToTray.Checked;
_config.InterfaceIp = ExtractIpFromDisplay(cmbInterface.SelectedItem as string ?? "") ?? ""; _config.SttEndpoint = txtSttEndpoint.Text;
_config.Save(); _config.Save();
} }
// ─── Logging ───────────────────────────────────────────────────────────
private void Log(string message) private void Log(string message)
{ {
if (IsDisposed) return; if (IsDisposed) return;
@@ -562,12 +529,13 @@ internal sealed partial class MainForm : Form
txtLog.ScrollToCaret(); txtLog.ScrollToCaret();
} }
// ─── Shutdown ──────────────────────────────────────────────────────────
private void OnFormClosing(object? sender, FormClosingEventArgs e) private void OnFormClosing(object? sender, FormClosingEventArgs e)
{ {
_pttHotkey?.Dispose(); _pttHotkey?.Dispose();
_trayIcon!.Visible = false; _trayIcon!.Visible = false;
_orchestrator?.DisposeAsync().AsTask().Wait(2000); ReleaseModel();
_sttSource?.DisposeAsync().AsTask().Wait(2000);
SaveConfig(); SaveConfig();
} }
} }
-283
View File
@@ -1,283 +0,0 @@
using Robovoice.Core;
using Robovoice.Tts.LibPiper;
namespace Robovoice.App;
internal sealed class Orchestrator : IAsyncDisposable
{
private readonly LibPiperTtsEngine _tts;
private readonly AudioOutput _audioOutput;
private readonly ISttSource _sttSource;
private readonly Action<string> _log;
private CancellationTokenSource? _currentCts;
private bool _disposed;
private readonly object _stateLock = new();
private readonly Queue<string> _pendingTexts = new();
private readonly List<float[]> _audioBuffer = new();
private int _bufferSampleRate;
private bool _playing;
private Task? _synthTask;
private CancellationTokenSource? _synthCts;
private System.Threading.Timer? _segmentTimer;
private static readonly TimeSpan SegmentTimeout = TimeSpan.FromMilliseconds(250);
public string OutputDeviceName { get; set; } = string.Empty;
public Orchestrator(
LibPiperTtsEngine tts,
AudioOutput audioOutput,
ISttSource sttSource,
Action<string> log)
{
_tts = tts;
_audioOutput = audioOutput;
_audioOutput.Log = log;
_sttSource = sttSource;
_log = log;
_sttSource.TranscriptReceived += OnTranscript;
}
private void OnTranscript(object? sender, TranscriptEventArgs e)
{
var msg = e.Message;
lock (_stateLock)
{
if (msg.Type == TranscriptType.Partial)
{
if (!string.IsNullOrWhiteSpace(msg.Text))
{
_log($"SEGMENT: \"{msg.Text}\" ({msg.Text.Length} chars)");
_pendingTexts.Enqueue(msg.Text);
EnsureSynthTask();
ResetSegmentTimer();
}
}
else
{
if (!string.IsNullOrWhiteSpace(msg.Text))
{
_log($"FINAL: \"{msg.Text}\" ({msg.Text.Length} chars)");
_pendingTexts.Enqueue(msg.Text);
EnsureSynthTask();
}
else
{
_log("FINAL: (empty)");
}
CancelSegmentTimer();
TransitionToPlaying();
}
}
}
private void ResetSegmentTimer()
{
_segmentTimer?.Dispose();
_segmentTimer = new System.Threading.Timer(_ => OnSegmentTimeout(), null, SegmentTimeout, Timeout.InfiniteTimeSpan);
}
private void CancelSegmentTimer()
{
_segmentTimer?.Dispose();
_segmentTimer = null;
}
private void OnSegmentTimeout()
{
_log("STT: segment timeout, starting playback early");
lock (_stateLock)
{
TransitionToPlaying();
}
}
private void TransitionToPlaying()
{
if (_playing)
return;
if (_audioBuffer.Count == 0)
{
if (_synthTask is null || _synthTask.IsCompleted)
{
_log("TTS: nothing to play");
CancelSegmentTimer();
}
return;
}
_playing = true;
CancelSegmentTimer();
int sampleRate = _bufferSampleRate;
var chunks = _audioBuffer.ToList();
_audioBuffer.Clear();
_log($"TTS: playing {chunks.Count} buffered chunks ({sampleRate} Hz)");
_audioOutput.Start(sampleRate, OutputDeviceName);
foreach (var samples in chunks)
{
_audioOutput.WriteSamples(samples);
}
}
private void EnsureSynthTask()
{
if (_synthTask is not null && !_synthTask.IsCompleted)
return;
_synthCts?.Cancel();
_synthCts = new CancellationTokenSource();
_synthTask = SynthLoopAsync(_synthCts.Token);
}
private async Task SynthLoopAsync(CancellationToken ct)
{
while (true)
{
string text;
lock (_stateLock)
{
if (_pendingTexts.Count == 0)
break;
text = _pendingTexts.Dequeue();
}
try
{
await foreach (var chunk in _tts.SynthesizeAsync(text, ct))
{
lock (_stateLock)
{
if (_playing)
{
_audioOutput.WriteSamples(chunk.Samples);
}
else
{
_bufferSampleRate = chunk.SampleRate;
_audioBuffer.Add(chunk.Samples);
}
}
}
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
_log($"TTS error: {ex.Message}");
}
}
lock (_stateLock)
{
if (_playing)
{
_audioOutput.Flush();
_log("TTS: synthesis complete, flushed");
}
}
}
public void NotifyPttPressed()
{
lock (_stateLock)
{
CancelSegmentTimer();
_synthCts?.Cancel();
_synthCts?.Dispose();
_synthTask = null;
if (_playing)
{
_audioOutput.Stop();
_playing = false;
}
_pendingTexts.Clear();
_audioBuffer.Clear();
_bufferSampleRate = 0;
}
}
public async Task InitializeTtsAsync()
{
_log("Initializing TTS engine...");
await _tts.InitializeAsync();
_log($"TTS ready (sample rate: {_tts.SampleRate} Hz)");
}
public async Task SynthesizeAsync(string text)
{
_currentCts?.Cancel();
_currentCts = new CancellationTokenSource();
var ct = _currentCts.Token;
var sw = System.Diagnostics.Stopwatch.StartNew();
_log($"Speak: \"{text}\" ({text.Length} chars)");
try
{
bool started = false;
int chunkCount = 0;
int totalSamples = 0;
await foreach (var chunk in _tts.SynthesizeAsync(text, ct))
{
if (!started)
{
started = true;
_audioOutput.Start(chunk.SampleRate, OutputDeviceName);
_log($"TTS: first chunk ({sw.ElapsedMilliseconds}ms)");
}
_audioOutput.WriteSamples(chunk.Samples);
chunkCount++;
totalSamples += chunk.Samples.Length;
}
if (!started)
{
_log("TTS: no audio produced");
}
else
{
_audioOutput.Flush();
double durationSec = (double)totalSamples / _tts.SampleRate;
_log($"TTS: done ({chunkCount} chunks, {durationSec:F2}s audio, {sw.ElapsedMilliseconds}ms)");
}
}
catch (OperationCanceledException)
{
_log("TTS: cancelled");
_audioOutput.Stop();
}
catch (Exception ex)
{
_log($"TTS error: {ex.Message}");
}
sw.Stop();
}
public async ValueTask DisposeAsync()
{
if (_disposed) return;
_currentCts?.Cancel();
_currentCts?.Dispose();
_synthCts?.Cancel();
_synthCts?.Dispose();
CancelSegmentTimer();
_sttSource.TranscriptReceived -= OnTranscript;
await _sttSource.DisposeAsync();
await _tts.DisposeAsync();
_audioOutput.Dispose();
_disposed = true;
}
}
+1 -1
View File
@@ -3,7 +3,7 @@
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\Robovoice.Core\Robovoice.Core.csproj" /> <ProjectReference Include="..\Robovoice.Core\Robovoice.Core.csproj" />
<ProjectReference Include="..\Robovoice.Tts.LibPiper\Robovoice.Tts.LibPiper.csproj" /> <ProjectReference Include="..\Robovoice.Tts.LibPiper\Robovoice.Tts.LibPiper.csproj" />
<ProjectReference Include="..\Robovoice.Stt.Dhcp\Robovoice.Stt.Dhcp.csproj" /> <ProjectReference Include="..\Robovoice.Stt.Tcp\Robovoice.Stt.Tcp.csproj" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
+187
View File
@@ -0,0 +1,187 @@
using System.Collections.Concurrent;
using Robovoice.Core;
using Robovoice.Tts.LibPiper;
namespace Robovoice.App;
internal sealed record TextItem(uint Session, string Text);
internal sealed record AudioItem(uint Session, float[]? Samples);
internal sealed class VoicePipeline : IDisposable
{
private readonly LibPiperTtsEngine _tts;
private readonly AudioOutput _audioOutput;
private readonly Action<string> _log;
private BlockingCollection<TextItem> _textQueue = new();
private BlockingCollection<AudioItem> _audioQueue = new();
private ManualResetEventSlim _gate = new(false);
private volatile uint _currentSession;
private volatile bool _running;
private Thread? _synthThread;
private Thread? _playerThread;
private bool _disposed;
public string OutputDeviceName { get; set; } = string.Empty;
public VoicePipeline(
LibPiperTtsEngine tts,
AudioOutput audioOutput,
Action<string> log)
{
_tts = tts;
_audioOutput = audioOutput;
_audioOutput.Log = log;
_log = log;
}
public void Start()
{
_running = true;
_synthThread = new Thread(SynthLoop) { IsBackground = true, Name = "VoicePipeline-Synth" };
_playerThread = new Thread(PlayerLoop) { IsBackground = true, Name = "VoicePipeline-Player" };
_synthThread.Start();
_playerThread.Start();
}
public void EnqueueSegment(string text)
{
if (!_running) return;
uint session = _currentSession;
_textQueue.Add(new TextItem(session, text));
}
public void EnqueueFinal(string text)
{
if (!_running) return;
uint session = _currentSession;
if (!string.IsNullOrEmpty(text))
_textQueue.Add(new TextItem(session, text));
_textQueue.Add(new TextItem(session, ""));
_gate.Reset();
}
public void OnPttPressed()
{
_currentSession++;
_gate.Reset();
_audioOutput.Stop();
}
public void OnPttReleased()
{
_gate.Set();
}
private void SynthLoop()
{
foreach (var item in _textQueue.GetConsumingEnumerable())
{
if (!_running) break;
if (item.Session != _currentSession)
continue;
if (item.Text.Length == 0)
{
// :F sentinel — signal end of utterance to player
_audioQueue.Add(new AudioItem(item.Session, null));
continue;
}
try
{
foreach (var chunk in _tts.SynthesizeSync(item.Text))
{
if (item.Session != _currentSession)
{
// Session changed mid-synthesis — drain piper cleanly
_tts.SynthesizeDrain();
break;
}
_audioQueue.Add(new AudioItem(item.Session, chunk.Samples));
}
}
catch (Exception ex)
{
_log($"TTS error: {ex.Message}");
}
}
}
private void PlayerLoop()
{
while (_running)
{
_gate.Wait();
if (!_running) break;
// Drain stale items, then wait for real audio for current session
AudioItem firstItem = default!;
while (_running)
{
try { firstItem = _audioQueue.Take(); }
catch (InvalidOperationException) { return; }
if (firstItem.Session != _currentSession)
continue;
break;
}
if (!_running) break;
if (firstItem.Samples == null)
{
// :F with no audio — nothing to play
_gate.Reset();
continue;
}
_audioOutput.Start(_tts.SampleRate, OutputDeviceName);
_audioOutput.WriteSamples(firstItem.Samples);
_log($"TTS: playback started (session {firstItem.Session})");
foreach (var item in _audioQueue.GetConsumingEnumerable())
{
if (!_running) break;
if (item.Session != _currentSession)
break;
if (item.Samples == null)
break;
_audioOutput.WriteSamples(item.Samples);
}
_audioOutput.Flush();
_log("TTS: playback finished");
_gate.Reset();
}
}
public void Dispose()
{
if (_disposed) return;
_disposed = true;
_running = false;
_gate.Set();
_textQueue.CompleteAdding();
_audioQueue.CompleteAdding();
_synthThread?.Join(5000);
_playerThread?.Join(5000);
_textQueue.Dispose();
_audioQueue.Dispose();
_gate.Dispose();
_tts.DisposeAsync().AsTask().Wait();
_audioOutput.Dispose();
}
}
-277
View File
@@ -1,277 +0,0 @@
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Text;
using Robovoice.Core;
namespace Robovoice.Stt.Dhcp;
public sealed class DhcpSttSource : ISttSource
{
private const string Magic = "HKMSTR";
private const int DhcpClientPort = 68;
private const int DhcpServerPort = 67;
private static readonly TimeSpan NopInterval = TimeSpan.FromMilliseconds(50);
private Socket? _sock;
private CancellationTokenSource? _cts;
private Task? _receiveTask;
private Task? _nopTask;
private EndPoint _broadcastEp = new IPEndPoint(IPAddress.Broadcast, DhcpServerPort);
private uint _nonce;
private uint _session;
private bool _disposed;
public string InterfaceIp { get; set; } = string.Empty;
public Action<string>? Log { get; set; }
public event TranscriptEventHandler? TranscriptReceived;
public Task StartAsync(CancellationToken ct = default)
{
ObjectDisposedException.ThrowIf(_disposed, this);
if (_cts is not null)
return Task.CompletedTask;
string ip = string.IsNullOrEmpty(InterfaceIp) ? AutoDetectInterfaceIp() ?? "0.0.0.0" : InterfaceIp;
_sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
_sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_sock.EnableBroadcast = true;
_sock.Bind(new IPEndPoint(IPAddress.Parse(ip), DhcpClientPort));
Log?.Invoke($"STT: bound to {ip}:{DhcpClientPort}, broadcasting to :{DhcpServerPort}");
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
_receiveTask = ReceiveLoopAsync(_cts.Token);
return Task.CompletedTask;
}
public async Task StopAsync(CancellationToken ct = default)
{
StopNop();
if (_cts is not null)
_cts.Cancel();
_sock?.Dispose();
_sock = null;
if (_receiveTask is not null)
{
try { await _receiveTask.WaitAsync(ct); }
catch { }
_receiveTask = null;
}
_cts?.Dispose();
_cts = null;
}
public void SendOn()
{
if (_cts is null)
return;
_session++;
_nonce = 0;
Log?.Invoke($"STT: session {_session} started");
SendNop();
_nopTask = NopLoopAsync(_cts.Token);
}
public void SendOff()
{
StopNop();
SendControl($"HKMSTR:OFF {_session} {_nonce}");
}
private void StopNop()
{
if (_nopTask is not null)
{
try { _nopTask.Wait(2000); } catch { }
_nopTask = null;
}
}
private async Task NopLoopAsync(CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
try { await Task.Delay(NopInterval, ct); }
catch (OperationCanceledException) { break; }
SendNop();
}
}
private void SendNop()
{
_nonce++;
SendControl($"HKMSTR {_session} {_nonce}");
}
private void SendControl(string message)
{
if (_sock is null)
return;
try
{
byte[] payload = Encoding.UTF8.GetBytes(message + "\n");
_sock.SendTo(payload, _broadcastEp);
}
catch (Exception ex)
{
Log?.Invoke($"STT: send failed: {ex.Message}");
}
}
private async Task ReceiveLoopAsync(CancellationToken ct)
{
byte[] buffer = new byte[4096];
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
while (!ct.IsCancellationRequested)
{
int received;
try
{
if (!_sock!.Poll(500_000, SelectMode.SelectRead))
continue;
received = _sock.ReceiveFrom(buffer, ref fromEp);
}
catch (OperationCanceledException)
{
break;
}
catch (ObjectDisposedException)
{
break;
}
catch (Exception ex)
{
Log?.Invoke($"STT: receive error: {ex.Message}");
continue;
}
string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');
if (!text.StartsWith(Magic))
continue;
TranscriptMessage? message = ParseReply(text);
if (message is null)
continue;
TranscriptReceived?.Invoke(this, new TranscriptEventArgs
{
Message = message,
});
}
}
private TranscriptMessage? ParseReply(string text)
{
// Format: HKMSTR:P <session> <text> or HKMSTR:F <session> <text>
// <text> may be empty.
string prefix;
TranscriptType type;
if (text.StartsWith("HKMSTR:P "))
{
prefix = "HKMSTR:P ";
type = TranscriptType.Partial;
}
else if (text.StartsWith("HKMSTR:F "))
{
prefix = "HKMSTR:F ";
type = TranscriptType.Final;
}
else
{
return null;
}
string rest = text[prefix.Length..];
int spaceIndex = rest.IndexOf(' ');
if (spaceIndex < 0)
{
if (uint.TryParse(rest, out uint sessionOnly))
{
if (sessionOnly != _session)
return null;
return new TranscriptMessage(type, string.Empty);
}
return null;
}
string sessionStr = rest[..spaceIndex];
if (!uint.TryParse(sessionStr, out uint session))
return null;
if (session != _session)
{
Log?.Invoke($"STT: dropping stale reply (session {session} != current {_session})");
return null;
}
string transcript = rest[(spaceIndex + 1)..];
return new TranscriptMessage(type, transcript);
}
public static List<(string Ip, string Name)> GetAvailableInterfaces()
{
var result = new List<(string, string)>();
foreach (var nic in NetworkInterface.GetAllNetworkInterfaces())
{
if (nic.OperationalStatus != OperationalStatus.Up)
continue;
if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback)
continue;
string desc = nic.Description;
if (desc.Contains("WireGuard", StringComparison.OrdinalIgnoreCase))
continue;
foreach (var addr in nic.GetIPProperties().UnicastAddresses)
{
if (addr.Address.AddressFamily != AddressFamily.InterNetwork)
continue;
string ip = addr.Address.ToString();
if (ip.StartsWith("192.168.") || ip.StartsWith("10.") ||
ip.StartsWith("172.16.") || ip.StartsWith("172.17.") ||
ip.StartsWith("172.18.") || ip.StartsWith("172.19.") ||
ip.StartsWith("172.20.") || ip.StartsWith("172.21.") ||
ip.StartsWith("172.22.") || ip.StartsWith("172.23.") ||
ip.StartsWith("172.24.") || ip.StartsWith("172.25.") ||
ip.StartsWith("172.26.") || ip.StartsWith("172.27.") ||
ip.StartsWith("172.28.") || ip.StartsWith("172.29.") ||
ip.StartsWith("172.30.") || ip.StartsWith("172.31."))
{
result.Add((ip, $"{nic.Name} ({ip})"));
}
}
}
return result;
}
private static string? AutoDetectInterfaceIp()
{
foreach (var (ip, _) in GetAvailableInterfaces())
return ip;
return null;
}
public async ValueTask DisposeAsync()
{
if (_disposed) return;
await StopAsync();
_disposed = true;
}
}
+291
View File
@@ -0,0 +1,291 @@
using System.Net;
using System.Net.Sockets;
using System.Text;
using Robovoice.Core;
namespace Robovoice.Stt.Tcp;
public sealed class TcpSttSource : ISttSource
{
private TcpClient? _tcp;
private NetworkStream? _stream;
private StreamWriter? _writer;
private Thread? _connectThread;
private volatile bool _running;
private readonly object _sendLock = new();
private uint _session;
private bool _disposed;
public string Endpoint { get; set; } = "127.0.0.1:6996";
public Action<string>? Log { get; set; }
public event TranscriptEventHandler? TranscriptReceived;
public Task StartAsync(CancellationToken ct = default)
{
ObjectDisposedException.ThrowIf(_disposed, this);
if (_running)
return Task.CompletedTask;
_running = true;
_connectThread = new Thread(ConnectLoop) { IsBackground = true, Name = "TcpSttSource-Connect" };
_connectThread.Start();
return Task.CompletedTask;
}
public Task StopAsync(CancellationToken ct = default)
{
_running = false;
lock (_sendLock)
{
_writer?.Dispose();
_stream?.Dispose();
_tcp?.Close();
_writer = null;
_stream = null;
_tcp = null;
}
// Threads are background — they'll die when the process exits.
// Closing the socket unblocks any pending Read.
_connectThread?.Join(1000);
return Task.CompletedTask;
}
private void ConnectLoop()
{
while (_running)
{
IPEndPoint? endpoint = ParseEndpoint(Endpoint);
if (endpoint is null)
{
Log?.Invoke($"STT: invalid endpoint '{Endpoint}'");
SleepInterruptible(3000);
continue;
}
try
{
var tcp = new TcpClient();
tcp.Connect(endpoint.Address, endpoint.Port);
tcp.NoDelay = true;
lock (_sendLock)
{
_tcp = tcp;
_stream = tcp.GetStream();
_writer = new StreamWriter(_stream, new UTF8Encoding(false)) { AutoFlush = true };
}
Log?.Invoke($"STT: connected to {Endpoint}");
// Blocking receive loop — runs until disconnected or stopped.
ReceiveLoop();
Log?.Invoke("STT: disconnected");
}
catch (Exception ex)
{
if (_running)
Log?.Invoke($"STT: connection failed ({ex.Message}), retrying...");
}
finally
{
lock (_sendLock)
{
_writer?.Dispose();
_stream?.Dispose();
_tcp?.Close();
_writer = null;
_stream = null;
_tcp = null;
}
}
if (_running)
SleepInterruptible(3000);
}
}
private void ReceiveLoop()
{
byte[] buffer = new byte[4096];
StringBuilder lineBuf = new();
while (_running)
{
NetworkStream? stream;
lock (_sendLock)
{
stream = _stream;
}
if (stream is null)
break;
int bytesRead;
try
{
bytesRead = stream.Read(buffer, 0, buffer.Length);
}
catch
{
break;
}
if (bytesRead == 0)
break;
for (int i = 0; i < bytesRead; i++)
{
byte b = buffer[i];
if (b == '\n')
{
string line = lineBuf.ToString().TrimEnd('\r');
lineBuf.Clear();
TranscriptMessage? message = ParseReply(line);
if (message is not null)
{
TranscriptReceived?.Invoke(this, new TranscriptEventArgs
{
Message = message,
});
}
}
else
{
lineBuf.Append((char)b);
}
}
}
}
private void SleepInterruptible(int ms)
{
int slice = 100;
int waited = 0;
while (_running && waited < ms)
{
int chunk = Math.Min(slice, ms - waited);
Thread.Sleep(chunk);
waited += chunk;
}
}
public void SendOn()
{
_session++;
Send($"ON {_session}");
}
public void SendOff()
{
Send($"OFF {_session}");
}
private void Send(string message)
{
lock (_sendLock)
{
if (_writer is null)
return;
try
{
_writer.WriteLine(message);
}
catch
{
Log?.Invoke($"STT: failed to send '{message}' (not connected?)");
}
}
}
private static IPEndPoint? ParseEndpoint(string endpoint)
{
int colon = endpoint.LastIndexOf(':');
if (colon <= 0)
return null;
string host = endpoint[..colon];
if (!int.TryParse(endpoint[(colon + 1)..], out int port))
return null;
if (IPAddress.TryParse(host, out var addr))
return new IPEndPoint(addr, port);
try
{
var addresses = Dns.GetHostAddresses(host);
addr = addresses.FirstOrDefault(a => a.AddressFamily == AddressFamily.InterNetwork);
if (addr is null)
return null;
return new IPEndPoint(addr, port);
}
catch
{
return null;
}
}
private TranscriptMessage? ParseReply(string line)
{
if (line.StartsWith("P ", StringComparison.Ordinal))
{
string rest = line["P ".Length..];
int space = rest.IndexOf(' ');
if (space < 0)
return null;
if (!uint.TryParse(rest[..space], out uint session))
return null;
if (session != _session)
{
Log?.Invoke($"STT: dropping stale reply (session {session} != current {_session})");
return null;
}
return new TranscriptMessage(TranscriptType.Partial, rest[(space + 1)..]);
}
if (line.StartsWith("F ", StringComparison.Ordinal))
{
string rest = line["F ".Length..];
int space = rest.IndexOf(' ');
if (space < 0)
{
if (uint.TryParse(rest, out uint session) && session == _session)
return new TranscriptMessage(TranscriptType.Final, string.Empty);
return null;
}
if (!uint.TryParse(rest[..space], out uint ses))
return null;
if (ses != _session)
{
Log?.Invoke($"STT: dropping stale reply (session {ses} != current {_session})");
return null;
}
return new TranscriptMessage(TranscriptType.Final, rest[(space + 1)..]);
}
return null;
}
public ValueTask DisposeAsync()
{
if (_disposed) return ValueTask.CompletedTask;
_disposed = true;
StopAsync();
return ValueTask.CompletedTask;
}
}
@@ -120,6 +120,71 @@ public sealed class LibPiperTtsEngine : ITtsEngine
} }
} }
public IEnumerable<AudioChunk> SynthesizeSync(string text)
{
ObjectDisposedException.ThrowIf(_disposed, this);
if (!_initialized)
throw new InvalidOperationException("Engine not initialized.");
var options = new PiperSynthesizeOptions
{
SpeakerId = 0,
LengthScale = _lengthScale,
NoiseScale = _noiseScale,
NoiseWScale = _noiseWScale,
};
byte[] textBytes = System.Text.Encoding.UTF8.GetBytes(EnsureTerminator(text) + "\0");
GCHandle textPin = GCHandle.Alloc(textBytes, GCHandleType.Pinned);
try
{
int startResult = PiperNative.piper_synthesize_start(
_synth, textPin.AddrOfPinnedObject(), in options);
if (startResult != PiperNative.PiperOk)
throw new InvalidOperationException($"piper_synthesize_start failed: {startResult}");
while (true)
{
PiperAudioChunk chunk = default;
int result = PiperNative.piper_synthesize_next(_synth, out chunk);
if (chunk.NumSamples > 0 && chunk.Samples != IntPtr.Zero)
{
int numSamples = (int)chunk.NumSamples;
float[] samples = new float[numSamples];
Marshal.Copy(chunk.Samples, samples, 0, numSamples);
if (chunk.SampleRate > 0)
_sampleRate = chunk.SampleRate;
yield return new AudioChunk(samples, _sampleRate);
}
if (result == PiperNative.PiperDone || chunk.IsLast)
break;
if (result < 0)
throw new InvalidOperationException($"piper_synthesize_next failed: {result}");
}
}
finally
{
textPin.Free();
}
}
public void SynthesizeDrain()
{
if (!_initialized || _synth == IntPtr.Zero)
return;
PiperAudioChunk chunk;
while (PiperNative.piper_synthesize_next(_synth, out chunk) != PiperNative.PiperDone)
{
if (chunk.IsLast) break;
}
}
private static string EnsureTerminator(string text) private static string EnsureTerminator(string text)
{ {
string trimmed = text.TrimEnd(); string trimmed = text.TrimEnd();
+1 -2
View File
@@ -2,7 +2,6 @@
<Project Path="Robovoice.App/Robovoice.App.csproj" /> <Project Path="Robovoice.App/Robovoice.App.csproj" />
<Project Path="Robovoice.Core/Robovoice.Core.csproj" /> <Project Path="Robovoice.Core/Robovoice.Core.csproj" />
<Project Path="Robovoice.Stt.File/Robovoice.Stt.File.csproj" /> <Project Path="Robovoice.Stt.File/Robovoice.Stt.File.csproj" />
<Project Path="Robovoice.Stt.Dhcp/Robovoice.Stt.Dhcp.csproj" /> <Project Path="Robovoice.Stt.Tcp/Robovoice.Stt.Tcp.csproj" />
<Project Path="Robovoice.Tts.LibPiper/Robovoice.Tts.LibPiper.csproj" /> <Project Path="Robovoice.Tts.LibPiper/Robovoice.Tts.LibPiper.csproj" />
<Project Path="DhcpTunnelTest/Robovoice.DhcpTunnelTest.csproj" />
</Solution> </Solution>
-136
View File
@@ -1,136 +0,0 @@
# Robovoice DHCP Tunnel Protocol
## Overview
Robovoice communicates with a remote STT server by tunneling through the
DHCP UDP ports (68→67). This exploits a common killswitch exception: VPN
software (e.g. WireGuard) blocks all traffic except DHCP, which is allowed
for network connectivity maintenance.
```
[Robovoice client] --broadcast UDP :68→:67--> [STT server]
[Robovoice client] <--unicast UDP :67→:68-- [STT server]
```
The client broadcasts NOP heartbeats while PTT is held. The server starts
recording on the first NOP and stops when it receives OFF or when 150ms
pass with no NOPs.
## Transport
- **Protocol:** UDP (connectionless, unreliable)
- **Client → Server:** broadcast, source port 68, dest port 67
- **Server → Client:** unicast, source port 67, dest port 68
- **Client binds:** to a specific LAN interface IP on port 68 (with
`SO_REUSEADDR` to coexist with the Windows DHCP service)
- **No connection state** — purely fire-and-forget datagrams
## Wire format
All messages are plain text, newline-terminated (`\n`). Every message starts
with the 6-byte magic `HKMSTR` to distinguish our traffic from real DHCP.
### Client → Server
**NOP (heartbeat while PTT held):**
```
HKMSTR <session> <nonce>\n
```
Sent every 50ms while PTT is held. The session is an incrementing unsigned
integer that identifies the current PTT utterance (incremented on each PTT
press). The nonce is an incrementing unsigned integer that makes each
datagram unique. The server should echo the session back in replies. Both
are discarded by the server for protocol logic — the server tracks liveness
via "did anything arrive recently."
**OFF (PTT released):**
```
HKMSTR:OFF <session> <nonce>\n
```
Sent once when PTT is released. This is the fast-stop signal. If lost, the
150ms timeout acts as a backstop.
### Server → Client
The server echoes the session ID from the NOPs in all replies. The client
drops any reply with a stale session ID.
**Partial segment (completed VAD segment):**
```
HKMSTR:P <session> <text>\n
```
A completed, VAD-separated utterance segment. The client starts TTS
synthesis immediately and buffers the audio output, but does **not** play
it yet. Playback starts when `:F` arrives (or timeout).
**Final (all done):**
```
HKMSTR:F <session> <text>\n
```
Signals that all segments have been sent. May be empty
(`HKMSTR:F <session>\n`). Triggers playback of all buffered audio on the
client. If `<text>` is non-empty, the client synthesizes it before playing.
The purpose of this design is to minimize latency: TTS synthesis runs in
parallel with recording, so by the time `:F` arrives, audio is already
buffered and playback starts immediately.
## Server state machine
```
┌──────────────────────────────────────────┐
│ │
▼ │
┌──────────┐ first NOP ┌──────────────┐ │
│ IDLE │ ──────────► │ RECORDING │ │
└──────────┘ └──────────────┘ │
│ │ │
OFF │ │ 150ms │
recv'd │ │ silence │
▼ ▼ │
┌─────────────┐ │
│ PROCESSING │ │
└─────────────┘ │
│ │
send │ │
:P/:F │ │
▼ │
back to IDLE ──────────┘
```
- **IDLE → RECORDING:** first NOP received, start mic capture
- **RECORDING:** VAD detects completed segments → send `HKMSTR:P <text>`
- **RECORDING → PROCESSING:** OFF received, OR 150ms since last NOP
- **PROCESSING → IDLE:** send remaining segments as `:P`, then `HKMSTR:F`
## Client playback model
1. `:P` arrives → start TTS synthesis immediately, buffer audio (don't play).
Reset 250ms segment timer.
2. More `:P` arrive → keep synthesizing and buffering, reset timer each time.
3. `:F` arrives → play all buffered audio immediately, cancel timer.
4. If `:F` doesn't arrive within 250ms of the last `:P` → play buffered audio
early. If more `:P` arrive after early playback, synthesis continues and
new audio is appended to the output — not a failure.
5. `:F` may be empty — it just signals "all segments sent, start/confirm playback."
## Timing
| Parameter | Value | Purpose |
|-----------|-------|---------|
| NOP interval | 50ms | Heartbeat frequency while PTT held |
| Silence timeout | 150ms | Stop recording if no NOPs (3 missed = lost OFF) |
| NOP bandwidth | ~20 msg/s × ~20 bytes | ~400 bytes/s — negligible |
## Why this works
1. **Outbound broadcast `:68→:67` to `255.255.255.255`** passes the
WireGuard WFP killswitch (DHCP exception matches this exact pattern)
2. **Inbound `:67→:68`** has no address restriction in the WFP rule, so
unicast replies pass through
3. **Binding to a specific interface IP** (not `0.0.0.0`) wins unicast
delivery over the Windows DHCP client service
4. **NOP spam** ensures the ON message gets through even at 5% packet loss
(3 consecutive NOPs = ~0.01% drop probability)
5. **150ms timeout** is the backstop for lost OFF — at 50ms intervals, 3
consecutive NOPs must all be lost to false-stop
+20
View File
@@ -0,0 +1,20 @@
[package]
name = "rvsttd"
version = "0.1.0"
edition = "2021"
[[bin]]
name = "rvsttd"
path = "src/main.rs"
[dependencies]
anyhow = "1"
cpal = "0.15"
serde = { version = "1", features = ["derive"] }
serde_json = "1"
[build-dependencies]
bindgen = "0.71"
[profile.release]
opt-level = 3
+28
View File
@@ -0,0 +1,28 @@
use std::path::PathBuf;
fn main() {
let home = std::env::var("HOME").unwrap_or_else(|_| "/root".to_string());
let rvsttd_dir = PathBuf::from(&home).join(".rvsttd");
let lib_dir = rvsttd_dir.join("lib");
let include_dir = rvsttd_dir.join("include");
let header = include_dir.join("moonshine-c-api.h");
println!("cargo:rerun-if-changed={}", header.display());
println!("cargo:rustc-link-search=native={}", lib_dir.display());
println!("cargo:rustc-link-lib=dylib=moonshine");
println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_dir.display());
let bindings = bindgen::Builder::default()
.header(header.to_str().unwrap())
.allowlist_function("moonshine_.*")
.allowlist_var("MOONSHINE_.*")
.allowlist_type("transcript.*|moonshine_option_t|speaker_span_t|transcript_word_t")
.derive_default(true)
.generate()
.expect("Unable to generate moonshine bindings");
let out_path = PathBuf::from(std::env::var("OUT_DIR").unwrap());
bindings
.write_to_file(out_path.join("moonshine_bindings.rs"))
.expect("Couldn't write bindings");
}
+102
View File
@@ -0,0 +1,102 @@
#!/usr/bin/env bash
set -euo pipefail
RVSTTD_DIR="${HOME}/.rvsttd"
LIB_DIR="${RVSTTD_DIR}/lib"
INCLUDE_DIR="${RVSTTD_DIR}/include"
BUILD_DIR="${RVSTTD_DIR}/build"
ARCH="linux-x86_64"
# Fetch latest release tag from GitHub API
VERSION=$(curl -s https://api.github.com/repos/moonshine-ai/moonshine/releases/latest | grep '"tag_name"' | sed -E 's/.*"([^"]+)".*/\1/')
if [ -z "${VERSION}" ]; then
echo "ERROR: could not fetch latest release tag from GitHub API"
exit 1
fi
PREBUILT_URL="https://github.com/moonshine-ai/moonshine/releases/download/${VERSION}/moonshine-voice-${ARCH}.tar.gz"
SOURCE_URL="https://github.com/moonshine-ai/moonshine/archive/refs/tags/${VERSION}.tar.gz"
echo "=== rvsttd setup ==="
echo "Latest Moonshine release: ${VERSION}"
echo "Target: ${RVSTTD_DIR}"
echo ""
mkdir -p "${LIB_DIR}" "${INCLUDE_DIR}" "${BUILD_DIR}"
# Step 1: Download prebuilt package (for libonnxruntime.so.1 + header)
PREBUILT_TGZ="${BUILD_DIR}/moonshine-voice-${ARCH}.tar.gz"
PREBUILT_EXTRACTED="${BUILD_DIR}/moonshine-voice-${ARCH}"
if [ ! -f "${LIB_DIR}/libonnxruntime.so.1" ]; then
echo ">>> Downloading prebuilt package (for libonnxruntime.so.1)..."
curl -L -o "${PREBUILT_TGZ}" "${PREBUILT_URL}"
mkdir -p "${PREBUILT_EXTRACTED}"
tar xzf "${PREBUILT_TGZ}" -C "${PREBUILT_EXTRACTED}" --strip-components=1
# Copy ONNX Runtime (prebuilt is fine — it has no glibc issue)
cp "${PREBUILT_EXTRACTED}/lib/libonnxruntime.so.1" "${LIB_DIR}/"
echo " Installed libonnxruntime.so.1"
# Copy the header (it's the same in source and prebuilt)
cp "${PREBUILT_EXTRACTED}/include/moonshine-c-api.h" "${INCLUDE_DIR}/"
echo " Installed moonshine-c-api.h"
else
echo ">>> libonnxruntime.so.1 already present, skipping prebuilt download"
fi
# Step 2: Download source
SOURCE_TGZ="${BUILD_DIR}/moonshine-source.tar.gz"
SOURCE_DIR="${BUILD_DIR}/moonshine-source"
if [ ! -d "${SOURCE_DIR}" ]; then
echo ">>> Downloading Moonshine source ${VERSION}..."
curl -L -o "${SOURCE_TGZ}" "${SOURCE_URL}"
mkdir -p "${SOURCE_DIR}"
tar xzf "${SOURCE_TGZ}" -C "${SOURCE_DIR}" --strip-components=1
fi
# Step 3: Build libmoonshine.so from source
CMAKE_BUILD="${BUILD_DIR}/cmake-build"
if [ ! -f "${LIB_DIR}/libmoonshine.so" ]; then
echo ">>> Building libmoonshine.so from source..."
# Point CMake at the prebuilt ONNX Runtime
ORT_LIB_DIR="${LIB_DIR}"
ORT_INCLUDE_DIR="${SOURCE_DIR}/core/third-party/onnxruntime/include"
# Patch CMake minimum version for older distros (Debian 11 ships 3.18)
find "${SOURCE_DIR}" -name CMakeLists.txt -exec sed -i 's/cmake_minimum_required(VERSION 3\.22\.1)/cmake_minimum_required(VERSION 3.18.4)/' {} +
# Remove -Werror (fails on third-party headers with older compilers)
sed -i 's/-Werror//' "${SOURCE_DIR}/core/CMakeLists.txt"
mkdir -p "${CMAKE_BUILD}"
cd "${CMAKE_BUILD}"
cmake "${SOURCE_DIR}/core" \
-DCMAKE_BUILD_TYPE=Release \
-DONNXRUNTIME_LIB_PATH="${ORT_LIB_DIR}/libonnxruntime.so.1" \
-DMOONSHINE_TTS_BUILD_ONNX=ON \
-DCMAKE_CXX_FLAGS="-I${SOURCE_DIR}/core/moonshine-tts/src"
make -j"$(nproc)" moonshine
cp "${CMAKE_BUILD}/libmoonshine.so" "${LIB_DIR}/"
cp "${SOURCE_DIR}/core/moonshine-c-api.h" "${INCLUDE_DIR}/"
echo " Installed libmoonshine.so (built from source)"
else
echo ">>> libmoonshine.so already present, skipping build"
fi
echo ""
echo "=== Setup complete ==="
echo "Library: ${LIB_DIR}/libmoonshine.so"
echo "Library: ${LIB_DIR}/libonnxruntime.so.1"
echo "Header: ${INCLUDE_DIR}/moonshine-c-api.h"
echo ""
echo "Next: cd rvsttd && cargo build --release"
echo "Then: ./target/release/rvsttd fetch # downloads the model"
echo "Then: ./target/release/rvsttd # starts the server"
+782
View File
@@ -0,0 +1,782 @@
use anyhow::{anyhow, Result};
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
use cpal::{SampleFormat, SampleRate};
use serde::Deserialize;
use std::collections::HashSet;
use std::ffi::CStr;
use std::io::{BufRead, BufReader, Write};
use std::net::{TcpListener, TcpStream};
use std::path::PathBuf;
use std::sync::atomic::{AtomicBool, Ordering};
use std::sync::mpsc;
use std::sync::{Arc, Mutex};
use std::thread;
use std::time::{Duration, SystemTime, UNIX_EPOCH};
include!(concat!(env!("OUT_DIR"), "/moonshine_bindings.rs"));
const SAMPLE_RATE: i32 = 16000;
const HEADER_VERSION: i32 = 30000;
const ARCH: u32 = 5; // MOONSHINE_MODEL_ARCH_MEDIUM_STREAMING
const BIND_ADDR: &str = "127.0.0.1:6996";
const MAX_TEXT_BYTES: usize = 1380;
// ─── helpers ──────────────────────────────────────────────────────────────
fn ts() -> String {
let now = SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default();
let secs = now.as_secs() % 86400;
let h = secs / 3600;
let m = (secs % 3600) / 60;
let s = secs % 60;
let ms = now.subsec_millis();
format!("{:02}:{:02}:{:02}.{:03}", h, m, s, ms)
}
fn log(msg: &str) {
eprintln!("[{}] {}", ts(), msg);
}
fn err_str(code: i32) -> String {
unsafe {
let s = moonshine_error_to_string(code);
if s.is_null() {
format!("error {}", code)
} else {
CStr::from_ptr(s).to_string_lossy().into_owned()
}
}
}
fn truncate_to_word(text: &str, max_bytes: usize) -> &str {
if text.len() <= max_bytes {
return text;
}
let cut = &text[..max_bytes.min(text.len())];
match cut.rfind(' ') {
Some(pos) => &text[..pos],
None => cut,
}
}
fn line_text(line: &transcript_line_t) -> String {
if line.text.is_null() {
return String::new();
}
unsafe { CStr::from_ptr(line.text) }
.to_string_lossy()
.into_owned()
}
fn unix_now() -> u64 {
SystemTime::now()
.duration_since(UNIX_EPOCH)
.unwrap_or_default()
.as_secs()
}
// ─── debug session recorder ──────────────────────────────────────────────
struct DebugRecorder {
dir: PathBuf,
audio: Vec<f32>,
log_lines: Vec<String>,
}
impl DebugRecorder {
fn new(base_dir: &PathBuf) -> Self {
let dir = base_dir.join(unix_now().to_string());
std::fs::create_dir_all(&dir).ok();
Self {
dir,
audio: Vec::new(),
log_lines: Vec::new(),
}
}
fn log_event(&mut self, event: &str) {
self.log_lines.push(format!("[{}] {}", ts(), event));
}
fn log_segment(&mut self, line: &transcript_line_t, prefix: &str) {
let text = line_text(line);
let event = format!(
"SEGMENT id={} prefix={} is_complete={} start={:.3}s duration={:.3}s text=\"{}\"",
line.id, prefix, line.is_complete, line.start_time, line.duration, text
);
self.log_event(&event);
}
fn add_audio(&mut self, samples: &[f32]) {
self.audio.extend_from_slice(samples);
}
fn save(self) {
// Write log
let log_path = self.dir.join("session.log");
match std::fs::File::create(&log_path) {
Ok(mut f) => {
for line in &self.log_lines {
writeln!(f, "{}", line).ok();
}
}
Err(e) => log(&format!("debug: failed to write log: {}", e)),
}
// Write WAV
let wav_path = self.dir.join("audio.wav");
match write_wav(&wav_path, &self.audio, SAMPLE_RATE as u32) {
Ok(()) => {
let secs = self.audio.len() as f64 / SAMPLE_RATE as f64;
log(&format!("debug: saved {} ({:.1}s, {} samples)", self.dir.display(), secs, self.audio.len()));
}
Err(e) => log(&format!("debug: failed to write wav: {}", e)),
}
}
}
fn write_wav(path: &PathBuf, samples: &[f32], sample_rate: u32) -> Result<()> {
let num_samples = samples.len() as u32;
let data_size = num_samples * 4; // f32 = 4 bytes
let file_size = 44 + data_size;
let mut f = std::fs::File::create(path)?;
// RIFF header
f.write_all(b"RIFF")?;
f.write_all(&(file_size - 8).to_le_bytes())?;
f.write_all(b"WAVE")?;
// fmt chunk
f.write_all(b"fmt ")?;
f.write_all(&16u32.to_le_bytes())?; // chunk size
f.write_all(&3u16.to_le_bytes())?; // IEEE float
f.write_all(&1u16.to_le_bytes())?; // mono
f.write_all(&sample_rate.to_le_bytes())?;
f.write_all(&(sample_rate * 4).to_le_bytes())?; // byte rate
f.write_all(&4u16.to_le_bytes())?; // block align
f.write_all(&32u16.to_le_bytes())?; // bits per sample
// data chunk
f.write_all(b"data")?;
f.write_all(&data_size.to_le_bytes())?;
// Convert f32 samples to little-endian bytes
let mut bytes = Vec::with_capacity(data_size as usize);
for &s in samples {
bytes.extend_from_slice(&s.to_le_bytes());
}
f.write_all(&bytes)?;
Ok(())
}
// ─── shared state ─────────────────────────────────────────────────────────
struct Shared {
writer: Mutex<TcpStream>,
session_id: u64,
transcriber_handle: i32,
debug_dir: Option<PathBuf>,
}
impl Shared {
fn send_msg(&self, prefix: &str, text: &str) {
let text = truncate_to_word(text, MAX_TEXT_BYTES);
let line = if text.is_empty() {
format!("{} {}\n", prefix, self.session_id)
} else {
format!("{} {} {}\n", prefix, self.session_id, text)
};
let mut writer = self.writer.lock().unwrap();
match writer.write_all(line.as_bytes()) {
Ok(_) => log(&format!("TX {} {} {}", prefix, self.session_id, text)),
Err(e) => log(&format!("TX failed: {}", e)),
}
}
}
// ─── session ──────────────────────────────────────────────────────────────
struct Session {
shared: Arc<Shared>,
stop_signal: Arc<AtomicBool>,
aborted: Arc<AtomicBool>,
transcriber: thread::JoinHandle<()>,
cpal_stream: Option<cpal::Stream>,
stream_handle: i32,
}
impl Session {
fn stop(mut self) {
// Signal transcriber to exit main loop, then wait for it to drain
// trailing audio + final flush. cpal stream stays alive during drain.
self.stop_signal.store(true, Ordering::SeqCst);
self.transcriber.join().ok();
// Now safe to kill ALSA — transcriber is done
self.cpal_stream.take();
unsafe { moonshine_free_stream(self.shared.transcriber_handle, self.stream_handle) };
}
fn abort(mut self) {
self.aborted.store(true, Ordering::SeqCst);
self.stop_signal.store(true, Ordering::SeqCst);
self.transcriber.join().ok();
self.cpal_stream.take();
unsafe { moonshine_free_stream(self.shared.transcriber_handle, self.stream_handle) };
}
}
fn start_session(shared: Arc<Shared>) -> Option<Session> {
let stream_handle = unsafe { moonshine_create_stream(shared.transcriber_handle, 0) };
if stream_handle < 0 {
log(&format!("create_stream failed: {}", err_str(stream_handle)));
return None;
}
let rc = unsafe { moonshine_start_stream(shared.transcriber_handle, stream_handle) };
if rc != 0 {
log(&format!("start_stream failed: {}", err_str(rc)));
unsafe { moonshine_free_stream(shared.transcriber_handle, stream_handle) };
return None;
}
let (tx, rx) = mpsc::channel::<Vec<f32>>();
let stop_signal = Arc::new(AtomicBool::new(false));
let aborted = Arc::new(AtomicBool::new(false));
let cpal_stream = match start_cpal(tx) {
Ok(s) => Some(s),
Err(e) => {
log(&format!("cpal failed: {}", e));
unsafe { moonshine_free_stream(shared.transcriber_handle, stream_handle) };
return None;
}
};
let shared_clone = shared.clone();
let stop_signal_clone = stop_signal.clone();
let aborted_clone = aborted.clone();
let transcriber = thread::spawn(move || {
transcriber_loop(shared_clone, rx, stop_signal_clone, aborted_clone, stream_handle);
});
Some(Session {
shared,
stop_signal,
aborted,
transcriber,
cpal_stream,
stream_handle,
})
}
fn transcriber_loop(
shared: Arc<Shared>,
rx: mpsc::Receiver<Vec<f32>>,
stop_signal: Arc<AtomicBool>,
aborted: Arc<AtomicBool>,
stream_handle: i32,
) {
let handle = shared.transcriber_handle;
let mut sent_ids: HashSet<u64> = HashSet::new();
// Debug recorder (if enabled)
let mut recorder = shared.debug_dir.as_ref().map(|_| DebugRecorder::new(
&shared.debug_dir.as_ref().unwrap().join(shared.session_id.to_string()),
));
if let Some(ref mut r) = recorder {
r.log_event(&format!("session {} started", shared.session_id));
}
// Main loop: process audio until stop_signal
while !stop_signal.load(Ordering::SeqCst) {
match rx.recv_timeout(Duration::from_millis(100)) {
Ok(chunk) => {
if let Some(ref mut r) = recorder {
r.add_audio(&chunk);
}
unsafe {
moonshine_transcribe_add_audio_to_stream(
handle, stream_handle,
chunk.as_ptr(), chunk.len() as u64,
SAMPLE_RATE, 0,
);
}
let mut t_ptr: *mut transcript_t = std::ptr::null_mut();
let rc = unsafe { moonshine_transcribe_stream(handle, stream_handle, 0, &mut t_ptr) };
if rc != 0 || t_ptr.is_null() {
continue;
}
if let Some(ref mut r) = recorder {
log_transcript_lines(r, t_ptr);
}
send_new_segments(&shared, t_ptr, &mut sent_ids, "P");
}
Err(mpsc::RecvTimeoutError::Timeout) => continue,
Err(mpsc::RecvTimeoutError::Disconnected) => break,
}
}
// Drain trailing audio from ALSA buffer (cpal stream still alive).
// Fixed 100ms window — cpal delivers every ~50ms (800 samples @ 16kHz),
// so this captures 1-2 more callbacks worth of trailing audio.
// Hold one chunk back so we can apply a fade-out to the very last one.
let drain_deadline = std::time::Instant::now() + Duration::from_millis(100);
let mut held_chunk: Option<Vec<f32>> = None;
while std::time::Instant::now() < drain_deadline {
match rx.recv_timeout(drain_deadline - std::time::Instant::now()) {
Ok(chunk) => {
// Feed previously held chunk to Moonshine + recorder (no fade)
if let Some(prev) = held_chunk.take() {
if let Some(ref mut r) = recorder {
r.add_audio(&prev);
}
unsafe {
moonshine_transcribe_add_audio_to_stream(
handle, stream_handle,
prev.as_ptr(), prev.len() as u64,
SAMPLE_RATE, 0,
);
}
}
held_chunk = Some(chunk);
}
Err(mpsc::RecvTimeoutError::Timeout) => break,
Err(mpsc::RecvTimeoutError::Disconnected) => break,
}
}
// Apply 5ms fade-out to the last chunk, then feed to both Moonshine and recorder
if let Some(mut chunk) = held_chunk.take() {
let fade_samples = (SAMPLE_RATE as usize * 25) / 1000; // 25ms
if chunk.len() > fade_samples {
let start = chunk.len() - fade_samples;
for i in 0..fade_samples {
let t = 1.0 - (i as f32 / fade_samples as f32);
chunk[start + i] *= t;
}
}
if let Some(ref mut r) = recorder {
r.add_audio(&chunk);
}
unsafe {
moonshine_transcribe_add_audio_to_stream(
handle, stream_handle,
chunk.as_ptr(), chunk.len() as u64,
SAMPLE_RATE, 0,
);
}
}
// If aborted (new session took over), skip final flush entirely
if aborted.load(Ordering::SeqCst) {
unsafe { moonshine_stop_stream(handle, stream_handle) };
log(&format!("Session {} aborted, skipping final flush", shared.session_id));
if let Some(mut r) = recorder {
r.log_event("aborted (new session took over)");
r.save();
}
return;
}
// Final flush
unsafe { moonshine_stop_stream(handle, stream_handle) };
let mut t_ptr: *mut transcript_t = std::ptr::null_mut();
let rc = unsafe { moonshine_transcribe_stream(handle, stream_handle, 0, &mut t_ptr) };
if rc == 0 && !t_ptr.is_null() {
let t = unsafe { &*t_ptr };
let mut new_segments: Vec<String> = Vec::new();
if let Some(ref mut r) = recorder {
log_transcript_lines(r, t_ptr);
}
for i in 0..t.line_count as usize {
let line = unsafe { &*t.lines.add(i) };
if line.text.is_null() || line.is_complete == 0 {
continue;
}
if !sent_ids.insert(line.id) {
continue;
}
let text = line_text(line);
if !text.is_empty() {
new_segments.push(text);
}
}
if new_segments.is_empty() {
shared.send_msg("F", "");
} else {
let last = new_segments.len() - 1;
for (i, text) in new_segments.iter().enumerate() {
let prefix = if i == last { "F" } else { "P" };
shared.send_msg(prefix, text);
if let Some(ref mut r) = recorder {
r.log_event(&format!("TX {} \"{}\"", prefix, text));
}
}
}
} else {
shared.send_msg("F", "");
if let Some(ref mut r) = recorder {
r.log_event("TX F (empty)");
}
}
if let Some(mut r) = recorder {
r.log_event("session ended");
r.save();
}
}
fn log_transcript_lines(recorder: &mut DebugRecorder, t_ptr: *const transcript_t) {
let t = unsafe { &*t_ptr };
for i in 0..t.line_count as usize {
let line = unsafe { &*t.lines.add(i) };
if line.text.is_null() {
continue;
}
recorder.log_segment(line, if line.is_complete != 0 { "complete" } else { "partial" });
}
}
fn send_new_segments(
shared: &Shared,
t_ptr: *const transcript_t,
sent_ids: &mut HashSet<u64>,
prefix: &str,
) {
let t = unsafe { &*t_ptr };
for i in 0..t.line_count as usize {
let line = unsafe { &*t.lines.add(i) };
if line.text.is_null() || line.is_complete == 0 {
continue;
}
if !sent_ids.insert(line.id) {
continue;
}
let text = line_text(line);
if text.is_empty() {
continue;
}
shared.send_msg(prefix, &text);
}
}
// ─── cpal ─────────────────────────────────────────────────────────────────
fn start_cpal(
tx: mpsc::Sender<Vec<f32>>,
) -> Result<cpal::Stream> {
let host = cpal::default_host();
let dev = host
.default_input_device()
.ok_or_else(|| anyhow!("no input device"))?;
let supported = dev
.supported_input_configs()?
.filter(|c| c.channels() <= 2 && c.min_sample_rate().0 <= 16000)
.min_by_key(|c| match c.sample_format() {
SampleFormat::F32 => 0,
SampleFormat::I16 => 1,
SampleFormat::U8 => 2,
_ => 99,
})
.ok_or_else(|| anyhow!("no suitable input config"))?;
let fmt = supported.sample_format();
let mut config = supported.with_max_sample_rate().config();
if config.channels > 1 {
config.channels = 1;
}
config.sample_rate = SampleRate(16000);
config.buffer_size = cpal::BufferSize::Fixed(800);
let err_fn = |e: cpal::StreamError| log(&format!("cpal error: {}", e));
let stream = match fmt {
SampleFormat::F32 => dev.build_input_stream(
&config,
move |data: &[f32], _: &_| {
let _ = tx.send(data.to_vec());
},
err_fn,
None,
)?,
SampleFormat::I16 => dev.build_input_stream(
&config,
move |data: &[i16], _: &_| {
let _ = tx.send(data.iter().map(|&x| x as f32 / 32768.0).collect());
},
err_fn,
None,
)?,
SampleFormat::U8 => dev.build_input_stream(
&config,
move |data: &[u8], _: &_| {
let _ = tx.send(data.iter().map(|&x| (x as f32 - 128.0) / 128.0).collect());
},
err_fn,
None,
)?,
_ => return Err(anyhow!("unsupported sample format {:?}", fmt)),
};
stream.play()?;
Ok(stream)
}
// ─── model fetch ──────────────────────────────────────────────────────────
#[derive(Deserialize)]
struct Manifest {
groups: Vec<ManifestGroup>,
}
#[derive(Deserialize)]
struct ManifestGroup {
#[allow(dead_code)]
base_url: String,
files: Vec<ManifestFile>,
}
#[derive(Deserialize)]
struct ManifestFile {
name: String,
url: String,
size: Option<u64>,
}
fn fetch_model(model_dir: &str) -> Result<()> {
let dest = PathBuf::from(model_dir);
log(&format!("Fetching medium-streaming-en model to {}", dest.display()));
let lang = std::ffi::CString::new("en").unwrap();
let opt_name = std::ffi::CString::new("model_arch").unwrap();
let opt_value = std::ffi::CString::new("5").unwrap();
let mut options = [moonshine_option_t {
name: opt_name.as_ptr(),
value: opt_value.as_ptr(),
}];
let mut json_ptr: *mut i8 = std::ptr::null_mut();
let rc = unsafe {
moonshine_get_stt_dependencies(
lang.as_ptr(),
options.as_mut_ptr(),
options.len() as u64,
&mut json_ptr,
)
};
if rc != 0 || json_ptr.is_null() {
return Err(anyhow!("moonshine_get_stt_dependencies failed: {}", err_str(rc)));
}
let json_str = unsafe { CStr::from_ptr(json_ptr) }
.to_string_lossy()
.into_owned();
unsafe { moonshine_free_buffer(json_ptr as *mut std::ffi::c_void) };
let manifest: Manifest = serde_json::from_str(&json_str)?;
std::fs::create_dir_all(&dest)?;
let mut total_files = 0;
let mut total_bytes: u64 = 0;
for group in &manifest.groups {
for file in &group.files {
let dest_path = dest.join(&file.name);
if dest_path.exists() {
log(&format!(" SKIP {} (already exists)", file.name));
continue;
}
if let Some(parent) = dest_path.parent() {
std::fs::create_dir_all(parent)?;
}
log(&format!(" GET {}", file.url));
if let Some(expected) = file.size {
log(&format!(" {} bytes", expected));
total_bytes += expected;
}
let status = std::process::Command::new("curl")
.arg("-sSL")
.arg("-o")
.arg(&dest_path)
.arg(&file.url)
.status()?;
if !status.success() {
return Err(anyhow!("curl failed for {}", file.url));
}
total_files += 1;
}
}
log(&format!("Done: {} files, ~{} MB", total_files, total_bytes / (1024 * 1024)));
log(&format!("Model directory: {}", dest.display()));
Ok(())
}
// ─── main ─────────────────────────────────────────────────────────────────
fn main() -> Result<()> {
let mut args = std::env::args().skip(1);
let home = std::env::var("HOME").unwrap_or_else(|_| "/root".to_string());
let default_model = format!("{}/.rvsttd/model", home);
let first = args.next();
if first.as_deref() == Some("fetch") {
let model_dir = args.next().unwrap_or_else(|| default_model.clone());
return fetch_model(&model_dir);
}
let mut args = first.into_iter().chain(args);
let mut model_dir = default_model;
let mut debug = false;
while let Some(a) = args.next() {
match a.as_str() {
"--model-dir" | "-m" => {
model_dir = args.next().unwrap_or(model_dir);
}
"--debug" => {
debug = true;
}
"--help" | "-h" => {
println!("Usage: rvsttd [--model-dir DIR] [--debug]");
println!(" rvsttd fetch [DIR]");
println!("Listens on TCP {}", BIND_ADDR);
println!("Model: medium-streaming (Moonshine)");
println!("--debug: save session audio + transcript log to ~/.rvsttd/debug/");
println!("'rvsttd fetch' downloads the English medium-streaming model");
return Ok(());
}
_ => return Err(anyhow!("unknown arg: {}", a)),
}
}
let debug_dir = if debug {
let d = PathBuf::from(&home).join(".rvsttd").join("debug");
std::fs::create_dir_all(&d).ok();
log(&format!("Debug mode enabled — sessions saved to {}", d.display()));
Some(d)
} else {
None
};
let model_path = std::fs::canonicalize(&model_dir)
.unwrap_or_else(|_| std::path::PathBuf::from(&model_dir));
log(&format!("Loading model from {}...", model_path.display()));
let c_dir = std::ffi::CString::new(model_path.to_str().unwrap()).unwrap();
let transcriber_handle = unsafe {
moonshine_load_transcriber_from_files(
c_dir.as_ptr(),
ARCH,
std::ptr::null(),
0,
HEADER_VERSION,
)
};
if transcriber_handle < 0 {
return Err(anyhow!("failed to load model: {}", err_str(transcriber_handle)));
}
log(&format!("Model loaded (handle {})", transcriber_handle));
let listener = TcpListener::bind(BIND_ADDR)?;
log(&format!("STT server listening on TCP {}", BIND_ADDR));
let mut current_session: Option<Session> = None;
for stream in listener.incoming() {
let stream = match stream {
Ok(s) => s,
Err(e) => {
log(&format!("accept failed: {}", e));
continue;
}
};
stream.set_nodelay(true).ok();
log(&format!("Client connected: {}", stream.peer_addr().map(|a| a.to_string()).unwrap_or_else(|_| "?".to_string())));
let writer_stream = stream.try_clone()?;
let reader = BufReader::new(stream);
for line in reader.lines() {
let line = match line {
Ok(l) => l,
Err(_) => break,
};
let line = line.trim();
log(&format!("RX {}", line));
// ON <session>
if let Some(rest) = line.strip_prefix("ON ") {
let new_session_id: u64 = rest.parse().unwrap_or(0);
if let Some(s) = current_session.take() {
log(&format!("Aborting session {} for new session {}", s.shared.session_id, new_session_id));
s.abort();
}
let shared = Arc::new(Shared {
writer: Mutex::new(writer_stream.try_clone()?),
session_id: new_session_id,
transcriber_handle,
debug_dir: debug_dir.clone(),
});
log(&format!("PTT on session {}", new_session_id));
match start_session(shared) {
Some(s) => current_session = Some(s),
None => log("Failed to start session"),
}
}
// OFF <session>
else if let Some(rest) = line.strip_prefix("OFF ") {
let off_session: u64 = rest.parse().unwrap_or(0);
if let Some(s) = current_session.as_ref() {
if s.shared.session_id == off_session {
log(&format!("OFF session {}", off_session));
if let Some(s) = current_session.take() {
s.stop();
}
} else {
log(&format!("OFF session {} (stale, current={}), ignoring", off_session, s.shared.session_id));
}
} else {
log(&format!("OFF session {} (no active session), ignoring", off_session));
}
}
else {
log(&format!("Unknown command: {}", line));
}
}
log("Client disconnected");
if let Some(s) = current_session.take() {
s.abort();
}
}
Ok(())
}