Compare commits
3 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| 4be00bc4f1 | |||
| 7765c6967c | |||
| facbfe6a5c |
@@ -27,3 +27,6 @@ Robovoice.Tts.LibPiper/piper.h
|
||||
## Temp files
|
||||
*.wav
|
||||
*.raw
|
||||
|
||||
## Rust build output
|
||||
target/
|
||||
|
||||
+21
@@ -0,0 +1,21 @@
|
||||
# Robovoice Backlog
|
||||
|
||||
## Second-order PTT
|
||||
|
||||
When a game is running, it has its own PTT button (e.g. a voip push-to-talk
|
||||
key). Robovoice should simulate pressing the game's PTT key before TTS audio
|
||||
output begins, and release it after playback finishes.
|
||||
|
||||
This lets the TTS audio be transmitted through the game's voip channel to
|
||||
other players.
|
||||
|
||||
### Considerations
|
||||
|
||||
- Needs a configurable "game PTT key" (separate from Robovoice's own PTT key)
|
||||
- Use `SendInput` or `keybd_event` to synthesize the keypress
|
||||
- Press the game PTT key right before buffered audio starts playing
|
||||
- Release it after `AudioOutput` finishes playback (need a playback-complete
|
||||
signal — currently `Flush()` doesn't provide one)
|
||||
- Edge cases: what if the user presses Robovoice PTT while game PTT is still
|
||||
held from a previous utterance? Flush should release game PTT too.
|
||||
- Should this be a per-output-device setting? (CABLE Output vs speakers)
|
||||
@@ -1,106 +0,0 @@
|
||||
using System.Net;
|
||||
using System.Net.NetworkInformation;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
|
||||
// Args: [interface_ip] [interval_ms]
|
||||
// Defaults: auto-detect LAN IP, 50ms
|
||||
|
||||
string localIp = args.Length > 0 ? args[0] : GetLanInterfaceIp() ?? "0.0.0.0";
|
||||
int intervalMs = args.Length > 1 && int.TryParse(args[1], out int iv) ? iv : 50;
|
||||
|
||||
using var sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
|
||||
sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
|
||||
sock.EnableBroadcast = true;
|
||||
|
||||
var localEp = new IPEndPoint(IPAddress.Parse(localIp), 68);
|
||||
sock.Bind(localEp);
|
||||
|
||||
Console.WriteLine($"Bound to {localEp} (SO_REUSEADDR)");
|
||||
Console.WriteLine("Sending NOPs + listening. Press Ctrl+C to stop.");
|
||||
Console.WriteLine();
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
var recvTask = Task.Run(() => ReceiveLoop(sock, cts.Token));
|
||||
|
||||
var destEp = new IPEndPoint(IPAddress.Broadcast, 67);
|
||||
uint session = 1;
|
||||
uint nonce = 0;
|
||||
|
||||
while (!cts.Token.IsCancellationRequested)
|
||||
{
|
||||
nonce++;
|
||||
string msg = $"HKMSTR {session} {nonce}\n";
|
||||
byte[] payload = Encoding.UTF8.GetBytes(msg);
|
||||
|
||||
try
|
||||
{
|
||||
int sent = sock.SendTo(payload, destEp);
|
||||
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] SENT s={session} n={nonce} {sent} bytes");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] SEND FAILED: {ex.Message}");
|
||||
}
|
||||
|
||||
try { Thread.Sleep(intervalMs); }
|
||||
catch (OperationCanceledException) { break; }
|
||||
}
|
||||
|
||||
cts.Cancel();
|
||||
await recvTask;
|
||||
|
||||
static string? GetLanInterfaceIp()
|
||||
{
|
||||
foreach (var nic in NetworkInterface.GetAllNetworkInterfaces())
|
||||
{
|
||||
if (nic.OperationalStatus != OperationalStatus.Up)
|
||||
continue;
|
||||
if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback)
|
||||
continue;
|
||||
if (nic.Description.Contains("WireGuard", StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
|
||||
foreach (var addr in nic.GetIPProperties().UnicastAddresses)
|
||||
{
|
||||
if (addr.Address.AddressFamily == AddressFamily.InterNetwork)
|
||||
{
|
||||
var ip = addr.Address.ToString();
|
||||
if (ip.StartsWith("192.168.") || ip.StartsWith("10.") || ip.StartsWith("172."))
|
||||
return ip;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static void ReceiveLoop(Socket sock, CancellationToken ct)
|
||||
{
|
||||
byte[] buffer = new byte[4096];
|
||||
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
|
||||
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
int received;
|
||||
try
|
||||
{
|
||||
if (!sock.Poll(500_000, SelectMode.SelectRead))
|
||||
continue;
|
||||
received = sock.ReceiveFrom(buffer, ref fromEp);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV ERROR: {ex.Message}");
|
||||
continue;
|
||||
}
|
||||
|
||||
string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');
|
||||
if (!text.StartsWith("HKMSTR"))
|
||||
{
|
||||
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV {received} bytes from {fromEp} (no magic)");
|
||||
continue;
|
||||
}
|
||||
|
||||
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV {received} bytes from {fromEp}: {text}");
|
||||
}
|
||||
}
|
||||
@@ -1,10 +0,0 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
@@ -12,7 +12,7 @@ public sealed class AppConfig
|
||||
public int LengthScale { get; set; } = 100;
|
||||
public int NoiseWScale { get; set; } = 800;
|
||||
public bool MinimizeToTray { get; set; } = true;
|
||||
public string InterfaceIp { get; set; } = string.Empty;
|
||||
public string SttEndpoint { get; set; } = "127.0.0.1:6996";
|
||||
|
||||
public static string AppDataDir => Path.Combine(
|
||||
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
|
||||
|
||||
Generated
+8
-9
@@ -17,7 +17,7 @@ partial class MainForm
|
||||
private Button btnBrowseFile = null!;
|
||||
private Label lblFileName = null!;
|
||||
private Label lblServer = null!;
|
||||
private ComboBox cmbInterface = null!;
|
||||
private TextBox txtSttEndpoint = null!;
|
||||
private Label lblLineStatus = null!;
|
||||
private RichTextBox txtLog = null!;
|
||||
private CheckBox chkMinimizeToTray = null!;
|
||||
@@ -57,7 +57,7 @@ partial class MainForm
|
||||
btnBrowseFile = new Button();
|
||||
lblFileName = new Label();
|
||||
lblServer = new Label();
|
||||
cmbInterface = new ComboBox();
|
||||
txtSttEndpoint = new TextBox();
|
||||
lblLineStatus = new Label();
|
||||
txtLog = new RichTextBox();
|
||||
chkMinimizeToTray = new CheckBox();
|
||||
@@ -143,15 +143,14 @@ partial class MainForm
|
||||
lblFileName.ForeColor = Color.Gray;
|
||||
|
||||
// lblServer
|
||||
lblServer.Text = "Interface:";
|
||||
lblServer.Text = "STT:";
|
||||
lblServer.Location = new Point(440, 48);
|
||||
lblServer.Size = new Size(55, 23);
|
||||
lblServer.Size = new Size(35, 23);
|
||||
lblServer.TextAlign = ContentAlignment.MiddleLeft;
|
||||
|
||||
// cmbInterface
|
||||
cmbInterface.Location = new Point(498, 45);
|
||||
cmbInterface.Size = new Size(165, 23);
|
||||
cmbInterface.DropDownStyle = ComboBoxStyle.DropDownList;
|
||||
// txtSttEndpoint
|
||||
txtSttEndpoint.Location = new Point(478, 45);
|
||||
txtSttEndpoint.Size = new Size(185, 23);
|
||||
|
||||
// lblLineStatus
|
||||
lblLineStatus.Text = "";
|
||||
@@ -270,7 +269,7 @@ partial class MainForm
|
||||
Controls.Add(btnBrowseFile);
|
||||
Controls.Add(lblFileName);
|
||||
Controls.Add(lblServer);
|
||||
Controls.Add(cmbInterface);
|
||||
Controls.Add(txtSttEndpoint);
|
||||
Controls.Add(lblLineStatus);
|
||||
Controls.Add(lblNoise);
|
||||
Controls.Add(trkNoise);
|
||||
|
||||
+24
-56
@@ -2,7 +2,7 @@ using NAudio.Wave;
|
||||
using Robovoice.App;
|
||||
using Robovoice.Core;
|
||||
using Robovoice.Core.Voices;
|
||||
using Robovoice.Stt.Dhcp;
|
||||
using Robovoice.Stt.Tcp;
|
||||
using Robovoice.Tts.LibPiper;
|
||||
using System.Diagnostics;
|
||||
|
||||
@@ -16,7 +16,7 @@ internal sealed partial class MainForm : Form
|
||||
|
||||
private LibPiperTtsEngine? _tts;
|
||||
private AudioOutput? _audioOutput;
|
||||
private DhcpSttSource? _sttSource;
|
||||
private TcpSttSource? _sttSource;
|
||||
private Orchestrator? _orchestrator;
|
||||
private PttHotkey? _pttHotkey;
|
||||
private NotifyIcon? _trayIcon;
|
||||
@@ -35,6 +35,8 @@ internal sealed partial class MainForm : Form
|
||||
}
|
||||
|
||||
private async void OnLoad(object? sender, EventArgs e)
|
||||
{
|
||||
try
|
||||
{
|
||||
ActiveControl = txtLog;
|
||||
PopulateVoices();
|
||||
@@ -61,7 +63,7 @@ internal sealed partial class MainForm : Form
|
||||
OnSliderScroll(null, EventArgs.Empty);
|
||||
|
||||
chkMinimizeToTray.Checked = _config.MinimizeToTray;
|
||||
PopulateInterfaces();
|
||||
txtSttEndpoint.Text = _config.SttEndpoint;
|
||||
|
||||
btnBrowseFile.Click += OnBrowseFile;
|
||||
btnTestVoice.Click += OnTestVoice;
|
||||
@@ -71,7 +73,7 @@ internal sealed partial class MainForm : Form
|
||||
txtPttKey.KeyDown += OnPttKeyDown;
|
||||
cmbOutput.SelectedIndexChanged += OnOutputChanged;
|
||||
cmbVoice.SelectedIndexChanged += OnVoiceChanged;
|
||||
cmbInterface.SelectedIndexChanged += OnInterfaceChanged;
|
||||
txtSttEndpoint.Leave += OnSttEndpointChanged;
|
||||
|
||||
trkNoise.Scroll += OnSliderScroll;
|
||||
trkSpeed.Scroll += OnSliderScroll;
|
||||
@@ -85,8 +87,13 @@ internal sealed partial class MainForm : Form
|
||||
Resize += OnResize;
|
||||
|
||||
SetupTray();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
System.Diagnostics.Debug.WriteLine($"OnLoad failed: {ex}");
|
||||
}
|
||||
|
||||
await InitializeEngineAsync();
|
||||
BeginInvoke(async () => await InitializeEngineAsync());
|
||||
}
|
||||
|
||||
private Keys _pttKey = Keys.F8;
|
||||
@@ -232,7 +239,8 @@ internal sealed partial class MainForm : Form
|
||||
|
||||
Log($"Loading voice: {voiceName}...");
|
||||
_audioOutput?.Dispose();
|
||||
_tts?.DisposeAsync().AsTask().Wait();
|
||||
if (_tts is not null)
|
||||
await _tts.DisposeAsync();
|
||||
|
||||
_tts = new LibPiperTtsEngine(
|
||||
modelPath,
|
||||
@@ -242,21 +250,21 @@ internal sealed partial class MainForm : Form
|
||||
noiseWScale: trkNoiseW.Value / 1000.0f);
|
||||
_audioOutput = new AudioOutput();
|
||||
|
||||
try
|
||||
{
|
||||
if (_sttSource is null)
|
||||
{
|
||||
string ifaceIp = cmbInterface.SelectedItem as string ?? "";
|
||||
_sttSource = new DhcpSttSource { InterfaceIp = ifaceIp, Log = Log };
|
||||
_sttSource = new TcpSttSource { Endpoint = txtSttEndpoint.Text, Log = Log };
|
||||
await _sttSource.StartAsync();
|
||||
}
|
||||
|
||||
_orchestrator?.DisposeAsync().AsTask().Wait();
|
||||
if (_orchestrator is not null)
|
||||
await _orchestrator.DisposeAsync();
|
||||
_orchestrator = new Orchestrator(_tts, _audioOutput, _sttSource, Log)
|
||||
{
|
||||
OutputDeviceName = cmbOutput.SelectedItem as string ?? string.Empty,
|
||||
};
|
||||
|
||||
try
|
||||
{
|
||||
await _orchestrator.InitializeTtsAsync();
|
||||
Log("Engine ready. Press PTT to send to STT server.");
|
||||
SetupHotkey();
|
||||
@@ -426,59 +434,19 @@ internal sealed partial class MainForm : Form
|
||||
SaveConfig();
|
||||
}
|
||||
|
||||
private void PopulateInterfaces()
|
||||
private async void OnSttEndpointChanged(object? sender, EventArgs e)
|
||||
{
|
||||
cmbInterface.Items.Clear();
|
||||
foreach (var (ip, name) in DhcpSttSource.GetAvailableInterfaces())
|
||||
{
|
||||
cmbInterface.Items.Add(name);
|
||||
cmbInterface.Items[^1] = name;
|
||||
cmbInterface.Items[cmbInterface.Items.Count - 1] = name;
|
||||
}
|
||||
|
||||
if (!string.IsNullOrEmpty(_config.InterfaceIp))
|
||||
{
|
||||
for (int i = 0; i < cmbInterface.Items.Count; i++)
|
||||
{
|
||||
if (cmbInterface.Items[i] is string s && s.Contains(_config.InterfaceIp))
|
||||
{
|
||||
cmbInterface.SelectedIndex = i;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (cmbInterface.Items.Count > 0)
|
||||
cmbInterface.SelectedIndex = 0;
|
||||
}
|
||||
|
||||
private void OnInterfaceChanged(object? sender, EventArgs e)
|
||||
{
|
||||
if (cmbInterface.SelectedItem is not string selected)
|
||||
return;
|
||||
|
||||
string? ip = ExtractIpFromDisplay(selected);
|
||||
if (ip is null) return;
|
||||
|
||||
_config.InterfaceIp = ip;
|
||||
_config.SttEndpoint = txtSttEndpoint.Text;
|
||||
SaveConfig();
|
||||
|
||||
if (_sttSource is not null)
|
||||
{
|
||||
_sttSource.DisposeAsync().AsTask().Wait(2000);
|
||||
await _sttSource.DisposeAsync();
|
||||
_sttSource = null;
|
||||
_ = InitializeEngineAsync();
|
||||
await InitializeEngineAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ExtractIpFromDisplay(string display)
|
||||
{
|
||||
int start = display.IndexOf('(');
|
||||
int end = display.IndexOf(')');
|
||||
if (start < 0 || end <= start) return null;
|
||||
return display[(start + 1)..end];
|
||||
}
|
||||
|
||||
private void SetupTray()
|
||||
{
|
||||
if (_trayInit) return;
|
||||
@@ -544,7 +512,7 @@ internal sealed partial class MainForm : Form
|
||||
_config.LengthScale = trkSpeed.Value;
|
||||
_config.NoiseWScale = trkNoiseW.Value;
|
||||
_config.MinimizeToTray = chkMinimizeToTray.Checked;
|
||||
_config.InterfaceIp = ExtractIpFromDisplay(cmbInterface.SelectedItem as string ?? "") ?? "";
|
||||
_config.SttEndpoint = txtSttEndpoint.Text;
|
||||
_config.Save();
|
||||
}
|
||||
|
||||
|
||||
@@ -19,8 +19,6 @@ internal sealed class Orchestrator : IAsyncDisposable
|
||||
private bool _playing;
|
||||
private Task? _synthTask;
|
||||
private CancellationTokenSource? _synthCts;
|
||||
private System.Threading.Timer? _segmentTimer;
|
||||
private static readonly TimeSpan SegmentTimeout = TimeSpan.FromMilliseconds(250);
|
||||
|
||||
public string OutputDeviceName { get; set; } = string.Empty;
|
||||
|
||||
@@ -51,7 +49,6 @@ internal sealed class Orchestrator : IAsyncDisposable
|
||||
_log($"SEGMENT: \"{msg.Text}\" ({msg.Text.Length} chars)");
|
||||
_pendingTexts.Enqueue(msg.Text);
|
||||
EnsureSynthTask();
|
||||
ResetSegmentTimer();
|
||||
}
|
||||
}
|
||||
else
|
||||
@@ -67,33 +64,11 @@ internal sealed class Orchestrator : IAsyncDisposable
|
||||
_log("FINAL: (empty)");
|
||||
}
|
||||
|
||||
CancelSegmentTimer();
|
||||
TransitionToPlaying();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void ResetSegmentTimer()
|
||||
{
|
||||
_segmentTimer?.Dispose();
|
||||
_segmentTimer = new System.Threading.Timer(_ => OnSegmentTimeout(), null, SegmentTimeout, Timeout.InfiniteTimeSpan);
|
||||
}
|
||||
|
||||
private void CancelSegmentTimer()
|
||||
{
|
||||
_segmentTimer?.Dispose();
|
||||
_segmentTimer = null;
|
||||
}
|
||||
|
||||
private void OnSegmentTimeout()
|
||||
{
|
||||
_log("STT: segment timeout, starting playback early");
|
||||
lock (_stateLock)
|
||||
{
|
||||
TransitionToPlaying();
|
||||
}
|
||||
}
|
||||
|
||||
private void TransitionToPlaying()
|
||||
{
|
||||
if (_playing)
|
||||
@@ -104,13 +79,11 @@ internal sealed class Orchestrator : IAsyncDisposable
|
||||
if (_synthTask is null || _synthTask.IsCompleted)
|
||||
{
|
||||
_log("TTS: nothing to play");
|
||||
CancelSegmentTimer();
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
_playing = true;
|
||||
CancelSegmentTimer();
|
||||
|
||||
int sampleRate = _bufferSampleRate;
|
||||
var chunks = _audioBuffer.ToList();
|
||||
@@ -189,7 +162,6 @@ internal sealed class Orchestrator : IAsyncDisposable
|
||||
{
|
||||
lock (_stateLock)
|
||||
{
|
||||
CancelSegmentTimer();
|
||||
_synthCts?.Cancel();
|
||||
_synthCts?.Dispose();
|
||||
_synthTask = null;
|
||||
@@ -209,7 +181,7 @@ internal sealed class Orchestrator : IAsyncDisposable
|
||||
public async Task InitializeTtsAsync()
|
||||
{
|
||||
_log("Initializing TTS engine...");
|
||||
await _tts.InitializeAsync();
|
||||
await Task.Run(() => _tts.InitializeAsync());
|
||||
_log($"TTS ready (sample rate: {_tts.SampleRate} Hz)");
|
||||
}
|
||||
|
||||
@@ -269,15 +241,17 @@ internal sealed class Orchestrator : IAsyncDisposable
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed) return;
|
||||
_currentCts?.Cancel();
|
||||
_disposed = true;
|
||||
|
||||
try { _currentCts?.Cancel(); } catch { }
|
||||
_currentCts?.Dispose();
|
||||
_synthCts?.Cancel();
|
||||
|
||||
try { _synthCts?.Cancel(); } catch { }
|
||||
_synthCts?.Dispose();
|
||||
CancelSegmentTimer();
|
||||
|
||||
_sttSource.TranscriptReceived -= OnTranscript;
|
||||
await _sttSource.DisposeAsync();
|
||||
await _tts.DisposeAsync();
|
||||
_audioOutput.Dispose();
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Robovoice.Core\Robovoice.Core.csproj" />
|
||||
<ProjectReference Include="..\Robovoice.Tts.LibPiper\Robovoice.Tts.LibPiper.csproj" />
|
||||
<ProjectReference Include="..\Robovoice.Stt.Dhcp\Robovoice.Stt.Dhcp.csproj" />
|
||||
<ProjectReference Include="..\Robovoice.Stt.Tcp\Robovoice.Stt.Tcp.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -1,277 +0,0 @@
|
||||
using System.Net;
|
||||
using System.Net.NetworkInformation;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using Robovoice.Core;
|
||||
|
||||
namespace Robovoice.Stt.Dhcp;
|
||||
|
||||
public sealed class DhcpSttSource : ISttSource
|
||||
{
|
||||
private const string Magic = "HKMSTR";
|
||||
private const int DhcpClientPort = 68;
|
||||
private const int DhcpServerPort = 67;
|
||||
private static readonly TimeSpan NopInterval = TimeSpan.FromMilliseconds(50);
|
||||
|
||||
private Socket? _sock;
|
||||
private CancellationTokenSource? _cts;
|
||||
private Task? _receiveTask;
|
||||
private Task? _nopTask;
|
||||
private EndPoint _broadcastEp = new IPEndPoint(IPAddress.Broadcast, DhcpServerPort);
|
||||
private uint _nonce;
|
||||
private uint _session;
|
||||
private bool _disposed;
|
||||
|
||||
public string InterfaceIp { get; set; } = string.Empty;
|
||||
|
||||
public Action<string>? Log { get; set; }
|
||||
|
||||
public event TranscriptEventHandler? TranscriptReceived;
|
||||
|
||||
public Task StartAsync(CancellationToken ct = default)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
if (_cts is not null)
|
||||
return Task.CompletedTask;
|
||||
|
||||
string ip = string.IsNullOrEmpty(InterfaceIp) ? AutoDetectInterfaceIp() ?? "0.0.0.0" : InterfaceIp;
|
||||
|
||||
_sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
|
||||
_sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
|
||||
_sock.EnableBroadcast = true;
|
||||
_sock.Bind(new IPEndPoint(IPAddress.Parse(ip), DhcpClientPort));
|
||||
|
||||
Log?.Invoke($"STT: bound to {ip}:{DhcpClientPort}, broadcasting to :{DhcpServerPort}");
|
||||
|
||||
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
_receiveTask = ReceiveLoopAsync(_cts.Token);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async Task StopAsync(CancellationToken ct = default)
|
||||
{
|
||||
StopNop();
|
||||
|
||||
if (_cts is not null)
|
||||
_cts.Cancel();
|
||||
|
||||
_sock?.Dispose();
|
||||
_sock = null;
|
||||
|
||||
if (_receiveTask is not null)
|
||||
{
|
||||
try { await _receiveTask.WaitAsync(ct); }
|
||||
catch { }
|
||||
_receiveTask = null;
|
||||
}
|
||||
|
||||
_cts?.Dispose();
|
||||
_cts = null;
|
||||
}
|
||||
|
||||
public void SendOn()
|
||||
{
|
||||
if (_cts is null)
|
||||
return;
|
||||
|
||||
_session++;
|
||||
_nonce = 0;
|
||||
Log?.Invoke($"STT: session {_session} started");
|
||||
SendNop();
|
||||
_nopTask = NopLoopAsync(_cts.Token);
|
||||
}
|
||||
|
||||
public void SendOff()
|
||||
{
|
||||
StopNop();
|
||||
SendControl($"HKMSTR:OFF {_session} {_nonce}");
|
||||
}
|
||||
|
||||
private void StopNop()
|
||||
{
|
||||
if (_nopTask is not null)
|
||||
{
|
||||
try { _nopTask.Wait(2000); } catch { }
|
||||
_nopTask = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task NopLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try { await Task.Delay(NopInterval, ct); }
|
||||
catch (OperationCanceledException) { break; }
|
||||
|
||||
SendNop();
|
||||
}
|
||||
}
|
||||
|
||||
private void SendNop()
|
||||
{
|
||||
_nonce++;
|
||||
SendControl($"HKMSTR {_session} {_nonce}");
|
||||
}
|
||||
|
||||
private void SendControl(string message)
|
||||
{
|
||||
if (_sock is null)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
byte[] payload = Encoding.UTF8.GetBytes(message + "\n");
|
||||
_sock.SendTo(payload, _broadcastEp);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log?.Invoke($"STT: send failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReceiveLoopAsync(CancellationToken ct)
|
||||
{
|
||||
byte[] buffer = new byte[4096];
|
||||
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
|
||||
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
int received;
|
||||
try
|
||||
{
|
||||
if (!_sock!.Poll(500_000, SelectMode.SelectRead))
|
||||
continue;
|
||||
received = _sock.ReceiveFrom(buffer, ref fromEp);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log?.Invoke($"STT: receive error: {ex.Message}");
|
||||
continue;
|
||||
}
|
||||
|
||||
string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');
|
||||
if (!text.StartsWith(Magic))
|
||||
continue;
|
||||
|
||||
TranscriptMessage? message = ParseReply(text);
|
||||
if (message is null)
|
||||
continue;
|
||||
|
||||
TranscriptReceived?.Invoke(this, new TranscriptEventArgs
|
||||
{
|
||||
Message = message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
private TranscriptMessage? ParseReply(string text)
|
||||
{
|
||||
// Format: HKMSTR:P <session> <text> or HKMSTR:F <session> <text>
|
||||
// <text> may be empty.
|
||||
string prefix;
|
||||
TranscriptType type;
|
||||
|
||||
if (text.StartsWith("HKMSTR:P "))
|
||||
{
|
||||
prefix = "HKMSTR:P ";
|
||||
type = TranscriptType.Partial;
|
||||
}
|
||||
else if (text.StartsWith("HKMSTR:F "))
|
||||
{
|
||||
prefix = "HKMSTR:F ";
|
||||
type = TranscriptType.Final;
|
||||
}
|
||||
else
|
||||
{
|
||||
return null;
|
||||
}
|
||||
|
||||
string rest = text[prefix.Length..];
|
||||
|
||||
int spaceIndex = rest.IndexOf(' ');
|
||||
if (spaceIndex < 0)
|
||||
{
|
||||
if (uint.TryParse(rest, out uint sessionOnly))
|
||||
{
|
||||
if (sessionOnly != _session)
|
||||
return null;
|
||||
return new TranscriptMessage(type, string.Empty);
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
string sessionStr = rest[..spaceIndex];
|
||||
if (!uint.TryParse(sessionStr, out uint session))
|
||||
return null;
|
||||
|
||||
if (session != _session)
|
||||
{
|
||||
Log?.Invoke($"STT: dropping stale reply (session {session} != current {_session})");
|
||||
return null;
|
||||
}
|
||||
|
||||
string transcript = rest[(spaceIndex + 1)..];
|
||||
return new TranscriptMessage(type, transcript);
|
||||
}
|
||||
|
||||
public static List<(string Ip, string Name)> GetAvailableInterfaces()
|
||||
{
|
||||
var result = new List<(string, string)>();
|
||||
|
||||
foreach (var nic in NetworkInterface.GetAllNetworkInterfaces())
|
||||
{
|
||||
if (nic.OperationalStatus != OperationalStatus.Up)
|
||||
continue;
|
||||
if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback)
|
||||
continue;
|
||||
|
||||
string desc = nic.Description;
|
||||
if (desc.Contains("WireGuard", StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
|
||||
foreach (var addr in nic.GetIPProperties().UnicastAddresses)
|
||||
{
|
||||
if (addr.Address.AddressFamily != AddressFamily.InterNetwork)
|
||||
continue;
|
||||
|
||||
string ip = addr.Address.ToString();
|
||||
if (ip.StartsWith("192.168.") || ip.StartsWith("10.") ||
|
||||
ip.StartsWith("172.16.") || ip.StartsWith("172.17.") ||
|
||||
ip.StartsWith("172.18.") || ip.StartsWith("172.19.") ||
|
||||
ip.StartsWith("172.20.") || ip.StartsWith("172.21.") ||
|
||||
ip.StartsWith("172.22.") || ip.StartsWith("172.23.") ||
|
||||
ip.StartsWith("172.24.") || ip.StartsWith("172.25.") ||
|
||||
ip.StartsWith("172.26.") || ip.StartsWith("172.27.") ||
|
||||
ip.StartsWith("172.28.") || ip.StartsWith("172.29.") ||
|
||||
ip.StartsWith("172.30.") || ip.StartsWith("172.31."))
|
||||
{
|
||||
result.Add((ip, $"{nic.Name} ({ip})"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string? AutoDetectInterfaceIp()
|
||||
{
|
||||
foreach (var (ip, _) in GetAvailableInterfaces())
|
||||
return ip;
|
||||
return null;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed) return;
|
||||
await StopAsync();
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,256 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using Robovoice.Core;
|
||||
|
||||
namespace Robovoice.Stt.Tcp;
|
||||
|
||||
public sealed class TcpSttSource : ISttSource
|
||||
{
|
||||
private TcpClient? _tcp;
|
||||
private NetworkStream? _stream;
|
||||
private StreamReader? _reader;
|
||||
private StreamWriter? _writer;
|
||||
private CancellationTokenSource? _cts;
|
||||
private Task? _runTask;
|
||||
private readonly object _sendLock = new();
|
||||
private uint _session;
|
||||
private bool _disposed;
|
||||
|
||||
public string Endpoint { get; set; } = "127.0.0.1:6996";
|
||||
|
||||
public Action<string>? Log { get; set; }
|
||||
|
||||
public event TranscriptEventHandler? TranscriptReceived;
|
||||
|
||||
public Task StartAsync(CancellationToken ct = default)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
if (_cts is not null)
|
||||
return Task.CompletedTask;
|
||||
|
||||
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
_runTask = RunAsync(_cts.Token);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async Task StopAsync(CancellationToken ct = default)
|
||||
{
|
||||
if (_cts is not null)
|
||||
_cts.Cancel();
|
||||
|
||||
CleanupConnection();
|
||||
|
||||
if (_runTask is not null)
|
||||
{
|
||||
try { await _runTask.WaitAsync(ct); }
|
||||
catch { }
|
||||
_runTask = null;
|
||||
}
|
||||
|
||||
_cts?.Dispose();
|
||||
_cts = null;
|
||||
}
|
||||
|
||||
private async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
IPEndPoint? endpoint = ParseEndpoint(Endpoint);
|
||||
if (endpoint is null)
|
||||
{
|
||||
Log?.Invoke($"STT: invalid endpoint '{Endpoint}'");
|
||||
try { await Task.Delay(3000, ct); } catch { break; }
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_tcp = new TcpClient();
|
||||
using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
connectCts.CancelAfter(TimeSpan.FromSeconds(5));
|
||||
await _tcp.ConnectAsync(endpoint.Address, endpoint.Port, connectCts.Token);
|
||||
|
||||
_stream = _tcp.GetStream();
|
||||
_reader = new StreamReader(_stream, Encoding.UTF8);
|
||||
_writer = new StreamWriter(_stream, Encoding.UTF8) { AutoFlush = true };
|
||||
|
||||
Log?.Invoke($"STT: connected to {Endpoint}");
|
||||
|
||||
await ReceiveLoopAsync(ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log?.Invoke($"STT: connection failed ({ex.Message}), retrying...");
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupConnection();
|
||||
}
|
||||
|
||||
if (!ct.IsCancellationRequested)
|
||||
{
|
||||
try { await Task.Delay(3000, ct); }
|
||||
catch (OperationCanceledException) { break; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReceiveLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested && _reader is not null)
|
||||
{
|
||||
string? line;
|
||||
try
|
||||
{
|
||||
line = await _reader.ReadLineAsync(ct);
|
||||
}
|
||||
catch
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (line is null)
|
||||
break;
|
||||
|
||||
TranscriptMessage? message = ParseReply(line);
|
||||
if (message is null)
|
||||
continue;
|
||||
|
||||
TranscriptReceived?.Invoke(this, new TranscriptEventArgs
|
||||
{
|
||||
Message = message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public void SendOn()
|
||||
{
|
||||
_session++;
|
||||
Send($"ON {_session}");
|
||||
}
|
||||
|
||||
public void SendOff()
|
||||
{
|
||||
Send($"OFF {_session}");
|
||||
}
|
||||
|
||||
private void Send(string message)
|
||||
{
|
||||
lock (_sendLock)
|
||||
{
|
||||
if (_writer is null)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
_writer.WriteLine(message);
|
||||
}
|
||||
catch
|
||||
{
|
||||
Log?.Invoke($"STT: failed to send '{message}' (not connected?)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void CleanupConnection()
|
||||
{
|
||||
lock (_sendLock)
|
||||
{
|
||||
_writer?.Dispose();
|
||||
_reader?.Dispose();
|
||||
_stream?.Dispose();
|
||||
_tcp?.Dispose();
|
||||
_writer = null;
|
||||
_reader = null;
|
||||
_stream = null;
|
||||
_tcp = null;
|
||||
}
|
||||
}
|
||||
|
||||
private static IPEndPoint? ParseEndpoint(string endpoint)
|
||||
{
|
||||
int colon = endpoint.LastIndexOf(':');
|
||||
if (colon <= 0)
|
||||
return null;
|
||||
|
||||
string host = endpoint[..colon];
|
||||
if (!int.TryParse(endpoint[(colon + 1)..], out int port))
|
||||
return null;
|
||||
|
||||
if (IPAddress.TryParse(host, out var addr))
|
||||
return new IPEndPoint(addr, port);
|
||||
|
||||
try
|
||||
{
|
||||
var addresses = Dns.GetHostAddresses(host);
|
||||
addr = addresses.FirstOrDefault(a => a.AddressFamily == AddressFamily.InterNetwork);
|
||||
if (addr is null)
|
||||
return null;
|
||||
return new IPEndPoint(addr, port);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private TranscriptMessage? ParseReply(string line)
|
||||
{
|
||||
if (line.StartsWith("P ", StringComparison.Ordinal))
|
||||
{
|
||||
string rest = line["P ".Length..];
|
||||
int space = rest.IndexOf(' ');
|
||||
if (space < 0)
|
||||
return null;
|
||||
|
||||
if (!uint.TryParse(rest[..space], out uint session))
|
||||
return null;
|
||||
|
||||
if (session != _session)
|
||||
{
|
||||
Log?.Invoke($"STT: dropping stale reply (session {session} != current {_session})");
|
||||
return null;
|
||||
}
|
||||
|
||||
return new TranscriptMessage(TranscriptType.Partial, rest[(space + 1)..]);
|
||||
}
|
||||
|
||||
if (line.StartsWith("F ", StringComparison.Ordinal))
|
||||
{
|
||||
string rest = line["F ".Length..];
|
||||
int space = rest.IndexOf(' ');
|
||||
if (space < 0)
|
||||
return null;
|
||||
|
||||
if (!uint.TryParse(rest[..space], out uint session))
|
||||
return null;
|
||||
|
||||
if (session != _session)
|
||||
{
|
||||
Log?.Invoke($"STT: dropping stale reply (session {session} != current {_session})");
|
||||
return null;
|
||||
}
|
||||
|
||||
return new TranscriptMessage(TranscriptType.Final, rest[(space + 1)..]);
|
||||
}
|
||||
|
||||
if (line == "F")
|
||||
{
|
||||
return new TranscriptMessage(TranscriptType.Final, string.Empty);
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed) return;
|
||||
await StopAsync();
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
+1
-2
@@ -2,7 +2,6 @@
|
||||
<Project Path="Robovoice.App/Robovoice.App.csproj" />
|
||||
<Project Path="Robovoice.Core/Robovoice.Core.csproj" />
|
||||
<Project Path="Robovoice.Stt.File/Robovoice.Stt.File.csproj" />
|
||||
<Project Path="Robovoice.Stt.Dhcp/Robovoice.Stt.Dhcp.csproj" />
|
||||
<Project Path="Robovoice.Stt.Tcp/Robovoice.Stt.Tcp.csproj" />
|
||||
<Project Path="Robovoice.Tts.LibPiper/Robovoice.Tts.LibPiper.csproj" />
|
||||
<Project Path="DhcpTunnelTest/Robovoice.DhcpTunnelTest.csproj" />
|
||||
</Solution>
|
||||
|
||||
+66
-96
@@ -1,136 +1,106 @@
|
||||
# Robovoice DHCP Tunnel Protocol
|
||||
# Robovoice STT Protocol
|
||||
|
||||
## Overview
|
||||
|
||||
Robovoice communicates with a remote STT server by tunneling through the
|
||||
DHCP UDP ports (68→67). This exploits a common killswitch exception: VPN
|
||||
software (e.g. WireGuard) blocks all traffic except DHCP, which is allowed
|
||||
for network connectivity maintenance.
|
||||
Robovoice connects to the STT server over TCP (typically through a gatuna
|
||||
L2 tunnel). The server captures audio, runs Moonshine STT, and sends
|
||||
transcript segments back. The client pre-synthesizes TTS on segments and
|
||||
plays audio on final.
|
||||
|
||||
```
|
||||
[Robovoice client] --broadcast UDP :68→:67--> [STT server]
|
||||
[Robovoice client] <--unicast UDP :67→:68-- [STT server]
|
||||
[Robovoice client] --TCP--> [STT server 127.0.0.1:6996]
|
||||
│ │
|
||||
├── ON <session>\n ────────►│ (abort old, start new session)
|
||||
├── OFF <session>\n ────────►│ (stop, final STT pass)
|
||||
│◄── P <session> <text>\n ──┤ (completed segment)
|
||||
│◄── F <session> <text>\n ──┤ (all done; text may be empty)
|
||||
```
|
||||
|
||||
The client broadcasts NOP heartbeats while PTT is held. The server starts
|
||||
recording on the first NOP and stops when it receives OFF or when 150ms
|
||||
pass with no NOPs.
|
||||
|
||||
## Transport
|
||||
|
||||
- **Protocol:** UDP (connectionless, unreliable)
|
||||
- **Client → Server:** broadcast, source port 68, dest port 67
|
||||
- **Server → Client:** unicast, source port 67, dest port 68
|
||||
- **Client binds:** to a specific LAN interface IP on port 68 (with
|
||||
`SO_REUSEADDR` to coexist with the Windows DHCP service)
|
||||
- **No connection state** — purely fire-and-forget datagrams
|
||||
- **Protocol:** TCP (reliable, ordered, connection-oriented)
|
||||
- **Server:** `127.0.0.1:6996` (hardcoded loopback)
|
||||
- **Framing:** newline-delimited text (`\n`), UTF-8
|
||||
- **Auto-reconnect:** client retries every 3s if connection drops
|
||||
|
||||
## Wire format
|
||||
|
||||
All messages are plain text, newline-terminated (`\n`). Every message starts
|
||||
with the 6-byte magic `HKMSTR` to distinguish our traffic from real DHCP.
|
||||
|
||||
### Client → Server
|
||||
|
||||
**NOP (heartbeat while PTT held):**
|
||||
**ON (PTT pressed):**
|
||||
```
|
||||
HKMSTR <session> <nonce>\n
|
||||
ON <session>\n
|
||||
```
|
||||
Sent every 50ms while PTT is held. The session is an incrementing unsigned
|
||||
integer that identifies the current PTT utterance (incremented on each PTT
|
||||
press). The nonce is an incrementing unsigned integer that makes each
|
||||
datagram unique. The server should echo the session back in replies. Both
|
||||
are discarded by the server for protocol logic — the server tracks liveness
|
||||
via "did anything arrive recently."
|
||||
Starts a new STT session. The server aborts any active session and starts
|
||||
recording. `<session>` is an incrementing unsigned integer chosen by the
|
||||
client. Replies from the server echo this session ID.
|
||||
|
||||
**OFF (PTT released):**
|
||||
```
|
||||
HKMSTR:OFF <session> <nonce>\n
|
||||
OFF <session>\n
|
||||
```
|
||||
Sent once when PTT is released. This is the fast-stop signal. If lost, the
|
||||
150ms timeout acts as a backstop.
|
||||
Stops the session. The server does a final STT pass on remaining audio and
|
||||
sends any new segments followed by `F`.
|
||||
|
||||
### Server → Client
|
||||
|
||||
The server echoes the session ID from the NOPs in all replies. The client
|
||||
drops any reply with a stale session ID.
|
||||
|
||||
**Partial segment (completed VAD segment):**
|
||||
**Segment (completed VAD segment):**
|
||||
```
|
||||
HKMSTR:P <session> <text>\n
|
||||
P <session> <text>\n
|
||||
```
|
||||
A completed, VAD-separated utterance segment. The client starts TTS
|
||||
synthesis immediately and buffers the audio output, but does **not** play
|
||||
it yet. Playback starts when `:F` arrives (or timeout).
|
||||
synthesis immediately and buffers the audio (does not play yet).
|
||||
|
||||
**Final (all done):**
|
||||
```
|
||||
HKMSTR:F <session> <text>\n
|
||||
F <session> <text>\n
|
||||
```
|
||||
Signals that all segments have been sent. May be empty
|
||||
(`HKMSTR:F <session>\n`). Triggers playback of all buffered audio on the
|
||||
client. If `<text>` is non-empty, the client synthesizes it before playing.
|
||||
Signals all segments have been sent. `<text>` may be empty (`F <session>\n`).
|
||||
Triggers playback of all buffered audio on the client. If text is non-empty,
|
||||
the client synthesizes it before playing.
|
||||
|
||||
The purpose of this design is to minimize latency: TTS synthesis runs in
|
||||
parallel with recording, so by the time `:F` arrives, audio is already
|
||||
buffered and playback starts immediately.
|
||||
## Session IDs
|
||||
|
||||
- Client increments session ID on each PTT press
|
||||
- Server echoes the session ID in all replies for that session
|
||||
- Client drops any reply with a stale session ID (handles the race where
|
||||
stale segments from an aborted session are still in the TCP buffer)
|
||||
- Server aborts old session on receiving `ON` with a new session ID
|
||||
|
||||
## Client playback model
|
||||
|
||||
1. `P` arrives → start TTS synthesis immediately, buffer audio (don't play)
|
||||
2. More `P` arrive → keep synthesizing and buffering
|
||||
3. `F` arrives → play all buffered audio immediately
|
||||
4. PTT pressed → flush: stop playback, cancel synthesis, clear buffers
|
||||
|
||||
The purpose of pre-synthesis is to minimize latency between PTT release
|
||||
and audio playback. By the time `F` arrives, audio is already buffered.
|
||||
|
||||
## Server state machine
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────┐
|
||||
┌──────────┐ ON <session> ┌──────────────┐
|
||||
│ IDLE │ ──────────────► │ RECORDING │
|
||||
└──────────┘ └──────────────┘
|
||||
│ │
|
||||
OFF │ │
|
||||
recv'd │ │
|
||||
▼ │
|
||||
┌──────────┐ first NOP ┌──────────────┐ │
|
||||
│ IDLE │ ──────────► │ RECORDING │ │
|
||||
└──────────┘ └──────────────┘ │
|
||||
│ │ │
|
||||
OFF │ │ 150ms │
|
||||
recv'd │ │ silence │
|
||||
▼ ▼ │
|
||||
┌─────────────┐ │
|
||||
│ PROCESSING │ │
|
||||
└─────────────┘ │
|
||||
│ │
|
||||
send │ │
|
||||
:P/:F │ │
|
||||
▼ │
|
||||
back to IDLE ──────────┘
|
||||
┌─────────────┐
|
||||
│ PROCESSING │
|
||||
└─────────────┘
|
||||
│
|
||||
send │
|
||||
P/F │
|
||||
▼
|
||||
back to IDLE
|
||||
```
|
||||
|
||||
- **IDLE → RECORDING:** first NOP received, start mic capture
|
||||
- **RECORDING:** VAD detects completed segments → send `HKMSTR:P <text>`
|
||||
- **RECORDING → PROCESSING:** OFF received, OR 150ms since last NOP
|
||||
- **PROCESSING → IDLE:** send remaining segments as `:P`, then `HKMSTR:F`
|
||||
- **IDLE → RECORDING:** `ON <session>` received, start mic capture
|
||||
- **RECORDING:** Moonshine streaming produces completed segments → send `P`
|
||||
- **RECORDING → PROCESSING:** `OFF <session>` received
|
||||
- **PROCESSING → IDLE:** final STT pass, send remaining `P` + `F`
|
||||
|
||||
## Client playback model
|
||||
|
||||
1. `:P` arrives → start TTS synthesis immediately, buffer audio (don't play).
|
||||
Reset 250ms segment timer.
|
||||
2. More `:P` arrive → keep synthesizing and buffering, reset timer each time.
|
||||
3. `:F` arrives → play all buffered audio immediately, cancel timer.
|
||||
4. If `:F` doesn't arrive within 250ms of the last `:P` → play buffered audio
|
||||
early. If more `:P` arrive after early playback, synthesis continues and
|
||||
new audio is appended to the output — not a failure.
|
||||
5. `:F` may be empty — it just signals "all segments sent, start/confirm playback."
|
||||
|
||||
## Timing
|
||||
|
||||
| Parameter | Value | Purpose |
|
||||
|-----------|-------|---------|
|
||||
| NOP interval | 50ms | Heartbeat frequency while PTT held |
|
||||
| Silence timeout | 150ms | Stop recording if no NOPs (3 missed = lost OFF) |
|
||||
| NOP bandwidth | ~20 msg/s × ~20 bytes | ~400 bytes/s — negligible |
|
||||
|
||||
## Why this works
|
||||
|
||||
1. **Outbound broadcast `:68→:67` to `255.255.255.255`** passes the
|
||||
WireGuard WFP killswitch (DHCP exception matches this exact pattern)
|
||||
2. **Inbound `:67→:68`** has no address restriction in the WFP rule, so
|
||||
unicast replies pass through
|
||||
3. **Binding to a specific interface IP** (not `0.0.0.0`) wins unicast
|
||||
delivery over the Windows DHCP client service
|
||||
4. **NOP spam** ensures the ON message gets through even at 5% packet loss
|
||||
(3 consecutive NOPs = ~0.01% drop probability)
|
||||
5. **150ms timeout** is the backstop for lost OFF — at 50ms intervals, 3
|
||||
consecutive NOPs must all be lost to false-stop
|
||||
If `ON` arrives while recording, the current session is aborted (no final
|
||||
flush) and a new session starts immediately.
|
||||
|
||||
@@ -0,0 +1,18 @@
|
||||
[package]
|
||||
name = "rvsttd"
|
||||
version = "0.1.0"
|
||||
edition = "2021"
|
||||
|
||||
[[bin]]
|
||||
name = "rvsttd"
|
||||
path = "src/main.rs"
|
||||
|
||||
[dependencies]
|
||||
anyhow = "1"
|
||||
cpal = "0.15"
|
||||
|
||||
[build-dependencies]
|
||||
bindgen = "0.71"
|
||||
|
||||
[profile.release]
|
||||
opt-level = 3
|
||||
@@ -0,0 +1,27 @@
|
||||
use std::path::PathBuf;
|
||||
|
||||
fn main() {
|
||||
let manifest_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR"));
|
||||
let lib_dir = manifest_dir.join("..").join("bench").join("moonshine-voice").join("lib");
|
||||
let include_dir = manifest_dir.join("..").join("bench").join("moonshine-voice").join("include");
|
||||
let header = include_dir.join("moonshine-c-api.h");
|
||||
|
||||
println!("cargo:rerun-if-changed={}", header.display());
|
||||
println!("cargo:rustc-link-search=native={}", lib_dir.display());
|
||||
println!("cargo:rustc-link-lib=dylib=moonshine");
|
||||
println!("cargo:rustc-link-arg=-Wl,-rpath,{}", lib_dir.display());
|
||||
|
||||
let bindings = bindgen::Builder::default()
|
||||
.header(header.to_str().unwrap())
|
||||
.allowlist_function("moonshine_.*")
|
||||
.allowlist_var("MOONSHINE_.*")
|
||||
.allowlist_type("transcript.*|moonshine_option_t|speaker_span_t|transcript_word_t")
|
||||
.derive_default(true)
|
||||
.generate()
|
||||
.expect("Unable to generate moonshine bindings");
|
||||
|
||||
let out_path = PathBuf::from(std::env::var("OUT_DIR").unwrap());
|
||||
bindings
|
||||
.write_to_file(out_path.join("moonshine_bindings.rs"))
|
||||
.expect("Couldn't write bindings");
|
||||
}
|
||||
@@ -0,0 +1,478 @@
|
||||
use anyhow::{anyhow, Result};
|
||||
use cpal::traits::{DeviceTrait, HostTrait, StreamTrait};
|
||||
use cpal::{SampleFormat, SampleRate};
|
||||
use std::collections::HashSet;
|
||||
use std::ffi::CStr;
|
||||
use std::io::{BufRead, BufReader, Write};
|
||||
use std::net::{TcpListener, TcpStream};
|
||||
use std::sync::atomic::{AtomicBool, Ordering};
|
||||
use std::sync::{Arc, Mutex};
|
||||
use std::thread;
|
||||
use std::time::{Duration, SystemTime, UNIX_EPOCH};
|
||||
|
||||
include!(concat!(env!("OUT_DIR"), "/moonshine_bindings.rs"));
|
||||
|
||||
const SAMPLE_RATE: i32 = 16000;
|
||||
const HEADER_VERSION: i32 = 30000;
|
||||
const ARCH: u32 = 5; // MOONSHINE_MODEL_ARCH_MEDIUM_STREAMING
|
||||
const BIND_ADDR: &str = "127.0.0.1:6996";
|
||||
const MAX_TEXT_BYTES: usize = 1380;
|
||||
|
||||
// ─── helpers ──────────────────────────────────────────────────────────────
|
||||
|
||||
fn ts() -> String {
|
||||
let now = SystemTime::now()
|
||||
.duration_since(UNIX_EPOCH)
|
||||
.unwrap_or_default();
|
||||
let secs = now.as_secs() % 86400;
|
||||
let h = secs / 3600;
|
||||
let m = (secs % 3600) / 60;
|
||||
let s = secs % 60;
|
||||
let ms = now.subsec_millis();
|
||||
format!("{:02}:{:02}:{:02}.{:03}", h, m, s, ms)
|
||||
}
|
||||
|
||||
fn log(msg: &str) {
|
||||
eprintln!("[{}] {}", ts(), msg);
|
||||
}
|
||||
|
||||
fn err_str(code: i32) -> String {
|
||||
unsafe {
|
||||
let s = moonshine_error_to_string(code);
|
||||
if s.is_null() {
|
||||
format!("error {}", code)
|
||||
} else {
|
||||
CStr::from_ptr(s).to_string_lossy().into_owned()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn truncate_to_word(text: &str, max_bytes: usize) -> &str {
|
||||
if text.len() <= max_bytes {
|
||||
return text;
|
||||
}
|
||||
let cut = &text[..max_bytes.min(text.len())];
|
||||
match cut.rfind(' ') {
|
||||
Some(pos) => &text[..pos],
|
||||
None => cut,
|
||||
}
|
||||
}
|
||||
|
||||
fn line_text(line: &transcript_line_t) -> String {
|
||||
if line.text.is_null() {
|
||||
return String::new();
|
||||
}
|
||||
unsafe { CStr::from_ptr(line.text) }
|
||||
.to_string_lossy()
|
||||
.into_owned()
|
||||
}
|
||||
|
||||
// ─── shared state ─────────────────────────────────────────────────────────
|
||||
|
||||
struct Shared {
|
||||
writer: Mutex<TcpStream>,
|
||||
session_id: u64,
|
||||
transcriber_handle: i32,
|
||||
}
|
||||
|
||||
impl Shared {
|
||||
fn send_msg(&self, prefix: &str, text: &str) {
|
||||
let text = truncate_to_word(text, MAX_TEXT_BYTES);
|
||||
let line = if text.is_empty() {
|
||||
format!("{} {}\n", prefix, self.session_id)
|
||||
} else {
|
||||
format!("{} {} {}\n", prefix, self.session_id, text)
|
||||
};
|
||||
let mut writer = self.writer.lock().unwrap();
|
||||
match writer.write_all(line.as_bytes()) {
|
||||
Ok(_) => log(&format!("TX {} {} {}", prefix, self.session_id, text)),
|
||||
Err(e) => log(&format!("TX failed: {}", e)),
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ─── session ──────────────────────────────────────────────────────────────
|
||||
|
||||
struct Session {
|
||||
shared: Arc<Shared>,
|
||||
stop_signal: Arc<AtomicBool>,
|
||||
aborted: Arc<AtomicBool>,
|
||||
transcriber: thread::JoinHandle<()>,
|
||||
cpal_stream: cpal::Stream,
|
||||
stream_handle: i32,
|
||||
}
|
||||
|
||||
impl Session {
|
||||
fn stop(self) {
|
||||
self.stop_signal.store(true, Ordering::SeqCst);
|
||||
drop(self.cpal_stream);
|
||||
self.transcriber.join().ok();
|
||||
unsafe { moonshine_free_stream(self.shared.transcriber_handle, self.stream_handle) };
|
||||
}
|
||||
|
||||
fn abort(self) {
|
||||
self.stop_signal.store(true, Ordering::SeqCst);
|
||||
self.aborted.store(true, Ordering::SeqCst);
|
||||
drop(self.cpal_stream);
|
||||
self.transcriber.join().ok();
|
||||
unsafe { moonshine_free_stream(self.shared.transcriber_handle, self.stream_handle) };
|
||||
}
|
||||
}
|
||||
|
||||
fn start_session(shared: Arc<Shared>) -> Option<Session> {
|
||||
let stream_handle = unsafe { moonshine_create_stream(shared.transcriber_handle, 0) };
|
||||
if stream_handle < 0 {
|
||||
log(&format!("create_stream failed: {}", err_str(stream_handle)));
|
||||
return None;
|
||||
}
|
||||
let rc = unsafe { moonshine_start_stream(shared.transcriber_handle, stream_handle) };
|
||||
if rc != 0 {
|
||||
log(&format!("start_stream failed: {}", err_str(rc)));
|
||||
unsafe { moonshine_free_stream(shared.transcriber_handle, stream_handle) };
|
||||
return None;
|
||||
}
|
||||
|
||||
let audio_buf: Arc<Mutex<Vec<f32>>> = Arc::new(Mutex::new(Vec::new()));
|
||||
let stop_signal = Arc::new(AtomicBool::new(false));
|
||||
let aborted = Arc::new(AtomicBool::new(false));
|
||||
|
||||
let cpal_stream = match start_cpal(audio_buf.clone(), stop_signal.clone()) {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
log(&format!("cpal failed: {}", e));
|
||||
unsafe { moonshine_free_stream(shared.transcriber_handle, stream_handle) };
|
||||
return None;
|
||||
}
|
||||
};
|
||||
|
||||
let shared_clone = shared.clone();
|
||||
let stop_signal_clone = stop_signal.clone();
|
||||
let aborted_clone = aborted.clone();
|
||||
|
||||
let transcriber = thread::spawn(move || {
|
||||
transcriber_loop(shared_clone, audio_buf, stop_signal_clone, aborted_clone, stream_handle);
|
||||
});
|
||||
|
||||
Some(Session {
|
||||
shared,
|
||||
stop_signal,
|
||||
aborted,
|
||||
transcriber,
|
||||
cpal_stream,
|
||||
stream_handle,
|
||||
})
|
||||
}
|
||||
|
||||
fn transcriber_loop(
|
||||
shared: Arc<Shared>,
|
||||
audio_buf: Arc<Mutex<Vec<f32>>>,
|
||||
stop_signal: Arc<AtomicBool>,
|
||||
aborted: Arc<AtomicBool>,
|
||||
stream_handle: i32,
|
||||
) {
|
||||
let handle = shared.transcriber_handle;
|
||||
let mut sent_ids: HashSet<u64> = HashSet::new();
|
||||
|
||||
while !stop_signal.load(Ordering::SeqCst) {
|
||||
let chunk = {
|
||||
let mut buf = audio_buf.lock().unwrap();
|
||||
if buf.is_empty() {
|
||||
drop(buf);
|
||||
thread::sleep(Duration::from_millis(5));
|
||||
continue;
|
||||
}
|
||||
std::mem::take(&mut *buf)
|
||||
};
|
||||
|
||||
unsafe {
|
||||
moonshine_transcribe_add_audio_to_stream(
|
||||
handle, stream_handle,
|
||||
chunk.as_ptr(), chunk.len() as u64,
|
||||
SAMPLE_RATE, 0,
|
||||
);
|
||||
}
|
||||
|
||||
let mut t_ptr: *mut transcript_t = std::ptr::null_mut();
|
||||
let rc = unsafe { moonshine_transcribe_stream(handle, stream_handle, 0, &mut t_ptr) };
|
||||
if rc != 0 || t_ptr.is_null() {
|
||||
continue;
|
||||
}
|
||||
|
||||
send_new_segments(&shared, t_ptr, &mut sent_ids, "P");
|
||||
}
|
||||
|
||||
// If aborted (new session took over), skip final flush entirely
|
||||
if aborted.load(Ordering::SeqCst) {
|
||||
unsafe { moonshine_stop_stream(handle, stream_handle) };
|
||||
log(&format!("Session {} aborted, skipping final flush", shared.session_id));
|
||||
return;
|
||||
}
|
||||
|
||||
// Drain remaining audio
|
||||
let remaining = {
|
||||
let mut buf = audio_buf.lock().unwrap();
|
||||
std::mem::take(&mut *buf)
|
||||
};
|
||||
if !remaining.is_empty() {
|
||||
unsafe {
|
||||
moonshine_transcribe_add_audio_to_stream(
|
||||
handle, stream_handle,
|
||||
remaining.as_ptr(), remaining.len() as u64,
|
||||
SAMPLE_RATE, 0,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// Final flush
|
||||
unsafe { moonshine_stop_stream(handle, stream_handle) };
|
||||
let mut t_ptr: *mut transcript_t = std::ptr::null_mut();
|
||||
let rc = unsafe { moonshine_transcribe_stream(handle, stream_handle, 0, &mut t_ptr) };
|
||||
|
||||
if rc == 0 && !t_ptr.is_null() {
|
||||
let t = unsafe { &*t_ptr };
|
||||
let mut new_segments: Vec<String> = Vec::new();
|
||||
|
||||
for i in 0..t.line_count as usize {
|
||||
let line = unsafe { &*t.lines.add(i) };
|
||||
if line.text.is_null() || line.is_complete == 0 {
|
||||
continue;
|
||||
}
|
||||
if !sent_ids.insert(line.id) {
|
||||
continue;
|
||||
}
|
||||
let text = line_text(line);
|
||||
if !text.is_empty() {
|
||||
new_segments.push(text);
|
||||
}
|
||||
}
|
||||
|
||||
if new_segments.is_empty() {
|
||||
shared.send_msg("F", "");
|
||||
} else {
|
||||
let last = new_segments.len() - 1;
|
||||
for (i, text) in new_segments.iter().enumerate() {
|
||||
let prefix = if i == last { "F" } else { "P" };
|
||||
shared.send_msg(prefix, text);
|
||||
}
|
||||
}
|
||||
} else {
|
||||
shared.send_msg("F", "");
|
||||
}
|
||||
}
|
||||
|
||||
fn send_new_segments(
|
||||
shared: &Shared,
|
||||
t_ptr: *const transcript_t,
|
||||
sent_ids: &mut HashSet<u64>,
|
||||
prefix: &str,
|
||||
) {
|
||||
let t = unsafe { &*t_ptr };
|
||||
|
||||
for i in 0..t.line_count as usize {
|
||||
let line = unsafe { &*t.lines.add(i) };
|
||||
if line.text.is_null() || line.is_complete == 0 {
|
||||
continue;
|
||||
}
|
||||
if !sent_ids.insert(line.id) {
|
||||
continue;
|
||||
}
|
||||
let text = line_text(line);
|
||||
if text.is_empty() {
|
||||
continue;
|
||||
}
|
||||
shared.send_msg(prefix, &text);
|
||||
}
|
||||
}
|
||||
|
||||
// ─── cpal ─────────────────────────────────────────────────────────────────
|
||||
|
||||
fn start_cpal(
|
||||
audio_buf: Arc<Mutex<Vec<f32>>>,
|
||||
stop_signal: Arc<AtomicBool>,
|
||||
) -> Result<cpal::Stream> {
|
||||
let host = cpal::default_host();
|
||||
let dev = host
|
||||
.default_input_device()
|
||||
.ok_or_else(|| anyhow!("no input device"))?;
|
||||
|
||||
let supported = dev
|
||||
.supported_input_configs()?
|
||||
.filter(|c| c.channels() <= 2 && c.min_sample_rate().0 <= 16000)
|
||||
.min_by_key(|c| match c.sample_format() {
|
||||
SampleFormat::F32 => 0,
|
||||
SampleFormat::I16 => 1,
|
||||
SampleFormat::U8 => 2,
|
||||
_ => 99,
|
||||
})
|
||||
.ok_or_else(|| anyhow!("no suitable input config"))?;
|
||||
|
||||
let fmt = supported.sample_format();
|
||||
let mut config = supported.with_max_sample_rate().config();
|
||||
if config.channels > 1 {
|
||||
config.channels = 1;
|
||||
}
|
||||
config.sample_rate = SampleRate(16000);
|
||||
|
||||
let err_fn = |e: cpal::StreamError| log(&format!("cpal error: {}", e));
|
||||
|
||||
let stream = match fmt {
|
||||
SampleFormat::F32 => dev.build_input_stream(
|
||||
&config,
|
||||
move |data: &[f32], _: &_| {
|
||||
if !stop_signal.load(Ordering::Relaxed) {
|
||||
audio_buf.lock().unwrap().extend_from_slice(data);
|
||||
}
|
||||
},
|
||||
err_fn,
|
||||
None,
|
||||
)?,
|
||||
SampleFormat::I16 => dev.build_input_stream(
|
||||
&config,
|
||||
move |data: &[i16], _: &_| {
|
||||
if !stop_signal.load(Ordering::Relaxed) {
|
||||
audio_buf.lock().unwrap().extend(data.iter().map(|&x| x as f32 / 32768.0));
|
||||
}
|
||||
},
|
||||
err_fn,
|
||||
None,
|
||||
)?,
|
||||
SampleFormat::U8 => dev.build_input_stream(
|
||||
&config,
|
||||
move |data: &[u8], _: &_| {
|
||||
if !stop_signal.load(Ordering::Relaxed) {
|
||||
audio_buf.lock().unwrap().extend(data.iter().map(|&x| (x as f32 - 128.0) / 128.0));
|
||||
}
|
||||
},
|
||||
err_fn,
|
||||
None,
|
||||
)?,
|
||||
_ => return Err(anyhow!("unsupported sample format {:?}", fmt)),
|
||||
};
|
||||
|
||||
stream.play()?;
|
||||
Ok(stream)
|
||||
}
|
||||
|
||||
// ─── main ─────────────────────────────────────────────────────────────────
|
||||
|
||||
fn main() -> Result<()> {
|
||||
let mut args = std::env::args().skip(1);
|
||||
let mut model_dir = String::from("../bench/medium-streaming-en");
|
||||
|
||||
while let Some(a) = args.next() {
|
||||
match a.as_str() {
|
||||
"--model-dir" | "-m" => {
|
||||
model_dir = args.next().unwrap_or(model_dir);
|
||||
}
|
||||
"--help" | "-h" => {
|
||||
println!("Usage: rvsttd [--model-dir DIR]");
|
||||
println!("Listens on TCP {}", BIND_ADDR);
|
||||
println!("Model: medium-streaming (Moonshine)");
|
||||
return Ok(());
|
||||
}
|
||||
_ => return Err(anyhow!("unknown arg: {}", a)),
|
||||
}
|
||||
}
|
||||
|
||||
let model_path = std::fs::canonicalize(&model_dir)
|
||||
.unwrap_or_else(|_| std::path::PathBuf::from(&model_dir));
|
||||
|
||||
log(&format!("Loading model from {}...", model_path.display()));
|
||||
let c_dir = std::ffi::CString::new(model_path.to_str().unwrap()).unwrap();
|
||||
let transcriber_handle = unsafe {
|
||||
moonshine_load_transcriber_from_files(
|
||||
c_dir.as_ptr(),
|
||||
ARCH,
|
||||
std::ptr::null(),
|
||||
0,
|
||||
HEADER_VERSION,
|
||||
)
|
||||
};
|
||||
if transcriber_handle < 0 {
|
||||
return Err(anyhow!("failed to load model: {}", err_str(transcriber_handle)));
|
||||
}
|
||||
log(&format!("Model loaded (handle {})", transcriber_handle));
|
||||
|
||||
let listener = TcpListener::bind(BIND_ADDR)?;
|
||||
log(&format!("STT server listening on TCP {}", BIND_ADDR));
|
||||
|
||||
let mut session_id_counter: u64 = 0;
|
||||
let mut current_session: Option<Session> = None;
|
||||
|
||||
for stream in listener.incoming() {
|
||||
let stream = match stream {
|
||||
Ok(s) => s,
|
||||
Err(e) => {
|
||||
log(&format!("accept failed: {}", e));
|
||||
continue;
|
||||
}
|
||||
};
|
||||
|
||||
stream.set_nodelay(true).ok();
|
||||
log(&format!("Client connected: {}", stream.peer_addr().unwrap_or_default()));
|
||||
|
||||
let writer_stream = stream.try_clone()?;
|
||||
let reader = BufReader::new(stream);
|
||||
|
||||
for line in reader.lines() {
|
||||
let line = match line {
|
||||
Ok(l) => l,
|
||||
Err(_) => break,
|
||||
};
|
||||
|
||||
let line = line.trim();
|
||||
log(&format!("RX {}", line));
|
||||
|
||||
// ON <session>
|
||||
if let Some(rest) = line.strip_prefix("ON ") {
|
||||
let new_session_id: u64 = rest.parse().unwrap_or(0);
|
||||
|
||||
if let Some(s) = current_session.take() {
|
||||
log(&format!("Aborting session {} for new session {}", s.shared.session_id, new_session_id));
|
||||
s.abort();
|
||||
}
|
||||
|
||||
session_id_counter = new_session_id;
|
||||
let shared = Arc::new(Shared {
|
||||
writer: Mutex::new(writer_stream.try_clone()?),
|
||||
session_id: session_id_counter,
|
||||
transcriber_handle,
|
||||
});
|
||||
|
||||
log(&format!("PTT on session {}", session_id_counter));
|
||||
match start_session(shared) {
|
||||
Some(s) => current_session = Some(s),
|
||||
None => log("Failed to start session"),
|
||||
}
|
||||
}
|
||||
// OFF <session>
|
||||
else if let Some(rest) = line.strip_prefix("OFF ") {
|
||||
let off_session: u64 = rest.parse().unwrap_or(0);
|
||||
|
||||
if let Some(s) = current_session.as_ref() {
|
||||
if s.shared.session_id == off_session {
|
||||
log(&format!("OFF session {}", off_session));
|
||||
if let Some(s) = current_session.take() {
|
||||
s.stop();
|
||||
}
|
||||
} else {
|
||||
log(&format!("OFF session {} (stale, current={}), ignoring", off_session, s.shared.session_id));
|
||||
}
|
||||
} else {
|
||||
log(&format!("OFF session {} (no active session), ignoring", off_session));
|
||||
}
|
||||
}
|
||||
else {
|
||||
log(&format!("Unknown command: {}", line));
|
||||
}
|
||||
}
|
||||
|
||||
log("Client disconnected");
|
||||
|
||||
if let Some(s) = current_session.take() {
|
||||
s.abort();
|
||||
}
|
||||
}
|
||||
|
||||
Ok(())
|
||||
}
|
||||
Reference in New Issue
Block a user