v0.6: DHCP tunnel transport with NOP heartbeat protocol

This commit is contained in:
2026-08-12 00:29:54 +00:00
parent 4a58b94f35
commit e1739059d9
12 changed files with 743 additions and 528 deletions
+105
View File
@@ -0,0 +1,105 @@
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Text;
// Args: [interface_ip] [interval_ms]
// Defaults: auto-detect LAN IP, 50ms
string localIp = args.Length > 0 ? args[0] : GetLanInterfaceIp() ?? "0.0.0.0";
int intervalMs = args.Length > 1 && int.TryParse(args[1], out int iv) ? iv : 50;
using var sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
sock.EnableBroadcast = true;
var localEp = new IPEndPoint(IPAddress.Parse(localIp), 68);
sock.Bind(localEp);
Console.WriteLine($"Bound to {localEp} (SO_REUSEADDR)");
Console.WriteLine("Sending NOPs + listening. Press Ctrl+C to stop.");
Console.WriteLine();
var cts = new CancellationTokenSource();
var recvTask = Task.Run(() => ReceiveLoop(sock, cts.Token));
var destEp = new IPEndPoint(IPAddress.Broadcast, 67);
uint nonce = 0;
while (!cts.Token.IsCancellationRequested)
{
nonce++;
string msg = $"HKMSTR {nonce}\n";
byte[] payload = Encoding.UTF8.GetBytes(msg);
try
{
int sent = sock.SendTo(payload, destEp);
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] SENT n={nonce} {sent} bytes");
}
catch (Exception ex)
{
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] SEND FAILED: {ex.Message}");
}
try { Thread.Sleep(intervalMs); }
catch (OperationCanceledException) { break; }
}
cts.Cancel();
await recvTask;
static string? GetLanInterfaceIp()
{
foreach (var nic in NetworkInterface.GetAllNetworkInterfaces())
{
if (nic.OperationalStatus != OperationalStatus.Up)
continue;
if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback)
continue;
if (nic.Description.Contains("WireGuard", StringComparison.OrdinalIgnoreCase))
continue;
foreach (var addr in nic.GetIPProperties().UnicastAddresses)
{
if (addr.Address.AddressFamily == AddressFamily.InterNetwork)
{
var ip = addr.Address.ToString();
if (ip.StartsWith("192.168.") || ip.StartsWith("10.") || ip.StartsWith("172."))
return ip;
}
}
}
return null;
}
static void ReceiveLoop(Socket sock, CancellationToken ct)
{
byte[] buffer = new byte[4096];
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
while (!ct.IsCancellationRequested)
{
int received;
try
{
if (!sock.Poll(500_000, SelectMode.SelectRead))
continue;
received = sock.ReceiveFrom(buffer, ref fromEp);
}
catch (Exception ex)
{
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV ERROR: {ex.Message}");
continue;
}
string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');
if (!text.StartsWith("HKMSTR"))
{
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV {received} bytes from {fromEp} (no magic)");
continue;
}
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV {received} bytes from {fromEp}: {text}");
}
}
@@ -0,0 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>
+1 -1
View File
@@ -12,7 +12,7 @@ public sealed class AppConfig
public int LengthScale { get; set; } = 100;
public int NoiseWScale { get; set; } = 800;
public bool MinimizeToTray { get; set; } = true;
public string ServerEndpoint { get; set; } = "127.0.0.1:5210";
public string InterfaceIp { get; set; } = string.Empty;
public static string AppDataDir => Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
+9 -17
View File
@@ -17,8 +17,7 @@ partial class MainForm
private Button btnBrowseFile = null!;
private Label lblFileName = null!;
private Label lblServer = null!;
private TextBox txtServer = null!;
private Button btnConnect = null!;
private ComboBox cmbInterface = null!;
private Label lblLineStatus = null!;
private RichTextBox txtLog = null!;
private CheckBox chkMinimizeToTray = null!;
@@ -58,8 +57,7 @@ partial class MainForm
btnBrowseFile = new Button();
lblFileName = new Label();
lblServer = new Label();
txtServer = new TextBox();
btnConnect = new Button();
cmbInterface = new ComboBox();
lblLineStatus = new Label();
txtLog = new RichTextBox();
chkMinimizeToTray = new CheckBox();
@@ -145,20 +143,15 @@ partial class MainForm
lblFileName.ForeColor = Color.Gray;
// lblServer
lblServer.Text = "Server:";
lblServer.Text = "Interface:";
lblServer.Location = new Point(440, 48);
lblServer.Size = new Size(45, 23);
lblServer.Size = new Size(55, 23);
lblServer.TextAlign = ContentAlignment.MiddleLeft;
// txtServer
txtServer.Location = new Point(488, 45);
txtServer.Size = new Size(110, 23);
// btnConnect
btnConnect.Text = "Connect";
btnConnect.Location = new Point(603, 44);
btnConnect.Size = new Size(60, 25);
btnConnect.UseVisualStyleBackColor = true;
// cmbInterface
cmbInterface.Location = new Point(498, 45);
cmbInterface.Size = new Size(165, 23);
cmbInterface.DropDownStyle = ComboBoxStyle.DropDownList;
// lblLineStatus
lblLineStatus.Text = "";
@@ -277,8 +270,7 @@ partial class MainForm
Controls.Add(btnBrowseFile);
Controls.Add(lblFileName);
Controls.Add(lblServer);
Controls.Add(txtServer);
Controls.Add(btnConnect);
Controls.Add(cmbInterface);
Controls.Add(lblLineStatus);
Controls.Add(lblNoise);
Controls.Add(trkNoise);
+46 -27
View File
@@ -2,7 +2,7 @@ using NAudio.Wave;
using Robovoice.App;
using Robovoice.Core;
using Robovoice.Core.Voices;
using Robovoice.Stt.Tcp;
using Robovoice.Stt.Dhcp;
using Robovoice.Tts.LibPiper;
using System.Diagnostics;
@@ -16,7 +16,7 @@ internal sealed partial class MainForm : Form
private LibPiperTtsEngine? _tts;
private AudioOutput? _audioOutput;
private TcpSttSource? _sttSource;
private DhcpSttSource? _sttSource;
private Orchestrator? _orchestrator;
private PttHotkey? _pttHotkey;
private NotifyIcon? _trayIcon;
@@ -61,7 +61,7 @@ internal sealed partial class MainForm : Form
OnSliderScroll(null, EventArgs.Empty);
chkMinimizeToTray.Checked = _config.MinimizeToTray;
txtServer.Text = _config.ServerEndpoint;
PopulateInterfaces();
btnBrowseFile.Click += OnBrowseFile;
btnTestVoice.Click += OnTestVoice;
@@ -71,8 +71,7 @@ internal sealed partial class MainForm : Form
txtPttKey.KeyDown += OnPttKeyDown;
cmbOutput.SelectedIndexChanged += OnOutputChanged;
cmbVoice.SelectedIndexChanged += OnVoiceChanged;
txtServer.Leave += OnServerChanged;
btnConnect.Click += OnConnect;
cmbInterface.SelectedIndexChanged += OnInterfaceChanged;
trkNoise.Scroll += OnSliderScroll;
trkSpeed.Scroll += OnSliderScroll;
@@ -245,8 +244,9 @@ internal sealed partial class MainForm : Form
if (_sttSource is null)
{
_sttSource = new TcpSttSource { ServerEndpoint = txtServer.Text, Log = Log };
Log($"STT endpoint: {_sttSource.ServerEndpoint} (press Connect)");
string ifaceIp = cmbInterface.SelectedItem as string ?? "";
_sttSource = new DhcpSttSource { InterfaceIp = ifaceIp, Log = Log };
await _sttSource.StartAsync();
}
_orchestrator?.DisposeAsync().AsTask().Wait();
@@ -425,38 +425,57 @@ internal sealed partial class MainForm : Form
SaveConfig();
}
private void OnServerChanged(object? sender, EventArgs e)
private void PopulateInterfaces()
{
if (_sttSource is not null)
cmbInterface.Items.Clear();
foreach (var (ip, name) in DhcpSttSource.GetAvailableInterfaces())
{
_sttSource.ServerEndpoint = txtServer.Text;
Log($"Server endpoint: {txtServer.Text}");
}
SaveConfig();
cmbInterface.Items.Add(name);
cmbInterface.Items[^1] = name;
cmbInterface.Items[cmbInterface.Items.Count - 1] = name;
}
private async void OnConnect(object? sender, EventArgs e)
if (!string.IsNullOrEmpty(_config.InterfaceIp))
{
if (_sttSource is null)
for (int i = 0; i < cmbInterface.Items.Count; i++)
{
Log("STT source not initialized.");
if (cmbInterface.Items[i] is string s && s.Contains(_config.InterfaceIp))
{
cmbInterface.SelectedIndex = i;
return;
}
}
}
_sttSource.ServerEndpoint = txtServer.Text;
if (cmbInterface.Items.Count > 0)
cmbInterface.SelectedIndex = 0;
}
private void OnInterfaceChanged(object? sender, EventArgs e)
{
if (cmbInterface.SelectedItem is not string selected)
return;
string? ip = ExtractIpFromDisplay(selected);
if (ip is null) return;
_config.InterfaceIp = ip;
SaveConfig();
btnConnect.Enabled = false;
try
if (_sttSource is not null)
{
if (_sttSource.IsRunning)
await _sttSource.ReconnectAsync();
else
await _sttSource.StartAsync();
_sttSource.DisposeAsync().AsTask().Wait(2000);
_sttSource = null;
_ = InitializeEngineAsync();
}
finally
}
private static string? ExtractIpFromDisplay(string display)
{
btnConnect.Enabled = true;
}
int start = display.IndexOf('(');
int end = display.IndexOf(')');
if (start < 0 || end <= start) return null;
return display[(start + 1)..end];
}
private void SetupTray()
@@ -524,7 +543,7 @@ internal sealed partial class MainForm : Form
_config.LengthScale = trkSpeed.Value;
_config.NoiseWScale = trkNoiseW.Value;
_config.MinimizeToTray = chkMinimizeToTray.Checked;
_config.ServerEndpoint = txtServer.Text;
_config.InterfaceIp = ExtractIpFromDisplay(cmbInterface.SelectedItem as string ?? "") ?? "";
_config.Save();
}
+1 -1
View File
@@ -3,7 +3,7 @@
<ItemGroup>
<ProjectReference Include="..\Robovoice.Core\Robovoice.Core.csproj" />
<ProjectReference Include="..\Robovoice.Tts.LibPiper\Robovoice.Tts.LibPiper.csproj" />
<ProjectReference Include="..\Robovoice.Stt.Tcp\Robovoice.Stt.Tcp.csproj" />
<ProjectReference Include="..\Robovoice.Stt.Dhcp\Robovoice.Stt.Dhcp.csproj" />
</ItemGroup>
<ItemGroup>
+232
View File
@@ -0,0 +1,232 @@
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Text;
using Robovoice.Core;
namespace Robovoice.Stt.Dhcp;
public sealed class DhcpSttSource : ISttSource
{
private const string Magic = "HKMSTR";
private const int DhcpClientPort = 68;
private const int DhcpServerPort = 67;
private static readonly TimeSpan NopInterval = TimeSpan.FromMilliseconds(50);
private Socket? _sock;
private CancellationTokenSource? _cts;
private Task? _receiveTask;
private Task? _nopTask;
private EndPoint _broadcastEp = new IPEndPoint(IPAddress.Broadcast, DhcpServerPort);
private uint _nonce;
private bool _disposed;
public string InterfaceIp { get; set; } = string.Empty;
public Action<string>? Log { get; set; }
public event TranscriptEventHandler? TranscriptReceived;
public Task StartAsync(CancellationToken ct = default)
{
ObjectDisposedException.ThrowIf(_disposed, this);
if (_cts is not null)
return Task.CompletedTask;
string ip = string.IsNullOrEmpty(InterfaceIp) ? AutoDetectInterfaceIp() ?? "0.0.0.0" : InterfaceIp;
_sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
_sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_sock.EnableBroadcast = true;
_sock.Bind(new IPEndPoint(IPAddress.Parse(ip), DhcpClientPort));
Log?.Invoke($"STT: bound to {ip}:{DhcpClientPort}, broadcasting to :{DhcpServerPort}");
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
_receiveTask = ReceiveLoopAsync(_cts.Token);
return Task.CompletedTask;
}
public async Task StopAsync(CancellationToken ct = default)
{
StopNop();
if (_cts is not null)
_cts.Cancel();
_sock?.Dispose();
_sock = null;
if (_receiveTask is not null)
{
try { await _receiveTask.WaitAsync(ct); }
catch { }
_receiveTask = null;
}
_cts?.Dispose();
_cts = null;
}
public void SendOn()
{
if (_cts is null)
return;
_nonce = 0;
SendNop();
_nopTask = NopLoopAsync(_cts.Token);
}
public void SendOff()
{
StopNop();
SendControl("HKMSTR:OFF");
}
private void StopNop()
{
if (_nopTask is not null)
{
try { _nopTask.Wait(2000); } catch { }
_nopTask = null;
}
}
private async Task NopLoopAsync(CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
try { await Task.Delay(NopInterval, ct); }
catch (OperationCanceledException) { break; }
SendNop();
}
}
private void SendNop()
{
_nonce++;
SendControl($"HKMSTR {_nonce}");
}
private void SendControl(string message)
{
if (_sock is null)
return;
try
{
byte[] payload = Encoding.UTF8.GetBytes(message + "\n");
_sock.SendTo(payload, _broadcastEp);
}
catch (Exception ex)
{
Log?.Invoke($"STT: send failed: {ex.Message}");
}
}
private async Task ReceiveLoopAsync(CancellationToken ct)
{
byte[] buffer = new byte[4096];
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
while (!ct.IsCancellationRequested)
{
int received;
try
{
if (!_sock!.Poll(500_000, SelectMode.SelectRead))
continue;
received = _sock.ReceiveFrom(buffer, ref fromEp);
}
catch (OperationCanceledException)
{
break;
}
catch (ObjectDisposedException)
{
break;
}
catch (Exception ex)
{
Log?.Invoke($"STT: receive error: {ex.Message}");
continue;
}
string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');
if (!text.StartsWith(Magic))
continue;
if (text.StartsWith("HKMSTR:P "))
{
string transcript = text["HKMSTR:P ".Length..];
TranscriptReceived?.Invoke(this, new TranscriptEventArgs
{
Message = new TranscriptMessage(TranscriptType.Partial, transcript),
});
}
else if (text.StartsWith("HKMSTR:F "))
{
string transcript = text["HKMSTR:F ".Length..];
TranscriptReceived?.Invoke(this, new TranscriptEventArgs
{
Message = new TranscriptMessage(TranscriptType.Final, transcript),
});
}
}
}
public static List<(string Ip, string Name)> GetAvailableInterfaces()
{
var result = new List<(string, string)>();
foreach (var nic in NetworkInterface.GetAllNetworkInterfaces())
{
if (nic.OperationalStatus != OperationalStatus.Up)
continue;
if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback)
continue;
string desc = nic.Description;
if (desc.Contains("WireGuard", StringComparison.OrdinalIgnoreCase))
continue;
foreach (var addr in nic.GetIPProperties().UnicastAddresses)
{
if (addr.Address.AddressFamily != AddressFamily.InterNetwork)
continue;
string ip = addr.Address.ToString();
if (ip.StartsWith("192.168.") || ip.StartsWith("10.") ||
ip.StartsWith("172.16.") || ip.StartsWith("172.17.") ||
ip.StartsWith("172.18.") || ip.StartsWith("172.19.") ||
ip.StartsWith("172.20.") || ip.StartsWith("172.21.") ||
ip.StartsWith("172.22.") || ip.StartsWith("172.23.") ||
ip.StartsWith("172.24.") || ip.StartsWith("172.25.") ||
ip.StartsWith("172.26.") || ip.StartsWith("172.27.") ||
ip.StartsWith("172.28.") || ip.StartsWith("172.29.") ||
ip.StartsWith("172.30.") || ip.StartsWith("172.31."))
{
result.Add((ip, $"{nic.Name} ({ip})"));
}
}
}
return result;
}
private static string? AutoDetectInterfaceIp()
{
foreach (var (ip, _) in GetAvailableInterfaces())
return ip;
return null;
}
public async ValueTask DisposeAsync()
{
if (_disposed) return;
await StopAsync();
_disposed = true;
}
}
-253
View File
@@ -1,253 +0,0 @@
using System.Net;
using System.Net.Sockets;
using System.Text.Json;
using System.Text.Json.Serialization;
using Robovoice.Core;
namespace Robovoice.Stt.Tcp;
public sealed class TcpSttSource : ISttSource
{
private TcpClient? _tcp;
private NetworkStream? _stream;
private StreamReader? _reader;
private StreamWriter? _writer;
private CancellationTokenSource? _cts;
private Task? _runTask;
private readonly object _sendLock = new();
private bool _disposed;
public string ServerEndpoint { get; set; } = "127.0.0.1:5210";
public bool IsRunning => _cts is not null;
public Action<string>? Log { get; set; }
public event TranscriptEventHandler? TranscriptReceived;
public Task StartAsync(CancellationToken ct = default)
{
ObjectDisposedException.ThrowIf(_disposed, this);
if (_cts is not null)
return Task.CompletedTask;
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
_runTask = RunAsync(_cts.Token);
return Task.CompletedTask;
}
public async Task StopAsync(CancellationToken ct = default)
{
if (_cts is not null)
_cts.Cancel();
CleanupConnection();
if (_runTask is not null)
{
try { await _runTask.WaitAsync(ct); }
catch { }
_runTask = null;
}
_cts?.Dispose();
_cts = null;
}
private async Task RunAsync(CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
IPEndPoint? endpoint = ParseEndpoint(ServerEndpoint);
if (endpoint is null)
{
Log?.Invoke($"STT: invalid endpoint '{ServerEndpoint}'");
try { await Task.Delay(3000, ct); } catch { break; }
continue;
}
try
{
_tcp = new TcpClient();
using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
connectCts.CancelAfter(TimeSpan.FromSeconds(5));
await _tcp.ConnectAsync(endpoint.Address, endpoint.Port, connectCts.Token);
_stream = _tcp.GetStream();
_reader = new StreamReader(_stream, System.Text.Encoding.UTF8);
_writer = new StreamWriter(_stream, System.Text.Encoding.UTF8) { AutoFlush = true };
Log?.Invoke($"STT: connected to {ServerEndpoint}");
await ReceiveLoopAsync(ct);
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
Log?.Invoke($"STT: connection failed ({ex.Message}), retrying...");
}
finally
{
CleanupConnection();
}
if (!ct.IsCancellationRequested)
{
try { await Task.Delay(3000, ct); }
catch (OperationCanceledException) { break; }
}
}
}
private async Task ReceiveLoopAsync(CancellationToken ct)
{
while (!ct.IsCancellationRequested && _reader is not null)
{
string? line;
try
{
line = await _reader.ReadLineAsync(ct);
}
catch
{
break;
}
if (line is null)
break;
TranscriptMessage? message = ParseTranscriptLine(line);
if (message is null)
continue;
TranscriptReceived?.Invoke(this, new TranscriptEventArgs
{
Message = message,
});
}
}
public void SendOn()
{
SendControl("on");
}
public void SendOff()
{
SendControl("off");
}
public async Task ReconnectAsync(CancellationToken ct = default)
{
if (_cts is null)
return;
Log?.Invoke("STT: reconnecting...");
CleanupConnection();
try { await Task.Delay(500, ct); }
catch (OperationCanceledException) { return; }
}
private void SendControl(string evt)
{
lock (_sendLock)
{
if (_writer is null)
return;
try
{
_writer.WriteLine(JsonSerializer.Serialize(new ControlDto { Event = evt }));
}
catch
{
Log?.Invoke($"STT: failed to send '{evt}' (not connected?)");
}
}
}
private void CleanupConnection()
{
lock (_sendLock)
{
_writer?.Dispose();
_reader?.Dispose();
_stream?.Dispose();
_tcp?.Dispose();
_writer = null;
_reader = null;
_stream = null;
_tcp = null;
}
}
private static IPEndPoint? ParseEndpoint(string endpoint)
{
int colon = endpoint.LastIndexOf(':');
if (colon <= 0)
return null;
string host = endpoint[..colon];
if (!int.TryParse(endpoint[(colon + 1)..], out int port))
return null;
if (IPAddress.TryParse(host, out var addr))
return new IPEndPoint(addr, port);
try
{
var addresses = Dns.GetHostAddresses(host);
addr = addresses.FirstOrDefault(a => a.AddressFamily == AddressFamily.InterNetwork);
if (addr is null)
return null;
return new IPEndPoint(addr, port);
}
catch
{
return null;
}
}
private static TranscriptMessage? ParseTranscriptLine(string line)
{
if (string.IsNullOrWhiteSpace(line))
return null;
try
{
var dto = JsonSerializer.Deserialize<TransmitDto>(line);
if (dto is null)
return null;
var type = dto.Final ? TranscriptType.Final : TranscriptType.Partial;
return new TranscriptMessage(type, dto.Text ?? string.Empty);
}
catch (JsonException)
{
return null;
}
}
public async ValueTask DisposeAsync()
{
if (_disposed) return;
await StopAsync();
_disposed = true;
}
}
internal sealed class ControlDto
{
[JsonPropertyName("event")]
public string Event { get; set; } = string.Empty;
}
internal sealed class TransmitDto
{
public bool Final { get; set; }
public string Text { get; set; } = string.Empty;
}
+2 -1
View File
@@ -2,6 +2,7 @@
<Project Path="Robovoice.App/Robovoice.App.csproj" />
<Project Path="Robovoice.Core/Robovoice.Core.csproj" />
<Project Path="Robovoice.Stt.File/Robovoice.Stt.File.csproj" />
<Project Path="Robovoice.Stt.Tcp/Robovoice.Stt.Tcp.csproj" />
<Project Path="Robovoice.Stt.Dhcp/Robovoice.Stt.Dhcp.csproj" />
<Project Path="Robovoice.Tts.LibPiper/Robovoice.Tts.LibPiper.csproj" />
<Project Path="DhcpTunnelTest/Robovoice.DhcpTunnelTest.csproj" />
</Solution>
+230 -162
View File
@@ -1,202 +1,232 @@
# Server implementation notes
The STT server listens for TCP connections from Robovoice, captures audio
from a microphone when `on` is received, runs speech recognition (Moonshine),
and sends transcript messages back over the same connection.
The STT server listens on UDP port 67, receives NOP heartbeats and OFF
messages from Robovoice, captures audio from a microphone, runs speech
recognition (Moonshine), and sends transcript messages back to the client's
source address.
## Architecture
```
┌── on/off (TCP, newline-delimited JSON)
┌── HKMSTR <nonce> (broadcast, every 50ms)
Robovoice ──────────────►│
│ STT Server
Robovoice ◄──────────────┤
└── partial/final (TCP, newline-delimited JSON)
└── HKMSTR:P/F <text> (unicast)
```
The server:
1. Listens on a TCP port (e.g. 5210)
2. Accepts a connection from Robovoice
3. Reads lines: waits for `{"event":"on"}`
4. Records audio from the microphone
5. Waits for `{"event":"off"}` (or a timeout)
6. Runs STT on the captured audio
7. Sends `{"final":true,"text":"..."}`\n back over the connection
1. Listens on UDP :67
2. First `HKMSTR <nonce>` → start recording
3. `HKMSTR:OFF <nonce>` → stop recording, run STT
4. 150ms with no NOPs → stop recording, run STT (backstop)
5. Send `HKMSTR:F <text>` back to the client's source address:port
## Framing
Every message is a single JSON object on one line, terminated by `\n`. No
length prefix, no binary framing. Use `readline()` / `StreamReader.ReadLineAsync()`.
All messages are newline-terminated UTF-8 text. No JSON, no binary framing.
- `HKMSTR <nonce>` — NOP heartbeat (client → server)
- `HKMSTR:OFF <nonce>` — stop signal (client → server)
- `HKMSTR:P <text>` — partial transcript (server → client)
- `HKMSTR:F <text>` — final transcript (server → client)
The nonce is an incrementing integer for packet uniqueness only. Discard it.
## Python server with Moonshine
[Moonshine](https://github.com/usefulsensors/moonshine) is a lightweight ASR
model by Useful Sensors. Install with `pip install moonshine`.
```python
import socket
import json
import threading
import time
import numpy as np
import sounddevice as sd
import moonshine
LISTEN_PORT = 5210
LISTEN_PORT = 67
CLIENT_PORT = 68
SAMPLE_RATE = 16000
SILENCE_TIMEOUT = 0.150 # 150ms
model = moonshine.MoonshineModel(model="moonshine/base")
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(("0.0.0.0", LISTEN_PORT))
server.listen(1)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
sock.bind(("0.0.0.0", LISTEN_PORT))
print(f"STT server listening on :{LISTEN_PORT}")
while True:
conn, addr = server.accept()
print(f"Client connected: {addr}")
recording = False
last_nop_time = 0
client_addr = None
audio_chunks = []
lock = threading.Lock()
buf = ""
with conn:
def monitor_silence():
"""Backstop: stop recording if no NOPs for 150ms."""
global recording
while True:
data = conn.recv(4096).decode("utf-8")
if not data:
break
buf += data
time.sleep(0.01)
with lock:
if recording and (time.monotonic() - last_nop_time) > SILENCE_TIMEOUT:
recording = False
threading.Thread(target=process_audio, daemon=True).start()
while "\n" in buf:
line, buf = buf.split("\n", 1)
msg = json.loads(line)
threading.Thread(target=monitor_silence, daemon=True).start()
if msg.get("event") == "on":
print("PTT on — recording")
def process_audio():
global audio_chunks
with lock:
chunks = audio_chunks
audio_chunks = []
addr = client_addr
# Record until "off" or timeout
conn.settimeout(0.1)
while True:
try:
data2 = conn.recv(4096).decode("utf-8")
if not data2:
break
buf += data2
while "\n" in buf:
line2, buf = buf.split("\n", 1)
msg2 = json.loads(line2)
if msg2.get("event") == "off":
break
except socket.timeout:
pass
if not chunks:
return
chunk = sd.rec(int(SAMPLE_RATE * 0.1),
samplerate=SAMPLE_RATE,
channels=1, dtype="float32")
sd.wait()
audio_chunks.append(chunk.flatten())
conn.settimeout(None)
if not audio_chunks:
continue
audio = np.concatenate(audio_chunks)
audio = np.concatenate(chunks)
print(f"Captured {len(audio)/SAMPLE_RATE:.1f}s")
text = moonshine.transcribe(model, audio).strip()
if text:
print(f"Transcript: {text}")
reply = json.dumps({"final": True, "text": text})
conn.sendall((reply + "\n").encode("utf-8"))
print(f"Final: {text}")
reply = f"HKMSTR:F {text}\n".encode("utf-8")
sock.sendto(reply, addr)
else:
print("Empty transcript")
while True:
data, addr = sock.recvfrom(4096)
text = data.decode("utf-8", errors="ignore").strip()
if not text.startswith("HKMSTR"):
continue
if text.startswith("HKMSTR:OFF"):
with lock:
if recording:
recording = False
threading.Thread(target=process_audio, daemon=True).start()
continue
if text.startswith("HKMSTR ") or text == "HKMSTR":
with lock:
client_addr = addr
last_nop_time = time.monotonic()
if not recording:
recording = True
audio_chunks = []
print(f"PTT on from {addr}")
# Capture 50ms of audio
chunk = sd.rec(int(SAMPLE_RATE * 0.05), samplerate=SAMPLE_RATE,
channels=1, dtype="float32")
sd.wait()
audio_chunks.append(chunk.flatten())
```
## Python server with streaming partials
For lower latency, send partial results while still recording:
For live feedback, send partials while recording:
```python
import socket
import json
import threading
import time
import numpy as np
import sounddevice as sd
import moonshine
LISTEN_PORT = 5210
LISTEN_PORT = 67
SAMPLE_RATE = 16000
CHUNK_DURATION = 0.5
SILENCE_TIMEOUT = 0.150
PARTIAL_INTERVAL = 0.5 # send partial every 500ms
model = moonshine.MoonshineModel(model="moonshine/base")
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
server.bind(("0.0.0.0", LISTEN_PORT))
server.listen(1)
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
sock.bind(("0.0.0.0", LISTEN_PORT))
print(f"STT server listening on :{LISTEN_PORT}")
while True:
conn, addr = server.accept()
print(f"Client connected: {addr}")
buf = ""
recording = False
last_nop_time = 0
last_partial_time = 0
client_addr = None
audio_chunks = []
lock = threading.Lock()
with conn:
while True:
data = conn.recv(4096).decode("utf-8")
if not data:
break
buf += data
def capture_and_maybe_partial():
global last_partial_time
with lock:
if not recording:
return
while "\n" in buf:
line, buf = buf.split("\n", 1)
msg = json.loads(line)
if msg.get("event") != "on":
continue
print("PTT on — recording")
audio_chunks = []
while True:
try:
conn.settimeout(CHUNK_DURATION)
data2 = conn.recv(4096).decode("utf-8")
if not data2:
break
buf += data2
while "\n" in buf:
line2, buf = buf.split("\n", 1)
msg2 = json.loads(line2)
if msg2.get("event") == "off":
break
except socket.timeout:
pass
chunk = sd.rec(int(SAMPLE_RATE * CHUNK_DURATION),
samplerate=SAMPLE_RATE,
chunk = sd.rec(int(SAMPLE_RATE * 0.05), samplerate=SAMPLE_RATE,
channels=1, dtype="float32")
sd.wait()
audio_chunks.append(chunk.flatten())
# Send partial every few chunks
if len(audio_chunks) % 4 == 0:
now = time.monotonic()
if now - last_partial_time > PARTIAL_INTERVAL:
last_partial_time = now
partial_audio = np.concatenate(audio_chunks)
partial_text = moonshine.transcribe(model, partial_audio).strip()
if partial_text:
reply = json.dumps({"final": False, "text": partial_text})
conn.sendall((reply + "\n").encode("utf-8"))
if partial_text and client_addr:
reply = f"HKMSTR:P {partial_text}\n".encode("utf-8")
sock.sendto(reply, client_addr)
conn.settimeout(None)
while True:
data, addr = sock.recvfrom(4096)
text = data.decode("utf-8", errors="ignore").strip()
if not audio_chunks:
if not text.startswith("HKMSTR"):
continue
audio = np.concatenate(audio_chunks)
text = moonshine.transcribe(model, audio).strip()
if text.startswith("HKMSTR:OFF"):
with lock:
if recording:
recording = False
chunks = audio_chunks
audio_chunks = []
if text:
print(f"Final: {text}")
reply = json.dumps({"final": True, "text": text})
conn.sendall((reply + "\n").encode("utf-8"))
if chunks:
audio = np.concatenate(chunks)
final_text = moonshine.transcribe(model, audio).strip()
if final_text:
reply = f"HKMSTR:F {final_text}\n".encode("utf-8")
sock.sendto(reply, addr)
continue
if text.startswith("HKMSTR") and not text.startswith("HKMSTR:"):
with lock:
client_addr = addr
last_nop_time = time.monotonic()
if not recording:
recording = True
audio_chunks = []
last_partial_time = time.monotonic()
print(f"PTT on from {addr}")
capture_and_maybe_partial()
# Check silence timeout
with lock:
if recording and (time.monotonic() - last_nop_time) > SILENCE_TIMEOUT:
recording = False
chunks = audio_chunks
audio_chunks = []
if 'chunks' in dir() and chunks:
audio = np.concatenate(chunks)
final_text = moonshine.transcribe(model, audio).strip()
if final_text:
reply = f"HKMSTR:F {final_text}\n".encode("utf-8")
sock.sendto(reply, addr)
```
## C# server skeleton
@@ -204,61 +234,99 @@ while True:
```csharp
using System.Net;
using System.Net.Sockets;
using System.Text.Json;
using System.Text;
var listener = new TcpListener(IPAddress.Any, 5210);
listener.Start();
var sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
sock.Bind(new IPEndPoint(IPAddress.Any, 67));
Console.WriteLine("STT server listening on :5210");
Console.WriteLine("STT server listening on :67");
byte[] buffer = new byte[4096];
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
bool recording = false;
DateTime lastNop = DateTime.MinValue;
List<float[]> audioChunks = new();
while (true)
{
var client = listener.AcceptTcpClient();
Console.WriteLine($"Client connected: {client.Client.RemoteEndPoint}");
using var stream = client.GetStream();
using var reader = new StreamReader(stream, Encoding.UTF8);
using var writer = new StreamWriter(stream, Encoding.UTF8) { AutoFlush = true };
string? line;
while ((line = reader.ReadLine()) is not null)
if (sock.Poll(50_000, SelectMode.SelectRead))
{
var msg = JsonSerializer.Deserialize<Dictionary<string, string>>(line);
if (msg?["event"] != "on")
int received = sock.ReceiveFrom(buffer, ref fromEp);
string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');
if (!text.StartsWith("HKMSTR"))
continue;
Console.WriteLine("PTT on — recording");
// Capture audio...
// Read until "off"
while ((line = reader.ReadLine()) is not null)
if (text.StartsWith("HKMSTR:OFF"))
{
msg = JsonSerializer.Deserialize<Dictionary<string, string>>(line);
if (msg?["event"] == "off")
break;
if (recording)
{
recording = false;
ProcessAndReply(audioChunks, fromEp);
audioChunks.Clear();
}
continue;
}
// Run STT...
// NOP
lastNop = DateTime.UtcNow;
if (!recording)
{
recording = true;
audioChunks.Clear();
Console.WriteLine($"PTT on from {fromEp}");
}
// Capture 50ms audio here...
// audioChunks.Add(capturedChunk);
// Check silence timeout
if (recording && (DateTime.UtcNow - lastNop).TotalMilliseconds > 150)
{
recording = false;
ProcessAndReply(audioChunks, fromEp);
audioChunks.Clear();
}
}
else
{
// Timeout check even without incoming data
if (recording && (DateTime.UtcNow - lastNop).TotalMilliseconds > 150)
{
recording = false;
ProcessAndReply(audioChunks, fromEp);
audioChunks.Clear();
}
}
}
void ProcessAndReply(List<float[]> chunks, EndPoint client)
{
if (chunks.Count == 0) return;
// Concatenate and run STT...
string text = "recognized text here";
var reply = JsonSerializer.Serialize(new { final = true, text });
writer.WriteLine(reply);
if (!string.IsNullOrEmpty(text))
{
byte[] reply = Encoding.UTF8.GetBytes($"HKMSTR:F {text}\n");
sock.SendTo(reply, client);
}
}
```
## Tips
- **One connection per client:** Robovoice maintains a single persistent TCP
connection. The server should handle one client at a time (or track
multiple if needed).
- **Timeout:** implement a recording timeout in case the `off` message is
delayed or the client disconnects. 1030 seconds is reasonable.
- **Partials:** optional but improve UX — Robovoice logs them so the user
sees live feedback. Only `final` triggers TTS.
- **Encoding:** always UTF-8. Every line is a UTF-8 JSON object terminated
by `\n`.
- **Reconnection:** Robovoice auto-reconnects every 3 seconds if the
connection drops. The server just needs to accept new connections.
- **Moonshine models:** `moonshine/base` (faster, less accurate) or
`moonshine/tiny` (fastest). Choose based on your hardware.
- **Reply address:** always reply to the source endpoint of the last NOP.
The client binds to a specific IP on :68.
- **Nonces:** discard them. They exist only to make each datagram unique.
Do not derive any meaning from nonce values.
- **Silence timeout:** 150ms = 3 missed NOPs at 50ms intervals. If you
change the NOP interval on the client, adjust this accordingly.
- **Partials:** optional. Send `HKMSTR:P <text>` while recording for live
feedback. Client logs them but only `HKMSTR:F` triggers TTS.
- **Broadcast only for C→S:** the WireGuard killswitch only allows
outbound broadcast to 255.255.255.255:67. Unicast from client won't pass.
- **Unicast OK for S→C:** the inbound WFP rule has no address restriction,
so unicast replies to :68 pass through.
- **Moonshine models:** `moonshine/base` or `moonshine/tiny`.
+93 -52
View File
@@ -1,70 +1,111 @@
# Robovoice TCP STT Protocol
# Robovoice DHCP Tunnel Protocol
## Overview
Robovoice acts as a **client**: it connects to a remote STT server over TCP,
sends control messages when the user presses/releases the PTT key, and
receives transcript messages back. The STT server captures audio from a
microphone, runs speech recognition (Moonshine), and sends transcripts back
over the same connection.
Robovoice communicates with a remote STT server by tunneling through the
DHCP UDP ports (68→67). This exploits a common killswitch exception: VPN
software (e.g. WireGuard) blocks all traffic except DHCP, which is allowed
for network connectivity maintenance.
```
[Robovoice client] --TCP--> [STT server :5210]
│ │
├── {"event":"on"}\n ──────►│
│ ├── capture audio
├── {"event":"off"}\n ──────►│
│ ├── run STT
│◄── {"final":true,...}\n ──┤
[Robovoice client] --broadcast UDP :68→:67--> [STT server]
[Robovoice client] <--unicast UDP :67→:68-- [STT server]
```
The client broadcasts NOP heartbeats while PTT is held. The server starts
recording on the first NOP and stops when it receives OFF or when 150ms
pass with no NOPs.
## Transport
- **Protocol:** TCP (reliable, ordered, connection-oriented)
- **Server endpoint:** configurable in Robovoice UI (default `127.0.0.1:5210`)
- **Framing:** newline-delimited JSON (NDJSON) — each message is a single
UTF-8 JSON object terminated by `\n`
- **Auto-reconnect:** if the connection drops, Robovoice retries every 3
seconds until the server is available
- **Protocol:** UDP (connectionless, unreliable)
- **Client → Server:** broadcast, source port 68, dest port 67
- **Server → Client:** unicast, source port 67, dest port 68
- **Client binds:** to a specific LAN interface IP on port 68 (with
`SO_REUSEADDR` to coexist with the Windows DHCP service)
- **No connection state** — purely fire-and-forget datagrams
## Control messages (client → server)
## Wire format
Sent by Robovoice when the user presses/releases the PTT key.
All messages are plain text, newline-terminated (`\n`). Every message starts
with the 6-byte magic `HKMSTR` to distinguish our traffic from real DHCP.
```json
{"event": "on"}
### Client → Server
**NOP (heartbeat while PTT held):**
```
HKMSTR <nonce>\n
```
Sent every 50ms while PTT is held. The nonce is an incrementing unsigned
integer that makes each datagram unique. The server discards it — it's
purely for packet uniqueness, not for any protocol logic.
**OFF (PTT released):**
```
HKMSTR:OFF <nonce>\n
```
Sent once when PTT is released. This is the fast-stop signal. If lost, the
150ms timeout acts as a backstop.
### Server → Client
**Partial transcript:**
```
HKMSTR:P <text>\n
```
Intermediate recognition result. Fire-and-forget. Client logs it but does
not act on it.
**Final transcript:**
```
HKMSTR:F <text>\n
```
Complete utterance. Client feeds this to the TTS engine.
## Server state machine
```
┌──────────────────────────────────────────┐
│ │
▼ │
┌──────────┐ first NOP ┌──────────────┐ │
│ IDLE │ ──────────► │ RECORDING │ │
└──────────┘ └──────────────┘ │
│ │ │
OFF │ │ 150ms │
recv'd │ │ silence │
▼ ▼ │
┌─────────────┐ │
│ PROCESSING │ │
└─────────────┘ │
│ │
STT │ │
done │ │
▼ │
send HKMSTR:F ─────────┘
```
```json
{"event": "off"}
```
- **IDLE → RECORDING:** first NOP received, start mic capture
- **RECORDING → PROCESSING:** OFF received, OR 150ms since last NOP
- **PROCESSING → IDLE:** STT done, send `HKMSTR:F <text>`
| Field | Type | Description |
|---------|--------|------------------------------------|
| `event` | string | `"on"` (PTT pressed) or `"off"` (PTT released) |
## Timing
## Transcript messages (server → client)
| Parameter | Value | Purpose |
|-----------|-------|---------|
| NOP interval | 50ms | Heartbeat frequency while PTT held |
| Silence timeout | 150ms | Stop recording if no NOPs (3 missed = lost OFF) |
| NOP bandwidth | ~20 msg/s × ~20 bytes | ~400 bytes/s — negligible |
Sent by the server back to Robovoice over the same TCP connection.
## Why this works
```json
{"final": false, "text": "hello world"}
```
```json
{"final": true, "text": "hello world how are you"}
```
| Field | Type | Required | Description |
|---------|---------|----------|--------------------------------------------------|
| `final` | bool | yes | `true` = final result, `false` = partial |
| `text` | string | yes | The transcript text (may be empty for partials) |
### Semantics
- **`final: false`** — intermediate recognition result (partial). Robovoice
logs these but does not act on them (only `final` triggers TTS).
- **`final: true`** — complete utterance. Robovoice feeds this to the TTS
engine and speaks it.
Malformed JSON or unknown field values are silently dropped by the client.
1. **Outbound broadcast `:68→:67` to `255.255.255.255`** passes the
WireGuard WFP killswitch (DHCP exception matches this exact pattern)
2. **Inbound `:67→:68`** has no address restriction in the WFP rule, so
unicast replies pass through
3. **Binding to a specific interface IP** (not `0.0.0.0`) wins unicast
delivery over the Windows DHCP client service
4. **NOP spam** ensures the ON message gets through even at 5% packet loss
(3 consecutive NOPs = ~0.01% drop probability)
5. **150ms timeout** is the backstop for lost OFF — at 50ms intervals, 3
consecutive NOPs must all be lost to false-stop