v0.6: DHCP tunnel transport with NOP heartbeat protocol

This commit is contained in:
2026-08-12 00:29:54 +00:00
parent 4a58b94f35
commit e1739059d9
12 changed files with 743 additions and 528 deletions
+105
View File
@@ -0,0 +1,105 @@
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Text;
// Args: [interface_ip] [interval_ms]
// Defaults: auto-detect LAN IP, 50ms
string localIp = args.Length > 0 ? args[0] : GetLanInterfaceIp() ?? "0.0.0.0";
int intervalMs = args.Length > 1 && int.TryParse(args[1], out int iv) ? iv : 50;
using var sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
sock.EnableBroadcast = true;
var localEp = new IPEndPoint(IPAddress.Parse(localIp), 68);
sock.Bind(localEp);
Console.WriteLine($"Bound to {localEp} (SO_REUSEADDR)");
Console.WriteLine("Sending NOPs + listening. Press Ctrl+C to stop.");
Console.WriteLine();
var cts = new CancellationTokenSource();
var recvTask = Task.Run(() => ReceiveLoop(sock, cts.Token));
var destEp = new IPEndPoint(IPAddress.Broadcast, 67);
uint nonce = 0;
while (!cts.Token.IsCancellationRequested)
{
nonce++;
string msg = $"HKMSTR {nonce}\n";
byte[] payload = Encoding.UTF8.GetBytes(msg);
try
{
int sent = sock.SendTo(payload, destEp);
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] SENT n={nonce} {sent} bytes");
}
catch (Exception ex)
{
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] SEND FAILED: {ex.Message}");
}
try { Thread.Sleep(intervalMs); }
catch (OperationCanceledException) { break; }
}
cts.Cancel();
await recvTask;
static string? GetLanInterfaceIp()
{
foreach (var nic in NetworkInterface.GetAllNetworkInterfaces())
{
if (nic.OperationalStatus != OperationalStatus.Up)
continue;
if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback)
continue;
if (nic.Description.Contains("WireGuard", StringComparison.OrdinalIgnoreCase))
continue;
foreach (var addr in nic.GetIPProperties().UnicastAddresses)
{
if (addr.Address.AddressFamily == AddressFamily.InterNetwork)
{
var ip = addr.Address.ToString();
if (ip.StartsWith("192.168.") || ip.StartsWith("10.") || ip.StartsWith("172."))
return ip;
}
}
}
return null;
}
static void ReceiveLoop(Socket sock, CancellationToken ct)
{
byte[] buffer = new byte[4096];
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
while (!ct.IsCancellationRequested)
{
int received;
try
{
if (!sock.Poll(500_000, SelectMode.SelectRead))
continue;
received = sock.ReceiveFrom(buffer, ref fromEp);
}
catch (Exception ex)
{
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV ERROR: {ex.Message}");
continue;
}
string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');
if (!text.StartsWith("HKMSTR"))
{
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV {received} bytes from {fromEp} (no magic)");
continue;
}
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV {received} bytes from {fromEp}: {text}");
}
}
@@ -0,0 +1,10 @@
<Project Sdk="Microsoft.NET.Sdk">
<PropertyGroup>
<OutputType>Exe</OutputType>
<TargetFramework>net10.0</TargetFramework>
<ImplicitUsings>enable</ImplicitUsings>
<Nullable>enable</Nullable>
</PropertyGroup>
</Project>
+1 -1
View File
@@ -12,7 +12,7 @@ public sealed class AppConfig
public int LengthScale { get; set; } = 100; public int LengthScale { get; set; } = 100;
public int NoiseWScale { get; set; } = 800; public int NoiseWScale { get; set; } = 800;
public bool MinimizeToTray { get; set; } = true; public bool MinimizeToTray { get; set; } = true;
public string ServerEndpoint { get; set; } = "127.0.0.1:5210"; public string InterfaceIp { get; set; } = string.Empty;
public static string AppDataDir => Path.Combine( public static string AppDataDir => Path.Combine(
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
+9 -17
View File
@@ -17,8 +17,7 @@ partial class MainForm
private Button btnBrowseFile = null!; private Button btnBrowseFile = null!;
private Label lblFileName = null!; private Label lblFileName = null!;
private Label lblServer = null!; private Label lblServer = null!;
private TextBox txtServer = null!; private ComboBox cmbInterface = null!;
private Button btnConnect = null!;
private Label lblLineStatus = null!; private Label lblLineStatus = null!;
private RichTextBox txtLog = null!; private RichTextBox txtLog = null!;
private CheckBox chkMinimizeToTray = null!; private CheckBox chkMinimizeToTray = null!;
@@ -58,8 +57,7 @@ partial class MainForm
btnBrowseFile = new Button(); btnBrowseFile = new Button();
lblFileName = new Label(); lblFileName = new Label();
lblServer = new Label(); lblServer = new Label();
txtServer = new TextBox(); cmbInterface = new ComboBox();
btnConnect = new Button();
lblLineStatus = new Label(); lblLineStatus = new Label();
txtLog = new RichTextBox(); txtLog = new RichTextBox();
chkMinimizeToTray = new CheckBox(); chkMinimizeToTray = new CheckBox();
@@ -145,20 +143,15 @@ partial class MainForm
lblFileName.ForeColor = Color.Gray; lblFileName.ForeColor = Color.Gray;
// lblServer // lblServer
lblServer.Text = "Server:"; lblServer.Text = "Interface:";
lblServer.Location = new Point(440, 48); lblServer.Location = new Point(440, 48);
lblServer.Size = new Size(45, 23); lblServer.Size = new Size(55, 23);
lblServer.TextAlign = ContentAlignment.MiddleLeft; lblServer.TextAlign = ContentAlignment.MiddleLeft;
// txtServer // cmbInterface
txtServer.Location = new Point(488, 45); cmbInterface.Location = new Point(498, 45);
txtServer.Size = new Size(110, 23); cmbInterface.Size = new Size(165, 23);
cmbInterface.DropDownStyle = ComboBoxStyle.DropDownList;
// btnConnect
btnConnect.Text = "Connect";
btnConnect.Location = new Point(603, 44);
btnConnect.Size = new Size(60, 25);
btnConnect.UseVisualStyleBackColor = true;
// lblLineStatus // lblLineStatus
lblLineStatus.Text = ""; lblLineStatus.Text = "";
@@ -277,8 +270,7 @@ partial class MainForm
Controls.Add(btnBrowseFile); Controls.Add(btnBrowseFile);
Controls.Add(lblFileName); Controls.Add(lblFileName);
Controls.Add(lblServer); Controls.Add(lblServer);
Controls.Add(txtServer); Controls.Add(cmbInterface);
Controls.Add(btnConnect);
Controls.Add(lblLineStatus); Controls.Add(lblLineStatus);
Controls.Add(lblNoise); Controls.Add(lblNoise);
Controls.Add(trkNoise); Controls.Add(trkNoise);
+48 -29
View File
@@ -2,7 +2,7 @@ using NAudio.Wave;
using Robovoice.App; using Robovoice.App;
using Robovoice.Core; using Robovoice.Core;
using Robovoice.Core.Voices; using Robovoice.Core.Voices;
using Robovoice.Stt.Tcp; using Robovoice.Stt.Dhcp;
using Robovoice.Tts.LibPiper; using Robovoice.Tts.LibPiper;
using System.Diagnostics; using System.Diagnostics;
@@ -16,7 +16,7 @@ internal sealed partial class MainForm : Form
private LibPiperTtsEngine? _tts; private LibPiperTtsEngine? _tts;
private AudioOutput? _audioOutput; private AudioOutput? _audioOutput;
private TcpSttSource? _sttSource; private DhcpSttSource? _sttSource;
private Orchestrator? _orchestrator; private Orchestrator? _orchestrator;
private PttHotkey? _pttHotkey; private PttHotkey? _pttHotkey;
private NotifyIcon? _trayIcon; private NotifyIcon? _trayIcon;
@@ -61,7 +61,7 @@ internal sealed partial class MainForm : Form
OnSliderScroll(null, EventArgs.Empty); OnSliderScroll(null, EventArgs.Empty);
chkMinimizeToTray.Checked = _config.MinimizeToTray; chkMinimizeToTray.Checked = _config.MinimizeToTray;
txtServer.Text = _config.ServerEndpoint; PopulateInterfaces();
btnBrowseFile.Click += OnBrowseFile; btnBrowseFile.Click += OnBrowseFile;
btnTestVoice.Click += OnTestVoice; btnTestVoice.Click += OnTestVoice;
@@ -71,8 +71,7 @@ internal sealed partial class MainForm : Form
txtPttKey.KeyDown += OnPttKeyDown; txtPttKey.KeyDown += OnPttKeyDown;
cmbOutput.SelectedIndexChanged += OnOutputChanged; cmbOutput.SelectedIndexChanged += OnOutputChanged;
cmbVoice.SelectedIndexChanged += OnVoiceChanged; cmbVoice.SelectedIndexChanged += OnVoiceChanged;
txtServer.Leave += OnServerChanged; cmbInterface.SelectedIndexChanged += OnInterfaceChanged;
btnConnect.Click += OnConnect;
trkNoise.Scroll += OnSliderScroll; trkNoise.Scroll += OnSliderScroll;
trkSpeed.Scroll += OnSliderScroll; trkSpeed.Scroll += OnSliderScroll;
@@ -245,8 +244,9 @@ internal sealed partial class MainForm : Form
if (_sttSource is null) if (_sttSource is null)
{ {
_sttSource = new TcpSttSource { ServerEndpoint = txtServer.Text, Log = Log }; string ifaceIp = cmbInterface.SelectedItem as string ?? "";
Log($"STT endpoint: {_sttSource.ServerEndpoint} (press Connect)"); _sttSource = new DhcpSttSource { InterfaceIp = ifaceIp, Log = Log };
await _sttSource.StartAsync();
} }
_orchestrator?.DisposeAsync().AsTask().Wait(); _orchestrator?.DisposeAsync().AsTask().Wait();
@@ -425,40 +425,59 @@ internal sealed partial class MainForm : Form
SaveConfig(); SaveConfig();
} }
private void OnServerChanged(object? sender, EventArgs e) private void PopulateInterfaces()
{ {
if (_sttSource is not null) cmbInterface.Items.Clear();
foreach (var (ip, name) in DhcpSttSource.GetAvailableInterfaces())
{ {
_sttSource.ServerEndpoint = txtServer.Text; cmbInterface.Items.Add(name);
Log($"Server endpoint: {txtServer.Text}"); cmbInterface.Items[^1] = name;
cmbInterface.Items[cmbInterface.Items.Count - 1] = name;
} }
SaveConfig();
if (!string.IsNullOrEmpty(_config.InterfaceIp))
{
for (int i = 0; i < cmbInterface.Items.Count; i++)
{
if (cmbInterface.Items[i] is string s && s.Contains(_config.InterfaceIp))
{
cmbInterface.SelectedIndex = i;
return;
}
}
}
if (cmbInterface.Items.Count > 0)
cmbInterface.SelectedIndex = 0;
} }
private async void OnConnect(object? sender, EventArgs e) private void OnInterfaceChanged(object? sender, EventArgs e)
{ {
if (_sttSource is null) if (cmbInterface.SelectedItem is not string selected)
{
Log("STT source not initialized.");
return; return;
}
_sttSource.ServerEndpoint = txtServer.Text; string? ip = ExtractIpFromDisplay(selected);
if (ip is null) return;
_config.InterfaceIp = ip;
SaveConfig(); SaveConfig();
btnConnect.Enabled = false;
try if (_sttSource is not null)
{ {
if (_sttSource.IsRunning) _sttSource.DisposeAsync().AsTask().Wait(2000);
await _sttSource.ReconnectAsync(); _sttSource = null;
else _ = InitializeEngineAsync();
await _sttSource.StartAsync();
}
finally
{
btnConnect.Enabled = true;
} }
} }
private static string? ExtractIpFromDisplay(string display)
{
int start = display.IndexOf('(');
int end = display.IndexOf(')');
if (start < 0 || end <= start) return null;
return display[(start + 1)..end];
}
private void SetupTray() private void SetupTray()
{ {
if (_trayInit) return; if (_trayInit) return;
@@ -524,7 +543,7 @@ internal sealed partial class MainForm : Form
_config.LengthScale = trkSpeed.Value; _config.LengthScale = trkSpeed.Value;
_config.NoiseWScale = trkNoiseW.Value; _config.NoiseWScale = trkNoiseW.Value;
_config.MinimizeToTray = chkMinimizeToTray.Checked; _config.MinimizeToTray = chkMinimizeToTray.Checked;
_config.ServerEndpoint = txtServer.Text; _config.InterfaceIp = ExtractIpFromDisplay(cmbInterface.SelectedItem as string ?? "") ?? "";
_config.Save(); _config.Save();
} }
+1 -1
View File
@@ -3,7 +3,7 @@
<ItemGroup> <ItemGroup>
<ProjectReference Include="..\Robovoice.Core\Robovoice.Core.csproj" /> <ProjectReference Include="..\Robovoice.Core\Robovoice.Core.csproj" />
<ProjectReference Include="..\Robovoice.Tts.LibPiper\Robovoice.Tts.LibPiper.csproj" /> <ProjectReference Include="..\Robovoice.Tts.LibPiper\Robovoice.Tts.LibPiper.csproj" />
<ProjectReference Include="..\Robovoice.Stt.Tcp\Robovoice.Stt.Tcp.csproj" /> <ProjectReference Include="..\Robovoice.Stt.Dhcp\Robovoice.Stt.Dhcp.csproj" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
+232
View File
@@ -0,0 +1,232 @@
using System.Net;
using System.Net.NetworkInformation;
using System.Net.Sockets;
using System.Text;
using Robovoice.Core;
namespace Robovoice.Stt.Dhcp;
public sealed class DhcpSttSource : ISttSource
{
private const string Magic = "HKMSTR";
private const int DhcpClientPort = 68;
private const int DhcpServerPort = 67;
private static readonly TimeSpan NopInterval = TimeSpan.FromMilliseconds(50);
private Socket? _sock;
private CancellationTokenSource? _cts;
private Task? _receiveTask;
private Task? _nopTask;
private EndPoint _broadcastEp = new IPEndPoint(IPAddress.Broadcast, DhcpServerPort);
private uint _nonce;
private bool _disposed;
public string InterfaceIp { get; set; } = string.Empty;
public Action<string>? Log { get; set; }
public event TranscriptEventHandler? TranscriptReceived;
public Task StartAsync(CancellationToken ct = default)
{
ObjectDisposedException.ThrowIf(_disposed, this);
if (_cts is not null)
return Task.CompletedTask;
string ip = string.IsNullOrEmpty(InterfaceIp) ? AutoDetectInterfaceIp() ?? "0.0.0.0" : InterfaceIp;
_sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
_sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
_sock.EnableBroadcast = true;
_sock.Bind(new IPEndPoint(IPAddress.Parse(ip), DhcpClientPort));
Log?.Invoke($"STT: bound to {ip}:{DhcpClientPort}, broadcasting to :{DhcpServerPort}");
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
_receiveTask = ReceiveLoopAsync(_cts.Token);
return Task.CompletedTask;
}
public async Task StopAsync(CancellationToken ct = default)
{
StopNop();
if (_cts is not null)
_cts.Cancel();
_sock?.Dispose();
_sock = null;
if (_receiveTask is not null)
{
try { await _receiveTask.WaitAsync(ct); }
catch { }
_receiveTask = null;
}
_cts?.Dispose();
_cts = null;
}
public void SendOn()
{
if (_cts is null)
return;
_nonce = 0;
SendNop();
_nopTask = NopLoopAsync(_cts.Token);
}
public void SendOff()
{
StopNop();
SendControl("HKMSTR:OFF");
}
private void StopNop()
{
if (_nopTask is not null)
{
try { _nopTask.Wait(2000); } catch { }
_nopTask = null;
}
}
private async Task NopLoopAsync(CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
try { await Task.Delay(NopInterval, ct); }
catch (OperationCanceledException) { break; }
SendNop();
}
}
private void SendNop()
{
_nonce++;
SendControl($"HKMSTR {_nonce}");
}
private void SendControl(string message)
{
if (_sock is null)
return;
try
{
byte[] payload = Encoding.UTF8.GetBytes(message + "\n");
_sock.SendTo(payload, _broadcastEp);
}
catch (Exception ex)
{
Log?.Invoke($"STT: send failed: {ex.Message}");
}
}
private async Task ReceiveLoopAsync(CancellationToken ct)
{
byte[] buffer = new byte[4096];
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
while (!ct.IsCancellationRequested)
{
int received;
try
{
if (!_sock!.Poll(500_000, SelectMode.SelectRead))
continue;
received = _sock.ReceiveFrom(buffer, ref fromEp);
}
catch (OperationCanceledException)
{
break;
}
catch (ObjectDisposedException)
{
break;
}
catch (Exception ex)
{
Log?.Invoke($"STT: receive error: {ex.Message}");
continue;
}
string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');
if (!text.StartsWith(Magic))
continue;
if (text.StartsWith("HKMSTR:P "))
{
string transcript = text["HKMSTR:P ".Length..];
TranscriptReceived?.Invoke(this, new TranscriptEventArgs
{
Message = new TranscriptMessage(TranscriptType.Partial, transcript),
});
}
else if (text.StartsWith("HKMSTR:F "))
{
string transcript = text["HKMSTR:F ".Length..];
TranscriptReceived?.Invoke(this, new TranscriptEventArgs
{
Message = new TranscriptMessage(TranscriptType.Final, transcript),
});
}
}
}
public static List<(string Ip, string Name)> GetAvailableInterfaces()
{
var result = new List<(string, string)>();
foreach (var nic in NetworkInterface.GetAllNetworkInterfaces())
{
if (nic.OperationalStatus != OperationalStatus.Up)
continue;
if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback)
continue;
string desc = nic.Description;
if (desc.Contains("WireGuard", StringComparison.OrdinalIgnoreCase))
continue;
foreach (var addr in nic.GetIPProperties().UnicastAddresses)
{
if (addr.Address.AddressFamily != AddressFamily.InterNetwork)
continue;
string ip = addr.Address.ToString();
if (ip.StartsWith("192.168.") || ip.StartsWith("10.") ||
ip.StartsWith("172.16.") || ip.StartsWith("172.17.") ||
ip.StartsWith("172.18.") || ip.StartsWith("172.19.") ||
ip.StartsWith("172.20.") || ip.StartsWith("172.21.") ||
ip.StartsWith("172.22.") || ip.StartsWith("172.23.") ||
ip.StartsWith("172.24.") || ip.StartsWith("172.25.") ||
ip.StartsWith("172.26.") || ip.StartsWith("172.27.") ||
ip.StartsWith("172.28.") || ip.StartsWith("172.29.") ||
ip.StartsWith("172.30.") || ip.StartsWith("172.31."))
{
result.Add((ip, $"{nic.Name} ({ip})"));
}
}
}
return result;
}
private static string? AutoDetectInterfaceIp()
{
foreach (var (ip, _) in GetAvailableInterfaces())
return ip;
return null;
}
public async ValueTask DisposeAsync()
{
if (_disposed) return;
await StopAsync();
_disposed = true;
}
}
-253
View File
@@ -1,253 +0,0 @@
using System.Net;
using System.Net.Sockets;
using System.Text.Json;
using System.Text.Json.Serialization;
using Robovoice.Core;
namespace Robovoice.Stt.Tcp;
public sealed class TcpSttSource : ISttSource
{
private TcpClient? _tcp;
private NetworkStream? _stream;
private StreamReader? _reader;
private StreamWriter? _writer;
private CancellationTokenSource? _cts;
private Task? _runTask;
private readonly object _sendLock = new();
private bool _disposed;
public string ServerEndpoint { get; set; } = "127.0.0.1:5210";
public bool IsRunning => _cts is not null;
public Action<string>? Log { get; set; }
public event TranscriptEventHandler? TranscriptReceived;
public Task StartAsync(CancellationToken ct = default)
{
ObjectDisposedException.ThrowIf(_disposed, this);
if (_cts is not null)
return Task.CompletedTask;
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
_runTask = RunAsync(_cts.Token);
return Task.CompletedTask;
}
public async Task StopAsync(CancellationToken ct = default)
{
if (_cts is not null)
_cts.Cancel();
CleanupConnection();
if (_runTask is not null)
{
try { await _runTask.WaitAsync(ct); }
catch { }
_runTask = null;
}
_cts?.Dispose();
_cts = null;
}
private async Task RunAsync(CancellationToken ct)
{
while (!ct.IsCancellationRequested)
{
IPEndPoint? endpoint = ParseEndpoint(ServerEndpoint);
if (endpoint is null)
{
Log?.Invoke($"STT: invalid endpoint '{ServerEndpoint}'");
try { await Task.Delay(3000, ct); } catch { break; }
continue;
}
try
{
_tcp = new TcpClient();
using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
connectCts.CancelAfter(TimeSpan.FromSeconds(5));
await _tcp.ConnectAsync(endpoint.Address, endpoint.Port, connectCts.Token);
_stream = _tcp.GetStream();
_reader = new StreamReader(_stream, System.Text.Encoding.UTF8);
_writer = new StreamWriter(_stream, System.Text.Encoding.UTF8) { AutoFlush = true };
Log?.Invoke($"STT: connected to {ServerEndpoint}");
await ReceiveLoopAsync(ct);
}
catch (OperationCanceledException)
{
break;
}
catch (Exception ex)
{
Log?.Invoke($"STT: connection failed ({ex.Message}), retrying...");
}
finally
{
CleanupConnection();
}
if (!ct.IsCancellationRequested)
{
try { await Task.Delay(3000, ct); }
catch (OperationCanceledException) { break; }
}
}
}
private async Task ReceiveLoopAsync(CancellationToken ct)
{
while (!ct.IsCancellationRequested && _reader is not null)
{
string? line;
try
{
line = await _reader.ReadLineAsync(ct);
}
catch
{
break;
}
if (line is null)
break;
TranscriptMessage? message = ParseTranscriptLine(line);
if (message is null)
continue;
TranscriptReceived?.Invoke(this, new TranscriptEventArgs
{
Message = message,
});
}
}
public void SendOn()
{
SendControl("on");
}
public void SendOff()
{
SendControl("off");
}
public async Task ReconnectAsync(CancellationToken ct = default)
{
if (_cts is null)
return;
Log?.Invoke("STT: reconnecting...");
CleanupConnection();
try { await Task.Delay(500, ct); }
catch (OperationCanceledException) { return; }
}
private void SendControl(string evt)
{
lock (_sendLock)
{
if (_writer is null)
return;
try
{
_writer.WriteLine(JsonSerializer.Serialize(new ControlDto { Event = evt }));
}
catch
{
Log?.Invoke($"STT: failed to send '{evt}' (not connected?)");
}
}
}
private void CleanupConnection()
{
lock (_sendLock)
{
_writer?.Dispose();
_reader?.Dispose();
_stream?.Dispose();
_tcp?.Dispose();
_writer = null;
_reader = null;
_stream = null;
_tcp = null;
}
}
private static IPEndPoint? ParseEndpoint(string endpoint)
{
int colon = endpoint.LastIndexOf(':');
if (colon <= 0)
return null;
string host = endpoint[..colon];
if (!int.TryParse(endpoint[(colon + 1)..], out int port))
return null;
if (IPAddress.TryParse(host, out var addr))
return new IPEndPoint(addr, port);
try
{
var addresses = Dns.GetHostAddresses(host);
addr = addresses.FirstOrDefault(a => a.AddressFamily == AddressFamily.InterNetwork);
if (addr is null)
return null;
return new IPEndPoint(addr, port);
}
catch
{
return null;
}
}
private static TranscriptMessage? ParseTranscriptLine(string line)
{
if (string.IsNullOrWhiteSpace(line))
return null;
try
{
var dto = JsonSerializer.Deserialize<TransmitDto>(line);
if (dto is null)
return null;
var type = dto.Final ? TranscriptType.Final : TranscriptType.Partial;
return new TranscriptMessage(type, dto.Text ?? string.Empty);
}
catch (JsonException)
{
return null;
}
}
public async ValueTask DisposeAsync()
{
if (_disposed) return;
await StopAsync();
_disposed = true;
}
}
internal sealed class ControlDto
{
[JsonPropertyName("event")]
public string Event { get; set; } = string.Empty;
}
internal sealed class TransmitDto
{
public bool Final { get; set; }
public string Text { get; set; } = string.Empty;
}
+2 -1
View File
@@ -2,6 +2,7 @@
<Project Path="Robovoice.App/Robovoice.App.csproj" /> <Project Path="Robovoice.App/Robovoice.App.csproj" />
<Project Path="Robovoice.Core/Robovoice.Core.csproj" /> <Project Path="Robovoice.Core/Robovoice.Core.csproj" />
<Project Path="Robovoice.Stt.File/Robovoice.Stt.File.csproj" /> <Project Path="Robovoice.Stt.File/Robovoice.Stt.File.csproj" />
<Project Path="Robovoice.Stt.Tcp/Robovoice.Stt.Tcp.csproj" /> <Project Path="Robovoice.Stt.Dhcp/Robovoice.Stt.Dhcp.csproj" />
<Project Path="Robovoice.Tts.LibPiper/Robovoice.Tts.LibPiper.csproj" /> <Project Path="Robovoice.Tts.LibPiper/Robovoice.Tts.LibPiper.csproj" />
<Project Path="DhcpTunnelTest/Robovoice.DhcpTunnelTest.csproj" />
</Solution> </Solution>
+242 -174
View File
@@ -1,202 +1,232 @@
# Server implementation notes # Server implementation notes
The STT server listens for TCP connections from Robovoice, captures audio The STT server listens on UDP port 67, receives NOP heartbeats and OFF
from a microphone when `on` is received, runs speech recognition (Moonshine), messages from Robovoice, captures audio from a microphone, runs speech
and sends transcript messages back over the same connection. recognition (Moonshine), and sends transcript messages back to the client's
source address.
## Architecture ## Architecture
``` ```
┌── on/off (TCP, newline-delimited JSON) ┌── HKMSTR <nonce> (broadcast, every 50ms)
Robovoice ──────────────►│ Robovoice ──────────────►│
│ STT Server │ STT Server
Robovoice ◄──────────────┤ Robovoice ◄──────────────┤
└── partial/final (TCP, newline-delimited JSON) └── HKMSTR:P/F <text> (unicast)
``` ```
The server: The server:
1. Listens on a TCP port (e.g. 5210) 1. Listens on UDP :67
2. Accepts a connection from Robovoice 2. First `HKMSTR <nonce>` → start recording
3. Reads lines: waits for `{"event":"on"}` 3. `HKMSTR:OFF <nonce>` → stop recording, run STT
4. Records audio from the microphone 4. 150ms with no NOPs → stop recording, run STT (backstop)
5. Waits for `{"event":"off"}` (or a timeout) 5. Send `HKMSTR:F <text>` back to the client's source address:port
6. Runs STT on the captured audio
7. Sends `{"final":true,"text":"..."}`\n back over the connection
## Framing ## Framing
Every message is a single JSON object on one line, terminated by `\n`. No All messages are newline-terminated UTF-8 text. No JSON, no binary framing.
length prefix, no binary framing. Use `readline()` / `StreamReader.ReadLineAsync()`.
- `HKMSTR <nonce>` — NOP heartbeat (client → server)
- `HKMSTR:OFF <nonce>` — stop signal (client → server)
- `HKMSTR:P <text>` — partial transcript (server → client)
- `HKMSTR:F <text>` — final transcript (server → client)
The nonce is an incrementing integer for packet uniqueness only. Discard it.
## Python server with Moonshine ## Python server with Moonshine
[Moonshine](https://github.com/usefulsensors/moonshine) is a lightweight ASR
model by Useful Sensors. Install with `pip install moonshine`.
```python ```python
import socket import socket
import json import threading
import time
import numpy as np import numpy as np
import sounddevice as sd import sounddevice as sd
import moonshine import moonshine
LISTEN_PORT = 5210 LISTEN_PORT = 67
CLIENT_PORT = 68
SAMPLE_RATE = 16000 SAMPLE_RATE = 16000
SILENCE_TIMEOUT = 0.150 # 150ms
model = moonshine.MoonshineModel(model="moonshine/base") model = moonshine.MoonshineModel(model="moonshine/base")
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server.bind(("0.0.0.0", LISTEN_PORT)) sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
server.listen(1) sock.bind(("0.0.0.0", LISTEN_PORT))
print(f"STT server listening on :{LISTEN_PORT}") print(f"STT server listening on :{LISTEN_PORT}")
recording = False
last_nop_time = 0
client_addr = None
audio_chunks = []
lock = threading.Lock()
def monitor_silence():
"""Backstop: stop recording if no NOPs for 150ms."""
global recording
while True:
time.sleep(0.01)
with lock:
if recording and (time.monotonic() - last_nop_time) > SILENCE_TIMEOUT:
recording = False
threading.Thread(target=process_audio, daemon=True).start()
threading.Thread(target=monitor_silence, daemon=True).start()
def process_audio():
global audio_chunks
with lock:
chunks = audio_chunks
audio_chunks = []
addr = client_addr
if not chunks:
return
audio = np.concatenate(chunks)
print(f"Captured {len(audio)/SAMPLE_RATE:.1f}s")
text = moonshine.transcribe(model, audio).strip()
if text:
print(f"Final: {text}")
reply = f"HKMSTR:F {text}\n".encode("utf-8")
sock.sendto(reply, addr)
else:
print("Empty transcript")
while True: while True:
conn, addr = server.accept() data, addr = sock.recvfrom(4096)
print(f"Client connected: {addr}") text = data.decode("utf-8", errors="ignore").strip()
buf = "" if not text.startswith("HKMSTR"):
with conn: continue
while True:
data = conn.recv(4096).decode("utf-8")
if not data:
break
buf += data
while "\n" in buf: if text.startswith("HKMSTR:OFF"):
line, buf = buf.split("\n", 1) with lock:
msg = json.loads(line) if recording:
recording = False
threading.Thread(target=process_audio, daemon=True).start()
continue
if msg.get("event") == "on": if text.startswith("HKMSTR ") or text == "HKMSTR":
print("PTT on — recording") with lock:
audio_chunks = [] client_addr = addr
last_nop_time = time.monotonic()
# Record until "off" or timeout if not recording:
conn.settimeout(0.1) recording = True
while True: audio_chunks = []
try: print(f"PTT on from {addr}")
data2 = conn.recv(4096).decode("utf-8")
if not data2:
break
buf += data2
while "\n" in buf:
line2, buf = buf.split("\n", 1)
msg2 = json.loads(line2)
if msg2.get("event") == "off":
break
except socket.timeout:
pass
chunk = sd.rec(int(SAMPLE_RATE * 0.1), # Capture 50ms of audio
samplerate=SAMPLE_RATE, chunk = sd.rec(int(SAMPLE_RATE * 0.05), samplerate=SAMPLE_RATE,
channels=1, dtype="float32") channels=1, dtype="float32")
sd.wait() sd.wait()
audio_chunks.append(chunk.flatten()) audio_chunks.append(chunk.flatten())
conn.settimeout(None)
if not audio_chunks:
continue
audio = np.concatenate(audio_chunks)
print(f"Captured {len(audio)/SAMPLE_RATE:.1f}s")
text = moonshine.transcribe(model, audio).strip()
if text:
print(f"Transcript: {text}")
reply = json.dumps({"final": True, "text": text})
conn.sendall((reply + "\n").encode("utf-8"))
else:
print("Empty transcript")
``` ```
## Python server with streaming partials ## Python server with streaming partials
For lower latency, send partial results while still recording: For live feedback, send partials while recording:
```python ```python
import socket import socket
import json import threading
import time
import numpy as np import numpy as np
import sounddevice as sd import sounddevice as sd
import moonshine import moonshine
LISTEN_PORT = 5210 LISTEN_PORT = 67
SAMPLE_RATE = 16000 SAMPLE_RATE = 16000
CHUNK_DURATION = 0.5 SILENCE_TIMEOUT = 0.150
PARTIAL_INTERVAL = 0.5 # send partial every 500ms
model = moonshine.MoonshineModel(model="moonshine/base") model = moonshine.MoonshineModel(model="moonshine/base")
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
server.bind(("0.0.0.0", LISTEN_PORT)) sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
server.listen(1) sock.bind(("0.0.0.0", LISTEN_PORT))
print(f"STT server listening on :{LISTEN_PORT}") print(f"STT server listening on :{LISTEN_PORT}")
recording = False
last_nop_time = 0
last_partial_time = 0
client_addr = None
audio_chunks = []
lock = threading.Lock()
def capture_and_maybe_partial():
global last_partial_time
with lock:
if not recording:
return
chunk = sd.rec(int(SAMPLE_RATE * 0.05), samplerate=SAMPLE_RATE,
channels=1, dtype="float32")
sd.wait()
audio_chunks.append(chunk.flatten())
now = time.monotonic()
if now - last_partial_time > PARTIAL_INTERVAL:
last_partial_time = now
partial_audio = np.concatenate(audio_chunks)
partial_text = moonshine.transcribe(model, partial_audio).strip()
if partial_text and client_addr:
reply = f"HKMSTR:P {partial_text}\n".encode("utf-8")
sock.sendto(reply, client_addr)
while True: while True:
conn, addr = server.accept() data, addr = sock.recvfrom(4096)
print(f"Client connected: {addr}") text = data.decode("utf-8", errors="ignore").strip()
buf = ""
with conn: if not text.startswith("HKMSTR"):
while True: continue
data = conn.recv(4096).decode("utf-8")
if not data:
break
buf += data
while "\n" in buf: if text.startswith("HKMSTR:OFF"):
line, buf = buf.split("\n", 1) with lock:
msg = json.loads(line) if recording:
recording = False
if msg.get("event") != "on": chunks = audio_chunks
continue
print("PTT on — recording")
audio_chunks = [] audio_chunks = []
while True: if chunks:
try: audio = np.concatenate(chunks)
conn.settimeout(CHUNK_DURATION) final_text = moonshine.transcribe(model, audio).strip()
data2 = conn.recv(4096).decode("utf-8") if final_text:
if not data2: reply = f"HKMSTR:F {final_text}\n".encode("utf-8")
break sock.sendto(reply, addr)
buf += data2 continue
while "\n" in buf:
line2, buf = buf.split("\n", 1)
msg2 = json.loads(line2)
if msg2.get("event") == "off":
break
except socket.timeout:
pass
chunk = sd.rec(int(SAMPLE_RATE * CHUNK_DURATION), if text.startswith("HKMSTR") and not text.startswith("HKMSTR:"):
samplerate=SAMPLE_RATE, with lock:
channels=1, dtype="float32") client_addr = addr
sd.wait() last_nop_time = time.monotonic()
audio_chunks.append(chunk.flatten())
# Send partial every few chunks if not recording:
if len(audio_chunks) % 4 == 0: recording = True
partial_audio = np.concatenate(audio_chunks) audio_chunks = []
partial_text = moonshine.transcribe(model, partial_audio).strip() last_partial_time = time.monotonic()
if partial_text: print(f"PTT on from {addr}")
reply = json.dumps({"final": False, "text": partial_text})
conn.sendall((reply + "\n").encode("utf-8"))
conn.settimeout(None) capture_and_maybe_partial()
if not audio_chunks: # Check silence timeout
continue with lock:
if recording and (time.monotonic() - last_nop_time) > SILENCE_TIMEOUT:
recording = False
chunks = audio_chunks
audio_chunks = []
audio = np.concatenate(audio_chunks) if 'chunks' in dir() and chunks:
text = moonshine.transcribe(model, audio).strip() audio = np.concatenate(chunks)
final_text = moonshine.transcribe(model, audio).strip()
if text: if final_text:
print(f"Final: {text}") reply = f"HKMSTR:F {final_text}\n".encode("utf-8")
reply = json.dumps({"final": True, "text": text}) sock.sendto(reply, addr)
conn.sendall((reply + "\n").encode("utf-8"))
``` ```
## C# server skeleton ## C# server skeleton
@@ -204,61 +234,99 @@ while True:
```csharp ```csharp
using System.Net; using System.Net;
using System.Net.Sockets; using System.Net.Sockets;
using System.Text.Json; using System.Text;
var listener = new TcpListener(IPAddress.Any, 5210); var sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
listener.Start(); sock.Bind(new IPEndPoint(IPAddress.Any, 67));
Console.WriteLine("STT server listening on :5210"); Console.WriteLine("STT server listening on :67");
byte[] buffer = new byte[4096];
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
bool recording = false;
DateTime lastNop = DateTime.MinValue;
List<float[]> audioChunks = new();
while (true) while (true)
{ {
var client = listener.AcceptTcpClient(); if (sock.Poll(50_000, SelectMode.SelectRead))
Console.WriteLine($"Client connected: {client.Client.RemoteEndPoint}");
using var stream = client.GetStream();
using var reader = new StreamReader(stream, Encoding.UTF8);
using var writer = new StreamWriter(stream, Encoding.UTF8) { AutoFlush = true };
string? line;
while ((line = reader.ReadLine()) is not null)
{ {
var msg = JsonSerializer.Deserialize<Dictionary<string, string>>(line); int received = sock.ReceiveFrom(buffer, ref fromEp);
if (msg?["event"] != "on") string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');
if (!text.StartsWith("HKMSTR"))
continue; continue;
Console.WriteLine("PTT on — recording"); if (text.StartsWith("HKMSTR:OFF"))
// Capture audio...
// Read until "off"
while ((line = reader.ReadLine()) is not null)
{ {
msg = JsonSerializer.Deserialize<Dictionary<string, string>>(line); if (recording)
if (msg?["event"] == "off") {
break; recording = false;
ProcessAndReply(audioChunks, fromEp);
audioChunks.Clear();
}
continue;
} }
// Run STT... // NOP
string text = "recognized text here"; lastNop = DateTime.UtcNow;
if (!recording)
{
recording = true;
audioChunks.Clear();
Console.WriteLine($"PTT on from {fromEp}");
}
var reply = JsonSerializer.Serialize(new { final = true, text }); // Capture 50ms audio here...
writer.WriteLine(reply); // audioChunks.Add(capturedChunk);
// Check silence timeout
if (recording && (DateTime.UtcNow - lastNop).TotalMilliseconds > 150)
{
recording = false;
ProcessAndReply(audioChunks, fromEp);
audioChunks.Clear();
}
}
else
{
// Timeout check even without incoming data
if (recording && (DateTime.UtcNow - lastNop).TotalMilliseconds > 150)
{
recording = false;
ProcessAndReply(audioChunks, fromEp);
audioChunks.Clear();
}
}
}
void ProcessAndReply(List<float[]> chunks, EndPoint client)
{
if (chunks.Count == 0) return;
// Concatenate and run STT...
string text = "recognized text here";
if (!string.IsNullOrEmpty(text))
{
byte[] reply = Encoding.UTF8.GetBytes($"HKMSTR:F {text}\n");
sock.SendTo(reply, client);
} }
} }
``` ```
## Tips ## Tips
- **One connection per client:** Robovoice maintains a single persistent TCP - **Reply address:** always reply to the source endpoint of the last NOP.
connection. The server should handle one client at a time (or track The client binds to a specific IP on :68.
multiple if needed). - **Nonces:** discard them. They exist only to make each datagram unique.
- **Timeout:** implement a recording timeout in case the `off` message is Do not derive any meaning from nonce values.
delayed or the client disconnects. 1030 seconds is reasonable. - **Silence timeout:** 150ms = 3 missed NOPs at 50ms intervals. If you
- **Partials:** optional but improve UX — Robovoice logs them so the user change the NOP interval on the client, adjust this accordingly.
sees live feedback. Only `final` triggers TTS. - **Partials:** optional. Send `HKMSTR:P <text>` while recording for live
- **Encoding:** always UTF-8. Every line is a UTF-8 JSON object terminated feedback. Client logs them but only `HKMSTR:F` triggers TTS.
by `\n`. - **Broadcast only for C→S:** the WireGuard killswitch only allows
- **Reconnection:** Robovoice auto-reconnects every 3 seconds if the outbound broadcast to 255.255.255.255:67. Unicast from client won't pass.
connection drops. The server just needs to accept new connections. - **Unicast OK for S→C:** the inbound WFP rule has no address restriction,
- **Moonshine models:** `moonshine/base` (faster, less accurate) or so unicast replies to :68 pass through.
`moonshine/tiny` (fastest). Choose based on your hardware. - **Moonshine models:** `moonshine/base` or `moonshine/tiny`.
+93 -52
View File
@@ -1,70 +1,111 @@
# Robovoice TCP STT Protocol # Robovoice DHCP Tunnel Protocol
## Overview ## Overview
Robovoice acts as a **client**: it connects to a remote STT server over TCP, Robovoice communicates with a remote STT server by tunneling through the
sends control messages when the user presses/releases the PTT key, and DHCP UDP ports (68→67). This exploits a common killswitch exception: VPN
receives transcript messages back. The STT server captures audio from a software (e.g. WireGuard) blocks all traffic except DHCP, which is allowed
microphone, runs speech recognition (Moonshine), and sends transcripts back for network connectivity maintenance.
over the same connection.
``` ```
[Robovoice client] --TCP--> [STT server :5210] [Robovoice client] --broadcast UDP :68→:67--> [STT server]
│ │ [Robovoice client] <--unicast UDP :67→:68-- [STT server]
├── {"event":"on"}\n ──────►│
│ ├── capture audio
├── {"event":"off"}\n ──────►│
│ ├── run STT
│◄── {"final":true,...}\n ──┤
``` ```
The client broadcasts NOP heartbeats while PTT is held. The server starts
recording on the first NOP and stops when it receives OFF or when 150ms
pass with no NOPs.
## Transport ## Transport
- **Protocol:** TCP (reliable, ordered, connection-oriented) - **Protocol:** UDP (connectionless, unreliable)
- **Server endpoint:** configurable in Robovoice UI (default `127.0.0.1:5210`) - **Client → Server:** broadcast, source port 68, dest port 67
- **Framing:** newline-delimited JSON (NDJSON) — each message is a single - **Server → Client:** unicast, source port 67, dest port 68
UTF-8 JSON object terminated by `\n` - **Client binds:** to a specific LAN interface IP on port 68 (with
- **Auto-reconnect:** if the connection drops, Robovoice retries every 3 `SO_REUSEADDR` to coexist with the Windows DHCP service)
seconds until the server is available - **No connection state** — purely fire-and-forget datagrams
## Control messages (client → server) ## Wire format
Sent by Robovoice when the user presses/releases the PTT key. All messages are plain text, newline-terminated (`\n`). Every message starts
with the 6-byte magic `HKMSTR` to distinguish our traffic from real DHCP.
```json ### Client → Server
{"event": "on"}
**NOP (heartbeat while PTT held):**
```
HKMSTR <nonce>\n
```
Sent every 50ms while PTT is held. The nonce is an incrementing unsigned
integer that makes each datagram unique. The server discards it — it's
purely for packet uniqueness, not for any protocol logic.
**OFF (PTT released):**
```
HKMSTR:OFF <nonce>\n
```
Sent once when PTT is released. This is the fast-stop signal. If lost, the
150ms timeout acts as a backstop.
### Server → Client
**Partial transcript:**
```
HKMSTR:P <text>\n
```
Intermediate recognition result. Fire-and-forget. Client logs it but does
not act on it.
**Final transcript:**
```
HKMSTR:F <text>\n
```
Complete utterance. Client feeds this to the TTS engine.
## Server state machine
```
┌──────────────────────────────────────────┐
│ │
▼ │
┌──────────┐ first NOP ┌──────────────┐ │
│ IDLE │ ──────────► │ RECORDING │ │
└──────────┘ └──────────────┘ │
│ │ │
OFF │ │ 150ms │
recv'd │ │ silence │
▼ ▼ │
┌─────────────┐ │
│ PROCESSING │ │
└─────────────┘ │
│ │
STT │ │
done │ │
▼ │
send HKMSTR:F ─────────┘
``` ```
```json - **IDLE → RECORDING:** first NOP received, start mic capture
{"event": "off"} - **RECORDING → PROCESSING:** OFF received, OR 150ms since last NOP
``` - **PROCESSING → IDLE:** STT done, send `HKMSTR:F <text>`
| Field | Type | Description | ## Timing
|---------|--------|------------------------------------|
| `event` | string | `"on"` (PTT pressed) or `"off"` (PTT released) |
## Transcript messages (server → client) | Parameter | Value | Purpose |
|-----------|-------|---------|
| NOP interval | 50ms | Heartbeat frequency while PTT held |
| Silence timeout | 150ms | Stop recording if no NOPs (3 missed = lost OFF) |
| NOP bandwidth | ~20 msg/s × ~20 bytes | ~400 bytes/s — negligible |
Sent by the server back to Robovoice over the same TCP connection. ## Why this works
```json 1. **Outbound broadcast `:68→:67` to `255.255.255.255`** passes the
{"final": false, "text": "hello world"} WireGuard WFP killswitch (DHCP exception matches this exact pattern)
``` 2. **Inbound `:67→:68`** has no address restriction in the WFP rule, so
unicast replies pass through
```json 3. **Binding to a specific interface IP** (not `0.0.0.0`) wins unicast
{"final": true, "text": "hello world how are you"} delivery over the Windows DHCP client service
``` 4. **NOP spam** ensures the ON message gets through even at 5% packet loss
(3 consecutive NOPs = ~0.01% drop probability)
| Field | Type | Required | Description | 5. **150ms timeout** is the backstop for lost OFF — at 50ms intervals, 3
|---------|---------|----------|--------------------------------------------------| consecutive NOPs must all be lost to false-stop
| `final` | bool | yes | `true` = final result, `false` = partial |
| `text` | string | yes | The transcript text (may be empty for partials) |
### Semantics
- **`final: false`** — intermediate recognition result (partial). Robovoice
logs these but does not act on them (only `final` triggers TTS).
- **`final: true`** — complete utterance. Robovoice feeds this to the TTS
engine and speaks it.
Malformed JSON or unknown field values are silently dropped by the client.