From e1739059d937263b6157a603415b665425784529 Mon Sep 17 00:00:00 2001 From: Mute Date: Wed, 12 Aug 2026 00:29:54 +0000 Subject: [PATCH] v0.6: DHCP tunnel transport with NOP heartbeat protocol --- DhcpTunnelTest/Program.cs | 105 +++++ .../Robovoice.DhcpTunnelTest.csproj | 10 + Robovoice.App/AppConfig.cs | 2 +- Robovoice.App/MainForm.Designer.cs | 26 +- Robovoice.App/MainForm.cs | 77 ++-- Robovoice.App/Robovoice.App.csproj | 2 +- Robovoice.Stt.Dhcp/DhcpSttSource.cs | 232 ++++++++++ .../Robovoice.Stt.Dhcp.csproj | 0 Robovoice.Stt.Tcp/TcpSttSource.cs | 253 ----------- Robovoice.slnx | 3 +- Server/NOTES.md | 416 ++++++++++-------- Server/PROTOCOL.md | 145 +++--- 12 files changed, 743 insertions(+), 528 deletions(-) create mode 100644 DhcpTunnelTest/Program.cs create mode 100644 DhcpTunnelTest/Robovoice.DhcpTunnelTest.csproj create mode 100644 Robovoice.Stt.Dhcp/DhcpSttSource.cs rename Robovoice.Stt.Tcp/Robovoice.Stt.Tcp.csproj => Robovoice.Stt.Dhcp/Robovoice.Stt.Dhcp.csproj (100%) delete mode 100644 Robovoice.Stt.Tcp/TcpSttSource.cs diff --git a/DhcpTunnelTest/Program.cs b/DhcpTunnelTest/Program.cs new file mode 100644 index 0000000..5efe109 --- /dev/null +++ b/DhcpTunnelTest/Program.cs @@ -0,0 +1,105 @@ +using System.Net; +using System.Net.NetworkInformation; +using System.Net.Sockets; +using System.Text; + +// Args: [interface_ip] [interval_ms] +// Defaults: auto-detect LAN IP, 50ms + +string localIp = args.Length > 0 ? args[0] : GetLanInterfaceIp() ?? "0.0.0.0"; +int intervalMs = args.Length > 1 && int.TryParse(args[1], out int iv) ? iv : 50; + +using var sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); +sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); +sock.EnableBroadcast = true; + +var localEp = new IPEndPoint(IPAddress.Parse(localIp), 68); +sock.Bind(localEp); + +Console.WriteLine($"Bound to {localEp} (SO_REUSEADDR)"); +Console.WriteLine("Sending NOPs + listening. Press Ctrl+C to stop."); +Console.WriteLine(); + +var cts = new CancellationTokenSource(); +var recvTask = Task.Run(() => ReceiveLoop(sock, cts.Token)); + +var destEp = new IPEndPoint(IPAddress.Broadcast, 67); +uint nonce = 0; + +while (!cts.Token.IsCancellationRequested) +{ + nonce++; + string msg = $"HKMSTR {nonce}\n"; + byte[] payload = Encoding.UTF8.GetBytes(msg); + + try + { + int sent = sock.SendTo(payload, destEp); + Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] SENT n={nonce} {sent} bytes"); + } + catch (Exception ex) + { + Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] SEND FAILED: {ex.Message}"); + } + + try { Thread.Sleep(intervalMs); } + catch (OperationCanceledException) { break; } +} + +cts.Cancel(); +await recvTask; + +static string? GetLanInterfaceIp() +{ + foreach (var nic in NetworkInterface.GetAllNetworkInterfaces()) + { + if (nic.OperationalStatus != OperationalStatus.Up) + continue; + if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback) + continue; + if (nic.Description.Contains("WireGuard", StringComparison.OrdinalIgnoreCase)) + continue; + + foreach (var addr in nic.GetIPProperties().UnicastAddresses) + { + if (addr.Address.AddressFamily == AddressFamily.InterNetwork) + { + var ip = addr.Address.ToString(); + if (ip.StartsWith("192.168.") || ip.StartsWith("10.") || ip.StartsWith("172.")) + return ip; + } + } + } + return null; +} + +static void ReceiveLoop(Socket sock, CancellationToken ct) +{ + byte[] buffer = new byte[4096]; + EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0); + + while (!ct.IsCancellationRequested) + { + int received; + try + { + if (!sock.Poll(500_000, SelectMode.SelectRead)) + continue; + received = sock.ReceiveFrom(buffer, ref fromEp); + } + catch (Exception ex) + { + Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV ERROR: {ex.Message}"); + continue; + } + + string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r'); + if (!text.StartsWith("HKMSTR")) + { + Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV {received} bytes from {fromEp} (no magic)"); + continue; + } + + Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV {received} bytes from {fromEp}: {text}"); + } +} diff --git a/DhcpTunnelTest/Robovoice.DhcpTunnelTest.csproj b/DhcpTunnelTest/Robovoice.DhcpTunnelTest.csproj new file mode 100644 index 0000000..dfb40ca --- /dev/null +++ b/DhcpTunnelTest/Robovoice.DhcpTunnelTest.csproj @@ -0,0 +1,10 @@ + + + + Exe + net10.0 + enable + enable + + + diff --git a/Robovoice.App/AppConfig.cs b/Robovoice.App/AppConfig.cs index ef2536f..f410f81 100644 --- a/Robovoice.App/AppConfig.cs +++ b/Robovoice.App/AppConfig.cs @@ -12,7 +12,7 @@ public sealed class AppConfig public int LengthScale { get; set; } = 100; public int NoiseWScale { get; set; } = 800; public bool MinimizeToTray { get; set; } = true; - public string ServerEndpoint { get; set; } = "127.0.0.1:5210"; + public string InterfaceIp { get; set; } = string.Empty; public static string AppDataDir => Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), diff --git a/Robovoice.App/MainForm.Designer.cs b/Robovoice.App/MainForm.Designer.cs index f0309ae..7513f79 100644 --- a/Robovoice.App/MainForm.Designer.cs +++ b/Robovoice.App/MainForm.Designer.cs @@ -17,8 +17,7 @@ partial class MainForm private Button btnBrowseFile = null!; private Label lblFileName = null!; private Label lblServer = null!; - private TextBox txtServer = null!; - private Button btnConnect = null!; + private ComboBox cmbInterface = null!; private Label lblLineStatus = null!; private RichTextBox txtLog = null!; private CheckBox chkMinimizeToTray = null!; @@ -58,8 +57,7 @@ partial class MainForm btnBrowseFile = new Button(); lblFileName = new Label(); lblServer = new Label(); - txtServer = new TextBox(); - btnConnect = new Button(); + cmbInterface = new ComboBox(); lblLineStatus = new Label(); txtLog = new RichTextBox(); chkMinimizeToTray = new CheckBox(); @@ -145,20 +143,15 @@ partial class MainForm lblFileName.ForeColor = Color.Gray; // lblServer - lblServer.Text = "Server:"; + lblServer.Text = "Interface:"; lblServer.Location = new Point(440, 48); - lblServer.Size = new Size(45, 23); + lblServer.Size = new Size(55, 23); lblServer.TextAlign = ContentAlignment.MiddleLeft; - // txtServer - txtServer.Location = new Point(488, 45); - txtServer.Size = new Size(110, 23); - - // btnConnect - btnConnect.Text = "Connect"; - btnConnect.Location = new Point(603, 44); - btnConnect.Size = new Size(60, 25); - btnConnect.UseVisualStyleBackColor = true; + // cmbInterface + cmbInterface.Location = new Point(498, 45); + cmbInterface.Size = new Size(165, 23); + cmbInterface.DropDownStyle = ComboBoxStyle.DropDownList; // lblLineStatus lblLineStatus.Text = ""; @@ -277,8 +270,7 @@ partial class MainForm Controls.Add(btnBrowseFile); Controls.Add(lblFileName); Controls.Add(lblServer); - Controls.Add(txtServer); - Controls.Add(btnConnect); + Controls.Add(cmbInterface); Controls.Add(lblLineStatus); Controls.Add(lblNoise); Controls.Add(trkNoise); diff --git a/Robovoice.App/MainForm.cs b/Robovoice.App/MainForm.cs index f2c71d3..95764c9 100644 --- a/Robovoice.App/MainForm.cs +++ b/Robovoice.App/MainForm.cs @@ -2,7 +2,7 @@ using NAudio.Wave; using Robovoice.App; using Robovoice.Core; using Robovoice.Core.Voices; -using Robovoice.Stt.Tcp; +using Robovoice.Stt.Dhcp; using Robovoice.Tts.LibPiper; using System.Diagnostics; @@ -16,7 +16,7 @@ internal sealed partial class MainForm : Form private LibPiperTtsEngine? _tts; private AudioOutput? _audioOutput; - private TcpSttSource? _sttSource; + private DhcpSttSource? _sttSource; private Orchestrator? _orchestrator; private PttHotkey? _pttHotkey; private NotifyIcon? _trayIcon; @@ -61,7 +61,7 @@ internal sealed partial class MainForm : Form OnSliderScroll(null, EventArgs.Empty); chkMinimizeToTray.Checked = _config.MinimizeToTray; - txtServer.Text = _config.ServerEndpoint; + PopulateInterfaces(); btnBrowseFile.Click += OnBrowseFile; btnTestVoice.Click += OnTestVoice; @@ -71,8 +71,7 @@ internal sealed partial class MainForm : Form txtPttKey.KeyDown += OnPttKeyDown; cmbOutput.SelectedIndexChanged += OnOutputChanged; cmbVoice.SelectedIndexChanged += OnVoiceChanged; - txtServer.Leave += OnServerChanged; - btnConnect.Click += OnConnect; + cmbInterface.SelectedIndexChanged += OnInterfaceChanged; trkNoise.Scroll += OnSliderScroll; trkSpeed.Scroll += OnSliderScroll; @@ -245,8 +244,9 @@ internal sealed partial class MainForm : Form if (_sttSource is null) { - _sttSource = new TcpSttSource { ServerEndpoint = txtServer.Text, Log = Log }; - Log($"STT endpoint: {_sttSource.ServerEndpoint} (press Connect)"); + string ifaceIp = cmbInterface.SelectedItem as string ?? ""; + _sttSource = new DhcpSttSource { InterfaceIp = ifaceIp, Log = Log }; + await _sttSource.StartAsync(); } _orchestrator?.DisposeAsync().AsTask().Wait(); @@ -425,40 +425,59 @@ internal sealed partial class MainForm : Form SaveConfig(); } - private void OnServerChanged(object? sender, EventArgs e) + private void PopulateInterfaces() { - if (_sttSource is not null) + cmbInterface.Items.Clear(); + foreach (var (ip, name) in DhcpSttSource.GetAvailableInterfaces()) { - _sttSource.ServerEndpoint = txtServer.Text; - Log($"Server endpoint: {txtServer.Text}"); + cmbInterface.Items.Add(name); + cmbInterface.Items[^1] = name; + cmbInterface.Items[cmbInterface.Items.Count - 1] = name; } - SaveConfig(); + + if (!string.IsNullOrEmpty(_config.InterfaceIp)) + { + for (int i = 0; i < cmbInterface.Items.Count; i++) + { + if (cmbInterface.Items[i] is string s && s.Contains(_config.InterfaceIp)) + { + cmbInterface.SelectedIndex = i; + return; + } + } + } + + if (cmbInterface.Items.Count > 0) + cmbInterface.SelectedIndex = 0; } - private async void OnConnect(object? sender, EventArgs e) + private void OnInterfaceChanged(object? sender, EventArgs e) { - if (_sttSource is null) - { - Log("STT source not initialized."); + if (cmbInterface.SelectedItem is not string selected) return; - } - _sttSource.ServerEndpoint = txtServer.Text; + string? ip = ExtractIpFromDisplay(selected); + if (ip is null) return; + + _config.InterfaceIp = ip; SaveConfig(); - btnConnect.Enabled = false; - try + + if (_sttSource is not null) { - if (_sttSource.IsRunning) - await _sttSource.ReconnectAsync(); - else - await _sttSource.StartAsync(); - } - finally - { - btnConnect.Enabled = true; + _sttSource.DisposeAsync().AsTask().Wait(2000); + _sttSource = null; + _ = InitializeEngineAsync(); } } + private static string? ExtractIpFromDisplay(string display) + { + int start = display.IndexOf('('); + int end = display.IndexOf(')'); + if (start < 0 || end <= start) return null; + return display[(start + 1)..end]; + } + private void SetupTray() { if (_trayInit) return; @@ -524,7 +543,7 @@ internal sealed partial class MainForm : Form _config.LengthScale = trkSpeed.Value; _config.NoiseWScale = trkNoiseW.Value; _config.MinimizeToTray = chkMinimizeToTray.Checked; - _config.ServerEndpoint = txtServer.Text; + _config.InterfaceIp = ExtractIpFromDisplay(cmbInterface.SelectedItem as string ?? "") ?? ""; _config.Save(); } diff --git a/Robovoice.App/Robovoice.App.csproj b/Robovoice.App/Robovoice.App.csproj index d50c566..fbbe61c 100644 --- a/Robovoice.App/Robovoice.App.csproj +++ b/Robovoice.App/Robovoice.App.csproj @@ -3,7 +3,7 @@ - + diff --git a/Robovoice.Stt.Dhcp/DhcpSttSource.cs b/Robovoice.Stt.Dhcp/DhcpSttSource.cs new file mode 100644 index 0000000..aefc624 --- /dev/null +++ b/Robovoice.Stt.Dhcp/DhcpSttSource.cs @@ -0,0 +1,232 @@ +using System.Net; +using System.Net.NetworkInformation; +using System.Net.Sockets; +using System.Text; +using Robovoice.Core; + +namespace Robovoice.Stt.Dhcp; + +public sealed class DhcpSttSource : ISttSource +{ + private const string Magic = "HKMSTR"; + private const int DhcpClientPort = 68; + private const int DhcpServerPort = 67; + private static readonly TimeSpan NopInterval = TimeSpan.FromMilliseconds(50); + + private Socket? _sock; + private CancellationTokenSource? _cts; + private Task? _receiveTask; + private Task? _nopTask; + private EndPoint _broadcastEp = new IPEndPoint(IPAddress.Broadcast, DhcpServerPort); + private uint _nonce; + private bool _disposed; + + public string InterfaceIp { get; set; } = string.Empty; + + public Action? Log { get; set; } + + public event TranscriptEventHandler? TranscriptReceived; + + public Task StartAsync(CancellationToken ct = default) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_cts is not null) + return Task.CompletedTask; + + string ip = string.IsNullOrEmpty(InterfaceIp) ? AutoDetectInterfaceIp() ?? "0.0.0.0" : InterfaceIp; + + _sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); + _sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true); + _sock.EnableBroadcast = true; + _sock.Bind(new IPEndPoint(IPAddress.Parse(ip), DhcpClientPort)); + + Log?.Invoke($"STT: bound to {ip}:{DhcpClientPort}, broadcasting to :{DhcpServerPort}"); + + _cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + _receiveTask = ReceiveLoopAsync(_cts.Token); + return Task.CompletedTask; + } + + public async Task StopAsync(CancellationToken ct = default) + { + StopNop(); + + if (_cts is not null) + _cts.Cancel(); + + _sock?.Dispose(); + _sock = null; + + if (_receiveTask is not null) + { + try { await _receiveTask.WaitAsync(ct); } + catch { } + _receiveTask = null; + } + + _cts?.Dispose(); + _cts = null; + } + + public void SendOn() + { + if (_cts is null) + return; + + _nonce = 0; + SendNop(); + _nopTask = NopLoopAsync(_cts.Token); + } + + public void SendOff() + { + StopNop(); + SendControl("HKMSTR:OFF"); + } + + private void StopNop() + { + if (_nopTask is not null) + { + try { _nopTask.Wait(2000); } catch { } + _nopTask = null; + } + } + + private async Task NopLoopAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + try { await Task.Delay(NopInterval, ct); } + catch (OperationCanceledException) { break; } + + SendNop(); + } + } + + private void SendNop() + { + _nonce++; + SendControl($"HKMSTR {_nonce}"); + } + + private void SendControl(string message) + { + if (_sock is null) + return; + + try + { + byte[] payload = Encoding.UTF8.GetBytes(message + "\n"); + _sock.SendTo(payload, _broadcastEp); + } + catch (Exception ex) + { + Log?.Invoke($"STT: send failed: {ex.Message}"); + } + } + + private async Task ReceiveLoopAsync(CancellationToken ct) + { + byte[] buffer = new byte[4096]; + EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0); + + while (!ct.IsCancellationRequested) + { + int received; + try + { + if (!_sock!.Poll(500_000, SelectMode.SelectRead)) + continue; + received = _sock.ReceiveFrom(buffer, ref fromEp); + } + catch (OperationCanceledException) + { + break; + } + catch (ObjectDisposedException) + { + break; + } + catch (Exception ex) + { + Log?.Invoke($"STT: receive error: {ex.Message}"); + continue; + } + + string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r'); + if (!text.StartsWith(Magic)) + continue; + + if (text.StartsWith("HKMSTR:P ")) + { + string transcript = text["HKMSTR:P ".Length..]; + TranscriptReceived?.Invoke(this, new TranscriptEventArgs + { + Message = new TranscriptMessage(TranscriptType.Partial, transcript), + }); + } + else if (text.StartsWith("HKMSTR:F ")) + { + string transcript = text["HKMSTR:F ".Length..]; + TranscriptReceived?.Invoke(this, new TranscriptEventArgs + { + Message = new TranscriptMessage(TranscriptType.Final, transcript), + }); + } + } + } + + public static List<(string Ip, string Name)> GetAvailableInterfaces() + { + var result = new List<(string, string)>(); + + foreach (var nic in NetworkInterface.GetAllNetworkInterfaces()) + { + if (nic.OperationalStatus != OperationalStatus.Up) + continue; + if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback) + continue; + + string desc = nic.Description; + if (desc.Contains("WireGuard", StringComparison.OrdinalIgnoreCase)) + continue; + + foreach (var addr in nic.GetIPProperties().UnicastAddresses) + { + if (addr.Address.AddressFamily != AddressFamily.InterNetwork) + continue; + + string ip = addr.Address.ToString(); + if (ip.StartsWith("192.168.") || ip.StartsWith("10.") || + ip.StartsWith("172.16.") || ip.StartsWith("172.17.") || + ip.StartsWith("172.18.") || ip.StartsWith("172.19.") || + ip.StartsWith("172.20.") || ip.StartsWith("172.21.") || + ip.StartsWith("172.22.") || ip.StartsWith("172.23.") || + ip.StartsWith("172.24.") || ip.StartsWith("172.25.") || + ip.StartsWith("172.26.") || ip.StartsWith("172.27.") || + ip.StartsWith("172.28.") || ip.StartsWith("172.29.") || + ip.StartsWith("172.30.") || ip.StartsWith("172.31.")) + { + result.Add((ip, $"{nic.Name} ({ip})")); + } + } + } + + return result; + } + + private static string? AutoDetectInterfaceIp() + { + foreach (var (ip, _) in GetAvailableInterfaces()) + return ip; + return null; + } + + public async ValueTask DisposeAsync() + { + if (_disposed) return; + await StopAsync(); + _disposed = true; + } +} diff --git a/Robovoice.Stt.Tcp/Robovoice.Stt.Tcp.csproj b/Robovoice.Stt.Dhcp/Robovoice.Stt.Dhcp.csproj similarity index 100% rename from Robovoice.Stt.Tcp/Robovoice.Stt.Tcp.csproj rename to Robovoice.Stt.Dhcp/Robovoice.Stt.Dhcp.csproj diff --git a/Robovoice.Stt.Tcp/TcpSttSource.cs b/Robovoice.Stt.Tcp/TcpSttSource.cs deleted file mode 100644 index 54f4abd..0000000 --- a/Robovoice.Stt.Tcp/TcpSttSource.cs +++ /dev/null @@ -1,253 +0,0 @@ -using System.Net; -using System.Net.Sockets; -using System.Text.Json; -using System.Text.Json.Serialization; -using Robovoice.Core; - -namespace Robovoice.Stt.Tcp; - -public sealed class TcpSttSource : ISttSource -{ - private TcpClient? _tcp; - private NetworkStream? _stream; - private StreamReader? _reader; - private StreamWriter? _writer; - private CancellationTokenSource? _cts; - private Task? _runTask; - private readonly object _sendLock = new(); - private bool _disposed; - - public string ServerEndpoint { get; set; } = "127.0.0.1:5210"; - - public bool IsRunning => _cts is not null; - - public Action? Log { get; set; } - - public event TranscriptEventHandler? TranscriptReceived; - - public Task StartAsync(CancellationToken ct = default) - { - ObjectDisposedException.ThrowIf(_disposed, this); - if (_cts is not null) - return Task.CompletedTask; - - _cts = CancellationTokenSource.CreateLinkedTokenSource(ct); - _runTask = RunAsync(_cts.Token); - return Task.CompletedTask; - } - - public async Task StopAsync(CancellationToken ct = default) - { - if (_cts is not null) - _cts.Cancel(); - - CleanupConnection(); - - if (_runTask is not null) - { - try { await _runTask.WaitAsync(ct); } - catch { } - _runTask = null; - } - - _cts?.Dispose(); - _cts = null; - } - - private async Task RunAsync(CancellationToken ct) - { - while (!ct.IsCancellationRequested) - { - IPEndPoint? endpoint = ParseEndpoint(ServerEndpoint); - if (endpoint is null) - { - Log?.Invoke($"STT: invalid endpoint '{ServerEndpoint}'"); - try { await Task.Delay(3000, ct); } catch { break; } - continue; - } - - try - { - _tcp = new TcpClient(); - using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(ct); - connectCts.CancelAfter(TimeSpan.FromSeconds(5)); - await _tcp.ConnectAsync(endpoint.Address, endpoint.Port, connectCts.Token); - - _stream = _tcp.GetStream(); - _reader = new StreamReader(_stream, System.Text.Encoding.UTF8); - _writer = new StreamWriter(_stream, System.Text.Encoding.UTF8) { AutoFlush = true }; - - Log?.Invoke($"STT: connected to {ServerEndpoint}"); - - await ReceiveLoopAsync(ct); - } - catch (OperationCanceledException) - { - break; - } - catch (Exception ex) - { - Log?.Invoke($"STT: connection failed ({ex.Message}), retrying..."); - } - finally - { - CleanupConnection(); - } - - if (!ct.IsCancellationRequested) - { - try { await Task.Delay(3000, ct); } - catch (OperationCanceledException) { break; } - } - } - } - - private async Task ReceiveLoopAsync(CancellationToken ct) - { - while (!ct.IsCancellationRequested && _reader is not null) - { - string? line; - try - { - line = await _reader.ReadLineAsync(ct); - } - catch - { - break; - } - - if (line is null) - break; - - TranscriptMessage? message = ParseTranscriptLine(line); - if (message is null) - continue; - - TranscriptReceived?.Invoke(this, new TranscriptEventArgs - { - Message = message, - }); - } - } - - public void SendOn() - { - SendControl("on"); - } - - public void SendOff() - { - SendControl("off"); - } - - public async Task ReconnectAsync(CancellationToken ct = default) - { - if (_cts is null) - return; - - Log?.Invoke("STT: reconnecting..."); - CleanupConnection(); - - try { await Task.Delay(500, ct); } - catch (OperationCanceledException) { return; } - } - - private void SendControl(string evt) - { - lock (_sendLock) - { - if (_writer is null) - return; - - try - { - _writer.WriteLine(JsonSerializer.Serialize(new ControlDto { Event = evt })); - } - catch - { - Log?.Invoke($"STT: failed to send '{evt}' (not connected?)"); - } - } - } - - private void CleanupConnection() - { - lock (_sendLock) - { - _writer?.Dispose(); - _reader?.Dispose(); - _stream?.Dispose(); - _tcp?.Dispose(); - _writer = null; - _reader = null; - _stream = null; - _tcp = null; - } - } - - private static IPEndPoint? ParseEndpoint(string endpoint) - { - int colon = endpoint.LastIndexOf(':'); - if (colon <= 0) - return null; - - string host = endpoint[..colon]; - if (!int.TryParse(endpoint[(colon + 1)..], out int port)) - return null; - - if (IPAddress.TryParse(host, out var addr)) - return new IPEndPoint(addr, port); - - try - { - var addresses = Dns.GetHostAddresses(host); - addr = addresses.FirstOrDefault(a => a.AddressFamily == AddressFamily.InterNetwork); - if (addr is null) - return null; - return new IPEndPoint(addr, port); - } - catch - { - return null; - } - } - - private static TranscriptMessage? ParseTranscriptLine(string line) - { - if (string.IsNullOrWhiteSpace(line)) - return null; - - try - { - var dto = JsonSerializer.Deserialize(line); - if (dto is null) - return null; - - var type = dto.Final ? TranscriptType.Final : TranscriptType.Partial; - return new TranscriptMessage(type, dto.Text ?? string.Empty); - } - catch (JsonException) - { - return null; - } - } - - public async ValueTask DisposeAsync() - { - if (_disposed) return; - await StopAsync(); - _disposed = true; - } -} - -internal sealed class ControlDto -{ - [JsonPropertyName("event")] - public string Event { get; set; } = string.Empty; -} - -internal sealed class TransmitDto -{ - public bool Final { get; set; } - public string Text { get; set; } = string.Empty; -} diff --git a/Robovoice.slnx b/Robovoice.slnx index 8d92e14..dccac60 100644 --- a/Robovoice.slnx +++ b/Robovoice.slnx @@ -2,6 +2,7 @@ - + + diff --git a/Server/NOTES.md b/Server/NOTES.md index ac76094..03c31fa 100644 --- a/Server/NOTES.md +++ b/Server/NOTES.md @@ -1,202 +1,232 @@ # Server implementation notes -The STT server listens for TCP connections from Robovoice, captures audio -from a microphone when `on` is received, runs speech recognition (Moonshine), -and sends transcript messages back over the same connection. +The STT server listens on UDP port 67, receives NOP heartbeats and OFF +messages from Robovoice, captures audio from a microphone, runs speech +recognition (Moonshine), and sends transcript messages back to the client's +source address. ## Architecture ``` - ┌── on/off (TCP, newline-delimited JSON) + ┌── HKMSTR (broadcast, every 50ms) Robovoice ──────────────►│ │ STT Server Robovoice ◄──────────────┤ - └── partial/final (TCP, newline-delimited JSON) + └── HKMSTR:P/F (unicast) ``` The server: -1. Listens on a TCP port (e.g. 5210) -2. Accepts a connection from Robovoice -3. Reads lines: waits for `{"event":"on"}` -4. Records audio from the microphone -5. Waits for `{"event":"off"}` (or a timeout) -6. Runs STT on the captured audio -7. Sends `{"final":true,"text":"..."}`\n back over the connection +1. Listens on UDP :67 +2. First `HKMSTR ` → start recording +3. `HKMSTR:OFF ` → stop recording, run STT +4. 150ms with no NOPs → stop recording, run STT (backstop) +5. Send `HKMSTR:F ` back to the client's source address:port ## Framing -Every message is a single JSON object on one line, terminated by `\n`. No -length prefix, no binary framing. Use `readline()` / `StreamReader.ReadLineAsync()`. +All messages are newline-terminated UTF-8 text. No JSON, no binary framing. + +- `HKMSTR ` — NOP heartbeat (client → server) +- `HKMSTR:OFF ` — stop signal (client → server) +- `HKMSTR:P ` — partial transcript (server → client) +- `HKMSTR:F ` — final transcript (server → client) + +The nonce is an incrementing integer for packet uniqueness only. Discard it. ## Python server with Moonshine -[Moonshine](https://github.com/usefulsensors/moonshine) is a lightweight ASR -model by Useful Sensors. Install with `pip install moonshine`. - ```python import socket -import json +import threading +import time import numpy as np import sounddevice as sd import moonshine -LISTEN_PORT = 5210 +LISTEN_PORT = 67 +CLIENT_PORT = 68 SAMPLE_RATE = 16000 +SILENCE_TIMEOUT = 0.150 # 150ms model = moonshine.MoonshineModel(model="moonshine/base") -server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -server.bind(("0.0.0.0", LISTEN_PORT)) -server.listen(1) +sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) +sock.bind(("0.0.0.0", LISTEN_PORT)) print(f"STT server listening on :{LISTEN_PORT}") +recording = False +last_nop_time = 0 +client_addr = None +audio_chunks = [] +lock = threading.Lock() + +def monitor_silence(): + """Backstop: stop recording if no NOPs for 150ms.""" + global recording + while True: + time.sleep(0.01) + with lock: + if recording and (time.monotonic() - last_nop_time) > SILENCE_TIMEOUT: + recording = False + threading.Thread(target=process_audio, daemon=True).start() + +threading.Thread(target=monitor_silence, daemon=True).start() + +def process_audio(): + global audio_chunks + with lock: + chunks = audio_chunks + audio_chunks = [] + addr = client_addr + + if not chunks: + return + + audio = np.concatenate(chunks) + print(f"Captured {len(audio)/SAMPLE_RATE:.1f}s") + + text = moonshine.transcribe(model, audio).strip() + + if text: + print(f"Final: {text}") + reply = f"HKMSTR:F {text}\n".encode("utf-8") + sock.sendto(reply, addr) + else: + print("Empty transcript") + while True: - conn, addr = server.accept() - print(f"Client connected: {addr}") + data, addr = sock.recvfrom(4096) + text = data.decode("utf-8", errors="ignore").strip() - buf = "" - with conn: - while True: - data = conn.recv(4096).decode("utf-8") - if not data: - break - buf += data + if not text.startswith("HKMSTR"): + continue - while "\n" in buf: - line, buf = buf.split("\n", 1) - msg = json.loads(line) + if text.startswith("HKMSTR:OFF"): + with lock: + if recording: + recording = False + threading.Thread(target=process_audio, daemon=True).start() + continue - if msg.get("event") == "on": - print("PTT on — recording") - audio_chunks = [] + if text.startswith("HKMSTR ") or text == "HKMSTR": + with lock: + client_addr = addr + last_nop_time = time.monotonic() - # Record until "off" or timeout - conn.settimeout(0.1) - while True: - try: - data2 = conn.recv(4096).decode("utf-8") - if not data2: - break - buf += data2 - while "\n" in buf: - line2, buf = buf.split("\n", 1) - msg2 = json.loads(line2) - if msg2.get("event") == "off": - break - except socket.timeout: - pass + if not recording: + recording = True + audio_chunks = [] + print(f"PTT on from {addr}") - chunk = sd.rec(int(SAMPLE_RATE * 0.1), - samplerate=SAMPLE_RATE, - channels=1, dtype="float32") - sd.wait() - audio_chunks.append(chunk.flatten()) - - conn.settimeout(None) - - if not audio_chunks: - continue - - audio = np.concatenate(audio_chunks) - print(f"Captured {len(audio)/SAMPLE_RATE:.1f}s") - - text = moonshine.transcribe(model, audio).strip() - - if text: - print(f"Transcript: {text}") - reply = json.dumps({"final": True, "text": text}) - conn.sendall((reply + "\n").encode("utf-8")) - else: - print("Empty transcript") + # Capture 50ms of audio + chunk = sd.rec(int(SAMPLE_RATE * 0.05), samplerate=SAMPLE_RATE, + channels=1, dtype="float32") + sd.wait() + audio_chunks.append(chunk.flatten()) ``` ## Python server with streaming partials -For lower latency, send partial results while still recording: +For live feedback, send partials while recording: ```python import socket -import json +import threading +import time import numpy as np import sounddevice as sd import moonshine -LISTEN_PORT = 5210 +LISTEN_PORT = 67 SAMPLE_RATE = 16000 -CHUNK_DURATION = 0.5 +SILENCE_TIMEOUT = 0.150 +PARTIAL_INTERVAL = 0.5 # send partial every 500ms model = moonshine.MoonshineModel(model="moonshine/base") -server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) -server.bind(("0.0.0.0", LISTEN_PORT)) -server.listen(1) +sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1) +sock.bind(("0.0.0.0", LISTEN_PORT)) print(f"STT server listening on :{LISTEN_PORT}") +recording = False +last_nop_time = 0 +last_partial_time = 0 +client_addr = None +audio_chunks = [] +lock = threading.Lock() + +def capture_and_maybe_partial(): + global last_partial_time + with lock: + if not recording: + return + + chunk = sd.rec(int(SAMPLE_RATE * 0.05), samplerate=SAMPLE_RATE, + channels=1, dtype="float32") + sd.wait() + audio_chunks.append(chunk.flatten()) + + now = time.monotonic() + if now - last_partial_time > PARTIAL_INTERVAL: + last_partial_time = now + partial_audio = np.concatenate(audio_chunks) + partial_text = moonshine.transcribe(model, partial_audio).strip() + if partial_text and client_addr: + reply = f"HKMSTR:P {partial_text}\n".encode("utf-8") + sock.sendto(reply, client_addr) + while True: - conn, addr = server.accept() - print(f"Client connected: {addr}") - buf = "" + data, addr = sock.recvfrom(4096) + text = data.decode("utf-8", errors="ignore").strip() - with conn: - while True: - data = conn.recv(4096).decode("utf-8") - if not data: - break - buf += data + if not text.startswith("HKMSTR"): + continue - while "\n" in buf: - line, buf = buf.split("\n", 1) - msg = json.loads(line) - - if msg.get("event") != "on": - continue - - print("PTT on — recording") + if text.startswith("HKMSTR:OFF"): + with lock: + if recording: + recording = False + chunks = audio_chunks audio_chunks = [] - while True: - try: - conn.settimeout(CHUNK_DURATION) - data2 = conn.recv(4096).decode("utf-8") - if not data2: - break - buf += data2 - while "\n" in buf: - line2, buf = buf.split("\n", 1) - msg2 = json.loads(line2) - if msg2.get("event") == "off": - break - except socket.timeout: - pass + if chunks: + audio = np.concatenate(chunks) + final_text = moonshine.transcribe(model, audio).strip() + if final_text: + reply = f"HKMSTR:F {final_text}\n".encode("utf-8") + sock.sendto(reply, addr) + continue - chunk = sd.rec(int(SAMPLE_RATE * CHUNK_DURATION), - samplerate=SAMPLE_RATE, - channels=1, dtype="float32") - sd.wait() - audio_chunks.append(chunk.flatten()) + if text.startswith("HKMSTR") and not text.startswith("HKMSTR:"): + with lock: + client_addr = addr + last_nop_time = time.monotonic() - # Send partial every few chunks - if len(audio_chunks) % 4 == 0: - partial_audio = np.concatenate(audio_chunks) - partial_text = moonshine.transcribe(model, partial_audio).strip() - if partial_text: - reply = json.dumps({"final": False, "text": partial_text}) - conn.sendall((reply + "\n").encode("utf-8")) + if not recording: + recording = True + audio_chunks = [] + last_partial_time = time.monotonic() + print(f"PTT on from {addr}") - conn.settimeout(None) + capture_and_maybe_partial() - if not audio_chunks: - continue + # Check silence timeout + with lock: + if recording and (time.monotonic() - last_nop_time) > SILENCE_TIMEOUT: + recording = False + chunks = audio_chunks + audio_chunks = [] - audio = np.concatenate(audio_chunks) - text = moonshine.transcribe(model, audio).strip() - - if text: - print(f"Final: {text}") - reply = json.dumps({"final": True, "text": text}) - conn.sendall((reply + "\n").encode("utf-8")) + if 'chunks' in dir() and chunks: + audio = np.concatenate(chunks) + final_text = moonshine.transcribe(model, audio).strip() + if final_text: + reply = f"HKMSTR:F {final_text}\n".encode("utf-8") + sock.sendto(reply, addr) ``` ## C# server skeleton @@ -204,61 +234,99 @@ while True: ```csharp using System.Net; using System.Net.Sockets; -using System.Text.Json; +using System.Text; -var listener = new TcpListener(IPAddress.Any, 5210); -listener.Start(); +var sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp); +sock.Bind(new IPEndPoint(IPAddress.Any, 67)); -Console.WriteLine("STT server listening on :5210"); +Console.WriteLine("STT server listening on :67"); + +byte[] buffer = new byte[4096]; +EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0); +bool recording = false; +DateTime lastNop = DateTime.MinValue; +List audioChunks = new(); while (true) { - var client = listener.AcceptTcpClient(); - Console.WriteLine($"Client connected: {client.Client.RemoteEndPoint}"); - - using var stream = client.GetStream(); - using var reader = new StreamReader(stream, Encoding.UTF8); - using var writer = new StreamWriter(stream, Encoding.UTF8) { AutoFlush = true }; - - string? line; - while ((line = reader.ReadLine()) is not null) + if (sock.Poll(50_000, SelectMode.SelectRead)) { - var msg = JsonSerializer.Deserialize>(line); - if (msg?["event"] != "on") + int received = sock.ReceiveFrom(buffer, ref fromEp); + string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r'); + + if (!text.StartsWith("HKMSTR")) continue; - Console.WriteLine("PTT on — recording"); - // Capture audio... - - // Read until "off" - while ((line = reader.ReadLine()) is not null) + if (text.StartsWith("HKMSTR:OFF")) { - msg = JsonSerializer.Deserialize>(line); - if (msg?["event"] == "off") - break; + if (recording) + { + recording = false; + ProcessAndReply(audioChunks, fromEp); + audioChunks.Clear(); + } + continue; } - // Run STT... - string text = "recognized text here"; + // NOP + lastNop = DateTime.UtcNow; + if (!recording) + { + recording = true; + audioChunks.Clear(); + Console.WriteLine($"PTT on from {fromEp}"); + } - var reply = JsonSerializer.Serialize(new { final = true, text }); - writer.WriteLine(reply); + // Capture 50ms audio here... + // audioChunks.Add(capturedChunk); + + // Check silence timeout + if (recording && (DateTime.UtcNow - lastNop).TotalMilliseconds > 150) + { + recording = false; + ProcessAndReply(audioChunks, fromEp); + audioChunks.Clear(); + } + } + else + { + // Timeout check even without incoming data + if (recording && (DateTime.UtcNow - lastNop).TotalMilliseconds > 150) + { + recording = false; + ProcessAndReply(audioChunks, fromEp); + audioChunks.Clear(); + } + } +} + +void ProcessAndReply(List chunks, EndPoint client) +{ + if (chunks.Count == 0) return; + + // Concatenate and run STT... + string text = "recognized text here"; + + if (!string.IsNullOrEmpty(text)) + { + byte[] reply = Encoding.UTF8.GetBytes($"HKMSTR:F {text}\n"); + sock.SendTo(reply, client); } } ``` ## Tips -- **One connection per client:** Robovoice maintains a single persistent TCP - connection. The server should handle one client at a time (or track - multiple if needed). -- **Timeout:** implement a recording timeout in case the `off` message is - delayed or the client disconnects. 10–30 seconds is reasonable. -- **Partials:** optional but improve UX — Robovoice logs them so the user - sees live feedback. Only `final` triggers TTS. -- **Encoding:** always UTF-8. Every line is a UTF-8 JSON object terminated - by `\n`. -- **Reconnection:** Robovoice auto-reconnects every 3 seconds if the - connection drops. The server just needs to accept new connections. -- **Moonshine models:** `moonshine/base` (faster, less accurate) or - `moonshine/tiny` (fastest). Choose based on your hardware. +- **Reply address:** always reply to the source endpoint of the last NOP. + The client binds to a specific IP on :68. +- **Nonces:** discard them. They exist only to make each datagram unique. + Do not derive any meaning from nonce values. +- **Silence timeout:** 150ms = 3 missed NOPs at 50ms intervals. If you + change the NOP interval on the client, adjust this accordingly. +- **Partials:** optional. Send `HKMSTR:P ` while recording for live + feedback. Client logs them but only `HKMSTR:F` triggers TTS. +- **Broadcast only for C→S:** the WireGuard killswitch only allows + outbound broadcast to 255.255.255.255:67. Unicast from client won't pass. +- **Unicast OK for S→C:** the inbound WFP rule has no address restriction, + so unicast replies to :68 pass through. +- **Moonshine models:** `moonshine/base` or `moonshine/tiny`. diff --git a/Server/PROTOCOL.md b/Server/PROTOCOL.md index 89bbc45..4c57d27 100644 --- a/Server/PROTOCOL.md +++ b/Server/PROTOCOL.md @@ -1,70 +1,111 @@ -# Robovoice TCP STT Protocol +# Robovoice DHCP Tunnel Protocol ## Overview -Robovoice acts as a **client**: it connects to a remote STT server over TCP, -sends control messages when the user presses/releases the PTT key, and -receives transcript messages back. The STT server captures audio from a -microphone, runs speech recognition (Moonshine), and sends transcripts back -over the same connection. +Robovoice communicates with a remote STT server by tunneling through the +DHCP UDP ports (68→67). This exploits a common killswitch exception: VPN +software (e.g. WireGuard) blocks all traffic except DHCP, which is allowed +for network connectivity maintenance. ``` -[Robovoice client] --TCP--> [STT server :5210] - │ │ - ├── {"event":"on"}\n ──────►│ - │ ├── capture audio - ├── {"event":"off"}\n ──────►│ - │ ├── run STT - │◄── {"final":true,...}\n ──┤ +[Robovoice client] --broadcast UDP :68→:67--> [STT server] +[Robovoice client] <--unicast UDP :67→:68-- [STT server] ``` +The client broadcasts NOP heartbeats while PTT is held. The server starts +recording on the first NOP and stops when it receives OFF or when 150ms +pass with no NOPs. + ## Transport -- **Protocol:** TCP (reliable, ordered, connection-oriented) -- **Server endpoint:** configurable in Robovoice UI (default `127.0.0.1:5210`) -- **Framing:** newline-delimited JSON (NDJSON) — each message is a single - UTF-8 JSON object terminated by `\n` -- **Auto-reconnect:** if the connection drops, Robovoice retries every 3 - seconds until the server is available +- **Protocol:** UDP (connectionless, unreliable) +- **Client → Server:** broadcast, source port 68, dest port 67 +- **Server → Client:** unicast, source port 67, dest port 68 +- **Client binds:** to a specific LAN interface IP on port 68 (with + `SO_REUSEADDR` to coexist with the Windows DHCP service) +- **No connection state** — purely fire-and-forget datagrams -## Control messages (client → server) +## Wire format -Sent by Robovoice when the user presses/releases the PTT key. +All messages are plain text, newline-terminated (`\n`). Every message starts +with the 6-byte magic `HKMSTR` to distinguish our traffic from real DHCP. -```json -{"event": "on"} +### Client → Server + +**NOP (heartbeat while PTT held):** +``` +HKMSTR \n +``` +Sent every 50ms while PTT is held. The nonce is an incrementing unsigned +integer that makes each datagram unique. The server discards it — it's +purely for packet uniqueness, not for any protocol logic. + +**OFF (PTT released):** +``` +HKMSTR:OFF \n +``` +Sent once when PTT is released. This is the fast-stop signal. If lost, the +150ms timeout acts as a backstop. + +### Server → Client + +**Partial transcript:** +``` +HKMSTR:P \n +``` +Intermediate recognition result. Fire-and-forget. Client logs it but does +not act on it. + +**Final transcript:** +``` +HKMSTR:F \n +``` +Complete utterance. Client feeds this to the TTS engine. + +## Server state machine + +``` + ┌──────────────────────────────────────────┐ + │ │ + ▼ │ + ┌──────────┐ first NOP ┌──────────────┐ │ + │ IDLE │ ──────────► │ RECORDING │ │ + └──────────┘ └──────────────┘ │ + │ │ │ + OFF │ │ 150ms │ + recv'd │ │ silence │ + ▼ ▼ │ + ┌─────────────┐ │ + │ PROCESSING │ │ + └─────────────┘ │ + │ │ + STT │ │ + done │ │ + ▼ │ + send HKMSTR:F ─────────┘ ``` -```json -{"event": "off"} -``` +- **IDLE → RECORDING:** first NOP received, start mic capture +- **RECORDING → PROCESSING:** OFF received, OR 150ms since last NOP +- **PROCESSING → IDLE:** STT done, send `HKMSTR:F ` -| Field | Type | Description | -|---------|--------|------------------------------------| -| `event` | string | `"on"` (PTT pressed) or `"off"` (PTT released) | +## Timing -## Transcript messages (server → client) +| Parameter | Value | Purpose | +|-----------|-------|---------| +| NOP interval | 50ms | Heartbeat frequency while PTT held | +| Silence timeout | 150ms | Stop recording if no NOPs (3 missed = lost OFF) | +| NOP bandwidth | ~20 msg/s × ~20 bytes | ~400 bytes/s — negligible | -Sent by the server back to Robovoice over the same TCP connection. +## Why this works -```json -{"final": false, "text": "hello world"} -``` - -```json -{"final": true, "text": "hello world how are you"} -``` - -| Field | Type | Required | Description | -|---------|---------|----------|--------------------------------------------------| -| `final` | bool | yes | `true` = final result, `false` = partial | -| `text` | string | yes | The transcript text (may be empty for partials) | - -### Semantics - -- **`final: false`** — intermediate recognition result (partial). Robovoice - logs these but does not act on them (only `final` triggers TTS). -- **`final: true`** — complete utterance. Robovoice feeds this to the TTS - engine and speaks it. - -Malformed JSON or unknown field values are silently dropped by the client. +1. **Outbound broadcast `:68→:67` to `255.255.255.255`** passes the + WireGuard WFP killswitch (DHCP exception matches this exact pattern) +2. **Inbound `:67→:68`** has no address restriction in the WFP rule, so + unicast replies pass through +3. **Binding to a specific interface IP** (not `0.0.0.0`) wins unicast + delivery over the Windows DHCP client service +4. **NOP spam** ensures the ON message gets through even at 5% packet loss + (3 consecutive NOPs = ~0.01% drop probability) +5. **150ms timeout** is the backstop for lost OFF — at 50ms intervals, 3 + consecutive NOPs must all be lost to false-stop