v0.6: DHCP tunnel transport with NOP heartbeat protocol
This commit is contained in:
@@ -0,0 +1,105 @@
|
||||
using System.Net;
|
||||
using System.Net.NetworkInformation;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
|
||||
// Args: [interface_ip] [interval_ms]
|
||||
// Defaults: auto-detect LAN IP, 50ms
|
||||
|
||||
string localIp = args.Length > 0 ? args[0] : GetLanInterfaceIp() ?? "0.0.0.0";
|
||||
int intervalMs = args.Length > 1 && int.TryParse(args[1], out int iv) ? iv : 50;
|
||||
|
||||
using var sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
|
||||
sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
|
||||
sock.EnableBroadcast = true;
|
||||
|
||||
var localEp = new IPEndPoint(IPAddress.Parse(localIp), 68);
|
||||
sock.Bind(localEp);
|
||||
|
||||
Console.WriteLine($"Bound to {localEp} (SO_REUSEADDR)");
|
||||
Console.WriteLine("Sending NOPs + listening. Press Ctrl+C to stop.");
|
||||
Console.WriteLine();
|
||||
|
||||
var cts = new CancellationTokenSource();
|
||||
var recvTask = Task.Run(() => ReceiveLoop(sock, cts.Token));
|
||||
|
||||
var destEp = new IPEndPoint(IPAddress.Broadcast, 67);
|
||||
uint nonce = 0;
|
||||
|
||||
while (!cts.Token.IsCancellationRequested)
|
||||
{
|
||||
nonce++;
|
||||
string msg = $"HKMSTR {nonce}\n";
|
||||
byte[] payload = Encoding.UTF8.GetBytes(msg);
|
||||
|
||||
try
|
||||
{
|
||||
int sent = sock.SendTo(payload, destEp);
|
||||
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] SENT n={nonce} {sent} bytes");
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] SEND FAILED: {ex.Message}");
|
||||
}
|
||||
|
||||
try { Thread.Sleep(intervalMs); }
|
||||
catch (OperationCanceledException) { break; }
|
||||
}
|
||||
|
||||
cts.Cancel();
|
||||
await recvTask;
|
||||
|
||||
static string? GetLanInterfaceIp()
|
||||
{
|
||||
foreach (var nic in NetworkInterface.GetAllNetworkInterfaces())
|
||||
{
|
||||
if (nic.OperationalStatus != OperationalStatus.Up)
|
||||
continue;
|
||||
if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback)
|
||||
continue;
|
||||
if (nic.Description.Contains("WireGuard", StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
|
||||
foreach (var addr in nic.GetIPProperties().UnicastAddresses)
|
||||
{
|
||||
if (addr.Address.AddressFamily == AddressFamily.InterNetwork)
|
||||
{
|
||||
var ip = addr.Address.ToString();
|
||||
if (ip.StartsWith("192.168.") || ip.StartsWith("10.") || ip.StartsWith("172."))
|
||||
return ip;
|
||||
}
|
||||
}
|
||||
}
|
||||
return null;
|
||||
}
|
||||
|
||||
static void ReceiveLoop(Socket sock, CancellationToken ct)
|
||||
{
|
||||
byte[] buffer = new byte[4096];
|
||||
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
|
||||
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
int received;
|
||||
try
|
||||
{
|
||||
if (!sock.Poll(500_000, SelectMode.SelectRead))
|
||||
continue;
|
||||
received = sock.ReceiveFrom(buffer, ref fromEp);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV ERROR: {ex.Message}");
|
||||
continue;
|
||||
}
|
||||
|
||||
string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');
|
||||
if (!text.StartsWith("HKMSTR"))
|
||||
{
|
||||
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV {received} bytes from {fromEp} (no magic)");
|
||||
continue;
|
||||
}
|
||||
|
||||
Console.WriteLine($"[{DateTime.Now:HH:mm:ss.fff}] RECV {received} bytes from {fromEp}: {text}");
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,10 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<PropertyGroup>
|
||||
<OutputType>Exe</OutputType>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
@@ -12,7 +12,7 @@ public sealed class AppConfig
|
||||
public int LengthScale { get; set; } = 100;
|
||||
public int NoiseWScale { get; set; } = 800;
|
||||
public bool MinimizeToTray { get; set; } = true;
|
||||
public string ServerEndpoint { get; set; } = "127.0.0.1:5210";
|
||||
public string InterfaceIp { get; set; } = string.Empty;
|
||||
|
||||
public static string AppDataDir => Path.Combine(
|
||||
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
|
||||
|
||||
Generated
+9
-17
@@ -17,8 +17,7 @@ partial class MainForm
|
||||
private Button btnBrowseFile = null!;
|
||||
private Label lblFileName = null!;
|
||||
private Label lblServer = null!;
|
||||
private TextBox txtServer = null!;
|
||||
private Button btnConnect = null!;
|
||||
private ComboBox cmbInterface = null!;
|
||||
private Label lblLineStatus = null!;
|
||||
private RichTextBox txtLog = null!;
|
||||
private CheckBox chkMinimizeToTray = null!;
|
||||
@@ -58,8 +57,7 @@ partial class MainForm
|
||||
btnBrowseFile = new Button();
|
||||
lblFileName = new Label();
|
||||
lblServer = new Label();
|
||||
txtServer = new TextBox();
|
||||
btnConnect = new Button();
|
||||
cmbInterface = new ComboBox();
|
||||
lblLineStatus = new Label();
|
||||
txtLog = new RichTextBox();
|
||||
chkMinimizeToTray = new CheckBox();
|
||||
@@ -145,20 +143,15 @@ partial class MainForm
|
||||
lblFileName.ForeColor = Color.Gray;
|
||||
|
||||
// lblServer
|
||||
lblServer.Text = "Server:";
|
||||
lblServer.Text = "Interface:";
|
||||
lblServer.Location = new Point(440, 48);
|
||||
lblServer.Size = new Size(45, 23);
|
||||
lblServer.Size = new Size(55, 23);
|
||||
lblServer.TextAlign = ContentAlignment.MiddleLeft;
|
||||
|
||||
// txtServer
|
||||
txtServer.Location = new Point(488, 45);
|
||||
txtServer.Size = new Size(110, 23);
|
||||
|
||||
// btnConnect
|
||||
btnConnect.Text = "Connect";
|
||||
btnConnect.Location = new Point(603, 44);
|
||||
btnConnect.Size = new Size(60, 25);
|
||||
btnConnect.UseVisualStyleBackColor = true;
|
||||
// cmbInterface
|
||||
cmbInterface.Location = new Point(498, 45);
|
||||
cmbInterface.Size = new Size(165, 23);
|
||||
cmbInterface.DropDownStyle = ComboBoxStyle.DropDownList;
|
||||
|
||||
// lblLineStatus
|
||||
lblLineStatus.Text = "";
|
||||
@@ -277,8 +270,7 @@ partial class MainForm
|
||||
Controls.Add(btnBrowseFile);
|
||||
Controls.Add(lblFileName);
|
||||
Controls.Add(lblServer);
|
||||
Controls.Add(txtServer);
|
||||
Controls.Add(btnConnect);
|
||||
Controls.Add(cmbInterface);
|
||||
Controls.Add(lblLineStatus);
|
||||
Controls.Add(lblNoise);
|
||||
Controls.Add(trkNoise);
|
||||
|
||||
+48
-29
@@ -2,7 +2,7 @@ using NAudio.Wave;
|
||||
using Robovoice.App;
|
||||
using Robovoice.Core;
|
||||
using Robovoice.Core.Voices;
|
||||
using Robovoice.Stt.Tcp;
|
||||
using Robovoice.Stt.Dhcp;
|
||||
using Robovoice.Tts.LibPiper;
|
||||
using System.Diagnostics;
|
||||
|
||||
@@ -16,7 +16,7 @@ internal sealed partial class MainForm : Form
|
||||
|
||||
private LibPiperTtsEngine? _tts;
|
||||
private AudioOutput? _audioOutput;
|
||||
private TcpSttSource? _sttSource;
|
||||
private DhcpSttSource? _sttSource;
|
||||
private Orchestrator? _orchestrator;
|
||||
private PttHotkey? _pttHotkey;
|
||||
private NotifyIcon? _trayIcon;
|
||||
@@ -61,7 +61,7 @@ internal sealed partial class MainForm : Form
|
||||
OnSliderScroll(null, EventArgs.Empty);
|
||||
|
||||
chkMinimizeToTray.Checked = _config.MinimizeToTray;
|
||||
txtServer.Text = _config.ServerEndpoint;
|
||||
PopulateInterfaces();
|
||||
|
||||
btnBrowseFile.Click += OnBrowseFile;
|
||||
btnTestVoice.Click += OnTestVoice;
|
||||
@@ -71,8 +71,7 @@ internal sealed partial class MainForm : Form
|
||||
txtPttKey.KeyDown += OnPttKeyDown;
|
||||
cmbOutput.SelectedIndexChanged += OnOutputChanged;
|
||||
cmbVoice.SelectedIndexChanged += OnVoiceChanged;
|
||||
txtServer.Leave += OnServerChanged;
|
||||
btnConnect.Click += OnConnect;
|
||||
cmbInterface.SelectedIndexChanged += OnInterfaceChanged;
|
||||
|
||||
trkNoise.Scroll += OnSliderScroll;
|
||||
trkSpeed.Scroll += OnSliderScroll;
|
||||
@@ -245,8 +244,9 @@ internal sealed partial class MainForm : Form
|
||||
|
||||
if (_sttSource is null)
|
||||
{
|
||||
_sttSource = new TcpSttSource { ServerEndpoint = txtServer.Text, Log = Log };
|
||||
Log($"STT endpoint: {_sttSource.ServerEndpoint} (press Connect)");
|
||||
string ifaceIp = cmbInterface.SelectedItem as string ?? "";
|
||||
_sttSource = new DhcpSttSource { InterfaceIp = ifaceIp, Log = Log };
|
||||
await _sttSource.StartAsync();
|
||||
}
|
||||
|
||||
_orchestrator?.DisposeAsync().AsTask().Wait();
|
||||
@@ -425,40 +425,59 @@ internal sealed partial class MainForm : Form
|
||||
SaveConfig();
|
||||
}
|
||||
|
||||
private void OnServerChanged(object? sender, EventArgs e)
|
||||
private void PopulateInterfaces()
|
||||
{
|
||||
if (_sttSource is not null)
|
||||
cmbInterface.Items.Clear();
|
||||
foreach (var (ip, name) in DhcpSttSource.GetAvailableInterfaces())
|
||||
{
|
||||
_sttSource.ServerEndpoint = txtServer.Text;
|
||||
Log($"Server endpoint: {txtServer.Text}");
|
||||
cmbInterface.Items.Add(name);
|
||||
cmbInterface.Items[^1] = name;
|
||||
cmbInterface.Items[cmbInterface.Items.Count - 1] = name;
|
||||
}
|
||||
SaveConfig();
|
||||
|
||||
if (!string.IsNullOrEmpty(_config.InterfaceIp))
|
||||
{
|
||||
for (int i = 0; i < cmbInterface.Items.Count; i++)
|
||||
{
|
||||
if (cmbInterface.Items[i] is string s && s.Contains(_config.InterfaceIp))
|
||||
{
|
||||
cmbInterface.SelectedIndex = i;
|
||||
return;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if (cmbInterface.Items.Count > 0)
|
||||
cmbInterface.SelectedIndex = 0;
|
||||
}
|
||||
|
||||
private async void OnConnect(object? sender, EventArgs e)
|
||||
private void OnInterfaceChanged(object? sender, EventArgs e)
|
||||
{
|
||||
if (_sttSource is null)
|
||||
{
|
||||
Log("STT source not initialized.");
|
||||
if (cmbInterface.SelectedItem is not string selected)
|
||||
return;
|
||||
}
|
||||
|
||||
_sttSource.ServerEndpoint = txtServer.Text;
|
||||
string? ip = ExtractIpFromDisplay(selected);
|
||||
if (ip is null) return;
|
||||
|
||||
_config.InterfaceIp = ip;
|
||||
SaveConfig();
|
||||
btnConnect.Enabled = false;
|
||||
try
|
||||
|
||||
if (_sttSource is not null)
|
||||
{
|
||||
if (_sttSource.IsRunning)
|
||||
await _sttSource.ReconnectAsync();
|
||||
else
|
||||
await _sttSource.StartAsync();
|
||||
}
|
||||
finally
|
||||
{
|
||||
btnConnect.Enabled = true;
|
||||
_sttSource.DisposeAsync().AsTask().Wait(2000);
|
||||
_sttSource = null;
|
||||
_ = InitializeEngineAsync();
|
||||
}
|
||||
}
|
||||
|
||||
private static string? ExtractIpFromDisplay(string display)
|
||||
{
|
||||
int start = display.IndexOf('(');
|
||||
int end = display.IndexOf(')');
|
||||
if (start < 0 || end <= start) return null;
|
||||
return display[(start + 1)..end];
|
||||
}
|
||||
|
||||
private void SetupTray()
|
||||
{
|
||||
if (_trayInit) return;
|
||||
@@ -524,7 +543,7 @@ internal sealed partial class MainForm : Form
|
||||
_config.LengthScale = trkSpeed.Value;
|
||||
_config.NoiseWScale = trkNoiseW.Value;
|
||||
_config.MinimizeToTray = chkMinimizeToTray.Checked;
|
||||
_config.ServerEndpoint = txtServer.Text;
|
||||
_config.InterfaceIp = ExtractIpFromDisplay(cmbInterface.SelectedItem as string ?? "") ?? "";
|
||||
_config.Save();
|
||||
}
|
||||
|
||||
|
||||
@@ -3,7 +3,7 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Robovoice.Core\Robovoice.Core.csproj" />
|
||||
<ProjectReference Include="..\Robovoice.Tts.LibPiper\Robovoice.Tts.LibPiper.csproj" />
|
||||
<ProjectReference Include="..\Robovoice.Stt.Tcp\Robovoice.Stt.Tcp.csproj" />
|
||||
<ProjectReference Include="..\Robovoice.Stt.Dhcp\Robovoice.Stt.Dhcp.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -0,0 +1,232 @@
|
||||
using System.Net;
|
||||
using System.Net.NetworkInformation;
|
||||
using System.Net.Sockets;
|
||||
using System.Text;
|
||||
using Robovoice.Core;
|
||||
|
||||
namespace Robovoice.Stt.Dhcp;
|
||||
|
||||
public sealed class DhcpSttSource : ISttSource
|
||||
{
|
||||
private const string Magic = "HKMSTR";
|
||||
private const int DhcpClientPort = 68;
|
||||
private const int DhcpServerPort = 67;
|
||||
private static readonly TimeSpan NopInterval = TimeSpan.FromMilliseconds(50);
|
||||
|
||||
private Socket? _sock;
|
||||
private CancellationTokenSource? _cts;
|
||||
private Task? _receiveTask;
|
||||
private Task? _nopTask;
|
||||
private EndPoint _broadcastEp = new IPEndPoint(IPAddress.Broadcast, DhcpServerPort);
|
||||
private uint _nonce;
|
||||
private bool _disposed;
|
||||
|
||||
public string InterfaceIp { get; set; } = string.Empty;
|
||||
|
||||
public Action<string>? Log { get; set; }
|
||||
|
||||
public event TranscriptEventHandler? TranscriptReceived;
|
||||
|
||||
public Task StartAsync(CancellationToken ct = default)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
if (_cts is not null)
|
||||
return Task.CompletedTask;
|
||||
|
||||
string ip = string.IsNullOrEmpty(InterfaceIp) ? AutoDetectInterfaceIp() ?? "0.0.0.0" : InterfaceIp;
|
||||
|
||||
_sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
|
||||
_sock.SetSocketOption(SocketOptionLevel.Socket, SocketOptionName.ReuseAddress, true);
|
||||
_sock.EnableBroadcast = true;
|
||||
_sock.Bind(new IPEndPoint(IPAddress.Parse(ip), DhcpClientPort));
|
||||
|
||||
Log?.Invoke($"STT: bound to {ip}:{DhcpClientPort}, broadcasting to :{DhcpServerPort}");
|
||||
|
||||
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
_receiveTask = ReceiveLoopAsync(_cts.Token);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async Task StopAsync(CancellationToken ct = default)
|
||||
{
|
||||
StopNop();
|
||||
|
||||
if (_cts is not null)
|
||||
_cts.Cancel();
|
||||
|
||||
_sock?.Dispose();
|
||||
_sock = null;
|
||||
|
||||
if (_receiveTask is not null)
|
||||
{
|
||||
try { await _receiveTask.WaitAsync(ct); }
|
||||
catch { }
|
||||
_receiveTask = null;
|
||||
}
|
||||
|
||||
_cts?.Dispose();
|
||||
_cts = null;
|
||||
}
|
||||
|
||||
public void SendOn()
|
||||
{
|
||||
if (_cts is null)
|
||||
return;
|
||||
|
||||
_nonce = 0;
|
||||
SendNop();
|
||||
_nopTask = NopLoopAsync(_cts.Token);
|
||||
}
|
||||
|
||||
public void SendOff()
|
||||
{
|
||||
StopNop();
|
||||
SendControl("HKMSTR:OFF");
|
||||
}
|
||||
|
||||
private void StopNop()
|
||||
{
|
||||
if (_nopTask is not null)
|
||||
{
|
||||
try { _nopTask.Wait(2000); } catch { }
|
||||
_nopTask = null;
|
||||
}
|
||||
}
|
||||
|
||||
private async Task NopLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
try { await Task.Delay(NopInterval, ct); }
|
||||
catch (OperationCanceledException) { break; }
|
||||
|
||||
SendNop();
|
||||
}
|
||||
}
|
||||
|
||||
private void SendNop()
|
||||
{
|
||||
_nonce++;
|
||||
SendControl($"HKMSTR {_nonce}");
|
||||
}
|
||||
|
||||
private void SendControl(string message)
|
||||
{
|
||||
if (_sock is null)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
byte[] payload = Encoding.UTF8.GetBytes(message + "\n");
|
||||
_sock.SendTo(payload, _broadcastEp);
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log?.Invoke($"STT: send failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReceiveLoopAsync(CancellationToken ct)
|
||||
{
|
||||
byte[] buffer = new byte[4096];
|
||||
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
|
||||
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
int received;
|
||||
try
|
||||
{
|
||||
if (!_sock!.Poll(500_000, SelectMode.SelectRead))
|
||||
continue;
|
||||
received = _sock.ReceiveFrom(buffer, ref fromEp);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (ObjectDisposedException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log?.Invoke($"STT: receive error: {ex.Message}");
|
||||
continue;
|
||||
}
|
||||
|
||||
string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');
|
||||
if (!text.StartsWith(Magic))
|
||||
continue;
|
||||
|
||||
if (text.StartsWith("HKMSTR:P "))
|
||||
{
|
||||
string transcript = text["HKMSTR:P ".Length..];
|
||||
TranscriptReceived?.Invoke(this, new TranscriptEventArgs
|
||||
{
|
||||
Message = new TranscriptMessage(TranscriptType.Partial, transcript),
|
||||
});
|
||||
}
|
||||
else if (text.StartsWith("HKMSTR:F "))
|
||||
{
|
||||
string transcript = text["HKMSTR:F ".Length..];
|
||||
TranscriptReceived?.Invoke(this, new TranscriptEventArgs
|
||||
{
|
||||
Message = new TranscriptMessage(TranscriptType.Final, transcript),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
public static List<(string Ip, string Name)> GetAvailableInterfaces()
|
||||
{
|
||||
var result = new List<(string, string)>();
|
||||
|
||||
foreach (var nic in NetworkInterface.GetAllNetworkInterfaces())
|
||||
{
|
||||
if (nic.OperationalStatus != OperationalStatus.Up)
|
||||
continue;
|
||||
if (nic.NetworkInterfaceType == NetworkInterfaceType.Loopback)
|
||||
continue;
|
||||
|
||||
string desc = nic.Description;
|
||||
if (desc.Contains("WireGuard", StringComparison.OrdinalIgnoreCase))
|
||||
continue;
|
||||
|
||||
foreach (var addr in nic.GetIPProperties().UnicastAddresses)
|
||||
{
|
||||
if (addr.Address.AddressFamily != AddressFamily.InterNetwork)
|
||||
continue;
|
||||
|
||||
string ip = addr.Address.ToString();
|
||||
if (ip.StartsWith("192.168.") || ip.StartsWith("10.") ||
|
||||
ip.StartsWith("172.16.") || ip.StartsWith("172.17.") ||
|
||||
ip.StartsWith("172.18.") || ip.StartsWith("172.19.") ||
|
||||
ip.StartsWith("172.20.") || ip.StartsWith("172.21.") ||
|
||||
ip.StartsWith("172.22.") || ip.StartsWith("172.23.") ||
|
||||
ip.StartsWith("172.24.") || ip.StartsWith("172.25.") ||
|
||||
ip.StartsWith("172.26.") || ip.StartsWith("172.27.") ||
|
||||
ip.StartsWith("172.28.") || ip.StartsWith("172.29.") ||
|
||||
ip.StartsWith("172.30.") || ip.StartsWith("172.31."))
|
||||
{
|
||||
result.Add((ip, $"{nic.Name} ({ip})"));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return result;
|
||||
}
|
||||
|
||||
private static string? AutoDetectInterfaceIp()
|
||||
{
|
||||
foreach (var (ip, _) in GetAvailableInterfaces())
|
||||
return ip;
|
||||
return null;
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed) return;
|
||||
await StopAsync();
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
@@ -1,253 +0,0 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Robovoice.Core;
|
||||
|
||||
namespace Robovoice.Stt.Tcp;
|
||||
|
||||
public sealed class TcpSttSource : ISttSource
|
||||
{
|
||||
private TcpClient? _tcp;
|
||||
private NetworkStream? _stream;
|
||||
private StreamReader? _reader;
|
||||
private StreamWriter? _writer;
|
||||
private CancellationTokenSource? _cts;
|
||||
private Task? _runTask;
|
||||
private readonly object _sendLock = new();
|
||||
private bool _disposed;
|
||||
|
||||
public string ServerEndpoint { get; set; } = "127.0.0.1:5210";
|
||||
|
||||
public bool IsRunning => _cts is not null;
|
||||
|
||||
public Action<string>? Log { get; set; }
|
||||
|
||||
public event TranscriptEventHandler? TranscriptReceived;
|
||||
|
||||
public Task StartAsync(CancellationToken ct = default)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
if (_cts is not null)
|
||||
return Task.CompletedTask;
|
||||
|
||||
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
_runTask = RunAsync(_cts.Token);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async Task StopAsync(CancellationToken ct = default)
|
||||
{
|
||||
if (_cts is not null)
|
||||
_cts.Cancel();
|
||||
|
||||
CleanupConnection();
|
||||
|
||||
if (_runTask is not null)
|
||||
{
|
||||
try { await _runTask.WaitAsync(ct); }
|
||||
catch { }
|
||||
_runTask = null;
|
||||
}
|
||||
|
||||
_cts?.Dispose();
|
||||
_cts = null;
|
||||
}
|
||||
|
||||
private async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
IPEndPoint? endpoint = ParseEndpoint(ServerEndpoint);
|
||||
if (endpoint is null)
|
||||
{
|
||||
Log?.Invoke($"STT: invalid endpoint '{ServerEndpoint}'");
|
||||
try { await Task.Delay(3000, ct); } catch { break; }
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_tcp = new TcpClient();
|
||||
using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
connectCts.CancelAfter(TimeSpan.FromSeconds(5));
|
||||
await _tcp.ConnectAsync(endpoint.Address, endpoint.Port, connectCts.Token);
|
||||
|
||||
_stream = _tcp.GetStream();
|
||||
_reader = new StreamReader(_stream, System.Text.Encoding.UTF8);
|
||||
_writer = new StreamWriter(_stream, System.Text.Encoding.UTF8) { AutoFlush = true };
|
||||
|
||||
Log?.Invoke($"STT: connected to {ServerEndpoint}");
|
||||
|
||||
await ReceiveLoopAsync(ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log?.Invoke($"STT: connection failed ({ex.Message}), retrying...");
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupConnection();
|
||||
}
|
||||
|
||||
if (!ct.IsCancellationRequested)
|
||||
{
|
||||
try { await Task.Delay(3000, ct); }
|
||||
catch (OperationCanceledException) { break; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReceiveLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested && _reader is not null)
|
||||
{
|
||||
string? line;
|
||||
try
|
||||
{
|
||||
line = await _reader.ReadLineAsync(ct);
|
||||
}
|
||||
catch
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (line is null)
|
||||
break;
|
||||
|
||||
TranscriptMessage? message = ParseTranscriptLine(line);
|
||||
if (message is null)
|
||||
continue;
|
||||
|
||||
TranscriptReceived?.Invoke(this, new TranscriptEventArgs
|
||||
{
|
||||
Message = message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public void SendOn()
|
||||
{
|
||||
SendControl("on");
|
||||
}
|
||||
|
||||
public void SendOff()
|
||||
{
|
||||
SendControl("off");
|
||||
}
|
||||
|
||||
public async Task ReconnectAsync(CancellationToken ct = default)
|
||||
{
|
||||
if (_cts is null)
|
||||
return;
|
||||
|
||||
Log?.Invoke("STT: reconnecting...");
|
||||
CleanupConnection();
|
||||
|
||||
try { await Task.Delay(500, ct); }
|
||||
catch (OperationCanceledException) { return; }
|
||||
}
|
||||
|
||||
private void SendControl(string evt)
|
||||
{
|
||||
lock (_sendLock)
|
||||
{
|
||||
if (_writer is null)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
_writer.WriteLine(JsonSerializer.Serialize(new ControlDto { Event = evt }));
|
||||
}
|
||||
catch
|
||||
{
|
||||
Log?.Invoke($"STT: failed to send '{evt}' (not connected?)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void CleanupConnection()
|
||||
{
|
||||
lock (_sendLock)
|
||||
{
|
||||
_writer?.Dispose();
|
||||
_reader?.Dispose();
|
||||
_stream?.Dispose();
|
||||
_tcp?.Dispose();
|
||||
_writer = null;
|
||||
_reader = null;
|
||||
_stream = null;
|
||||
_tcp = null;
|
||||
}
|
||||
}
|
||||
|
||||
private static IPEndPoint? ParseEndpoint(string endpoint)
|
||||
{
|
||||
int colon = endpoint.LastIndexOf(':');
|
||||
if (colon <= 0)
|
||||
return null;
|
||||
|
||||
string host = endpoint[..colon];
|
||||
if (!int.TryParse(endpoint[(colon + 1)..], out int port))
|
||||
return null;
|
||||
|
||||
if (IPAddress.TryParse(host, out var addr))
|
||||
return new IPEndPoint(addr, port);
|
||||
|
||||
try
|
||||
{
|
||||
var addresses = Dns.GetHostAddresses(host);
|
||||
addr = addresses.FirstOrDefault(a => a.AddressFamily == AddressFamily.InterNetwork);
|
||||
if (addr is null)
|
||||
return null;
|
||||
return new IPEndPoint(addr, port);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static TranscriptMessage? ParseTranscriptLine(string line)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
return null;
|
||||
|
||||
try
|
||||
{
|
||||
var dto = JsonSerializer.Deserialize<TransmitDto>(line);
|
||||
if (dto is null)
|
||||
return null;
|
||||
|
||||
var type = dto.Final ? TranscriptType.Final : TranscriptType.Partial;
|
||||
return new TranscriptMessage(type, dto.Text ?? string.Empty);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed) return;
|
||||
await StopAsync();
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class ControlDto
|
||||
{
|
||||
[JsonPropertyName("event")]
|
||||
public string Event { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
internal sealed class TransmitDto
|
||||
{
|
||||
public bool Final { get; set; }
|
||||
public string Text { get; set; } = string.Empty;
|
||||
}
|
||||
+2
-1
@@ -2,6 +2,7 @@
|
||||
<Project Path="Robovoice.App/Robovoice.App.csproj" />
|
||||
<Project Path="Robovoice.Core/Robovoice.Core.csproj" />
|
||||
<Project Path="Robovoice.Stt.File/Robovoice.Stt.File.csproj" />
|
||||
<Project Path="Robovoice.Stt.Tcp/Robovoice.Stt.Tcp.csproj" />
|
||||
<Project Path="Robovoice.Stt.Dhcp/Robovoice.Stt.Dhcp.csproj" />
|
||||
<Project Path="Robovoice.Tts.LibPiper/Robovoice.Tts.LibPiper.csproj" />
|
||||
<Project Path="DhcpTunnelTest/Robovoice.DhcpTunnelTest.csproj" />
|
||||
</Solution>
|
||||
|
||||
+242
-174
@@ -1,202 +1,232 @@
|
||||
# Server implementation notes
|
||||
|
||||
The STT server listens for TCP connections from Robovoice, captures audio
|
||||
from a microphone when `on` is received, runs speech recognition (Moonshine),
|
||||
and sends transcript messages back over the same connection.
|
||||
The STT server listens on UDP port 67, receives NOP heartbeats and OFF
|
||||
messages from Robovoice, captures audio from a microphone, runs speech
|
||||
recognition (Moonshine), and sends transcript messages back to the client's
|
||||
source address.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌── on/off (TCP, newline-delimited JSON)
|
||||
┌── HKMSTR <nonce> (broadcast, every 50ms)
|
||||
Robovoice ──────────────►│
|
||||
│ STT Server
|
||||
Robovoice ◄──────────────┤
|
||||
└── partial/final (TCP, newline-delimited JSON)
|
||||
└── HKMSTR:P/F <text> (unicast)
|
||||
```
|
||||
|
||||
The server:
|
||||
1. Listens on a TCP port (e.g. 5210)
|
||||
2. Accepts a connection from Robovoice
|
||||
3. Reads lines: waits for `{"event":"on"}`
|
||||
4. Records audio from the microphone
|
||||
5. Waits for `{"event":"off"}` (or a timeout)
|
||||
6. Runs STT on the captured audio
|
||||
7. Sends `{"final":true,"text":"..."}`\n back over the connection
|
||||
1. Listens on UDP :67
|
||||
2. First `HKMSTR <nonce>` → start recording
|
||||
3. `HKMSTR:OFF <nonce>` → stop recording, run STT
|
||||
4. 150ms with no NOPs → stop recording, run STT (backstop)
|
||||
5. Send `HKMSTR:F <text>` back to the client's source address:port
|
||||
|
||||
## Framing
|
||||
|
||||
Every message is a single JSON object on one line, terminated by `\n`. No
|
||||
length prefix, no binary framing. Use `readline()` / `StreamReader.ReadLineAsync()`.
|
||||
All messages are newline-terminated UTF-8 text. No JSON, no binary framing.
|
||||
|
||||
- `HKMSTR <nonce>` — NOP heartbeat (client → server)
|
||||
- `HKMSTR:OFF <nonce>` — stop signal (client → server)
|
||||
- `HKMSTR:P <text>` — partial transcript (server → client)
|
||||
- `HKMSTR:F <text>` — final transcript (server → client)
|
||||
|
||||
The nonce is an incrementing integer for packet uniqueness only. Discard it.
|
||||
|
||||
## Python server with Moonshine
|
||||
|
||||
[Moonshine](https://github.com/usefulsensors/moonshine) is a lightweight ASR
|
||||
model by Useful Sensors. Install with `pip install moonshine`.
|
||||
|
||||
```python
|
||||
import socket
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
import moonshine
|
||||
|
||||
LISTEN_PORT = 5210
|
||||
LISTEN_PORT = 67
|
||||
CLIENT_PORT = 68
|
||||
SAMPLE_RATE = 16000
|
||||
SILENCE_TIMEOUT = 0.150 # 150ms
|
||||
|
||||
model = moonshine.MoonshineModel(model="moonshine/base")
|
||||
|
||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
server.bind(("0.0.0.0", LISTEN_PORT))
|
||||
server.listen(1)
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
|
||||
sock.bind(("0.0.0.0", LISTEN_PORT))
|
||||
|
||||
print(f"STT server listening on :{LISTEN_PORT}")
|
||||
|
||||
recording = False
|
||||
last_nop_time = 0
|
||||
client_addr = None
|
||||
audio_chunks = []
|
||||
lock = threading.Lock()
|
||||
|
||||
def monitor_silence():
|
||||
"""Backstop: stop recording if no NOPs for 150ms."""
|
||||
global recording
|
||||
while True:
|
||||
time.sleep(0.01)
|
||||
with lock:
|
||||
if recording and (time.monotonic() - last_nop_time) > SILENCE_TIMEOUT:
|
||||
recording = False
|
||||
threading.Thread(target=process_audio, daemon=True).start()
|
||||
|
||||
threading.Thread(target=monitor_silence, daemon=True).start()
|
||||
|
||||
def process_audio():
|
||||
global audio_chunks
|
||||
with lock:
|
||||
chunks = audio_chunks
|
||||
audio_chunks = []
|
||||
addr = client_addr
|
||||
|
||||
if not chunks:
|
||||
return
|
||||
|
||||
audio = np.concatenate(chunks)
|
||||
print(f"Captured {len(audio)/SAMPLE_RATE:.1f}s")
|
||||
|
||||
text = moonshine.transcribe(model, audio).strip()
|
||||
|
||||
if text:
|
||||
print(f"Final: {text}")
|
||||
reply = f"HKMSTR:F {text}\n".encode("utf-8")
|
||||
sock.sendto(reply, addr)
|
||||
else:
|
||||
print("Empty transcript")
|
||||
|
||||
while True:
|
||||
conn, addr = server.accept()
|
||||
print(f"Client connected: {addr}")
|
||||
data, addr = sock.recvfrom(4096)
|
||||
text = data.decode("utf-8", errors="ignore").strip()
|
||||
|
||||
buf = ""
|
||||
with conn:
|
||||
while True:
|
||||
data = conn.recv(4096).decode("utf-8")
|
||||
if not data:
|
||||
break
|
||||
buf += data
|
||||
if not text.startswith("HKMSTR"):
|
||||
continue
|
||||
|
||||
while "\n" in buf:
|
||||
line, buf = buf.split("\n", 1)
|
||||
msg = json.loads(line)
|
||||
if text.startswith("HKMSTR:OFF"):
|
||||
with lock:
|
||||
if recording:
|
||||
recording = False
|
||||
threading.Thread(target=process_audio, daemon=True).start()
|
||||
continue
|
||||
|
||||
if msg.get("event") == "on":
|
||||
print("PTT on — recording")
|
||||
audio_chunks = []
|
||||
if text.startswith("HKMSTR ") or text == "HKMSTR":
|
||||
with lock:
|
||||
client_addr = addr
|
||||
last_nop_time = time.monotonic()
|
||||
|
||||
# Record until "off" or timeout
|
||||
conn.settimeout(0.1)
|
||||
while True:
|
||||
try:
|
||||
data2 = conn.recv(4096).decode("utf-8")
|
||||
if not data2:
|
||||
break
|
||||
buf += data2
|
||||
while "\n" in buf:
|
||||
line2, buf = buf.split("\n", 1)
|
||||
msg2 = json.loads(line2)
|
||||
if msg2.get("event") == "off":
|
||||
break
|
||||
except socket.timeout:
|
||||
pass
|
||||
if not recording:
|
||||
recording = True
|
||||
audio_chunks = []
|
||||
print(f"PTT on from {addr}")
|
||||
|
||||
chunk = sd.rec(int(SAMPLE_RATE * 0.1),
|
||||
samplerate=SAMPLE_RATE,
|
||||
channels=1, dtype="float32")
|
||||
sd.wait()
|
||||
audio_chunks.append(chunk.flatten())
|
||||
|
||||
conn.settimeout(None)
|
||||
|
||||
if not audio_chunks:
|
||||
continue
|
||||
|
||||
audio = np.concatenate(audio_chunks)
|
||||
print(f"Captured {len(audio)/SAMPLE_RATE:.1f}s")
|
||||
|
||||
text = moonshine.transcribe(model, audio).strip()
|
||||
|
||||
if text:
|
||||
print(f"Transcript: {text}")
|
||||
reply = json.dumps({"final": True, "text": text})
|
||||
conn.sendall((reply + "\n").encode("utf-8"))
|
||||
else:
|
||||
print("Empty transcript")
|
||||
# Capture 50ms of audio
|
||||
chunk = sd.rec(int(SAMPLE_RATE * 0.05), samplerate=SAMPLE_RATE,
|
||||
channels=1, dtype="float32")
|
||||
sd.wait()
|
||||
audio_chunks.append(chunk.flatten())
|
||||
```
|
||||
|
||||
## Python server with streaming partials
|
||||
|
||||
For lower latency, send partial results while still recording:
|
||||
For live feedback, send partials while recording:
|
||||
|
||||
```python
|
||||
import socket
|
||||
import json
|
||||
import threading
|
||||
import time
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
import moonshine
|
||||
|
||||
LISTEN_PORT = 5210
|
||||
LISTEN_PORT = 67
|
||||
SAMPLE_RATE = 16000
|
||||
CHUNK_DURATION = 0.5
|
||||
SILENCE_TIMEOUT = 0.150
|
||||
PARTIAL_INTERVAL = 0.5 # send partial every 500ms
|
||||
|
||||
model = moonshine.MoonshineModel(model="moonshine/base")
|
||||
|
||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
server.bind(("0.0.0.0", LISTEN_PORT))
|
||||
server.listen(1)
|
||||
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
||||
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
|
||||
sock.bind(("0.0.0.0", LISTEN_PORT))
|
||||
|
||||
print(f"STT server listening on :{LISTEN_PORT}")
|
||||
|
||||
recording = False
|
||||
last_nop_time = 0
|
||||
last_partial_time = 0
|
||||
client_addr = None
|
||||
audio_chunks = []
|
||||
lock = threading.Lock()
|
||||
|
||||
def capture_and_maybe_partial():
|
||||
global last_partial_time
|
||||
with lock:
|
||||
if not recording:
|
||||
return
|
||||
|
||||
chunk = sd.rec(int(SAMPLE_RATE * 0.05), samplerate=SAMPLE_RATE,
|
||||
channels=1, dtype="float32")
|
||||
sd.wait()
|
||||
audio_chunks.append(chunk.flatten())
|
||||
|
||||
now = time.monotonic()
|
||||
if now - last_partial_time > PARTIAL_INTERVAL:
|
||||
last_partial_time = now
|
||||
partial_audio = np.concatenate(audio_chunks)
|
||||
partial_text = moonshine.transcribe(model, partial_audio).strip()
|
||||
if partial_text and client_addr:
|
||||
reply = f"HKMSTR:P {partial_text}\n".encode("utf-8")
|
||||
sock.sendto(reply, client_addr)
|
||||
|
||||
while True:
|
||||
conn, addr = server.accept()
|
||||
print(f"Client connected: {addr}")
|
||||
buf = ""
|
||||
data, addr = sock.recvfrom(4096)
|
||||
text = data.decode("utf-8", errors="ignore").strip()
|
||||
|
||||
with conn:
|
||||
while True:
|
||||
data = conn.recv(4096).decode("utf-8")
|
||||
if not data:
|
||||
break
|
||||
buf += data
|
||||
if not text.startswith("HKMSTR"):
|
||||
continue
|
||||
|
||||
while "\n" in buf:
|
||||
line, buf = buf.split("\n", 1)
|
||||
msg = json.loads(line)
|
||||
|
||||
if msg.get("event") != "on":
|
||||
continue
|
||||
|
||||
print("PTT on — recording")
|
||||
if text.startswith("HKMSTR:OFF"):
|
||||
with lock:
|
||||
if recording:
|
||||
recording = False
|
||||
chunks = audio_chunks
|
||||
audio_chunks = []
|
||||
|
||||
while True:
|
||||
try:
|
||||
conn.settimeout(CHUNK_DURATION)
|
||||
data2 = conn.recv(4096).decode("utf-8")
|
||||
if not data2:
|
||||
break
|
||||
buf += data2
|
||||
while "\n" in buf:
|
||||
line2, buf = buf.split("\n", 1)
|
||||
msg2 = json.loads(line2)
|
||||
if msg2.get("event") == "off":
|
||||
break
|
||||
except socket.timeout:
|
||||
pass
|
||||
if chunks:
|
||||
audio = np.concatenate(chunks)
|
||||
final_text = moonshine.transcribe(model, audio).strip()
|
||||
if final_text:
|
||||
reply = f"HKMSTR:F {final_text}\n".encode("utf-8")
|
||||
sock.sendto(reply, addr)
|
||||
continue
|
||||
|
||||
chunk = sd.rec(int(SAMPLE_RATE * CHUNK_DURATION),
|
||||
samplerate=SAMPLE_RATE,
|
||||
channels=1, dtype="float32")
|
||||
sd.wait()
|
||||
audio_chunks.append(chunk.flatten())
|
||||
if text.startswith("HKMSTR") and not text.startswith("HKMSTR:"):
|
||||
with lock:
|
||||
client_addr = addr
|
||||
last_nop_time = time.monotonic()
|
||||
|
||||
# Send partial every few chunks
|
||||
if len(audio_chunks) % 4 == 0:
|
||||
partial_audio = np.concatenate(audio_chunks)
|
||||
partial_text = moonshine.transcribe(model, partial_audio).strip()
|
||||
if partial_text:
|
||||
reply = json.dumps({"final": False, "text": partial_text})
|
||||
conn.sendall((reply + "\n").encode("utf-8"))
|
||||
if not recording:
|
||||
recording = True
|
||||
audio_chunks = []
|
||||
last_partial_time = time.monotonic()
|
||||
print(f"PTT on from {addr}")
|
||||
|
||||
conn.settimeout(None)
|
||||
capture_and_maybe_partial()
|
||||
|
||||
if not audio_chunks:
|
||||
continue
|
||||
# Check silence timeout
|
||||
with lock:
|
||||
if recording and (time.monotonic() - last_nop_time) > SILENCE_TIMEOUT:
|
||||
recording = False
|
||||
chunks = audio_chunks
|
||||
audio_chunks = []
|
||||
|
||||
audio = np.concatenate(audio_chunks)
|
||||
text = moonshine.transcribe(model, audio).strip()
|
||||
|
||||
if text:
|
||||
print(f"Final: {text}")
|
||||
reply = json.dumps({"final": True, "text": text})
|
||||
conn.sendall((reply + "\n").encode("utf-8"))
|
||||
if 'chunks' in dir() and chunks:
|
||||
audio = np.concatenate(chunks)
|
||||
final_text = moonshine.transcribe(model, audio).strip()
|
||||
if final_text:
|
||||
reply = f"HKMSTR:F {final_text}\n".encode("utf-8")
|
||||
sock.sendto(reply, addr)
|
||||
```
|
||||
|
||||
## C# server skeleton
|
||||
@@ -204,61 +234,99 @@ while True:
|
||||
```csharp
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text.Json;
|
||||
using System.Text;
|
||||
|
||||
var listener = new TcpListener(IPAddress.Any, 5210);
|
||||
listener.Start();
|
||||
var sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
|
||||
sock.Bind(new IPEndPoint(IPAddress.Any, 67));
|
||||
|
||||
Console.WriteLine("STT server listening on :5210");
|
||||
Console.WriteLine("STT server listening on :67");
|
||||
|
||||
byte[] buffer = new byte[4096];
|
||||
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
|
||||
bool recording = false;
|
||||
DateTime lastNop = DateTime.MinValue;
|
||||
List<float[]> audioChunks = new();
|
||||
|
||||
while (true)
|
||||
{
|
||||
var client = listener.AcceptTcpClient();
|
||||
Console.WriteLine($"Client connected: {client.Client.RemoteEndPoint}");
|
||||
|
||||
using var stream = client.GetStream();
|
||||
using var reader = new StreamReader(stream, Encoding.UTF8);
|
||||
using var writer = new StreamWriter(stream, Encoding.UTF8) { AutoFlush = true };
|
||||
|
||||
string? line;
|
||||
while ((line = reader.ReadLine()) is not null)
|
||||
if (sock.Poll(50_000, SelectMode.SelectRead))
|
||||
{
|
||||
var msg = JsonSerializer.Deserialize<Dictionary<string, string>>(line);
|
||||
if (msg?["event"] != "on")
|
||||
int received = sock.ReceiveFrom(buffer, ref fromEp);
|
||||
string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');
|
||||
|
||||
if (!text.StartsWith("HKMSTR"))
|
||||
continue;
|
||||
|
||||
Console.WriteLine("PTT on — recording");
|
||||
// Capture audio...
|
||||
|
||||
// Read until "off"
|
||||
while ((line = reader.ReadLine()) is not null)
|
||||
if (text.StartsWith("HKMSTR:OFF"))
|
||||
{
|
||||
msg = JsonSerializer.Deserialize<Dictionary<string, string>>(line);
|
||||
if (msg?["event"] == "off")
|
||||
break;
|
||||
if (recording)
|
||||
{
|
||||
recording = false;
|
||||
ProcessAndReply(audioChunks, fromEp);
|
||||
audioChunks.Clear();
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Run STT...
|
||||
string text = "recognized text here";
|
||||
// NOP
|
||||
lastNop = DateTime.UtcNow;
|
||||
if (!recording)
|
||||
{
|
||||
recording = true;
|
||||
audioChunks.Clear();
|
||||
Console.WriteLine($"PTT on from {fromEp}");
|
||||
}
|
||||
|
||||
var reply = JsonSerializer.Serialize(new { final = true, text });
|
||||
writer.WriteLine(reply);
|
||||
// Capture 50ms audio here...
|
||||
// audioChunks.Add(capturedChunk);
|
||||
|
||||
// Check silence timeout
|
||||
if (recording && (DateTime.UtcNow - lastNop).TotalMilliseconds > 150)
|
||||
{
|
||||
recording = false;
|
||||
ProcessAndReply(audioChunks, fromEp);
|
||||
audioChunks.Clear();
|
||||
}
|
||||
}
|
||||
else
|
||||
{
|
||||
// Timeout check even without incoming data
|
||||
if (recording && (DateTime.UtcNow - lastNop).TotalMilliseconds > 150)
|
||||
{
|
||||
recording = false;
|
||||
ProcessAndReply(audioChunks, fromEp);
|
||||
audioChunks.Clear();
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void ProcessAndReply(List<float[]> chunks, EndPoint client)
|
||||
{
|
||||
if (chunks.Count == 0) return;
|
||||
|
||||
// Concatenate and run STT...
|
||||
string text = "recognized text here";
|
||||
|
||||
if (!string.IsNullOrEmpty(text))
|
||||
{
|
||||
byte[] reply = Encoding.UTF8.GetBytes($"HKMSTR:F {text}\n");
|
||||
sock.SendTo(reply, client);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Tips
|
||||
|
||||
- **One connection per client:** Robovoice maintains a single persistent TCP
|
||||
connection. The server should handle one client at a time (or track
|
||||
multiple if needed).
|
||||
- **Timeout:** implement a recording timeout in case the `off` message is
|
||||
delayed or the client disconnects. 10–30 seconds is reasonable.
|
||||
- **Partials:** optional but improve UX — Robovoice logs them so the user
|
||||
sees live feedback. Only `final` triggers TTS.
|
||||
- **Encoding:** always UTF-8. Every line is a UTF-8 JSON object terminated
|
||||
by `\n`.
|
||||
- **Reconnection:** Robovoice auto-reconnects every 3 seconds if the
|
||||
connection drops. The server just needs to accept new connections.
|
||||
- **Moonshine models:** `moonshine/base` (faster, less accurate) or
|
||||
`moonshine/tiny` (fastest). Choose based on your hardware.
|
||||
- **Reply address:** always reply to the source endpoint of the last NOP.
|
||||
The client binds to a specific IP on :68.
|
||||
- **Nonces:** discard them. They exist only to make each datagram unique.
|
||||
Do not derive any meaning from nonce values.
|
||||
- **Silence timeout:** 150ms = 3 missed NOPs at 50ms intervals. If you
|
||||
change the NOP interval on the client, adjust this accordingly.
|
||||
- **Partials:** optional. Send `HKMSTR:P <text>` while recording for live
|
||||
feedback. Client logs them but only `HKMSTR:F` triggers TTS.
|
||||
- **Broadcast only for C→S:** the WireGuard killswitch only allows
|
||||
outbound broadcast to 255.255.255.255:67. Unicast from client won't pass.
|
||||
- **Unicast OK for S→C:** the inbound WFP rule has no address restriction,
|
||||
so unicast replies to :68 pass through.
|
||||
- **Moonshine models:** `moonshine/base` or `moonshine/tiny`.
|
||||
|
||||
+93
-52
@@ -1,70 +1,111 @@
|
||||
# Robovoice TCP STT Protocol
|
||||
# Robovoice DHCP Tunnel Protocol
|
||||
|
||||
## Overview
|
||||
|
||||
Robovoice acts as a **client**: it connects to a remote STT server over TCP,
|
||||
sends control messages when the user presses/releases the PTT key, and
|
||||
receives transcript messages back. The STT server captures audio from a
|
||||
microphone, runs speech recognition (Moonshine), and sends transcripts back
|
||||
over the same connection.
|
||||
Robovoice communicates with a remote STT server by tunneling through the
|
||||
DHCP UDP ports (68→67). This exploits a common killswitch exception: VPN
|
||||
software (e.g. WireGuard) blocks all traffic except DHCP, which is allowed
|
||||
for network connectivity maintenance.
|
||||
|
||||
```
|
||||
[Robovoice client] --TCP--> [STT server :5210]
|
||||
│ │
|
||||
├── {"event":"on"}\n ──────►│
|
||||
│ ├── capture audio
|
||||
├── {"event":"off"}\n ──────►│
|
||||
│ ├── run STT
|
||||
│◄── {"final":true,...}\n ──┤
|
||||
[Robovoice client] --broadcast UDP :68→:67--> [STT server]
|
||||
[Robovoice client] <--unicast UDP :67→:68-- [STT server]
|
||||
```
|
||||
|
||||
The client broadcasts NOP heartbeats while PTT is held. The server starts
|
||||
recording on the first NOP and stops when it receives OFF or when 150ms
|
||||
pass with no NOPs.
|
||||
|
||||
## Transport
|
||||
|
||||
- **Protocol:** TCP (reliable, ordered, connection-oriented)
|
||||
- **Server endpoint:** configurable in Robovoice UI (default `127.0.0.1:5210`)
|
||||
- **Framing:** newline-delimited JSON (NDJSON) — each message is a single
|
||||
UTF-8 JSON object terminated by `\n`
|
||||
- **Auto-reconnect:** if the connection drops, Robovoice retries every 3
|
||||
seconds until the server is available
|
||||
- **Protocol:** UDP (connectionless, unreliable)
|
||||
- **Client → Server:** broadcast, source port 68, dest port 67
|
||||
- **Server → Client:** unicast, source port 67, dest port 68
|
||||
- **Client binds:** to a specific LAN interface IP on port 68 (with
|
||||
`SO_REUSEADDR` to coexist with the Windows DHCP service)
|
||||
- **No connection state** — purely fire-and-forget datagrams
|
||||
|
||||
## Control messages (client → server)
|
||||
## Wire format
|
||||
|
||||
Sent by Robovoice when the user presses/releases the PTT key.
|
||||
All messages are plain text, newline-terminated (`\n`). Every message starts
|
||||
with the 6-byte magic `HKMSTR` to distinguish our traffic from real DHCP.
|
||||
|
||||
```json
|
||||
{"event": "on"}
|
||||
### Client → Server
|
||||
|
||||
**NOP (heartbeat while PTT held):**
|
||||
```
|
||||
HKMSTR <nonce>\n
|
||||
```
|
||||
Sent every 50ms while PTT is held. The nonce is an incrementing unsigned
|
||||
integer that makes each datagram unique. The server discards it — it's
|
||||
purely for packet uniqueness, not for any protocol logic.
|
||||
|
||||
**OFF (PTT released):**
|
||||
```
|
||||
HKMSTR:OFF <nonce>\n
|
||||
```
|
||||
Sent once when PTT is released. This is the fast-stop signal. If lost, the
|
||||
150ms timeout acts as a backstop.
|
||||
|
||||
### Server → Client
|
||||
|
||||
**Partial transcript:**
|
||||
```
|
||||
HKMSTR:P <text>\n
|
||||
```
|
||||
Intermediate recognition result. Fire-and-forget. Client logs it but does
|
||||
not act on it.
|
||||
|
||||
**Final transcript:**
|
||||
```
|
||||
HKMSTR:F <text>\n
|
||||
```
|
||||
Complete utterance. Client feeds this to the TTS engine.
|
||||
|
||||
## Server state machine
|
||||
|
||||
```
|
||||
┌──────────────────────────────────────────┐
|
||||
│ │
|
||||
▼ │
|
||||
┌──────────┐ first NOP ┌──────────────┐ │
|
||||
│ IDLE │ ──────────► │ RECORDING │ │
|
||||
└──────────┘ └──────────────┘ │
|
||||
│ │ │
|
||||
OFF │ │ 150ms │
|
||||
recv'd │ │ silence │
|
||||
▼ ▼ │
|
||||
┌─────────────┐ │
|
||||
│ PROCESSING │ │
|
||||
└─────────────┘ │
|
||||
│ │
|
||||
STT │ │
|
||||
done │ │
|
||||
▼ │
|
||||
send HKMSTR:F ─────────┘
|
||||
```
|
||||
|
||||
```json
|
||||
{"event": "off"}
|
||||
```
|
||||
- **IDLE → RECORDING:** first NOP received, start mic capture
|
||||
- **RECORDING → PROCESSING:** OFF received, OR 150ms since last NOP
|
||||
- **PROCESSING → IDLE:** STT done, send `HKMSTR:F <text>`
|
||||
|
||||
| Field | Type | Description |
|
||||
|---------|--------|------------------------------------|
|
||||
| `event` | string | `"on"` (PTT pressed) or `"off"` (PTT released) |
|
||||
## Timing
|
||||
|
||||
## Transcript messages (server → client)
|
||||
| Parameter | Value | Purpose |
|
||||
|-----------|-------|---------|
|
||||
| NOP interval | 50ms | Heartbeat frequency while PTT held |
|
||||
| Silence timeout | 150ms | Stop recording if no NOPs (3 missed = lost OFF) |
|
||||
| NOP bandwidth | ~20 msg/s × ~20 bytes | ~400 bytes/s — negligible |
|
||||
|
||||
Sent by the server back to Robovoice over the same TCP connection.
|
||||
## Why this works
|
||||
|
||||
```json
|
||||
{"final": false, "text": "hello world"}
|
||||
```
|
||||
|
||||
```json
|
||||
{"final": true, "text": "hello world how are you"}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|---------|---------|----------|--------------------------------------------------|
|
||||
| `final` | bool | yes | `true` = final result, `false` = partial |
|
||||
| `text` | string | yes | The transcript text (may be empty for partials) |
|
||||
|
||||
### Semantics
|
||||
|
||||
- **`final: false`** — intermediate recognition result (partial). Robovoice
|
||||
logs these but does not act on them (only `final` triggers TTS).
|
||||
- **`final: true`** — complete utterance. Robovoice feeds this to the TTS
|
||||
engine and speaks it.
|
||||
|
||||
Malformed JSON or unknown field values are silently dropped by the client.
|
||||
1. **Outbound broadcast `:68→:67` to `255.255.255.255`** passes the
|
||||
WireGuard WFP killswitch (DHCP exception matches this exact pattern)
|
||||
2. **Inbound `:67→:68`** has no address restriction in the WFP rule, so
|
||||
unicast replies pass through
|
||||
3. **Binding to a specific interface IP** (not `0.0.0.0`) wins unicast
|
||||
delivery over the Windows DHCP client service
|
||||
4. **NOP spam** ensures the ON message gets through even at 5% packet loss
|
||||
(3 consecutive NOPs = ~0.01% drop probability)
|
||||
5. **150ms timeout** is the backstop for lost OFF — at 50ms intervals, 3
|
||||
consecutive NOPs must all be lost to false-stop
|
||||
|
||||
Reference in New Issue
Block a user