From 4a58b94f35d08efe3dcf79806c9955adb19c0e27 Mon Sep 17 00:00:00 2001 From: Mute Date: Tue, 11 Aug 2026 09:03:50 +0000 Subject: [PATCH] v0.5: TCP STT client with PTT on/off, server endpoint in config + UI --- Robovoice.App/AppConfig.cs | 1 + Robovoice.App/MainForm.Designer.cs | 29 +- Robovoice.App/MainForm.cs | 85 ++++-- Robovoice.App/Orchestrator.cs | 14 +- Robovoice.App/Robovoice.App.csproj | 3 +- Robovoice.Core/ISttSource.cs | 2 + .../Robovoice.Stt.Tcp.csproj | 26 +- Robovoice.Stt.Tcp/TcpSttSource.cs | 253 +++++++++++++++++ Robovoice.slnx | 2 +- Server/NOTES.md | 264 ++++++++++++++++++ Server/PROTOCOL.md | 70 +++++ 11 files changed, 696 insertions(+), 53 deletions(-) rename Robovoice.Stt.Udp/Robovoice.Stt.Udp.csproj => Robovoice.Stt.Tcp/Robovoice.Stt.Tcp.csproj (84%) create mode 100644 Robovoice.Stt.Tcp/TcpSttSource.cs create mode 100644 Server/NOTES.md create mode 100644 Server/PROTOCOL.md diff --git a/Robovoice.App/AppConfig.cs b/Robovoice.App/AppConfig.cs index f0ad4a7..ef2536f 100644 --- a/Robovoice.App/AppConfig.cs +++ b/Robovoice.App/AppConfig.cs @@ -12,6 +12,7 @@ public sealed class AppConfig public int LengthScale { get; set; } = 100; public int NoiseWScale { get; set; } = 800; public bool MinimizeToTray { get; set; } = true; + public string ServerEndpoint { get; set; } = "127.0.0.1:5210"; public static string AppDataDir => Path.Combine( Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData), diff --git a/Robovoice.App/MainForm.Designer.cs b/Robovoice.App/MainForm.Designer.cs index b11f1cd..f0309ae 100644 --- a/Robovoice.App/MainForm.Designer.cs +++ b/Robovoice.App/MainForm.Designer.cs @@ -16,6 +16,9 @@ partial class MainForm private Label lblFile = null!; private Button btnBrowseFile = null!; private Label lblFileName = null!; + private Label lblServer = null!; + private TextBox txtServer = null!; + private Button btnConnect = null!; private Label lblLineStatus = null!; private RichTextBox txtLog = null!; private CheckBox chkMinimizeToTray = null!; @@ -54,6 +57,9 @@ partial class MainForm lblFile = new Label(); btnBrowseFile = new Button(); lblFileName = new Label(); + lblServer = new Label(); + txtServer = new TextBox(); + btnConnect = new Button(); lblLineStatus = new Label(); txtLog = new RichTextBox(); chkMinimizeToTray = new CheckBox(); @@ -134,13 +140,29 @@ partial class MainForm // lblFileName lblFileName.Text = "(none)"; lblFileName.Location = new Point(155, 48); - lblFileName.Size = new Size(490, 23); + lblFileName.Size = new Size(280, 23); lblFileName.TextAlign = ContentAlignment.MiddleLeft; lblFileName.ForeColor = Color.Gray; + // lblServer + lblServer.Text = "Server:"; + lblServer.Location = new Point(440, 48); + lblServer.Size = new Size(45, 23); + lblServer.TextAlign = ContentAlignment.MiddleLeft; + + // txtServer + txtServer.Location = new Point(488, 45); + txtServer.Size = new Size(110, 23); + + // btnConnect + btnConnect.Text = "Connect"; + btnConnect.Location = new Point(603, 44); + btnConnect.Size = new Size(60, 25); + btnConnect.UseVisualStyleBackColor = true; + // lblLineStatus lblLineStatus.Text = ""; - lblLineStatus.Location = new Point(650, 48); + lblLineStatus.Location = new Point(645, 48); lblLineStatus.Size = new Size(160, 23); lblLineStatus.TextAlign = ContentAlignment.MiddleRight; lblLineStatus.ForeColor = Color.DarkBlue; @@ -254,6 +276,9 @@ partial class MainForm Controls.Add(lblFile); Controls.Add(btnBrowseFile); Controls.Add(lblFileName); + Controls.Add(lblServer); + Controls.Add(txtServer); + Controls.Add(btnConnect); Controls.Add(lblLineStatus); Controls.Add(lblNoise); Controls.Add(trkNoise); diff --git a/Robovoice.App/MainForm.cs b/Robovoice.App/MainForm.cs index 3eab176..f2c71d3 100644 --- a/Robovoice.App/MainForm.cs +++ b/Robovoice.App/MainForm.cs @@ -2,7 +2,7 @@ using NAudio.Wave; using Robovoice.App; using Robovoice.Core; using Robovoice.Core.Voices; -using Robovoice.Stt.File; +using Robovoice.Stt.Tcp; using Robovoice.Tts.LibPiper; using System.Diagnostics; @@ -16,7 +16,7 @@ internal sealed partial class MainForm : Form private LibPiperTtsEngine? _tts; private AudioOutput? _audioOutput; - private FileSttSource? _sttSource; + private TcpSttSource? _sttSource; private Orchestrator? _orchestrator; private PttHotkey? _pttHotkey; private NotifyIcon? _trayIcon; @@ -61,6 +61,7 @@ internal sealed partial class MainForm : Form OnSliderScroll(null, EventArgs.Empty); chkMinimizeToTray.Checked = _config.MinimizeToTray; + txtServer.Text = _config.ServerEndpoint; btnBrowseFile.Click += OnBrowseFile; btnTestVoice.Click += OnTestVoice; @@ -70,6 +71,8 @@ internal sealed partial class MainForm : Form txtPttKey.KeyDown += OnPttKeyDown; cmbOutput.SelectedIndexChanged += OnOutputChanged; cmbVoice.SelectedIndexChanged += OnVoiceChanged; + txtServer.Leave += OnServerChanged; + btnConnect.Click += OnConnect; trkNoise.Scroll += OnSliderScroll; trkSpeed.Scroll += OnSliderScroll; @@ -239,7 +242,13 @@ internal sealed partial class MainForm : Form lengthScale: trkSpeed.Value / 100.0f, noiseWScale: trkNoiseW.Value / 1000.0f); _audioOutput = new AudioOutput(); - _sttSource = new FileSttSource(); + + if (_sttSource is null) + { + _sttSource = new TcpSttSource { ServerEndpoint = txtServer.Text, Log = Log }; + Log($"STT endpoint: {_sttSource.ServerEndpoint} (press Connect)"); + } + _orchestrator?.DisposeAsync().AsTask().Wait(); _orchestrator = new Orchestrator(_tts, _audioOutput, _sttSource, Log) { @@ -249,7 +258,7 @@ internal sealed partial class MainForm : Form try { await _orchestrator.InitializeTtsAsync(); - Log("Engine ready. Select a text file and press PTT key."); + Log("Engine ready. Press PTT to send to STT server."); SetupHotkey(); } catch (Exception ex) @@ -271,22 +280,27 @@ internal sealed partial class MainForm : Form private void OnPttPressed(object? sender, EventArgs e) { Log("PTT pressed"); + try + { + _sttSource?.SendOn(); + } + catch (Exception ex) + { + Log($"SendOn failed: {ex.Message}"); + } } private void OnPttReleased(object? sender, EventArgs e) { Log("PTT released"); - if (_orchestrator is null || _sttSource is null) return; - - if (_sttSource.LineCount == 0) + try { - Log("No text file loaded."); - return; + _sttSource?.SendOff(); + } + catch (Exception ex) + { + Log($"SendOff failed: {ex.Message}"); } - - UpdateLineStatus(); - _orchestrator.TriggerNextLine(); - UpdateLineStatus(); } private void OnBrowseFile(object? sender, EventArgs e) @@ -294,16 +308,16 @@ internal sealed partial class MainForm : Form using var dlg = new OpenFileDialog { Filter = "Text files (*.txt)|*.txt|All files (*.*)|*.*", - Title = "Select a text file to read sequentially", + Title = "Select a text file to speak", }; if (dlg.ShowDialog() == DialogResult.OK) { - _sttSource?.LoadFile(dlg.FileName); + string text = File.ReadAllText(dlg.FileName); lblFileName.Text = Path.GetFileName(dlg.FileName); lblFileName.ForeColor = Color.Black; - UpdateLineStatus(); - Log($"Loaded: {dlg.FileName} ({_sttSource?.LineCount} lines)"); + Log($"Loaded: {dlg.FileName} ({text.Length} chars)"); + _ = _orchestrator?.SynthesizeAsync(text); } } @@ -411,15 +425,38 @@ internal sealed partial class MainForm : Form SaveConfig(); } - private void UpdateLineStatus() + private void OnServerChanged(object? sender, EventArgs e) { - if (_sttSource is null || _sttSource.LineCount == 0) + if (_sttSource is not null) { - lblLineStatus.Text = ""; + _sttSource.ServerEndpoint = txtServer.Text; + Log($"Server endpoint: {txtServer.Text}"); + } + SaveConfig(); + } + + private async void OnConnect(object? sender, EventArgs e) + { + if (_sttSource is null) + { + Log("STT source not initialized."); return; } - int displayIdx = (_sttSource.CurrentIndex % _sttSource.LineCount) + 1; - lblLineStatus.Text = $"Line {displayIdx}/{_sttSource.LineCount}"; + + _sttSource.ServerEndpoint = txtServer.Text; + SaveConfig(); + btnConnect.Enabled = false; + try + { + if (_sttSource.IsRunning) + await _sttSource.ReconnectAsync(); + else + await _sttSource.StartAsync(); + } + finally + { + btnConnect.Enabled = true; + } } private void SetupTray() @@ -487,6 +524,7 @@ internal sealed partial class MainForm : Form _config.LengthScale = trkSpeed.Value; _config.NoiseWScale = trkNoiseW.Value; _config.MinimizeToTray = chkMinimizeToTray.Checked; + _config.ServerEndpoint = txtServer.Text; _config.Save(); } @@ -508,7 +546,8 @@ internal sealed partial class MainForm : Form { _pttHotkey?.Dispose(); _trayIcon!.Visible = false; - _orchestrator?.DisposeAsync().AsTask().Wait(); + _orchestrator?.DisposeAsync().AsTask().Wait(2000); + _sttSource?.DisposeAsync().AsTask().Wait(2000); SaveConfig(); } } diff --git a/Robovoice.App/Orchestrator.cs b/Robovoice.App/Orchestrator.cs index c1d9756..5ac2e92 100644 --- a/Robovoice.App/Orchestrator.cs +++ b/Robovoice.App/Orchestrator.cs @@ -1,5 +1,4 @@ using Robovoice.Core; -using Robovoice.Stt.File; using Robovoice.Tts.LibPiper; namespace Robovoice.App; @@ -8,7 +7,7 @@ internal sealed class Orchestrator : IAsyncDisposable { private readonly LibPiperTtsEngine _tts; private readonly AudioOutput _audioOutput; - private readonly FileSttSource _sttSource; + private readonly ISttSource _sttSource; private readonly Action _log; private CancellationTokenSource? _currentCts; private bool _disposed; @@ -18,7 +17,7 @@ internal sealed class Orchestrator : IAsyncDisposable public Orchestrator( LibPiperTtsEngine tts, AudioOutput audioOutput, - FileSttSource sttSource, + ISttSource sttSource, Action log) { _tts = tts; @@ -97,15 +96,6 @@ internal sealed class Orchestrator : IAsyncDisposable sw.Stop(); } - public void TriggerNextLine() - { - _sttSource.EmitNext(); - } - - public string? GetPendingLine() => _sttSource.GetPendingLine(); - public int GetCurrentLineIndex() => _sttSource.CurrentIndex; - public int GetLineCount() => _sttSource.LineCount; - public async ValueTask DisposeAsync() { if (_disposed) return; diff --git a/Robovoice.App/Robovoice.App.csproj b/Robovoice.App/Robovoice.App.csproj index 6bf6aa7..d50c566 100644 --- a/Robovoice.App/Robovoice.App.csproj +++ b/Robovoice.App/Robovoice.App.csproj @@ -3,8 +3,7 @@ - - + diff --git a/Robovoice.Core/ISttSource.cs b/Robovoice.Core/ISttSource.cs index f91c362..703449d 100644 --- a/Robovoice.Core/ISttSource.cs +++ b/Robovoice.Core/ISttSource.cs @@ -2,6 +2,8 @@ namespace Robovoice.Core; public interface ISttSource : IAsyncDisposable { + event TranscriptEventHandler? TranscriptReceived; + Task StartAsync(CancellationToken ct = default); Task StopAsync(CancellationToken ct = default); diff --git a/Robovoice.Stt.Udp/Robovoice.Stt.Udp.csproj b/Robovoice.Stt.Tcp/Robovoice.Stt.Tcp.csproj similarity index 84% rename from Robovoice.Stt.Udp/Robovoice.Stt.Udp.csproj rename to Robovoice.Stt.Tcp/Robovoice.Stt.Tcp.csproj index a946c55..a74cfd0 100644 --- a/Robovoice.Stt.Udp/Robovoice.Stt.Udp.csproj +++ b/Robovoice.Stt.Tcp/Robovoice.Stt.Tcp.csproj @@ -1,13 +1,13 @@ - - - - - - - - net10.0 - enable - enable - - - + + + + + + + + net10.0 + enable + enable + + + diff --git a/Robovoice.Stt.Tcp/TcpSttSource.cs b/Robovoice.Stt.Tcp/TcpSttSource.cs new file mode 100644 index 0000000..54f4abd --- /dev/null +++ b/Robovoice.Stt.Tcp/TcpSttSource.cs @@ -0,0 +1,253 @@ +using System.Net; +using System.Net.Sockets; +using System.Text.Json; +using System.Text.Json.Serialization; +using Robovoice.Core; + +namespace Robovoice.Stt.Tcp; + +public sealed class TcpSttSource : ISttSource +{ + private TcpClient? _tcp; + private NetworkStream? _stream; + private StreamReader? _reader; + private StreamWriter? _writer; + private CancellationTokenSource? _cts; + private Task? _runTask; + private readonly object _sendLock = new(); + private bool _disposed; + + public string ServerEndpoint { get; set; } = "127.0.0.1:5210"; + + public bool IsRunning => _cts is not null; + + public Action? Log { get; set; } + + public event TranscriptEventHandler? TranscriptReceived; + + public Task StartAsync(CancellationToken ct = default) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (_cts is not null) + return Task.CompletedTask; + + _cts = CancellationTokenSource.CreateLinkedTokenSource(ct); + _runTask = RunAsync(_cts.Token); + return Task.CompletedTask; + } + + public async Task StopAsync(CancellationToken ct = default) + { + if (_cts is not null) + _cts.Cancel(); + + CleanupConnection(); + + if (_runTask is not null) + { + try { await _runTask.WaitAsync(ct); } + catch { } + _runTask = null; + } + + _cts?.Dispose(); + _cts = null; + } + + private async Task RunAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested) + { + IPEndPoint? endpoint = ParseEndpoint(ServerEndpoint); + if (endpoint is null) + { + Log?.Invoke($"STT: invalid endpoint '{ServerEndpoint}'"); + try { await Task.Delay(3000, ct); } catch { break; } + continue; + } + + try + { + _tcp = new TcpClient(); + using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(ct); + connectCts.CancelAfter(TimeSpan.FromSeconds(5)); + await _tcp.ConnectAsync(endpoint.Address, endpoint.Port, connectCts.Token); + + _stream = _tcp.GetStream(); + _reader = new StreamReader(_stream, System.Text.Encoding.UTF8); + _writer = new StreamWriter(_stream, System.Text.Encoding.UTF8) { AutoFlush = true }; + + Log?.Invoke($"STT: connected to {ServerEndpoint}"); + + await ReceiveLoopAsync(ct); + } + catch (OperationCanceledException) + { + break; + } + catch (Exception ex) + { + Log?.Invoke($"STT: connection failed ({ex.Message}), retrying..."); + } + finally + { + CleanupConnection(); + } + + if (!ct.IsCancellationRequested) + { + try { await Task.Delay(3000, ct); } + catch (OperationCanceledException) { break; } + } + } + } + + private async Task ReceiveLoopAsync(CancellationToken ct) + { + while (!ct.IsCancellationRequested && _reader is not null) + { + string? line; + try + { + line = await _reader.ReadLineAsync(ct); + } + catch + { + break; + } + + if (line is null) + break; + + TranscriptMessage? message = ParseTranscriptLine(line); + if (message is null) + continue; + + TranscriptReceived?.Invoke(this, new TranscriptEventArgs + { + Message = message, + }); + } + } + + public void SendOn() + { + SendControl("on"); + } + + public void SendOff() + { + SendControl("off"); + } + + public async Task ReconnectAsync(CancellationToken ct = default) + { + if (_cts is null) + return; + + Log?.Invoke("STT: reconnecting..."); + CleanupConnection(); + + try { await Task.Delay(500, ct); } + catch (OperationCanceledException) { return; } + } + + private void SendControl(string evt) + { + lock (_sendLock) + { + if (_writer is null) + return; + + try + { + _writer.WriteLine(JsonSerializer.Serialize(new ControlDto { Event = evt })); + } + catch + { + Log?.Invoke($"STT: failed to send '{evt}' (not connected?)"); + } + } + } + + private void CleanupConnection() + { + lock (_sendLock) + { + _writer?.Dispose(); + _reader?.Dispose(); + _stream?.Dispose(); + _tcp?.Dispose(); + _writer = null; + _reader = null; + _stream = null; + _tcp = null; + } + } + + private static IPEndPoint? ParseEndpoint(string endpoint) + { + int colon = endpoint.LastIndexOf(':'); + if (colon <= 0) + return null; + + string host = endpoint[..colon]; + if (!int.TryParse(endpoint[(colon + 1)..], out int port)) + return null; + + if (IPAddress.TryParse(host, out var addr)) + return new IPEndPoint(addr, port); + + try + { + var addresses = Dns.GetHostAddresses(host); + addr = addresses.FirstOrDefault(a => a.AddressFamily == AddressFamily.InterNetwork); + if (addr is null) + return null; + return new IPEndPoint(addr, port); + } + catch + { + return null; + } + } + + private static TranscriptMessage? ParseTranscriptLine(string line) + { + if (string.IsNullOrWhiteSpace(line)) + return null; + + try + { + var dto = JsonSerializer.Deserialize(line); + if (dto is null) + return null; + + var type = dto.Final ? TranscriptType.Final : TranscriptType.Partial; + return new TranscriptMessage(type, dto.Text ?? string.Empty); + } + catch (JsonException) + { + return null; + } + } + + public async ValueTask DisposeAsync() + { + if (_disposed) return; + await StopAsync(); + _disposed = true; + } +} + +internal sealed class ControlDto +{ + [JsonPropertyName("event")] + public string Event { get; set; } = string.Empty; +} + +internal sealed class TransmitDto +{ + public bool Final { get; set; } + public string Text { get; set; } = string.Empty; +} diff --git a/Robovoice.slnx b/Robovoice.slnx index 7206f29..8d92e14 100644 --- a/Robovoice.slnx +++ b/Robovoice.slnx @@ -2,6 +2,6 @@ - + diff --git a/Server/NOTES.md b/Server/NOTES.md new file mode 100644 index 0000000..ac76094 --- /dev/null +++ b/Server/NOTES.md @@ -0,0 +1,264 @@ +# Server implementation notes + +The STT server listens for TCP connections from Robovoice, captures audio +from a microphone when `on` is received, runs speech recognition (Moonshine), +and sends transcript messages back over the same connection. + +## Architecture + +``` + ┌── on/off (TCP, newline-delimited JSON) +Robovoice ──────────────►│ + │ STT Server +Robovoice ◄──────────────┤ + └── partial/final (TCP, newline-delimited JSON) +``` + +The server: +1. Listens on a TCP port (e.g. 5210) +2. Accepts a connection from Robovoice +3. Reads lines: waits for `{"event":"on"}` +4. Records audio from the microphone +5. Waits for `{"event":"off"}` (or a timeout) +6. Runs STT on the captured audio +7. Sends `{"final":true,"text":"..."}`\n back over the connection + +## Framing + +Every message is a single JSON object on one line, terminated by `\n`. No +length prefix, no binary framing. Use `readline()` / `StreamReader.ReadLineAsync()`. + +## Python server with Moonshine + +[Moonshine](https://github.com/usefulsensors/moonshine) is a lightweight ASR +model by Useful Sensors. Install with `pip install moonshine`. + +```python +import socket +import json +import numpy as np +import sounddevice as sd +import moonshine + +LISTEN_PORT = 5210 +SAMPLE_RATE = 16000 + +model = moonshine.MoonshineModel(model="moonshine/base") + +server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +server.bind(("0.0.0.0", LISTEN_PORT)) +server.listen(1) + +print(f"STT server listening on :{LISTEN_PORT}") + +while True: + conn, addr = server.accept() + print(f"Client connected: {addr}") + + buf = "" + with conn: + while True: + data = conn.recv(4096).decode("utf-8") + if not data: + break + buf += data + + while "\n" in buf: + line, buf = buf.split("\n", 1) + msg = json.loads(line) + + if msg.get("event") == "on": + print("PTT on — recording") + audio_chunks = [] + + # Record until "off" or timeout + conn.settimeout(0.1) + while True: + try: + data2 = conn.recv(4096).decode("utf-8") + if not data2: + break + buf += data2 + while "\n" in buf: + line2, buf = buf.split("\n", 1) + msg2 = json.loads(line2) + if msg2.get("event") == "off": + break + except socket.timeout: + pass + + chunk = sd.rec(int(SAMPLE_RATE * 0.1), + samplerate=SAMPLE_RATE, + channels=1, dtype="float32") + sd.wait() + audio_chunks.append(chunk.flatten()) + + conn.settimeout(None) + + if not audio_chunks: + continue + + audio = np.concatenate(audio_chunks) + print(f"Captured {len(audio)/SAMPLE_RATE:.1f}s") + + text = moonshine.transcribe(model, audio).strip() + + if text: + print(f"Transcript: {text}") + reply = json.dumps({"final": True, "text": text}) + conn.sendall((reply + "\n").encode("utf-8")) + else: + print("Empty transcript") +``` + +## Python server with streaming partials + +For lower latency, send partial results while still recording: + +```python +import socket +import json +import numpy as np +import sounddevice as sd +import moonshine + +LISTEN_PORT = 5210 +SAMPLE_RATE = 16000 +CHUNK_DURATION = 0.5 + +model = moonshine.MoonshineModel(model="moonshine/base") + +server = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +server.bind(("0.0.0.0", LISTEN_PORT)) +server.listen(1) + +print(f"STT server listening on :{LISTEN_PORT}") + +while True: + conn, addr = server.accept() + print(f"Client connected: {addr}") + buf = "" + + with conn: + while True: + data = conn.recv(4096).decode("utf-8") + if not data: + break + buf += data + + while "\n" in buf: + line, buf = buf.split("\n", 1) + msg = json.loads(line) + + if msg.get("event") != "on": + continue + + print("PTT on — recording") + audio_chunks = [] + + while True: + try: + conn.settimeout(CHUNK_DURATION) + data2 = conn.recv(4096).decode("utf-8") + if not data2: + break + buf += data2 + while "\n" in buf: + line2, buf = buf.split("\n", 1) + msg2 = json.loads(line2) + if msg2.get("event") == "off": + break + except socket.timeout: + pass + + chunk = sd.rec(int(SAMPLE_RATE * CHUNK_DURATION), + samplerate=SAMPLE_RATE, + channels=1, dtype="float32") + sd.wait() + audio_chunks.append(chunk.flatten()) + + # Send partial every few chunks + if len(audio_chunks) % 4 == 0: + partial_audio = np.concatenate(audio_chunks) + partial_text = moonshine.transcribe(model, partial_audio).strip() + if partial_text: + reply = json.dumps({"final": False, "text": partial_text}) + conn.sendall((reply + "\n").encode("utf-8")) + + conn.settimeout(None) + + if not audio_chunks: + continue + + audio = np.concatenate(audio_chunks) + text = moonshine.transcribe(model, audio).strip() + + if text: + print(f"Final: {text}") + reply = json.dumps({"final": True, "text": text}) + conn.sendall((reply + "\n").encode("utf-8")) +``` + +## C# server skeleton + +```csharp +using System.Net; +using System.Net.Sockets; +using System.Text.Json; + +var listener = new TcpListener(IPAddress.Any, 5210); +listener.Start(); + +Console.WriteLine("STT server listening on :5210"); + +while (true) +{ + var client = listener.AcceptTcpClient(); + Console.WriteLine($"Client connected: {client.Client.RemoteEndPoint}"); + + using var stream = client.GetStream(); + using var reader = new StreamReader(stream, Encoding.UTF8); + using var writer = new StreamWriter(stream, Encoding.UTF8) { AutoFlush = true }; + + string? line; + while ((line = reader.ReadLine()) is not null) + { + var msg = JsonSerializer.Deserialize>(line); + if (msg?["event"] != "on") + continue; + + Console.WriteLine("PTT on — recording"); + // Capture audio... + + // Read until "off" + while ((line = reader.ReadLine()) is not null) + { + msg = JsonSerializer.Deserialize>(line); + if (msg?["event"] == "off") + break; + } + + // Run STT... + string text = "recognized text here"; + + var reply = JsonSerializer.Serialize(new { final = true, text }); + writer.WriteLine(reply); + } +} +``` + +## Tips + +- **One connection per client:** Robovoice maintains a single persistent TCP + connection. The server should handle one client at a time (or track + multiple if needed). +- **Timeout:** implement a recording timeout in case the `off` message is + delayed or the client disconnects. 10–30 seconds is reasonable. +- **Partials:** optional but improve UX — Robovoice logs them so the user + sees live feedback. Only `final` triggers TTS. +- **Encoding:** always UTF-8. Every line is a UTF-8 JSON object terminated + by `\n`. +- **Reconnection:** Robovoice auto-reconnects every 3 seconds if the + connection drops. The server just needs to accept new connections. +- **Moonshine models:** `moonshine/base` (faster, less accurate) or + `moonshine/tiny` (fastest). Choose based on your hardware. diff --git a/Server/PROTOCOL.md b/Server/PROTOCOL.md new file mode 100644 index 0000000..89bbc45 --- /dev/null +++ b/Server/PROTOCOL.md @@ -0,0 +1,70 @@ +# Robovoice TCP STT Protocol + +## Overview + +Robovoice acts as a **client**: it connects to a remote STT server over TCP, +sends control messages when the user presses/releases the PTT key, and +receives transcript messages back. The STT server captures audio from a +microphone, runs speech recognition (Moonshine), and sends transcripts back +over the same connection. + +``` +[Robovoice client] --TCP--> [STT server :5210] + │ │ + ├── {"event":"on"}\n ──────►│ + │ ├── capture audio + ├── {"event":"off"}\n ──────►│ + │ ├── run STT + │◄── {"final":true,...}\n ──┤ +``` + +## Transport + +- **Protocol:** TCP (reliable, ordered, connection-oriented) +- **Server endpoint:** configurable in Robovoice UI (default `127.0.0.1:5210`) +- **Framing:** newline-delimited JSON (NDJSON) — each message is a single + UTF-8 JSON object terminated by `\n` +- **Auto-reconnect:** if the connection drops, Robovoice retries every 3 + seconds until the server is available + +## Control messages (client → server) + +Sent by Robovoice when the user presses/releases the PTT key. + +```json +{"event": "on"} +``` + +```json +{"event": "off"} +``` + +| Field | Type | Description | +|---------|--------|------------------------------------| +| `event` | string | `"on"` (PTT pressed) or `"off"` (PTT released) | + +## Transcript messages (server → client) + +Sent by the server back to Robovoice over the same TCP connection. + +```json +{"final": false, "text": "hello world"} +``` + +```json +{"final": true, "text": "hello world how are you"} +``` + +| Field | Type | Required | Description | +|---------|---------|----------|--------------------------------------------------| +| `final` | bool | yes | `true` = final result, `false` = partial | +| `text` | string | yes | The transcript text (may be empty for partials) | + +### Semantics + +- **`final: false`** — intermediate recognition result (partial). Robovoice + logs these but does not act on them (only `final` triggers TTS). +- **`final: true`** — complete utterance. Robovoice feeds this to the TTS + engine and speaks it. + +Malformed JSON or unknown field values are silently dropped by the client.