v0.5: TCP STT client with PTT on/off, server endpoint in config + UI
This commit is contained in:
@@ -12,6 +12,7 @@ public sealed class AppConfig
|
||||
public int LengthScale { get; set; } = 100;
|
||||
public int NoiseWScale { get; set; } = 800;
|
||||
public bool MinimizeToTray { get; set; } = true;
|
||||
public string ServerEndpoint { get; set; } = "127.0.0.1:5210";
|
||||
|
||||
public static string AppDataDir => Path.Combine(
|
||||
Environment.GetFolderPath(Environment.SpecialFolder.LocalApplicationData),
|
||||
|
||||
Generated
+27
-2
@@ -16,6 +16,9 @@ partial class MainForm
|
||||
private Label lblFile = null!;
|
||||
private Button btnBrowseFile = null!;
|
||||
private Label lblFileName = null!;
|
||||
private Label lblServer = null!;
|
||||
private TextBox txtServer = null!;
|
||||
private Button btnConnect = null!;
|
||||
private Label lblLineStatus = null!;
|
||||
private RichTextBox txtLog = null!;
|
||||
private CheckBox chkMinimizeToTray = null!;
|
||||
@@ -54,6 +57,9 @@ partial class MainForm
|
||||
lblFile = new Label();
|
||||
btnBrowseFile = new Button();
|
||||
lblFileName = new Label();
|
||||
lblServer = new Label();
|
||||
txtServer = new TextBox();
|
||||
btnConnect = new Button();
|
||||
lblLineStatus = new Label();
|
||||
txtLog = new RichTextBox();
|
||||
chkMinimizeToTray = new CheckBox();
|
||||
@@ -134,13 +140,29 @@ partial class MainForm
|
||||
// lblFileName
|
||||
lblFileName.Text = "(none)";
|
||||
lblFileName.Location = new Point(155, 48);
|
||||
lblFileName.Size = new Size(490, 23);
|
||||
lblFileName.Size = new Size(280, 23);
|
||||
lblFileName.TextAlign = ContentAlignment.MiddleLeft;
|
||||
lblFileName.ForeColor = Color.Gray;
|
||||
|
||||
// lblServer
|
||||
lblServer.Text = "Server:";
|
||||
lblServer.Location = new Point(440, 48);
|
||||
lblServer.Size = new Size(45, 23);
|
||||
lblServer.TextAlign = ContentAlignment.MiddleLeft;
|
||||
|
||||
// txtServer
|
||||
txtServer.Location = new Point(488, 45);
|
||||
txtServer.Size = new Size(110, 23);
|
||||
|
||||
// btnConnect
|
||||
btnConnect.Text = "Connect";
|
||||
btnConnect.Location = new Point(603, 44);
|
||||
btnConnect.Size = new Size(60, 25);
|
||||
btnConnect.UseVisualStyleBackColor = true;
|
||||
|
||||
// lblLineStatus
|
||||
lblLineStatus.Text = "";
|
||||
lblLineStatus.Location = new Point(650, 48);
|
||||
lblLineStatus.Location = new Point(645, 48);
|
||||
lblLineStatus.Size = new Size(160, 23);
|
||||
lblLineStatus.TextAlign = ContentAlignment.MiddleRight;
|
||||
lblLineStatus.ForeColor = Color.DarkBlue;
|
||||
@@ -254,6 +276,9 @@ partial class MainForm
|
||||
Controls.Add(lblFile);
|
||||
Controls.Add(btnBrowseFile);
|
||||
Controls.Add(lblFileName);
|
||||
Controls.Add(lblServer);
|
||||
Controls.Add(txtServer);
|
||||
Controls.Add(btnConnect);
|
||||
Controls.Add(lblLineStatus);
|
||||
Controls.Add(lblNoise);
|
||||
Controls.Add(trkNoise);
|
||||
|
||||
+62
-23
@@ -2,7 +2,7 @@ using NAudio.Wave;
|
||||
using Robovoice.App;
|
||||
using Robovoice.Core;
|
||||
using Robovoice.Core.Voices;
|
||||
using Robovoice.Stt.File;
|
||||
using Robovoice.Stt.Tcp;
|
||||
using Robovoice.Tts.LibPiper;
|
||||
using System.Diagnostics;
|
||||
|
||||
@@ -16,7 +16,7 @@ internal sealed partial class MainForm : Form
|
||||
|
||||
private LibPiperTtsEngine? _tts;
|
||||
private AudioOutput? _audioOutput;
|
||||
private FileSttSource? _sttSource;
|
||||
private TcpSttSource? _sttSource;
|
||||
private Orchestrator? _orchestrator;
|
||||
private PttHotkey? _pttHotkey;
|
||||
private NotifyIcon? _trayIcon;
|
||||
@@ -61,6 +61,7 @@ internal sealed partial class MainForm : Form
|
||||
OnSliderScroll(null, EventArgs.Empty);
|
||||
|
||||
chkMinimizeToTray.Checked = _config.MinimizeToTray;
|
||||
txtServer.Text = _config.ServerEndpoint;
|
||||
|
||||
btnBrowseFile.Click += OnBrowseFile;
|
||||
btnTestVoice.Click += OnTestVoice;
|
||||
@@ -70,6 +71,8 @@ internal sealed partial class MainForm : Form
|
||||
txtPttKey.KeyDown += OnPttKeyDown;
|
||||
cmbOutput.SelectedIndexChanged += OnOutputChanged;
|
||||
cmbVoice.SelectedIndexChanged += OnVoiceChanged;
|
||||
txtServer.Leave += OnServerChanged;
|
||||
btnConnect.Click += OnConnect;
|
||||
|
||||
trkNoise.Scroll += OnSliderScroll;
|
||||
trkSpeed.Scroll += OnSliderScroll;
|
||||
@@ -239,7 +242,13 @@ internal sealed partial class MainForm : Form
|
||||
lengthScale: trkSpeed.Value / 100.0f,
|
||||
noiseWScale: trkNoiseW.Value / 1000.0f);
|
||||
_audioOutput = new AudioOutput();
|
||||
_sttSource = new FileSttSource();
|
||||
|
||||
if (_sttSource is null)
|
||||
{
|
||||
_sttSource = new TcpSttSource { ServerEndpoint = txtServer.Text, Log = Log };
|
||||
Log($"STT endpoint: {_sttSource.ServerEndpoint} (press Connect)");
|
||||
}
|
||||
|
||||
_orchestrator?.DisposeAsync().AsTask().Wait();
|
||||
_orchestrator = new Orchestrator(_tts, _audioOutput, _sttSource, Log)
|
||||
{
|
||||
@@ -249,7 +258,7 @@ internal sealed partial class MainForm : Form
|
||||
try
|
||||
{
|
||||
await _orchestrator.InitializeTtsAsync();
|
||||
Log("Engine ready. Select a text file and press PTT key.");
|
||||
Log("Engine ready. Press PTT to send to STT server.");
|
||||
SetupHotkey();
|
||||
}
|
||||
catch (Exception ex)
|
||||
@@ -271,22 +280,27 @@ internal sealed partial class MainForm : Form
|
||||
private void OnPttPressed(object? sender, EventArgs e)
|
||||
{
|
||||
Log("PTT pressed");
|
||||
try
|
||||
{
|
||||
_sttSource?.SendOn();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log($"SendOn failed: {ex.Message}");
|
||||
}
|
||||
}
|
||||
|
||||
private void OnPttReleased(object? sender, EventArgs e)
|
||||
{
|
||||
Log("PTT released");
|
||||
if (_orchestrator is null || _sttSource is null) return;
|
||||
|
||||
if (_sttSource.LineCount == 0)
|
||||
try
|
||||
{
|
||||
Log("No text file loaded.");
|
||||
return;
|
||||
_sttSource?.SendOff();
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log($"SendOff failed: {ex.Message}");
|
||||
}
|
||||
|
||||
UpdateLineStatus();
|
||||
_orchestrator.TriggerNextLine();
|
||||
UpdateLineStatus();
|
||||
}
|
||||
|
||||
private void OnBrowseFile(object? sender, EventArgs e)
|
||||
@@ -294,16 +308,16 @@ internal sealed partial class MainForm : Form
|
||||
using var dlg = new OpenFileDialog
|
||||
{
|
||||
Filter = "Text files (*.txt)|*.txt|All files (*.*)|*.*",
|
||||
Title = "Select a text file to read sequentially",
|
||||
Title = "Select a text file to speak",
|
||||
};
|
||||
|
||||
if (dlg.ShowDialog() == DialogResult.OK)
|
||||
{
|
||||
_sttSource?.LoadFile(dlg.FileName);
|
||||
string text = File.ReadAllText(dlg.FileName);
|
||||
lblFileName.Text = Path.GetFileName(dlg.FileName);
|
||||
lblFileName.ForeColor = Color.Black;
|
||||
UpdateLineStatus();
|
||||
Log($"Loaded: {dlg.FileName} ({_sttSource?.LineCount} lines)");
|
||||
Log($"Loaded: {dlg.FileName} ({text.Length} chars)");
|
||||
_ = _orchestrator?.SynthesizeAsync(text);
|
||||
}
|
||||
}
|
||||
|
||||
@@ -411,15 +425,38 @@ internal sealed partial class MainForm : Form
|
||||
SaveConfig();
|
||||
}
|
||||
|
||||
private void UpdateLineStatus()
|
||||
private void OnServerChanged(object? sender, EventArgs e)
|
||||
{
|
||||
if (_sttSource is null || _sttSource.LineCount == 0)
|
||||
if (_sttSource is not null)
|
||||
{
|
||||
lblLineStatus.Text = "";
|
||||
_sttSource.ServerEndpoint = txtServer.Text;
|
||||
Log($"Server endpoint: {txtServer.Text}");
|
||||
}
|
||||
SaveConfig();
|
||||
}
|
||||
|
||||
private async void OnConnect(object? sender, EventArgs e)
|
||||
{
|
||||
if (_sttSource is null)
|
||||
{
|
||||
Log("STT source not initialized.");
|
||||
return;
|
||||
}
|
||||
int displayIdx = (_sttSource.CurrentIndex % _sttSource.LineCount) + 1;
|
||||
lblLineStatus.Text = $"Line {displayIdx}/{_sttSource.LineCount}";
|
||||
|
||||
_sttSource.ServerEndpoint = txtServer.Text;
|
||||
SaveConfig();
|
||||
btnConnect.Enabled = false;
|
||||
try
|
||||
{
|
||||
if (_sttSource.IsRunning)
|
||||
await _sttSource.ReconnectAsync();
|
||||
else
|
||||
await _sttSource.StartAsync();
|
||||
}
|
||||
finally
|
||||
{
|
||||
btnConnect.Enabled = true;
|
||||
}
|
||||
}
|
||||
|
||||
private void SetupTray()
|
||||
@@ -487,6 +524,7 @@ internal sealed partial class MainForm : Form
|
||||
_config.LengthScale = trkSpeed.Value;
|
||||
_config.NoiseWScale = trkNoiseW.Value;
|
||||
_config.MinimizeToTray = chkMinimizeToTray.Checked;
|
||||
_config.ServerEndpoint = txtServer.Text;
|
||||
_config.Save();
|
||||
}
|
||||
|
||||
@@ -508,7 +546,8 @@ internal sealed partial class MainForm : Form
|
||||
{
|
||||
_pttHotkey?.Dispose();
|
||||
_trayIcon!.Visible = false;
|
||||
_orchestrator?.DisposeAsync().AsTask().Wait();
|
||||
_orchestrator?.DisposeAsync().AsTask().Wait(2000);
|
||||
_sttSource?.DisposeAsync().AsTask().Wait(2000);
|
||||
SaveConfig();
|
||||
}
|
||||
}
|
||||
|
||||
@@ -1,5 +1,4 @@
|
||||
using Robovoice.Core;
|
||||
using Robovoice.Stt.File;
|
||||
using Robovoice.Tts.LibPiper;
|
||||
|
||||
namespace Robovoice.App;
|
||||
@@ -8,7 +7,7 @@ internal sealed class Orchestrator : IAsyncDisposable
|
||||
{
|
||||
private readonly LibPiperTtsEngine _tts;
|
||||
private readonly AudioOutput _audioOutput;
|
||||
private readonly FileSttSource _sttSource;
|
||||
private readonly ISttSource _sttSource;
|
||||
private readonly Action<string> _log;
|
||||
private CancellationTokenSource? _currentCts;
|
||||
private bool _disposed;
|
||||
@@ -18,7 +17,7 @@ internal sealed class Orchestrator : IAsyncDisposable
|
||||
public Orchestrator(
|
||||
LibPiperTtsEngine tts,
|
||||
AudioOutput audioOutput,
|
||||
FileSttSource sttSource,
|
||||
ISttSource sttSource,
|
||||
Action<string> log)
|
||||
{
|
||||
_tts = tts;
|
||||
@@ -97,15 +96,6 @@ internal sealed class Orchestrator : IAsyncDisposable
|
||||
sw.Stop();
|
||||
}
|
||||
|
||||
public void TriggerNextLine()
|
||||
{
|
||||
_sttSource.EmitNext();
|
||||
}
|
||||
|
||||
public string? GetPendingLine() => _sttSource.GetPendingLine();
|
||||
public int GetCurrentLineIndex() => _sttSource.CurrentIndex;
|
||||
public int GetLineCount() => _sttSource.LineCount;
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed) return;
|
||||
|
||||
@@ -3,8 +3,7 @@
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Robovoice.Core\Robovoice.Core.csproj" />
|
||||
<ProjectReference Include="..\Robovoice.Tts.LibPiper\Robovoice.Tts.LibPiper.csproj" />
|
||||
<ProjectReference Include="..\Robovoice.Stt.File\Robovoice.Stt.File.csproj" />
|
||||
<ProjectReference Include="..\Robovoice.Stt.Udp\Robovoice.Stt.Udp.csproj" />
|
||||
<ProjectReference Include="..\Robovoice.Stt.Tcp\Robovoice.Stt.Tcp.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<ItemGroup>
|
||||
|
||||
@@ -2,6 +2,8 @@ namespace Robovoice.Core;
|
||||
|
||||
public interface ISttSource : IAsyncDisposable
|
||||
{
|
||||
event TranscriptEventHandler? TranscriptReceived;
|
||||
|
||||
Task StartAsync(CancellationToken ct = default);
|
||||
|
||||
Task StopAsync(CancellationToken ct = default);
|
||||
|
||||
+13
-13
@@ -1,13 +1,13 @@
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Robovoice.Core\Robovoice.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
<Project Sdk="Microsoft.NET.Sdk">
|
||||
|
||||
<ItemGroup>
|
||||
<ProjectReference Include="..\Robovoice.Core\Robovoice.Core.csproj" />
|
||||
</ItemGroup>
|
||||
|
||||
<PropertyGroup>
|
||||
<TargetFramework>net10.0</TargetFramework>
|
||||
<ImplicitUsings>enable</ImplicitUsings>
|
||||
<Nullable>enable</Nullable>
|
||||
</PropertyGroup>
|
||||
|
||||
</Project>
|
||||
@@ -0,0 +1,253 @@
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text.Json;
|
||||
using System.Text.Json.Serialization;
|
||||
using Robovoice.Core;
|
||||
|
||||
namespace Robovoice.Stt.Tcp;
|
||||
|
||||
public sealed class TcpSttSource : ISttSource
|
||||
{
|
||||
private TcpClient? _tcp;
|
||||
private NetworkStream? _stream;
|
||||
private StreamReader? _reader;
|
||||
private StreamWriter? _writer;
|
||||
private CancellationTokenSource? _cts;
|
||||
private Task? _runTask;
|
||||
private readonly object _sendLock = new();
|
||||
private bool _disposed;
|
||||
|
||||
public string ServerEndpoint { get; set; } = "127.0.0.1:5210";
|
||||
|
||||
public bool IsRunning => _cts is not null;
|
||||
|
||||
public Action<string>? Log { get; set; }
|
||||
|
||||
public event TranscriptEventHandler? TranscriptReceived;
|
||||
|
||||
public Task StartAsync(CancellationToken ct = default)
|
||||
{
|
||||
ObjectDisposedException.ThrowIf(_disposed, this);
|
||||
if (_cts is not null)
|
||||
return Task.CompletedTask;
|
||||
|
||||
_cts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
_runTask = RunAsync(_cts.Token);
|
||||
return Task.CompletedTask;
|
||||
}
|
||||
|
||||
public async Task StopAsync(CancellationToken ct = default)
|
||||
{
|
||||
if (_cts is not null)
|
||||
_cts.Cancel();
|
||||
|
||||
CleanupConnection();
|
||||
|
||||
if (_runTask is not null)
|
||||
{
|
||||
try { await _runTask.WaitAsync(ct); }
|
||||
catch { }
|
||||
_runTask = null;
|
||||
}
|
||||
|
||||
_cts?.Dispose();
|
||||
_cts = null;
|
||||
}
|
||||
|
||||
private async Task RunAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested)
|
||||
{
|
||||
IPEndPoint? endpoint = ParseEndpoint(ServerEndpoint);
|
||||
if (endpoint is null)
|
||||
{
|
||||
Log?.Invoke($"STT: invalid endpoint '{ServerEndpoint}'");
|
||||
try { await Task.Delay(3000, ct); } catch { break; }
|
||||
continue;
|
||||
}
|
||||
|
||||
try
|
||||
{
|
||||
_tcp = new TcpClient();
|
||||
using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(ct);
|
||||
connectCts.CancelAfter(TimeSpan.FromSeconds(5));
|
||||
await _tcp.ConnectAsync(endpoint.Address, endpoint.Port, connectCts.Token);
|
||||
|
||||
_stream = _tcp.GetStream();
|
||||
_reader = new StreamReader(_stream, System.Text.Encoding.UTF8);
|
||||
_writer = new StreamWriter(_stream, System.Text.Encoding.UTF8) { AutoFlush = true };
|
||||
|
||||
Log?.Invoke($"STT: connected to {ServerEndpoint}");
|
||||
|
||||
await ReceiveLoopAsync(ct);
|
||||
}
|
||||
catch (OperationCanceledException)
|
||||
{
|
||||
break;
|
||||
}
|
||||
catch (Exception ex)
|
||||
{
|
||||
Log?.Invoke($"STT: connection failed ({ex.Message}), retrying...");
|
||||
}
|
||||
finally
|
||||
{
|
||||
CleanupConnection();
|
||||
}
|
||||
|
||||
if (!ct.IsCancellationRequested)
|
||||
{
|
||||
try { await Task.Delay(3000, ct); }
|
||||
catch (OperationCanceledException) { break; }
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private async Task ReceiveLoopAsync(CancellationToken ct)
|
||||
{
|
||||
while (!ct.IsCancellationRequested && _reader is not null)
|
||||
{
|
||||
string? line;
|
||||
try
|
||||
{
|
||||
line = await _reader.ReadLineAsync(ct);
|
||||
}
|
||||
catch
|
||||
{
|
||||
break;
|
||||
}
|
||||
|
||||
if (line is null)
|
||||
break;
|
||||
|
||||
TranscriptMessage? message = ParseTranscriptLine(line);
|
||||
if (message is null)
|
||||
continue;
|
||||
|
||||
TranscriptReceived?.Invoke(this, new TranscriptEventArgs
|
||||
{
|
||||
Message = message,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
public void SendOn()
|
||||
{
|
||||
SendControl("on");
|
||||
}
|
||||
|
||||
public void SendOff()
|
||||
{
|
||||
SendControl("off");
|
||||
}
|
||||
|
||||
public async Task ReconnectAsync(CancellationToken ct = default)
|
||||
{
|
||||
if (_cts is null)
|
||||
return;
|
||||
|
||||
Log?.Invoke("STT: reconnecting...");
|
||||
CleanupConnection();
|
||||
|
||||
try { await Task.Delay(500, ct); }
|
||||
catch (OperationCanceledException) { return; }
|
||||
}
|
||||
|
||||
private void SendControl(string evt)
|
||||
{
|
||||
lock (_sendLock)
|
||||
{
|
||||
if (_writer is null)
|
||||
return;
|
||||
|
||||
try
|
||||
{
|
||||
_writer.WriteLine(JsonSerializer.Serialize(new ControlDto { Event = evt }));
|
||||
}
|
||||
catch
|
||||
{
|
||||
Log?.Invoke($"STT: failed to send '{evt}' (not connected?)");
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
private void CleanupConnection()
|
||||
{
|
||||
lock (_sendLock)
|
||||
{
|
||||
_writer?.Dispose();
|
||||
_reader?.Dispose();
|
||||
_stream?.Dispose();
|
||||
_tcp?.Dispose();
|
||||
_writer = null;
|
||||
_reader = null;
|
||||
_stream = null;
|
||||
_tcp = null;
|
||||
}
|
||||
}
|
||||
|
||||
private static IPEndPoint? ParseEndpoint(string endpoint)
|
||||
{
|
||||
int colon = endpoint.LastIndexOf(':');
|
||||
if (colon <= 0)
|
||||
return null;
|
||||
|
||||
string host = endpoint[..colon];
|
||||
if (!int.TryParse(endpoint[(colon + 1)..], out int port))
|
||||
return null;
|
||||
|
||||
if (IPAddress.TryParse(host, out var addr))
|
||||
return new IPEndPoint(addr, port);
|
||||
|
||||
try
|
||||
{
|
||||
var addresses = Dns.GetHostAddresses(host);
|
||||
addr = addresses.FirstOrDefault(a => a.AddressFamily == AddressFamily.InterNetwork);
|
||||
if (addr is null)
|
||||
return null;
|
||||
return new IPEndPoint(addr, port);
|
||||
}
|
||||
catch
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
private static TranscriptMessage? ParseTranscriptLine(string line)
|
||||
{
|
||||
if (string.IsNullOrWhiteSpace(line))
|
||||
return null;
|
||||
|
||||
try
|
||||
{
|
||||
var dto = JsonSerializer.Deserialize<TransmitDto>(line);
|
||||
if (dto is null)
|
||||
return null;
|
||||
|
||||
var type = dto.Final ? TranscriptType.Final : TranscriptType.Partial;
|
||||
return new TranscriptMessage(type, dto.Text ?? string.Empty);
|
||||
}
|
||||
catch (JsonException)
|
||||
{
|
||||
return null;
|
||||
}
|
||||
}
|
||||
|
||||
public async ValueTask DisposeAsync()
|
||||
{
|
||||
if (_disposed) return;
|
||||
await StopAsync();
|
||||
_disposed = true;
|
||||
}
|
||||
}
|
||||
|
||||
internal sealed class ControlDto
|
||||
{
|
||||
[JsonPropertyName("event")]
|
||||
public string Event { get; set; } = string.Empty;
|
||||
}
|
||||
|
||||
internal sealed class TransmitDto
|
||||
{
|
||||
public bool Final { get; set; }
|
||||
public string Text { get; set; } = string.Empty;
|
||||
}
|
||||
+1
-1
@@ -2,6 +2,6 @@
|
||||
<Project Path="Robovoice.App/Robovoice.App.csproj" />
|
||||
<Project Path="Robovoice.Core/Robovoice.Core.csproj" />
|
||||
<Project Path="Robovoice.Stt.File/Robovoice.Stt.File.csproj" />
|
||||
<Project Path="Robovoice.Stt.Udp/Robovoice.Stt.Udp.csproj" />
|
||||
<Project Path="Robovoice.Stt.Tcp/Robovoice.Stt.Tcp.csproj" />
|
||||
<Project Path="Robovoice.Tts.LibPiper/Robovoice.Tts.LibPiper.csproj" />
|
||||
</Solution>
|
||||
|
||||
+264
@@ -0,0 +1,264 @@
|
||||
# Server implementation notes
|
||||
|
||||
The STT server listens for TCP connections from Robovoice, captures audio
|
||||
from a microphone when `on` is received, runs speech recognition (Moonshine),
|
||||
and sends transcript messages back over the same connection.
|
||||
|
||||
## Architecture
|
||||
|
||||
```
|
||||
┌── on/off (TCP, newline-delimited JSON)
|
||||
Robovoice ──────────────►│
|
||||
│ STT Server
|
||||
Robovoice ◄──────────────┤
|
||||
└── partial/final (TCP, newline-delimited JSON)
|
||||
```
|
||||
|
||||
The server:
|
||||
1. Listens on a TCP port (e.g. 5210)
|
||||
2. Accepts a connection from Robovoice
|
||||
3. Reads lines: waits for `{"event":"on"}`
|
||||
4. Records audio from the microphone
|
||||
5. Waits for `{"event":"off"}` (or a timeout)
|
||||
6. Runs STT on the captured audio
|
||||
7. Sends `{"final":true,"text":"..."}`\n back over the connection
|
||||
|
||||
## Framing
|
||||
|
||||
Every message is a single JSON object on one line, terminated by `\n`. No
|
||||
length prefix, no binary framing. Use `readline()` / `StreamReader.ReadLineAsync()`.
|
||||
|
||||
## Python server with Moonshine
|
||||
|
||||
[Moonshine](https://github.com/usefulsensors/moonshine) is a lightweight ASR
|
||||
model by Useful Sensors. Install with `pip install moonshine`.
|
||||
|
||||
```python
|
||||
import socket
|
||||
import json
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
import moonshine
|
||||
|
||||
LISTEN_PORT = 5210
|
||||
SAMPLE_RATE = 16000
|
||||
|
||||
model = moonshine.MoonshineModel(model="moonshine/base")
|
||||
|
||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
server.bind(("0.0.0.0", LISTEN_PORT))
|
||||
server.listen(1)
|
||||
|
||||
print(f"STT server listening on :{LISTEN_PORT}")
|
||||
|
||||
while True:
|
||||
conn, addr = server.accept()
|
||||
print(f"Client connected: {addr}")
|
||||
|
||||
buf = ""
|
||||
with conn:
|
||||
while True:
|
||||
data = conn.recv(4096).decode("utf-8")
|
||||
if not data:
|
||||
break
|
||||
buf += data
|
||||
|
||||
while "\n" in buf:
|
||||
line, buf = buf.split("\n", 1)
|
||||
msg = json.loads(line)
|
||||
|
||||
if msg.get("event") == "on":
|
||||
print("PTT on — recording")
|
||||
audio_chunks = []
|
||||
|
||||
# Record until "off" or timeout
|
||||
conn.settimeout(0.1)
|
||||
while True:
|
||||
try:
|
||||
data2 = conn.recv(4096).decode("utf-8")
|
||||
if not data2:
|
||||
break
|
||||
buf += data2
|
||||
while "\n" in buf:
|
||||
line2, buf = buf.split("\n", 1)
|
||||
msg2 = json.loads(line2)
|
||||
if msg2.get("event") == "off":
|
||||
break
|
||||
except socket.timeout:
|
||||
pass
|
||||
|
||||
chunk = sd.rec(int(SAMPLE_RATE * 0.1),
|
||||
samplerate=SAMPLE_RATE,
|
||||
channels=1, dtype="float32")
|
||||
sd.wait()
|
||||
audio_chunks.append(chunk.flatten())
|
||||
|
||||
conn.settimeout(None)
|
||||
|
||||
if not audio_chunks:
|
||||
continue
|
||||
|
||||
audio = np.concatenate(audio_chunks)
|
||||
print(f"Captured {len(audio)/SAMPLE_RATE:.1f}s")
|
||||
|
||||
text = moonshine.transcribe(model, audio).strip()
|
||||
|
||||
if text:
|
||||
print(f"Transcript: {text}")
|
||||
reply = json.dumps({"final": True, "text": text})
|
||||
conn.sendall((reply + "\n").encode("utf-8"))
|
||||
else:
|
||||
print("Empty transcript")
|
||||
```
|
||||
|
||||
## Python server with streaming partials
|
||||
|
||||
For lower latency, send partial results while still recording:
|
||||
|
||||
```python
|
||||
import socket
|
||||
import json
|
||||
import numpy as np
|
||||
import sounddevice as sd
|
||||
import moonshine
|
||||
|
||||
LISTEN_PORT = 5210
|
||||
SAMPLE_RATE = 16000
|
||||
CHUNK_DURATION = 0.5
|
||||
|
||||
model = moonshine.MoonshineModel(model="moonshine/base")
|
||||
|
||||
server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
server.bind(("0.0.0.0", LISTEN_PORT))
|
||||
server.listen(1)
|
||||
|
||||
print(f"STT server listening on :{LISTEN_PORT}")
|
||||
|
||||
while True:
|
||||
conn, addr = server.accept()
|
||||
print(f"Client connected: {addr}")
|
||||
buf = ""
|
||||
|
||||
with conn:
|
||||
while True:
|
||||
data = conn.recv(4096).decode("utf-8")
|
||||
if not data:
|
||||
break
|
||||
buf += data
|
||||
|
||||
while "\n" in buf:
|
||||
line, buf = buf.split("\n", 1)
|
||||
msg = json.loads(line)
|
||||
|
||||
if msg.get("event") != "on":
|
||||
continue
|
||||
|
||||
print("PTT on — recording")
|
||||
audio_chunks = []
|
||||
|
||||
while True:
|
||||
try:
|
||||
conn.settimeout(CHUNK_DURATION)
|
||||
data2 = conn.recv(4096).decode("utf-8")
|
||||
if not data2:
|
||||
break
|
||||
buf += data2
|
||||
while "\n" in buf:
|
||||
line2, buf = buf.split("\n", 1)
|
||||
msg2 = json.loads(line2)
|
||||
if msg2.get("event") == "off":
|
||||
break
|
||||
except socket.timeout:
|
||||
pass
|
||||
|
||||
chunk = sd.rec(int(SAMPLE_RATE * CHUNK_DURATION),
|
||||
samplerate=SAMPLE_RATE,
|
||||
channels=1, dtype="float32")
|
||||
sd.wait()
|
||||
audio_chunks.append(chunk.flatten())
|
||||
|
||||
# Send partial every few chunks
|
||||
if len(audio_chunks) % 4 == 0:
|
||||
partial_audio = np.concatenate(audio_chunks)
|
||||
partial_text = moonshine.transcribe(model, partial_audio).strip()
|
||||
if partial_text:
|
||||
reply = json.dumps({"final": False, "text": partial_text})
|
||||
conn.sendall((reply + "\n").encode("utf-8"))
|
||||
|
||||
conn.settimeout(None)
|
||||
|
||||
if not audio_chunks:
|
||||
continue
|
||||
|
||||
audio = np.concatenate(audio_chunks)
|
||||
text = moonshine.transcribe(model, audio).strip()
|
||||
|
||||
if text:
|
||||
print(f"Final: {text}")
|
||||
reply = json.dumps({"final": True, "text": text})
|
||||
conn.sendall((reply + "\n").encode("utf-8"))
|
||||
```
|
||||
|
||||
## C# server skeleton
|
||||
|
||||
```csharp
|
||||
using System.Net;
|
||||
using System.Net.Sockets;
|
||||
using System.Text.Json;
|
||||
|
||||
var listener = new TcpListener(IPAddress.Any, 5210);
|
||||
listener.Start();
|
||||
|
||||
Console.WriteLine("STT server listening on :5210");
|
||||
|
||||
while (true)
|
||||
{
|
||||
var client = listener.AcceptTcpClient();
|
||||
Console.WriteLine($"Client connected: {client.Client.RemoteEndPoint}");
|
||||
|
||||
using var stream = client.GetStream();
|
||||
using var reader = new StreamReader(stream, Encoding.UTF8);
|
||||
using var writer = new StreamWriter(stream, Encoding.UTF8) { AutoFlush = true };
|
||||
|
||||
string? line;
|
||||
while ((line = reader.ReadLine()) is not null)
|
||||
{
|
||||
var msg = JsonSerializer.Deserialize<Dictionary<string, string>>(line);
|
||||
if (msg?["event"] != "on")
|
||||
continue;
|
||||
|
||||
Console.WriteLine("PTT on — recording");
|
||||
// Capture audio...
|
||||
|
||||
// Read until "off"
|
||||
while ((line = reader.ReadLine()) is not null)
|
||||
{
|
||||
msg = JsonSerializer.Deserialize<Dictionary<string, string>>(line);
|
||||
if (msg?["event"] == "off")
|
||||
break;
|
||||
}
|
||||
|
||||
// Run STT...
|
||||
string text = "recognized text here";
|
||||
|
||||
var reply = JsonSerializer.Serialize(new { final = true, text });
|
||||
writer.WriteLine(reply);
|
||||
}
|
||||
}
|
||||
```
|
||||
|
||||
## Tips
|
||||
|
||||
- **One connection per client:** Robovoice maintains a single persistent TCP
|
||||
connection. The server should handle one client at a time (or track
|
||||
multiple if needed).
|
||||
- **Timeout:** implement a recording timeout in case the `off` message is
|
||||
delayed or the client disconnects. 10–30 seconds is reasonable.
|
||||
- **Partials:** optional but improve UX — Robovoice logs them so the user
|
||||
sees live feedback. Only `final` triggers TTS.
|
||||
- **Encoding:** always UTF-8. Every line is a UTF-8 JSON object terminated
|
||||
by `\n`.
|
||||
- **Reconnection:** Robovoice auto-reconnects every 3 seconds if the
|
||||
connection drops. The server just needs to accept new connections.
|
||||
- **Moonshine models:** `moonshine/base` (faster, less accurate) or
|
||||
`moonshine/tiny` (fastest). Choose based on your hardware.
|
||||
@@ -0,0 +1,70 @@
|
||||
# Robovoice TCP STT Protocol
|
||||
|
||||
## Overview
|
||||
|
||||
Robovoice acts as a **client**: it connects to a remote STT server over TCP,
|
||||
sends control messages when the user presses/releases the PTT key, and
|
||||
receives transcript messages back. The STT server captures audio from a
|
||||
microphone, runs speech recognition (Moonshine), and sends transcripts back
|
||||
over the same connection.
|
||||
|
||||
```
|
||||
[Robovoice client] --TCP--> [STT server :5210]
|
||||
│ │
|
||||
├── {"event":"on"}\n ──────►│
|
||||
│ ├── capture audio
|
||||
├── {"event":"off"}\n ──────►│
|
||||
│ ├── run STT
|
||||
│◄── {"final":true,...}\n ──┤
|
||||
```
|
||||
|
||||
## Transport
|
||||
|
||||
- **Protocol:** TCP (reliable, ordered, connection-oriented)
|
||||
- **Server endpoint:** configurable in Robovoice UI (default `127.0.0.1:5210`)
|
||||
- **Framing:** newline-delimited JSON (NDJSON) — each message is a single
|
||||
UTF-8 JSON object terminated by `\n`
|
||||
- **Auto-reconnect:** if the connection drops, Robovoice retries every 3
|
||||
seconds until the server is available
|
||||
|
||||
## Control messages (client → server)
|
||||
|
||||
Sent by Robovoice when the user presses/releases the PTT key.
|
||||
|
||||
```json
|
||||
{"event": "on"}
|
||||
```
|
||||
|
||||
```json
|
||||
{"event": "off"}
|
||||
```
|
||||
|
||||
| Field | Type | Description |
|
||||
|---------|--------|------------------------------------|
|
||||
| `event` | string | `"on"` (PTT pressed) or `"off"` (PTT released) |
|
||||
|
||||
## Transcript messages (server → client)
|
||||
|
||||
Sent by the server back to Robovoice over the same TCP connection.
|
||||
|
||||
```json
|
||||
{"final": false, "text": "hello world"}
|
||||
```
|
||||
|
||||
```json
|
||||
{"final": true, "text": "hello world how are you"}
|
||||
```
|
||||
|
||||
| Field | Type | Required | Description |
|
||||
|---------|---------|----------|--------------------------------------------------|
|
||||
| `final` | bool | yes | `true` = final result, `false` = partial |
|
||||
| `text` | string | yes | The transcript text (may be empty for partials) |
|
||||
|
||||
### Semantics
|
||||
|
||||
- **`final: false`** — intermediate recognition result (partial). Robovoice
|
||||
logs these but does not act on them (only `final` triggers TTS).
|
||||
- **`final: true`** — complete utterance. Robovoice feeds this to the TTS
|
||||
engine and speaks it.
|
||||
|
||||
Malformed JSON or unknown field values are silently dropped by the client.
|
||||
Reference in New Issue
Block a user