From caad0ed50e512816b393577ba007cc9d7a99e08a Mon Sep 17 00:00:00 2001 From: Mute Date: Thu, 13 Aug 2026 12:38:27 +0000 Subject: [PATCH] =?UTF-8?q?v0.9:=20VoicePipeline=20refactor=20=E2=80=94=20?= =?UTF-8?q?queue-based=20pipeline,=20lock/release=20model,=20thread-based?= =?UTF-8?q?=20TCP?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- Robovoice.App/MainForm.Designer.cs | 95 ++---- Robovoice.App/MainForm.cs | 304 +++++++++++--------- Robovoice.App/Orchestrator.cs | 257 ----------------- Robovoice.App/VoicePipeline.cs | 190 ++++++++++++ Robovoice.Stt.Tcp/TcpSttSource.cs | 199 +++++++------ Robovoice.Tts.LibPiper/LibPiperTtsEngine.cs | 65 +++++ 6 files changed, 556 insertions(+), 554 deletions(-) delete mode 100644 Robovoice.App/Orchestrator.cs create mode 100644 Robovoice.App/VoicePipeline.cs diff --git a/Robovoice.App/MainForm.Designer.cs b/Robovoice.App/MainForm.Designer.cs index c81541a..fe122cd 100644 --- a/Robovoice.App/MainForm.Designer.cs +++ b/Robovoice.App/MainForm.Designer.cs @@ -9,21 +9,15 @@ partial class MainForm private TextBox txtPttKey = null!; private Label lblVoice = null!; private ComboBox cmbVoice = null!; - private Button btnTestVoice = null!; private Button btnManageVoices = null!; private Label lblOutput = null!; private ComboBox cmbOutput = null!; - private Label lblFile = null!; - private Button btnBrowseFile = null!; - private Label lblFileName = null!; private Label lblServer = null!; private TextBox txtSttEndpoint = null!; - private Label lblLineStatus = null!; + private Button btnLock = null!; private RichTextBox txtLog = null!; private CheckBox chkMinimizeToTray = null!; private Button btnClearLog = null!; - private Label lblTextInput = null!; - private TextBox txtTextInput = null!; private Label lblNoise = null!; private TrackBar trkNoise = null!; private Label lblNoiseVal = null!; @@ -49,21 +43,15 @@ partial class MainForm txtPttKey = new TextBox(); lblVoice = new Label(); cmbVoice = new ComboBox(); - btnTestVoice = new Button(); btnManageVoices = new Button(); lblOutput = new Label(); cmbOutput = new ComboBox(); - lblFile = new Label(); - btnBrowseFile = new Button(); - lblFileName = new Label(); lblServer = new Label(); txtSttEndpoint = new TextBox(); - lblLineStatus = new Label(); + btnLock = new Button(); txtLog = new RichTextBox(); chkMinimizeToTray = new CheckBox(); btnClearLog = new Button(); - lblTextInput = new Label(); - txtTextInput = new TextBox(); lblNoise = new Label(); trkNoise = new TrackBar(); lblNoiseVal = new Label(); @@ -100,64 +88,38 @@ partial class MainForm cmbVoice.Size = new Size(200, 23); cmbVoice.DropDownStyle = ComboBoxStyle.DropDownList; - // btnTestVoice - btnTestVoice.Text = "Test"; - btnTestVoice.Location = new Point(423, 11); - btnTestVoice.Size = new Size(45, 25); - btnTestVoice.UseVisualStyleBackColor = true; - // btnManageVoices btnManageVoices.Text = "Add/Remove..."; - btnManageVoices.Location = new Point(474, 11); + btnManageVoices.Location = new Point(423, 11); btnManageVoices.Size = new Size(95, 25); btnManageVoices.UseVisualStyleBackColor = true; // lblOutput lblOutput.Text = "Output:"; - lblOutput.Location = new Point(580, 15); + lblOutput.Location = new Point(530, 15); lblOutput.Size = new Size(50, 23); lblOutput.TextAlign = ContentAlignment.MiddleLeft; // cmbOutput - cmbOutput.Location = new Point(633, 12); - cmbOutput.Size = new Size(180, 23); + cmbOutput.Location = new Point(583, 12); + cmbOutput.Size = new Size(150, 23); cmbOutput.DropDownStyle = ComboBoxStyle.DropDownList; - // lblFile - lblFile.Text = "Text File:"; - lblFile.Location = new Point(12, 48); - lblFile.Size = new Size(60, 23); - lblFile.TextAlign = ContentAlignment.MiddleLeft; - - // btnBrowseFile - btnBrowseFile.Text = "Browse..."; - btnBrowseFile.Location = new Point(75, 45); - btnBrowseFile.Size = new Size(75, 25); - btnBrowseFile.UseVisualStyleBackColor = true; - - // lblFileName - lblFileName.Text = "(none)"; - lblFileName.Location = new Point(155, 48); - lblFileName.Size = new Size(280, 23); - lblFileName.TextAlign = ContentAlignment.MiddleLeft; - lblFileName.ForeColor = Color.Gray; - // lblServer lblServer.Text = "STT:"; - lblServer.Location = new Point(440, 48); + lblServer.Location = new Point(12, 48); lblServer.Size = new Size(35, 23); lblServer.TextAlign = ContentAlignment.MiddleLeft; // txtSttEndpoint - txtSttEndpoint.Location = new Point(478, 45); - txtSttEndpoint.Size = new Size(185, 23); + txtSttEndpoint.Location = new Point(50, 45); + txtSttEndpoint.Size = new Size(200, 23); - // lblLineStatus - lblLineStatus.Text = ""; - lblLineStatus.Location = new Point(645, 48); - lblLineStatus.Size = new Size(160, 23); - lblLineStatus.TextAlign = ContentAlignment.MiddleRight; - lblLineStatus.ForeColor = Color.DarkBlue; + // btnLock + btnLock.Text = "Lock Model"; + btnLock.Location = new Point(260, 44); + btnLock.Size = new Size(90, 25); + btnLock.UseVisualStyleBackColor = true; // lblNoise lblNoise.Text = "Noise:"; @@ -165,7 +127,7 @@ partial class MainForm lblNoise.Size = new Size(40, 23); lblNoise.TextAlign = ContentAlignment.MiddleLeft; - // trkNoise (0-1000 → 0.0-1.0, default 667) + // trkNoise trkNoise.Location = new Point(52, 78); trkNoise.Size = new Size(120, 45); trkNoise.Minimum = 0; @@ -186,7 +148,7 @@ partial class MainForm lblSpeed.Size = new Size(40, 23); lblSpeed.TextAlign = ContentAlignment.MiddleLeft; - // trkSpeed (50-300 → 0.5-3.0, default 100) + // trkSpeed trkSpeed.Location = new Point(260, 78); trkSpeed.Size = new Size(120, 45); trkSpeed.Minimum = 50; @@ -207,7 +169,7 @@ partial class MainForm lblNoiseW.Size = new Size(45, 23); lblNoiseW.TextAlign = ContentAlignment.MiddleLeft; - // trkNoiseW (0-1000 → 0.0-1.0, default 800) + // trkNoiseW trkNoiseW.Location = new Point(475, 78); trkNoiseW.Size = new Size(120, 45); trkNoiseW.Minimum = 0; @@ -223,19 +185,8 @@ partial class MainForm lblNoiseWVal.TextAlign = ContentAlignment.MiddleLeft; // txtLog - txtLog.Location = new Point(12, 149); - txtLog.Size = new Size(800, 310); - - // lblTextInput - lblTextInput.Text = "Text:"; - lblTextInput.Location = new Point(12, 121); - lblTextInput.Size = new Size(35, 23); - lblTextInput.TextAlign = ContentAlignment.MiddleLeft; - - // txtTextInput - txtTextInput.Location = new Point(50, 118); - txtTextInput.Size = new Size(762, 23); - + txtLog.Location = new Point(12, 121); + txtLog.Size = new Size(800, 338); txtLog.ReadOnly = true; txtLog.Font = new Font("Consolas", 9F); txtLog.BackColor = Color.FromArgb(30, 30, 30); @@ -261,16 +212,12 @@ partial class MainForm Controls.Add(txtPttKey); Controls.Add(lblVoice); Controls.Add(cmbVoice); - Controls.Add(btnTestVoice); Controls.Add(btnManageVoices); Controls.Add(lblOutput); Controls.Add(cmbOutput); - Controls.Add(lblFile); - Controls.Add(btnBrowseFile); - Controls.Add(lblFileName); Controls.Add(lblServer); Controls.Add(txtSttEndpoint); - Controls.Add(lblLineStatus); + Controls.Add(btnLock); Controls.Add(lblNoise); Controls.Add(trkNoise); Controls.Add(lblNoiseVal); @@ -280,8 +227,6 @@ partial class MainForm Controls.Add(lblNoiseW); Controls.Add(trkNoiseW); Controls.Add(lblNoiseWVal); - Controls.Add(lblTextInput); - Controls.Add(txtTextInput); Controls.Add(txtLog); Controls.Add(chkMinimizeToTray); Controls.Add(btnClearLog); diff --git a/Robovoice.App/MainForm.cs b/Robovoice.App/MainForm.cs index 930c2c9..83b53af 100644 --- a/Robovoice.App/MainForm.cs +++ b/Robovoice.App/MainForm.cs @@ -17,10 +17,11 @@ internal sealed partial class MainForm : Form private LibPiperTtsEngine? _tts; private AudioOutput? _audioOutput; private TcpSttSource? _sttSource; - private Orchestrator? _orchestrator; + private VoicePipeline? _pipeline; private PttHotkey? _pttHotkey; private NotifyIcon? _trayIcon; private bool _trayInit; + private bool _locked; public MainForm() { @@ -65,14 +66,11 @@ internal sealed partial class MainForm : Form chkMinimizeToTray.Checked = _config.MinimizeToTray; txtSttEndpoint.Text = _config.SttEndpoint; - btnBrowseFile.Click += OnBrowseFile; - btnTestVoice.Click += OnTestVoice; btnManageVoices.Click += OnManageVoices; btnClearLog.Click += (_, _) => txtLog.Clear(); + btnLock.Click += OnLockToggle; txtPttKey.Enter += OnPttKeyFocus; txtPttKey.KeyDown += OnPttKeyDown; - cmbOutput.SelectedIndexChanged += OnOutputChanged; - cmbVoice.SelectedIndexChanged += OnVoiceChanged; txtSttEndpoint.Leave += OnSttEndpointChanged; trkNoise.Scroll += OnSliderScroll; @@ -90,10 +88,8 @@ internal sealed partial class MainForm : Form } catch (Exception ex) { - System.Diagnostics.Debug.WriteLine($"OnLoad failed: {ex}"); + Debug.WriteLine($"OnLoad failed: {ex}"); } - - BeginInvoke(async () => await InitializeEngineAsync()); } private Keys _pttKey = Keys.F8; @@ -216,7 +212,21 @@ internal sealed partial class MainForm : Form cmbOutput.SelectedIndex = 0; } - private async Task InitializeEngineAsync() + // ─── Lock / Release ──────────────────────────────────────────────────── + + private async void OnLockToggle(object? sender, EventArgs e) + { + if (_locked) + { + ReleaseModel(); + } + else + { + await LockModelAsync(); + } + } + + private async Task LockModelAsync() { if (cmbVoice.SelectedItem is not string voiceName) { @@ -237,44 +247,98 @@ internal sealed partial class MainForm : Form return; } + btnLock.Enabled = false; Log($"Loading voice: {voiceName}..."); - _audioOutput?.Dispose(); - if (_tts is not null) - await _tts.DisposeAsync(); - - _tts = new LibPiperTtsEngine( - modelPath, - _espeakDataPath, - noiseScale: trkNoise.Value / 1000.0f, - lengthScale: trkSpeed.Value / 100.0f, - noiseWScale: trkNoiseW.Value / 1000.0f); - _audioOutput = new AudioOutput(); try { - if (_sttSource is null) - { - _sttSource = new TcpSttSource { Endpoint = txtSttEndpoint.Text, Log = Log }; - await _sttSource.StartAsync(); - } + _tts = new LibPiperTtsEngine( + modelPath, + _espeakDataPath, + noiseScale: trkNoise.Value / 1000.0f, + lengthScale: trkSpeed.Value / 100.0f, + noiseWScale: trkNoiseW.Value / 1000.0f); - if (_orchestrator is not null) - await _orchestrator.DisposeAsync(); - _orchestrator = new Orchestrator(_tts, _audioOutput, _sttSource, Log) + await Task.Run(() => _tts.InitializeAsync()); + Log($"TTS ready (sample rate: {_tts.SampleRate} Hz)"); + + _audioOutput = new AudioOutput(); + + _sttSource = new TcpSttSource { Endpoint = txtSttEndpoint.Text, Log = Log }; + _sttSource.TranscriptReceived += OnTranscript; + await _sttSource.StartAsync(); + + _pipeline = new VoicePipeline(_tts, _audioOutput, Log) { OutputDeviceName = cmbOutput.SelectedItem as string ?? string.Empty, }; + _pipeline.Start(); + + _locked = true; + SetControlsLocked(true); + btnLock.Text = "Release Model"; + btnLock.Enabled = true; - await _orchestrator.InitializeTtsAsync(); - Log("Engine ready. Press PTT to send to STT server."); SetupHotkey(); + Log("Model locked. Press PTT to talk."); } catch (Exception ex) { - Log($"Init failed: {ex.Message}"); + Log($"Lock failed: {ex.Message}"); + btnLock.Enabled = true; + _tts?.DisposeAsync().AsTask().Wait(); + _tts = null; + _audioOutput?.Dispose(); + _audioOutput = null; + if (_sttSource is not null) + { + _sttSource.TranscriptReceived -= OnTranscript; + _sttSource.DisposeAsync().AsTask().Wait(); + _sttSource = null; + } } } + private void ReleaseModel() + { + if (!_locked) return; + + _pttHotkey?.Dispose(); + _pttHotkey = null; + + if (_sttSource is not null) + { + _sttSource.TranscriptReceived -= OnTranscript; + _sttSource.DisposeAsync().AsTask().Wait(); + _sttSource = null; + } + + _pipeline?.Dispose(); + _pipeline = null; + + // VoicePipeline.Dispose calls _tts.DisposeAsync and _audioOutput.Dispose + _tts = null; + _audioOutput = null; + + _locked = false; + SetControlsLocked(false); + btnLock.Text = "Lock Model"; + Log("Model released."); + } + + private void SetControlsLocked(bool locked) + { + cmbVoice.Enabled = !locked; + cmbOutput.Enabled = !locked; + trkNoise.Enabled = !locked; + trkSpeed.Enabled = !locked; + trkNoiseW.Enabled = !locked; + txtSttEndpoint.Enabled = !locked; + btnManageVoices.Enabled = !locked; + } + + // ─── PTT ──────────────────────────────────────────────────────────────── + private void SetupHotkey() { _pttHotkey?.Dispose(); @@ -287,86 +351,56 @@ internal sealed partial class MainForm : Form private void OnPttPressed(object? sender, EventArgs e) { - Log("PTT pressed"); - _orchestrator?.NotifyPttPressed(); - try - { - _sttSource?.SendOn(); - } - catch (Exception ex) - { - Log($"SendOn failed: {ex.Message}"); - } + _pipeline?.OnPttPressed(); + _sttSource?.SendOn(); } private void OnPttReleased(object? sender, EventArgs e) { - Log("PTT released"); - try - { - _sttSource?.SendOff(); - } - catch (Exception ex) - { - Log($"SendOff failed: {ex.Message}"); - } + _sttSource?.SendOff(); + _pipeline?.OnPttReleased(); } - private void OnBrowseFile(object? sender, EventArgs e) + // ─── STT transcript → pipeline ────────────────────────────────────────── + + private void OnTranscript(object? sender, TranscriptEventArgs e) { - using var dlg = new OpenFileDialog - { - Filter = "Text files (*.txt)|*.txt|All files (*.*)|*.*", - Title = "Select a text file to speak", - }; + if (_pipeline is null || !_locked) return; - if (dlg.ShowDialog() == DialogResult.OK) + var msg = e.Message; + if (msg.Type == TranscriptType.Partial) { - string text = File.ReadAllText(dlg.FileName); - lblFileName.Text = Path.GetFileName(dlg.FileName); - lblFileName.ForeColor = Color.Black; - Log($"Loaded: {dlg.FileName} ({text.Length} chars)"); - _ = _orchestrator?.SynthesizeAsync(text); + if (!string.IsNullOrWhiteSpace(msg.Text)) + { + Log($"SEGMENT: \"{msg.Text}\""); + _pipeline.EnqueueSegment(msg.Text); + } + } + else + { + if (!string.IsNullOrWhiteSpace(msg.Text)) + { + Log($"FINAL: \"{msg.Text}\""); + _pipeline.EnqueueFinal(msg.Text); + } + else + { + Log("FINAL: (empty)"); + _pipeline.EnqueueFinal(""); + } } } - private void OnTestVoice(object? sender, EventArgs e) - { - if (_orchestrator is null) - { - Log("Engine not initialized."); - return; - } - - if (cmbVoice.SelectedItem is not string voiceName) - { - Log("No voice selected."); - return; - } - - if (!VoiceCatalogue.IsVoiceInstalled(_voicesDir, voiceName)) - { - Log($"Voice '{voiceName}' is not installed. Use Add/Remove to download it."); - return; - } - - string text = txtTextInput.Text.Trim(); - if (string.IsNullOrEmpty(text)) - text = "Hello, this is a voice test."; - - btnTestVoice.Enabled = false; - try - { - _ = _orchestrator.SynthesizeAsync(text); - } - finally - { - btnTestVoice.Enabled = true; - } - } + // ─── Voice manager ───────────────────────────────────────────────────── private void OnManageVoices(object? sender, EventArgs e) { + if (_locked) + { + Log("Release the model before managing voices."); + return; + } + string currentVoice = cmbVoice.SelectedItem as string ?? string.Empty; using var dlg = new VoiceManagerForm(_voicesDir, currentVoice); dlg.ShowDialog(this); @@ -384,31 +418,9 @@ internal sealed partial class MainForm : Form } SaveConfig(); - _ = InitializeEngineAsync(); } - private void OnVoiceChanged(object? sender, EventArgs e) - { - btnTestVoice.Enabled = cmbVoice.SelectedItem is string voiceName - && VoiceCatalogue.IsVoiceInstalled(_voicesDir, voiceName); - - if (cmbVoice.SelectedItem is string name) - { - SaveConfig(); - _ = ReinitializeEngineAsync(name); - } - } - - private async Task ReinitializeEngineAsync(string voiceName) - { - if (!VoiceCatalogue.IsVoiceInstalled(_voicesDir, voiceName)) - { - btnTestVoice.Enabled = false; - return; - } - - await InitializeEngineAsync(); - } + // ─── Slider / endpoint ───────────────────────────────────────────────── private void OnSliderScroll(object? sender, EventArgs e) { @@ -420,33 +432,16 @@ internal sealed partial class MainForm : Form private void OnSliderReleased(object? sender, MouseEventArgs e) { SaveConfig(); - if (cmbVoice.SelectedItem is string name && VoiceCatalogue.IsVoiceInstalled(_voicesDir, name)) - { - _ = InitializeEngineAsync(); - } } - private void OnOutputChanged(object? sender, EventArgs e) - { - if (_orchestrator is not null) - _orchestrator.OutputDeviceName = cmbOutput.SelectedItem as string ?? string.Empty; - Log($"Output device: {_orchestrator?.OutputDeviceName}"); - SaveConfig(); - } - - private async void OnSttEndpointChanged(object? sender, EventArgs e) + private void OnSttEndpointChanged(object? sender, EventArgs e) { _config.SttEndpoint = txtSttEndpoint.Text; SaveConfig(); - - if (_sttSource is not null) - { - await _sttSource.DisposeAsync(); - _sttSource = null; - await InitializeEngineAsync(); - } } + // ─── Tray ────────────────────────────────────────────────────────────── + private void SetupTray() { if (_trayInit) return; @@ -503,6 +498,8 @@ internal sealed partial class MainForm : Form Activate(); } + // ─── Config ──────────────────────────────────────────────────────────── + private void SaveConfig() { _config.Voice = cmbVoice.SelectedItem as string ?? string.Empty; @@ -516,6 +513,8 @@ internal sealed partial class MainForm : Form _config.Save(); } + // ─── Logging ─────────────────────────────────────────────────────────── + private void Log(string message) { if (IsDisposed) return; @@ -530,12 +529,35 @@ internal sealed partial class MainForm : Form txtLog.ScrollToCaret(); } + // ─── Shutdown ────────────────────────────────────────────────────────── + + private static string DisposeLogPath => Path.Combine(AppConfig.AppDataDir, "dispose.log"); + + private static void LogDispose(string label, long elapsedMs) + { + string line = $"[{DateTime.Now:HH:mm:ss.fff}] {label}: {elapsedMs}ms"; + try + { + Directory.CreateDirectory(AppConfig.AppDataDir); + File.AppendAllText(DisposeLogPath, line + "\n"); + } + catch { } + } + private void OnFormClosing(object? sender, FormClosingEventArgs e) { + var sw = Stopwatch.StartNew(); _pttHotkey?.Dispose(); + LogDispose("pttHotkey.Dispose", sw.ElapsedMilliseconds); + _trayIcon!.Visible = false; - _orchestrator?.DisposeAsync().AsTask().Wait(2000); - _sttSource?.DisposeAsync().AsTask().Wait(2000); + + sw.Restart(); + ReleaseModel(); + LogDispose("ReleaseModel", sw.ElapsedMilliseconds); + + sw.Restart(); SaveConfig(); + LogDispose("SaveConfig", sw.ElapsedMilliseconds); } } diff --git a/Robovoice.App/Orchestrator.cs b/Robovoice.App/Orchestrator.cs deleted file mode 100644 index c696619..0000000 --- a/Robovoice.App/Orchestrator.cs +++ /dev/null @@ -1,257 +0,0 @@ -using Robovoice.Core; -using Robovoice.Tts.LibPiper; - -namespace Robovoice.App; - -internal sealed class Orchestrator : IAsyncDisposable -{ - private readonly LibPiperTtsEngine _tts; - private readonly AudioOutput _audioOutput; - private readonly ISttSource _sttSource; - private readonly Action _log; - private CancellationTokenSource? _currentCts; - private bool _disposed; - - private readonly object _stateLock = new(); - private readonly Queue _pendingTexts = new(); - private readonly List _audioBuffer = new(); - private int _bufferSampleRate; - private bool _playing; - private Task? _synthTask; - private CancellationTokenSource? _synthCts; - - public string OutputDeviceName { get; set; } = string.Empty; - - public Orchestrator( - LibPiperTtsEngine tts, - AudioOutput audioOutput, - ISttSource sttSource, - Action log) - { - _tts = tts; - _audioOutput = audioOutput; - _audioOutput.Log = log; - _sttSource = sttSource; - _log = log; - _sttSource.TranscriptReceived += OnTranscript; - } - - private void OnTranscript(object? sender, TranscriptEventArgs e) - { - var msg = e.Message; - - lock (_stateLock) - { - if (msg.Type == TranscriptType.Partial) - { - if (!string.IsNullOrWhiteSpace(msg.Text)) - { - _log($"SEGMENT: \"{msg.Text}\" ({msg.Text.Length} chars)"); - _pendingTexts.Enqueue(msg.Text); - EnsureSynthTask(); - } - } - else - { - if (!string.IsNullOrWhiteSpace(msg.Text)) - { - _log($"FINAL: \"{msg.Text}\" ({msg.Text.Length} chars)"); - _pendingTexts.Enqueue(msg.Text); - EnsureSynthTask(); - } - else - { - _log("FINAL: (empty)"); - } - - TransitionToPlaying(); - } - } - } - - private void TransitionToPlaying() - { - if (_playing) - return; - - if (_audioBuffer.Count == 0) - { - if (_synthTask is null || _synthTask.IsCompleted) - { - _log("TTS: nothing to play"); - } - return; - } - - _playing = true; - - int sampleRate = _bufferSampleRate; - var chunks = _audioBuffer.ToList(); - _audioBuffer.Clear(); - - _log($"TTS: playing {chunks.Count} buffered chunks ({sampleRate} Hz)"); - - _audioOutput.Start(sampleRate, OutputDeviceName); - foreach (var samples in chunks) - { - _audioOutput.WriteSamples(samples); - } - } - - private void EnsureSynthTask() - { - if (_synthTask is not null && !_synthTask.IsCompleted) - return; - - _synthCts?.Cancel(); - _synthCts = new CancellationTokenSource(); - _synthTask = SynthLoopAsync(_synthCts.Token); - } - - private async Task SynthLoopAsync(CancellationToken ct) - { - while (true) - { - string text; - lock (_stateLock) - { - if (_pendingTexts.Count == 0) - break; - text = _pendingTexts.Dequeue(); - } - - try - { - await foreach (var chunk in _tts.SynthesizeAsync(text, ct)) - { - lock (_stateLock) - { - if (_playing) - { - _audioOutput.WriteSamples(chunk.Samples); - } - else - { - _bufferSampleRate = chunk.SampleRate; - _audioBuffer.Add(chunk.Samples); - } - } - } - } - catch (OperationCanceledException) - { - break; - } - catch (Exception ex) - { - _log($"TTS error: {ex.Message}"); - } - } - - lock (_stateLock) - { - if (_playing) - { - _audioOutput.Flush(); - _log("TTS: synthesis complete, flushed"); - } - } - } - - public void NotifyPttPressed() - { - lock (_stateLock) - { - _synthCts?.Cancel(); - _synthCts?.Dispose(); - _synthTask = null; - - if (_playing) - { - _audioOutput.Stop(); - _playing = false; - } - - _pendingTexts.Clear(); - _audioBuffer.Clear(); - _bufferSampleRate = 0; - } - } - - public async Task InitializeTtsAsync() - { - _log("Initializing TTS engine..."); - await Task.Run(() => _tts.InitializeAsync()); - _log($"TTS ready (sample rate: {_tts.SampleRate} Hz)"); - } - - public async Task SynthesizeAsync(string text) - { - _currentCts?.Cancel(); - _currentCts = new CancellationTokenSource(); - var ct = _currentCts.Token; - - var sw = System.Diagnostics.Stopwatch.StartNew(); - _log($"Speak: \"{text}\" ({text.Length} chars)"); - - try - { - bool started = false; - int chunkCount = 0; - int totalSamples = 0; - - await foreach (var chunk in _tts.SynthesizeAsync(text, ct)) - { - if (!started) - { - started = true; - _audioOutput.Start(chunk.SampleRate, OutputDeviceName); - _log($"TTS: first chunk ({sw.ElapsedMilliseconds}ms)"); - } - - _audioOutput.WriteSamples(chunk.Samples); - chunkCount++; - totalSamples += chunk.Samples.Length; - } - - if (!started) - { - _log("TTS: no audio produced"); - } - else - { - _audioOutput.Flush(); - double durationSec = (double)totalSamples / _tts.SampleRate; - _log($"TTS: done ({chunkCount} chunks, {durationSec:F2}s audio, {sw.ElapsedMilliseconds}ms)"); - } - } - catch (OperationCanceledException) - { - _log("TTS: cancelled"); - _audioOutput.Stop(); - } - catch (Exception ex) - { - _log($"TTS error: {ex.Message}"); - } - - sw.Stop(); - } - - public async ValueTask DisposeAsync() - { - if (_disposed) return; - _disposed = true; - - try { _currentCts?.Cancel(); } catch { } - _currentCts?.Dispose(); - - try { _synthCts?.Cancel(); } catch { } - _synthCts?.Dispose(); - - _sttSource.TranscriptReceived -= OnTranscript; - await _sttSource.DisposeAsync(); - await _tts.DisposeAsync(); - _audioOutput.Dispose(); - } -} diff --git a/Robovoice.App/VoicePipeline.cs b/Robovoice.App/VoicePipeline.cs new file mode 100644 index 0000000..ae25990 --- /dev/null +++ b/Robovoice.App/VoicePipeline.cs @@ -0,0 +1,190 @@ +using System.Collections.Concurrent; +using Robovoice.Core; +using Robovoice.Tts.LibPiper; + +namespace Robovoice.App; + +internal sealed record TextItem(uint Session, string Text); + +internal sealed record AudioItem(uint Session, float[]? Samples); + +internal sealed class VoicePipeline : IDisposable +{ + private readonly LibPiperTtsEngine _tts; + private readonly AudioOutput _audioOutput; + private readonly Action _log; + + private BlockingCollection _textQueue = new(); + private BlockingCollection _audioQueue = new(); + private ManualResetEventSlim _gate = new(false); + private volatile uint _currentSession; + private volatile bool _running; + + private Thread? _synthThread; + private Thread? _playerThread; + private bool _disposed; + + public string OutputDeviceName { get; set; } = string.Empty; + + public VoicePipeline( + LibPiperTtsEngine tts, + AudioOutput audioOutput, + Action log) + { + _tts = tts; + _audioOutput = audioOutput; + _audioOutput.Log = log; + _log = log; + } + + public void Start() + { + _running = true; + _synthThread = new Thread(SynthLoop) { IsBackground = true, Name = "VoicePipeline-Synth" }; + _playerThread = new Thread(PlayerLoop) { IsBackground = true, Name = "VoicePipeline-Player" }; + _synthThread.Start(); + _playerThread.Start(); + } + + public void EnqueueSegment(string text) + { + if (!_running) return; + uint session = _currentSession; + _textQueue.Add(new TextItem(session, text)); + } + + public void EnqueueFinal(string text) + { + if (!_running) return; + uint session = _currentSession; + if (!string.IsNullOrEmpty(text)) + _textQueue.Add(new TextItem(session, text)); + _textQueue.Add(new TextItem(session, "")); + _gate.Reset(); + } + + public void OnPttPressed() + { + _currentSession++; + _gate.Reset(); + _audioOutput.Stop(); + + // Unblock player thread if it's waiting on audioQueue.Take() + _audioQueue.Add(new AudioItem(_currentSession - 1, null)); + } + + public void OnPttReleased() + { + _gate.Set(); + } + + private void SynthLoop() + { + foreach (var item in _textQueue.GetConsumingEnumerable()) + { + if (!_running) break; + + if (item.Session != _currentSession) + continue; + + if (item.Text.Length == 0) + { + // :F sentinel — signal end of utterance to player + _audioQueue.Add(new AudioItem(item.Session, null)); + continue; + } + + try + { + foreach (var chunk in _tts.SynthesizeSync(item.Text)) + { + if (item.Session != _currentSession) + { + // Session changed mid-synthesis — drain piper cleanly + _tts.SynthesizeDrain(); + break; + } + + _audioQueue.Add(new AudioItem(item.Session, chunk.Samples)); + } + } + catch (Exception ex) + { + _log($"TTS error: {ex.Message}"); + } + } + } + + private void PlayerLoop() + { + while (_running) + { + _gate.Wait(); + if (!_running) break; + + // Block until first audio item is available + AudioItem firstItem; + try + { + firstItem = _audioQueue.Take(); + } + catch (InvalidOperationException) + { + break; + } + + if (!_running) break; + if (firstItem.Session != _currentSession) + continue; + if (firstItem.Samples == null) + { + // :F with no audio — nothing to play + _gate.Reset(); + continue; + } + + _audioOutput.Start(_tts.SampleRate, OutputDeviceName); + _audioOutput.WriteSamples(firstItem.Samples); + _log($"TTS: playback started (session {firstItem.Session})"); + + foreach (var item in _audioQueue.GetConsumingEnumerable()) + { + if (!_running) break; + + if (item.Session != _currentSession) + break; + + if (item.Samples == null) + break; + + _audioOutput.WriteSamples(item.Samples); + } + + _audioOutput.Flush(); + _audioOutput.Stop(); + _log("TTS: playback finished"); + _gate.Reset(); + } + } + + public void Dispose() + { + if (_disposed) return; + _disposed = true; + _running = false; + + _gate.Set(); + _textQueue.CompleteAdding(); + _audioQueue.CompleteAdding(); + + _synthThread?.Join(5000); + _playerThread?.Join(5000); + + _textQueue.Dispose(); + _audioQueue.Dispose(); + _gate.Dispose(); + + _tts.DisposeAsync().AsTask().Wait(); + _audioOutput.Dispose(); + } +} diff --git a/Robovoice.Stt.Tcp/TcpSttSource.cs b/Robovoice.Stt.Tcp/TcpSttSource.cs index 3af3864..af8fbb8 100644 --- a/Robovoice.Stt.Tcp/TcpSttSource.cs +++ b/Robovoice.Stt.Tcp/TcpSttSource.cs @@ -9,10 +9,10 @@ public sealed class TcpSttSource : ISttSource { private TcpClient? _tcp; private NetworkStream? _stream; - private StreamReader? _reader; private StreamWriter? _writer; - private CancellationTokenSource? _cts; - private Task? _runTask; + private Thread? _recvThread; + private Thread? _connectThread; + private volatile bool _running; private readonly object _sendLock = new(); private uint _session; private bool _disposed; @@ -26,105 +26,156 @@ public sealed class TcpSttSource : ISttSource public Task StartAsync(CancellationToken ct = default) { ObjectDisposedException.ThrowIf(_disposed, this); - if (_cts is not null) + if (_running) return Task.CompletedTask; - _cts = CancellationTokenSource.CreateLinkedTokenSource(ct); - _runTask = RunAsync(_cts.Token); + _running = true; + _connectThread = new Thread(ConnectLoop) { IsBackground = true, Name = "TcpSttSource-Connect" }; + _connectThread.Start(); + return Task.CompletedTask; } - public async Task StopAsync(CancellationToken ct = default) + public Task StopAsync(CancellationToken ct = default) { - if (_cts is not null) - _cts.Cancel(); + _running = false; - CleanupConnection(); - - if (_runTask is not null) + lock (_sendLock) { - try { await _runTask.WaitAsync(ct); } - catch { } - _runTask = null; + _writer?.Dispose(); + _stream?.Dispose(); + _tcp?.Close(); + _writer = null; + _stream = null; + _tcp = null; } - _cts?.Dispose(); - _cts = null; + // Threads are background — they'll die when the process exits. + // Closing the socket unblocks any pending Read. + _connectThread?.Join(1000); + _recvThread?.Join(1000); + + return Task.CompletedTask; } - private async Task RunAsync(CancellationToken ct) + private void ConnectLoop() { - while (!ct.IsCancellationRequested) + while (_running) { IPEndPoint? endpoint = ParseEndpoint(Endpoint); if (endpoint is null) { Log?.Invoke($"STT: invalid endpoint '{Endpoint}'"); - try { await Task.Delay(3000, ct); } catch { break; } + SleepInterruptible(3000); continue; } try { - _tcp = new TcpClient(); - using var connectCts = CancellationTokenSource.CreateLinkedTokenSource(ct); - connectCts.CancelAfter(TimeSpan.FromSeconds(5)); - await _tcp.ConnectAsync(endpoint.Address, endpoint.Port, connectCts.Token); + var tcp = new TcpClient(); + tcp.Connect(endpoint.Address, endpoint.Port); + tcp.NoDelay = true; - _stream = _tcp.GetStream(); - _reader = new StreamReader(_stream, Encoding.UTF8); - _writer = new StreamWriter(_stream, Encoding.UTF8) { AutoFlush = true }; + lock (_sendLock) + { + _tcp = tcp; + _stream = tcp.GetStream(); + _writer = new StreamWriter(_stream, new UTF8Encoding(false)) { AutoFlush = true }; + } Log?.Invoke($"STT: connected to {Endpoint}"); - await ReceiveLoopAsync(ct); - } - catch (OperationCanceledException) - { - break; + // Blocking receive loop — runs until disconnected or stopped. + ReceiveLoop(); + + Log?.Invoke("STT: disconnected"); } catch (Exception ex) { - Log?.Invoke($"STT: connection failed ({ex.Message}), retrying..."); + if (_running) + Log?.Invoke($"STT: connection failed ({ex.Message}), retrying..."); } finally { - CleanupConnection(); + lock (_sendLock) + { + _writer?.Dispose(); + _stream?.Dispose(); + _tcp?.Close(); + _writer = null; + _stream = null; + _tcp = null; + } } - if (!ct.IsCancellationRequested) - { - try { await Task.Delay(3000, ct); } - catch (OperationCanceledException) { break; } - } + if (_running) + SleepInterruptible(3000); } } - private async Task ReceiveLoopAsync(CancellationToken ct) + private void ReceiveLoop() { - while (!ct.IsCancellationRequested && _reader is not null) + byte[] buffer = new byte[4096]; + StringBuilder lineBuf = new(); + + while (_running) { - string? line; + NetworkStream? stream; + lock (_sendLock) + { + stream = _stream; + } + + if (stream is null) + break; + + int bytesRead; try { - line = await _reader.ReadLineAsync(ct); + bytesRead = stream.Read(buffer, 0, buffer.Length); } catch { break; } - if (line is null) + if (bytesRead == 0) break; - TranscriptMessage? message = ParseReply(line); - if (message is null) - continue; - - TranscriptReceived?.Invoke(this, new TranscriptEventArgs + for (int i = 0; i < bytesRead; i++) { - Message = message, - }); + byte b = buffer[i]; + if (b == '\n') + { + string line = lineBuf.ToString().TrimEnd('\r'); + lineBuf.Clear(); + + TranscriptMessage? message = ParseReply(line); + if (message is not null) + { + TranscriptReceived?.Invoke(this, new TranscriptEventArgs + { + Message = message, + }); + } + } + else + { + lineBuf.Append((char)b); + } + } + } + } + + private void SleepInterruptible(int ms) + { + int slice = 100; + int waited = 0; + while (_running && waited < ms) + { + int chunk = Math.Min(slice, ms - waited); + Thread.Sleep(chunk); + waited += chunk; } } @@ -157,21 +208,6 @@ public sealed class TcpSttSource : ISttSource } } - private void CleanupConnection() - { - lock (_sendLock) - { - _writer?.Dispose(); - _reader?.Dispose(); - _stream?.Dispose(); - _tcp?.Dispose(); - _writer = null; - _reader = null; - _stream = null; - _tcp = null; - } - } - private static IPEndPoint? ParseEndpoint(string endpoint) { int colon = endpoint.LastIndexOf(':'); @@ -224,33 +260,34 @@ public sealed class TcpSttSource : ISttSource { string rest = line["F ".Length..]; int space = rest.IndexOf(' '); + if (space < 0) - return null; - - if (!uint.TryParse(rest[..space], out uint session)) - return null; - - if (session != _session) { - Log?.Invoke($"STT: dropping stale reply (session {session} != current {_session})"); + if (uint.TryParse(rest, out uint session) && session == _session) + return new TranscriptMessage(TranscriptType.Final, string.Empty); + return null; + } + + if (!uint.TryParse(rest[..space], out uint ses)) + return null; + + if (ses != _session) + { + Log?.Invoke($"STT: dropping stale reply (session {ses} != current {_session})"); return null; } return new TranscriptMessage(TranscriptType.Final, rest[(space + 1)..]); } - if (line == "F") - { - return new TranscriptMessage(TranscriptType.Final, string.Empty); - } - return null; } - public async ValueTask DisposeAsync() + public ValueTask DisposeAsync() { - if (_disposed) return; - await StopAsync(); + if (_disposed) return ValueTask.CompletedTask; _disposed = true; + StopAsync(); + return ValueTask.CompletedTask; } } diff --git a/Robovoice.Tts.LibPiper/LibPiperTtsEngine.cs b/Robovoice.Tts.LibPiper/LibPiperTtsEngine.cs index c4db517..e4f5d65 100644 --- a/Robovoice.Tts.LibPiper/LibPiperTtsEngine.cs +++ b/Robovoice.Tts.LibPiper/LibPiperTtsEngine.cs @@ -120,6 +120,71 @@ public sealed class LibPiperTtsEngine : ITtsEngine } } + public IEnumerable SynthesizeSync(string text) + { + ObjectDisposedException.ThrowIf(_disposed, this); + if (!_initialized) + throw new InvalidOperationException("Engine not initialized."); + + var options = new PiperSynthesizeOptions + { + SpeakerId = 0, + LengthScale = _lengthScale, + NoiseScale = _noiseScale, + NoiseWScale = _noiseWScale, + }; + + byte[] textBytes = System.Text.Encoding.UTF8.GetBytes(EnsureTerminator(text) + "\0"); + GCHandle textPin = GCHandle.Alloc(textBytes, GCHandleType.Pinned); + try + { + int startResult = PiperNative.piper_synthesize_start( + _synth, textPin.AddrOfPinnedObject(), in options); + if (startResult != PiperNative.PiperOk) + throw new InvalidOperationException($"piper_synthesize_start failed: {startResult}"); + + while (true) + { + PiperAudioChunk chunk = default; + int result = PiperNative.piper_synthesize_next(_synth, out chunk); + + if (chunk.NumSamples > 0 && chunk.Samples != IntPtr.Zero) + { + int numSamples = (int)chunk.NumSamples; + float[] samples = new float[numSamples]; + Marshal.Copy(chunk.Samples, samples, 0, numSamples); + + if (chunk.SampleRate > 0) + _sampleRate = chunk.SampleRate; + + yield return new AudioChunk(samples, _sampleRate); + } + + if (result == PiperNative.PiperDone || chunk.IsLast) + break; + + if (result < 0) + throw new InvalidOperationException($"piper_synthesize_next failed: {result}"); + } + } + finally + { + textPin.Free(); + } + } + + public void SynthesizeDrain() + { + if (!_initialized || _synth == IntPtr.Zero) + return; + + PiperAudioChunk chunk; + while (PiperNative.piper_synthesize_next(_synth, out chunk) != PiperNative.PiperDone) + { + if (chunk.IsLast) break; + } + } + private static string EnsureTerminator(string text) { string trimmed = text.TrimEnd();