v0.7: pre-synth buffering, session IDs, segment timeout playback

2026-08-12 10:30:11 +00:00
parent e1739059d9
commit ecf00c1bc4
6 changed files with 285 additions and 372 deletions
@@ -12,6 +12,16 @@ internal sealed class Orchestrator : IAsyncDisposable
    private CancellationTokenSource? _currentCts;
    private bool _disposed;

+    private readonly object _stateLock = new();
+    private readonly Queue<string> _pendingTexts = new();
+    private readonly List<float[]> _audioBuffer = new();
+    private int _bufferSampleRate;
+    private bool _playing;
+    private Task? _synthTask;
+    private CancellationTokenSource? _synthCts;
+    private System.Threading.Timer? _segmentTimer;
+    private static readonly TimeSpan SegmentTimeout = TimeSpan.FromMilliseconds(250);
+
    public string OutputDeviceName { get; set; } = string.Empty;

    public Orchestrator(
@@ -30,9 +40,169 @@ internal sealed class Orchestrator : IAsyncDisposable

    private void OnTranscript(object? sender, TranscriptEventArgs e)
    {
-        if (e.Message.Type == TranscriptType.Final)
+        var msg = e.Message;
+
+        lock (_stateLock)
        {
-            _ = SynthesizeAsync(e.Message.Text);
+            if (msg.Type == TranscriptType.Partial)
+            {
+                if (!string.IsNullOrWhiteSpace(msg.Text))
+                {
+                    _log($"SEGMENT: \"{msg.Text}\" ({msg.Text.Length} chars)");
+                    _pendingTexts.Enqueue(msg.Text);
+                    EnsureSynthTask();
+                    ResetSegmentTimer();
+                }
+            }
+            else
+            {
+                if (!string.IsNullOrWhiteSpace(msg.Text))
+                {
+                    _log($"FINAL: \"{msg.Text}\" ({msg.Text.Length} chars)");
+                    _pendingTexts.Enqueue(msg.Text);
+                    EnsureSynthTask();
+                }
+                else
+                {
+                    _log("FINAL: (empty)");
+                }
+
+                CancelSegmentTimer();
+                TransitionToPlaying();
+            }
+        }
+    }
+
+    private void ResetSegmentTimer()
+    {
+        _segmentTimer?.Dispose();
+        _segmentTimer = new System.Threading.Timer(_ => OnSegmentTimeout(), null, SegmentTimeout, Timeout.InfiniteTimeSpan);
+    }
+
+    private void CancelSegmentTimer()
+    {
+        _segmentTimer?.Dispose();
+        _segmentTimer = null;
+    }
+
+    private void OnSegmentTimeout()
+    {
+        _log("STT: segment timeout, starting playback early");
+        lock (_stateLock)
+        {
+            TransitionToPlaying();
+        }
+    }
+
+    private void TransitionToPlaying()
+    {
+        if (_playing)
+            return;
+
+        if (_audioBuffer.Count == 0)
+        {
+            if (_synthTask is null || _synthTask.IsCompleted)
+            {
+                _log("TTS: nothing to play");
+                CancelSegmentTimer();
+            }
+            return;
+        }
+
+        _playing = true;
+        CancelSegmentTimer();
+
+        int sampleRate = _bufferSampleRate;
+        var chunks = _audioBuffer.ToList();
+        _audioBuffer.Clear();
+
+        _log($"TTS: playing {chunks.Count} buffered chunks ({sampleRate} Hz)");
+
+        _audioOutput.Start(sampleRate, OutputDeviceName);
+        foreach (var samples in chunks)
+        {
+            _audioOutput.WriteSamples(samples);
+        }
+    }
+
+    private void EnsureSynthTask()
+    {
+        if (_synthTask is not null && !_synthTask.IsCompleted)
+            return;
+
+        _synthCts?.Cancel();
+        _synthCts = new CancellationTokenSource();
+        _synthTask = SynthLoopAsync(_synthCts.Token);
+    }
+
+    private async Task SynthLoopAsync(CancellationToken ct)
+    {
+        while (true)
+        {
+            string text;
+            lock (_stateLock)
+            {
+                if (_pendingTexts.Count == 0)
+                    break;
+                text = _pendingTexts.Dequeue();
+            }
+
+            try
+            {
+                await foreach (var chunk in _tts.SynthesizeAsync(text, ct))
+                {
+                    lock (_stateLock)
+                    {
+                        if (_playing)
+                        {
+                            _audioOutput.WriteSamples(chunk.Samples);
+                        }
+                        else
+                        {
+                            _bufferSampleRate = chunk.SampleRate;
+                            _audioBuffer.Add(chunk.Samples);
+                        }
+                    }
+                }
+            }
+            catch (OperationCanceledException)
+            {
+                break;
+            }
+            catch (Exception ex)
+            {
+                _log($"TTS error: {ex.Message}");
+            }
+        }
+
+        lock (_stateLock)
+        {
+            if (_playing)
+            {
+                _audioOutput.Flush();
+                _log("TTS: synthesis complete, flushed");
+            }
+        }
+    }
+
+    public void NotifyPttPressed()
+    {
+        lock (_stateLock)
+        {
+            CancelSegmentTimer();
+            _synthCts?.Cancel();
+            _synthCts?.Dispose();
+            _synthTask = null;
+
+            if (_playing)
+            {
+                _audioOutput.Stop();
+                _playing = false;
+            }
+
+            _pendingTexts.Clear();
+            _audioBuffer.Clear();
+            _bufferSampleRate = 0;
        }
    }

@@ -50,7 +220,7 @@ internal sealed class Orchestrator : IAsyncDisposable
        var ct = _currentCts.Token;

        var sw = System.Diagnostics.Stopwatch.StartNew();
-        _log($"FINAL: \"{text}\" ({text.Length} chars)");
+        _log($"Speak: \"{text}\" ({text.Length} chars)");

        try
        {
@@ -101,6 +271,9 @@ internal sealed class Orchestrator : IAsyncDisposable
        if (_disposed) return;
        _currentCts?.Cancel();
        _currentCts?.Dispose();
+        _synthCts?.Cancel();
+        _synthCts?.Dispose();
+        CancelSegmentTimer();
        _sttSource.TranscriptReceived -= OnTranscript;
        await _sttSource.DisposeAsync();
        await _tts.DisposeAsync();