333 lines
9.5 KiB
Markdown
333 lines
9.5 KiB
Markdown
# Server implementation notes
|
|
|
|
The STT server listens on UDP port 67, receives NOP heartbeats and OFF
|
|
messages from Robovoice, captures audio from a microphone, runs speech
|
|
recognition (Moonshine), and sends transcript messages back to the client's
|
|
source address.
|
|
|
|
## Architecture
|
|
|
|
```
|
|
┌── HKMSTR <nonce> (broadcast, every 50ms)
|
|
Robovoice ──────────────►│
|
|
│ STT Server
|
|
Robovoice ◄──────────────┤
|
|
└── HKMSTR:P/F <text> (unicast)
|
|
```
|
|
|
|
The server:
|
|
1. Listens on UDP :67
|
|
2. First `HKMSTR <nonce>` → start recording
|
|
3. `HKMSTR:OFF <nonce>` → stop recording, run STT
|
|
4. 150ms with no NOPs → stop recording, run STT (backstop)
|
|
5. Send `HKMSTR:F <text>` back to the client's source address:port
|
|
|
|
## Framing
|
|
|
|
All messages are newline-terminated UTF-8 text. No JSON, no binary framing.
|
|
|
|
- `HKMSTR <nonce>` — NOP heartbeat (client → server)
|
|
- `HKMSTR:OFF <nonce>` — stop signal (client → server)
|
|
- `HKMSTR:P <text>` — partial transcript (server → client)
|
|
- `HKMSTR:F <text>` — final transcript (server → client)
|
|
|
|
The nonce is an incrementing integer for packet uniqueness only. Discard it.
|
|
|
|
## Python server with Moonshine
|
|
|
|
```python
|
|
import socket
|
|
import threading
|
|
import time
|
|
import numpy as np
|
|
import sounddevice as sd
|
|
import moonshine
|
|
|
|
LISTEN_PORT = 67
|
|
CLIENT_PORT = 68
|
|
SAMPLE_RATE = 16000
|
|
SILENCE_TIMEOUT = 0.150 # 150ms
|
|
|
|
model = moonshine.MoonshineModel(model="moonshine/base")
|
|
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
|
|
sock.bind(("0.0.0.0", LISTEN_PORT))
|
|
|
|
print(f"STT server listening on :{LISTEN_PORT}")
|
|
|
|
recording = False
|
|
last_nop_time = 0
|
|
client_addr = None
|
|
audio_chunks = []
|
|
lock = threading.Lock()
|
|
|
|
def monitor_silence():
|
|
"""Backstop: stop recording if no NOPs for 150ms."""
|
|
global recording
|
|
while True:
|
|
time.sleep(0.01)
|
|
with lock:
|
|
if recording and (time.monotonic() - last_nop_time) > SILENCE_TIMEOUT:
|
|
recording = False
|
|
threading.Thread(target=process_audio, daemon=True).start()
|
|
|
|
threading.Thread(target=monitor_silence, daemon=True).start()
|
|
|
|
def process_audio():
|
|
global audio_chunks
|
|
with lock:
|
|
chunks = audio_chunks
|
|
audio_chunks = []
|
|
addr = client_addr
|
|
|
|
if not chunks:
|
|
return
|
|
|
|
audio = np.concatenate(chunks)
|
|
print(f"Captured {len(audio)/SAMPLE_RATE:.1f}s")
|
|
|
|
text = moonshine.transcribe(model, audio).strip()
|
|
|
|
if text:
|
|
print(f"Final: {text}")
|
|
reply = f"HKMSTR:F {text}\n".encode("utf-8")
|
|
sock.sendto(reply, addr)
|
|
else:
|
|
print("Empty transcript")
|
|
|
|
while True:
|
|
data, addr = sock.recvfrom(4096)
|
|
text = data.decode("utf-8", errors="ignore").strip()
|
|
|
|
if not text.startswith("HKMSTR"):
|
|
continue
|
|
|
|
if text.startswith("HKMSTR:OFF"):
|
|
with lock:
|
|
if recording:
|
|
recording = False
|
|
threading.Thread(target=process_audio, daemon=True).start()
|
|
continue
|
|
|
|
if text.startswith("HKMSTR ") or text == "HKMSTR":
|
|
with lock:
|
|
client_addr = addr
|
|
last_nop_time = time.monotonic()
|
|
|
|
if not recording:
|
|
recording = True
|
|
audio_chunks = []
|
|
print(f"PTT on from {addr}")
|
|
|
|
# Capture 50ms of audio
|
|
chunk = sd.rec(int(SAMPLE_RATE * 0.05), samplerate=SAMPLE_RATE,
|
|
channels=1, dtype="float32")
|
|
sd.wait()
|
|
audio_chunks.append(chunk.flatten())
|
|
```
|
|
|
|
## Python server with streaming partials
|
|
|
|
For live feedback, send partials while recording:
|
|
|
|
```python
|
|
import socket
|
|
import threading
|
|
import time
|
|
import numpy as np
|
|
import sounddevice as sd
|
|
import moonshine
|
|
|
|
LISTEN_PORT = 67
|
|
SAMPLE_RATE = 16000
|
|
SILENCE_TIMEOUT = 0.150
|
|
PARTIAL_INTERVAL = 0.5 # send partial every 500ms
|
|
|
|
model = moonshine.MoonshineModel(model="moonshine/base")
|
|
|
|
sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
|
|
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
|
|
sock.bind(("0.0.0.0", LISTEN_PORT))
|
|
|
|
print(f"STT server listening on :{LISTEN_PORT}")
|
|
|
|
recording = False
|
|
last_nop_time = 0
|
|
last_partial_time = 0
|
|
client_addr = None
|
|
audio_chunks = []
|
|
lock = threading.Lock()
|
|
|
|
def capture_and_maybe_partial():
|
|
global last_partial_time
|
|
with lock:
|
|
if not recording:
|
|
return
|
|
|
|
chunk = sd.rec(int(SAMPLE_RATE * 0.05), samplerate=SAMPLE_RATE,
|
|
channels=1, dtype="float32")
|
|
sd.wait()
|
|
audio_chunks.append(chunk.flatten())
|
|
|
|
now = time.monotonic()
|
|
if now - last_partial_time > PARTIAL_INTERVAL:
|
|
last_partial_time = now
|
|
partial_audio = np.concatenate(audio_chunks)
|
|
partial_text = moonshine.transcribe(model, partial_audio).strip()
|
|
if partial_text and client_addr:
|
|
reply = f"HKMSTR:P {partial_text}\n".encode("utf-8")
|
|
sock.sendto(reply, client_addr)
|
|
|
|
while True:
|
|
data, addr = sock.recvfrom(4096)
|
|
text = data.decode("utf-8", errors="ignore").strip()
|
|
|
|
if not text.startswith("HKMSTR"):
|
|
continue
|
|
|
|
if text.startswith("HKMSTR:OFF"):
|
|
with lock:
|
|
if recording:
|
|
recording = False
|
|
chunks = audio_chunks
|
|
audio_chunks = []
|
|
|
|
if chunks:
|
|
audio = np.concatenate(chunks)
|
|
final_text = moonshine.transcribe(model, audio).strip()
|
|
if final_text:
|
|
reply = f"HKMSTR:F {final_text}\n".encode("utf-8")
|
|
sock.sendto(reply, addr)
|
|
continue
|
|
|
|
if text.startswith("HKMSTR") and not text.startswith("HKMSTR:"):
|
|
with lock:
|
|
client_addr = addr
|
|
last_nop_time = time.monotonic()
|
|
|
|
if not recording:
|
|
recording = True
|
|
audio_chunks = []
|
|
last_partial_time = time.monotonic()
|
|
print(f"PTT on from {addr}")
|
|
|
|
capture_and_maybe_partial()
|
|
|
|
# Check silence timeout
|
|
with lock:
|
|
if recording and (time.monotonic() - last_nop_time) > SILENCE_TIMEOUT:
|
|
recording = False
|
|
chunks = audio_chunks
|
|
audio_chunks = []
|
|
|
|
if 'chunks' in dir() and chunks:
|
|
audio = np.concatenate(chunks)
|
|
final_text = moonshine.transcribe(model, audio).strip()
|
|
if final_text:
|
|
reply = f"HKMSTR:F {final_text}\n".encode("utf-8")
|
|
sock.sendto(reply, addr)
|
|
```
|
|
|
|
## C# server skeleton
|
|
|
|
```csharp
|
|
using System.Net;
|
|
using System.Net.Sockets;
|
|
using System.Text;
|
|
|
|
var sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
|
|
sock.Bind(new IPEndPoint(IPAddress.Any, 67));
|
|
|
|
Console.WriteLine("STT server listening on :67");
|
|
|
|
byte[] buffer = new byte[4096];
|
|
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
|
|
bool recording = false;
|
|
DateTime lastNop = DateTime.MinValue;
|
|
List<float[]> audioChunks = new();
|
|
|
|
while (true)
|
|
{
|
|
if (sock.Poll(50_000, SelectMode.SelectRead))
|
|
{
|
|
int received = sock.ReceiveFrom(buffer, ref fromEp);
|
|
string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');
|
|
|
|
if (!text.StartsWith("HKMSTR"))
|
|
continue;
|
|
|
|
if (text.StartsWith("HKMSTR:OFF"))
|
|
{
|
|
if (recording)
|
|
{
|
|
recording = false;
|
|
ProcessAndReply(audioChunks, fromEp);
|
|
audioChunks.Clear();
|
|
}
|
|
continue;
|
|
}
|
|
|
|
// NOP
|
|
lastNop = DateTime.UtcNow;
|
|
if (!recording)
|
|
{
|
|
recording = true;
|
|
audioChunks.Clear();
|
|
Console.WriteLine($"PTT on from {fromEp}");
|
|
}
|
|
|
|
// Capture 50ms audio here...
|
|
// audioChunks.Add(capturedChunk);
|
|
|
|
// Check silence timeout
|
|
if (recording && (DateTime.UtcNow - lastNop).TotalMilliseconds > 150)
|
|
{
|
|
recording = false;
|
|
ProcessAndReply(audioChunks, fromEp);
|
|
audioChunks.Clear();
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// Timeout check even without incoming data
|
|
if (recording && (DateTime.UtcNow - lastNop).TotalMilliseconds > 150)
|
|
{
|
|
recording = false;
|
|
ProcessAndReply(audioChunks, fromEp);
|
|
audioChunks.Clear();
|
|
}
|
|
}
|
|
}
|
|
|
|
void ProcessAndReply(List<float[]> chunks, EndPoint client)
|
|
{
|
|
if (chunks.Count == 0) return;
|
|
|
|
// Concatenate and run STT...
|
|
string text = "recognized text here";
|
|
|
|
if (!string.IsNullOrEmpty(text))
|
|
{
|
|
byte[] reply = Encoding.UTF8.GetBytes($"HKMSTR:F {text}\n");
|
|
sock.SendTo(reply, client);
|
|
}
|
|
}
|
|
```
|
|
|
|
## Tips
|
|
|
|
- **Reply address:** always reply to the source endpoint of the last NOP.
|
|
The client binds to a specific IP on :68.
|
|
- **Nonces:** discard them. They exist only to make each datagram unique.
|
|
Do not derive any meaning from nonce values.
|
|
- **Silence timeout:** 150ms = 3 missed NOPs at 50ms intervals. If you
|
|
change the NOP interval on the client, adjust this accordingly.
|
|
- **Partials:** optional. Send `HKMSTR:P <text>` while recording for live
|
|
feedback. Client logs them but only `HKMSTR:F` triggers TTS.
|
|
- **Broadcast only for C→S:** the WireGuard killswitch only allows
|
|
outbound broadcast to 255.255.255.255:67. Unicast from client won't pass.
|
|
- **Unicast OK for S→C:** the inbound WFP rule has no address restriction,
|
|
so unicast replies to :68 pass through.
|
|
- **Moonshine models:** `moonshine/base` or `moonshine/tiny`.
|