Server implementation notes

The STT server listens on UDP port 67, receives NOP heartbeats and OFF messages from Robovoice, captures audio from a microphone, runs speech recognition (Moonshine), and sends transcript messages back to the client's source address.

Architecture

                         ┌── HKMSTR <nonce> (broadcast, every 50ms)
Robovoice  ──────────────►│
                         │  STT Server
Robovoice  ◄──────────────┤
                         └── HKMSTR:P/F <text> (unicast)

The server:

Listens on UDP :67
First HKMSTR <nonce> → start recording
HKMSTR:OFF <nonce> → stop recording, run STT
150ms with no NOPs → stop recording, run STT (backstop)
Send HKMSTR:F <text> back to the client's source address:port

Framing

All messages are newline-terminated UTF-8 text. No JSON, no binary framing.

HKMSTR <nonce> — NOP heartbeat (client → server)
HKMSTR:OFF <nonce> — stop signal (client → server)
HKMSTR:P <text> — partial transcript (server → client)
HKMSTR:F <text> — final transcript (server → client)

The nonce is an incrementing integer for packet uniqueness only. Discard it.

Python server with Moonshine

import socket
import threading
import time
import numpy as np
import sounddevice as sd
import moonshine

LISTEN_PORT = 67
CLIENT_PORT = 68
SAMPLE_RATE = 16000
SILENCE_TIMEOUT = 0.150  # 150ms

model = moonshine.MoonshineModel(model="moonshine/base")

sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
sock.bind(("0.0.0.0", LISTEN_PORT))

print(f"STT server listening on :{LISTEN_PORT}")

recording = False
last_nop_time = 0
client_addr = None
audio_chunks = []
lock = threading.Lock()

def monitor_silence():
    """Backstop: stop recording if no NOPs for 150ms."""
    global recording
    while True:
        time.sleep(0.01)
        with lock:
            if recording and (time.monotonic() - last_nop_time) > SILENCE_TIMEOUT:
                recording = False
                threading.Thread(target=process_audio, daemon=True).start()

threading.Thread(target=monitor_silence, daemon=True).start()

def process_audio():
    global audio_chunks
    with lock:
        chunks = audio_chunks
        audio_chunks = []
        addr = client_addr

    if not chunks:
        return

    audio = np.concatenate(chunks)
    print(f"Captured {len(audio)/SAMPLE_RATE:.1f}s")

    text = moonshine.transcribe(model, audio).strip()

    if text:
        print(f"Final: {text}")
        reply = f"HKMSTR:F {text}\n".encode("utf-8")
        sock.sendto(reply, addr)
    else:
        print("Empty transcript")

while True:
    data, addr = sock.recvfrom(4096)
    text = data.decode("utf-8", errors="ignore").strip()

    if not text.startswith("HKMSTR"):
        continue

    if text.startswith("HKMSTR:OFF"):
        with lock:
            if recording:
                recording = False
                threading.Thread(target=process_audio, daemon=True).start()
        continue

    if text.startswith("HKMSTR ") or text == "HKMSTR":
        with lock:
            client_addr = addr
            last_nop_time = time.monotonic()

            if not recording:
                recording = True
                audio_chunks = []
                print(f"PTT on from {addr}")

            # Capture 50ms of audio
            chunk = sd.rec(int(SAMPLE_RATE * 0.05), samplerate=SAMPLE_RATE,
                          channels=1, dtype="float32")
            sd.wait()
            audio_chunks.append(chunk.flatten())

Python server with streaming partials

For live feedback, send partials while recording:

import socket
import threading
import time
import numpy as np
import sounddevice as sd
import moonshine

LISTEN_PORT = 67
SAMPLE_RATE = 16000
SILENCE_TIMEOUT = 0.150
PARTIAL_INTERVAL = 0.5  # send partial every 500ms

model = moonshine.MoonshineModel(model="moonshine/base")

sock = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
sock.setsockopt(socket.SOL_SOCKET, socket.SO_BROADCAST, 1)
sock.bind(("0.0.0.0", LISTEN_PORT))

print(f"STT server listening on :{LISTEN_PORT}")

recording = False
last_nop_time = 0
last_partial_time = 0
client_addr = None
audio_chunks = []
lock = threading.Lock()

def capture_and_maybe_partial():
    global last_partial_time
    with lock:
        if not recording:
            return

        chunk = sd.rec(int(SAMPLE_RATE * 0.05), samplerate=SAMPLE_RATE,
                      channels=1, dtype="float32")
        sd.wait()
        audio_chunks.append(chunk.flatten())

        now = time.monotonic()
        if now - last_partial_time > PARTIAL_INTERVAL:
            last_partial_time = now
            partial_audio = np.concatenate(audio_chunks)
            partial_text = moonshine.transcribe(model, partial_audio).strip()
            if partial_text and client_addr:
                reply = f"HKMSTR:P {partial_text}\n".encode("utf-8")
                sock.sendto(reply, client_addr)

while True:
    data, addr = sock.recvfrom(4096)
    text = data.decode("utf-8", errors="ignore").strip()

    if not text.startswith("HKMSTR"):
        continue

    if text.startswith("HKMSTR:OFF"):
        with lock:
            if recording:
                recording = False
                chunks = audio_chunks
                audio_chunks = []

        if chunks:
            audio = np.concatenate(chunks)
            final_text = moonshine.transcribe(model, audio).strip()
            if final_text:
                reply = f"HKMSTR:F {final_text}\n".encode("utf-8")
                sock.sendto(reply, addr)
        continue

    if text.startswith("HKMSTR") and not text.startswith("HKMSTR:"):
        with lock:
            client_addr = addr
            last_nop_time = time.monotonic()

            if not recording:
                recording = True
                audio_chunks = []
                last_partial_time = time.monotonic()
                print(f"PTT on from {addr}")

        capture_and_maybe_partial()

        # Check silence timeout
        with lock:
            if recording and (time.monotonic() - last_nop_time) > SILENCE_TIMEOUT:
                recording = False
                chunks = audio_chunks
                audio_chunks = []

        if 'chunks' in dir() and chunks:
            audio = np.concatenate(chunks)
            final_text = moonshine.transcribe(model, audio).strip()
            if final_text:
                reply = f"HKMSTR:F {final_text}\n".encode("utf-8")
                sock.sendto(reply, addr)

C# server skeleton

using System.Net;
using System.Net.Sockets;
using System.Text;

var sock = new Socket(AddressFamily.InterNetwork, SocketType.Dgram, ProtocolType.Udp);
sock.Bind(new IPEndPoint(IPAddress.Any, 67));

Console.WriteLine("STT server listening on :67");

byte[] buffer = new byte[4096];
EndPoint fromEp = new IPEndPoint(IPAddress.Any, 0);
bool recording = false;
DateTime lastNop = DateTime.MinValue;
List<float[]> audioChunks = new();

while (true)
{
    if (sock.Poll(50_000, SelectMode.SelectRead))
    {
        int received = sock.ReceiveFrom(buffer, ref fromEp);
        string text = Encoding.UTF8.GetString(buffer, 0, received).TrimEnd('\n', '\r');

        if (!text.StartsWith("HKMSTR"))
            continue;

        if (text.StartsWith("HKMSTR:OFF"))
        {
            if (recording)
            {
                recording = false;
                ProcessAndReply(audioChunks, fromEp);
                audioChunks.Clear();
            }
            continue;
        }

        // NOP
        lastNop = DateTime.UtcNow;
        if (!recording)
        {
            recording = true;
            audioChunks.Clear();
            Console.WriteLine($"PTT on from {fromEp}");
        }

        // Capture 50ms audio here...
        // audioChunks.Add(capturedChunk);

        // Check silence timeout
        if (recording && (DateTime.UtcNow - lastNop).TotalMilliseconds > 150)
        {
            recording = false;
            ProcessAndReply(audioChunks, fromEp);
            audioChunks.Clear();
        }
    }
    else
    {
        // Timeout check even without incoming data
        if (recording && (DateTime.UtcNow - lastNop).TotalMilliseconds > 150)
        {
            recording = false;
            ProcessAndReply(audioChunks, fromEp);
            audioChunks.Clear();
        }
    }
}

void ProcessAndReply(List<float[]> chunks, EndPoint client)
{
    if (chunks.Count == 0) return;

    // Concatenate and run STT...
    string text = "recognized text here";

    if (!string.IsNullOrEmpty(text))
    {
        byte[] reply = Encoding.UTF8.GetBytes($"HKMSTR:F {text}\n");
        sock.SendTo(reply, client);
    }
}

Tips

Reply address: always reply to the source endpoint of the last NOP. The client binds to a specific IP on :68.
Nonces: discard them. They exist only to make each datagram unique. Do not derive any meaning from nonce values.
Silence timeout: 150ms = 3 missed NOPs at 50ms intervals. If you change the NOP interval on the client, adjust this accordingly.
Partials: optional. Send HKMSTR:P <text> while recording for live feedback. Client logs them but only HKMSTR:F triggers TTS.
Broadcast only for C→S: the WireGuard killswitch only allows outbound broadcast to 255.255.255.255:67. Unicast from client won't pass.
Unicast OK for S→C: the inbound WFP rule has no address restriction, so unicast replies to :68 pass through.
Moonshine models: moonshine/base or moonshine/tiny.

9.5 KiB Raw Blame History