Add live audio capture via WASAPI loopback

- LiveCapture.cs: real-time WASAPI loopback capture with rolling buffer
  and continuous FFT analysis, enqueues WaveFrames at ~10Hz
- MusicAnalyzer.cs: refactored ExtractFeatures/MapPitchToPeriod/BuildWaveFrame
  as public reusable methods with sampleRate parameter
- MainForm: audio device selector (defaults to system default), Live button
  starts/stops capture, Stop All stops live too
- Adaptive normalization (running max with slow decay) for live volume changes
This commit is contained in:
2026-08-08 19:20:46 +00:00
parent 933c7ede38
commit ccc5093ee2
3 changed files with 330 additions and 84 deletions
+151
View File
@@ -0,0 +1,151 @@
using System.Numerics;
using FftSharp;
using NAudio.CoreAudioApi;
using NAudio.Wave;
namespace Substation;
public class LiveCapture : IDisposable
{
readonly State _state;
readonly MMDevice _device;
WasapiLoopbackCapture? _capture;
Thread? _processThread;
volatile bool _running;
readonly Queue<double> _sampleBuffer = new();
readonly object _bufferLock = new();
double[]? _prevRhythmMag;
double _liveMaxFlux;
double _liveMaxMelodyEnergy;
int _sampleRate;
int _fftsPerTick;
int _fftIndexInTick;
public event Action? Stopped;
public LiveCapture(State state, MMDevice device)
{
_state = state;
_device = device;
}
public void Start()
{
_running = true;
_capture = new WasapiLoopbackCapture(_device);
_sampleRate = _capture.WaveFormat.SampleRate;
_fftsPerTick = Math.Max(1, (int)Math.Round(MusicAnalyzer.TickDuration * _sampleRate / MusicAnalyzer.HopSize));
_capture.DataAvailable += OnDataAvailable;
_capture.RecordingStopped += OnRecordingStopped;
_processThread = new Thread(ProcessLoop) { IsBackground = true, Name = "LiveCapture-FFT" };
_processThread.Start();
_capture.StartRecording();
Console.WriteLine($"[live] capture started: {_device.FriendlyName} ({_sampleRate}Hz, {_capture.WaveFormat.Channels}ch)");
}
public void Stop()
{
_running = false;
try { _capture?.StopRecording(); } catch { }
}
void OnDataAvailable(object? sender, WaveInEventArgs e)
{
int channels = _capture!.WaveFormat.Channels;
int bytesPerSample = _capture.WaveFormat.BitsPerSample / 8;
int frameSize = channels * bytesPerSample;
int sampleCount = e.BytesRecorded / frameSize;
lock (_bufferLock)
{
for (int i = 0; i < sampleCount; i++)
{
int offset = i * frameSize;
float left = BitConverter.ToSingle(e.Buffer, offset);
float right = channels >= 2
? BitConverter.ToSingle(e.Buffer, offset + bytesPerSample)
: left;
_sampleBuffer.Enqueue((left + right) * 0.5);
}
// Cap buffer size to prevent memory growth if processing falls behind
while (_sampleBuffer.Count > _sampleRate * 2)
_sampleBuffer.Dequeue();
}
}
void OnRecordingStopped(object? sender, StoppedEventArgs e)
{
_running = false;
Stopped?.Invoke();
Console.WriteLine("[live] capture stopped");
}
void ProcessLoop()
{
var window = new FftSharp.Windows.Hanning();
var buffer = new double[MusicAnalyzer.WindowSize];
var tf = new MusicAnalyzer.TickFeature();
while (_running)
{
double[]? windowData = null;
lock (_bufferLock)
{
if (_sampleBuffer.Count >= MusicAnalyzer.WindowSize)
{
for (int i = 0; i < MusicAnalyzer.WindowSize; i++)
buffer[i] = _sampleBuffer.Dequeue();
windowData = buffer;
}
}
if (windowData == null)
{
Thread.Sleep(5);
continue;
}
window.ApplyInPlace(windowData);
var spectrum = FFT.Forward(windowData);
var mag = FFT.Magnitude(spectrum);
var (rhythmEnergy, rhythmFlux, melodyEnergy, melodyFreq) =
MusicAnalyzer.ExtractFeatures(mag, _prevRhythmMag, _sampleRate);
_prevRhythmMag = mag;
// Adaptive normalization: running max with slow decay
_liveMaxFlux = Math.Max(rhythmFlux, _liveMaxFlux * 0.999);
_liveMaxMelodyEnergy = Math.Max(melodyEnergy, _liveMaxMelodyEnergy * 0.999);
tf.RhythmFlux = Math.Max(tf.RhythmFlux, rhythmFlux);
tf.MelodyEnergy += melodyEnergy;
tf.MelodyFreqSamples.Add(melodyFreq);
tf.MelodyCount++;
_fftIndexInTick++;
if (_fftIndexInTick >= _fftsPerTick)
{
var (frameA, frameB) = MusicAnalyzer.BuildWaveFrame(tf, _liveMaxFlux, _liveMaxMelodyEnergy);
_state.EnqueueStream('A', new[] { frameA });
_state.EnqueueStream('B', new[] { frameB });
tf = new MusicAnalyzer.TickFeature();
_fftIndexInTick = 0;
}
}
}
public void Dispose()
{
_running = false;
try { _capture?.Dispose(); } catch { }
}
}
+119 -26
View File
@@ -1,6 +1,7 @@
using System.ComponentModel;
using System.Drawing.Drawing2D;
using System.Diagnostics;
using NAudio.CoreAudioApi;
using NAudio.Wave;
namespace Substation;
@@ -48,10 +49,15 @@ public class MainForm : Form
string _trayState = "";
readonly Button _btnMusic;
readonly Button _btnLive;
readonly Label _lblTrack;
readonly Label _lblAudioDev;
readonly ComboBox _cboAudioDev;
bool _isMusicRunning;
bool _isLiveRunning;
WaveOutEvent? _audioOut;
Stopwatch? _musicStopwatch;
LiveCapture? _liveCapture;
public MainForm(CoyoteDevice device, State state, Server server)
{
@@ -61,7 +67,7 @@ public class MainForm : Form
_loopCts = new CancellationTokenSource();
Text = $"Substation {typeof(MainForm).Assembly.GetName().Version}";
ClientSize = new Size(420, 425);
ClientSize = new Size(420, 450);
FormBorderStyle = FormBorderStyle.FixedSingle;
MaximizeBox = false;
StartPosition = FormStartPosition.CenterScreen;
@@ -117,30 +123,52 @@ public class MainForm : Form
{
Text = "Test",
Location = new Point(16, 96),
Size = new Size(100, 32)
Size = new Size(80, 32)
};
_btnTest.Click += OnTest;
_btnMusic = new Button
{
Text = "Music",
Location = new Point(130, 96),
Size = new Size(100, 32)
Location = new Point(104, 96),
Size = new Size(80, 32)
};
_btnMusic.Click += OnMusic;
_btnLive = new Button
{
Text = "Live",
Location = new Point(192, 96),
Size = new Size(80, 32)
};
_btnLive.Click += OnLive;
_btnStop = new Button
{
Text = "Stop All",
Location = new Point(244, 96),
Size = new Size(100, 32)
Location = new Point(280, 96),
Size = new Size(80, 32)
};
_btnStop.Click += OnStop;
_lblAudioDev = new Label
{
Text = "Audio:",
Location = new Point(16, 134),
Size = new Size(40, 20)
};
_cboAudioDev = new ComboBox
{
DropDownStyle = ComboBoxStyle.DropDownList,
Location = new Point(60, 132),
Size = new Size(344, 22)
};
PopulateAudioDevices();
_lblTrack = new Label
{
Text = "",
Location = new Point(16, 134),
Location = new Point(16, 158),
Size = new Size(388, 16),
ForeColor = Color.DimGray
};
@@ -148,37 +176,37 @@ public class MainForm : Form
_lblSendA = new Label
{
Text = "Send A",
Location = new Point(16, 160),
Location = new Point(16, 184),
Size = new Size(48, 20),
Font = new Font(Font.FontFamily, 9, FontStyle.Bold)
};
_heatA = new HeatMap(60)
{
Location = new Point(72, 158),
Location = new Point(72, 182),
Size = new Size(332, 22)
};
_lblSendB = new Label
{
Text = "Send B",
Location = new Point(16, 188),
Location = new Point(16, 212),
Size = new Size(48, 20),
Font = new Font(Font.FontFamily, 9, FontStyle.Bold)
};
_heatB = new HeatMap(60)
{
Location = new Point(72, 186),
Location = new Point(72, 210),
Size = new Size(332, 22)
};
_lblRecvA = new Label
{
Text = "Recv A",
Location = new Point(16, 216),
Location = new Point(16, 240),
Size = new Size(48, 20),
Font = new Font(Font.FontFamily, 9, FontStyle.Bold)
};
_gaugeRecvA = new ProgressBar
{
Location = new Point(72, 216),
Location = new Point(72, 240),
Size = new Size(332, 20),
Minimum = 0,
Maximum = 200,
@@ -187,13 +215,13 @@ public class MainForm : Form
_lblRecvB = new Label
{
Text = "Recv B",
Location = new Point(16, 242),
Location = new Point(16, 266),
Size = new Size(48, 20),
Font = new Font(Font.FontFamily, 9, FontStyle.Bold)
};
_gaugeRecvB = new ProgressBar
{
Location = new Point(72, 242),
Location = new Point(72, 266),
Size = new Size(332, 20),
Minimum = 0,
Maximum = 200,
@@ -203,13 +231,13 @@ public class MainForm : Form
_lblLimitA = new Label
{
Text = "Lim A",
Location = new Point(16, 274),
Location = new Point(16, 298),
Size = new Size(40, 20),
Font = new Font(Font.FontFamily, 9, FontStyle.Bold)
};
_limitBarA = new TrackBar
{
Location = new Point(60, 270),
Location = new Point(60, 294),
Size = new Size(220, 45),
Minimum = 0,
Maximum = 200,
@@ -221,14 +249,14 @@ public class MainForm : Form
_lblLimitValA = new Label
{
Text = "30",
Location = new Point(286, 274),
Location = new Point(286, 298),
Size = new Size(32, 20),
TextAlign = ContentAlignment.MiddleRight
};
_chkScaleA = new CheckBox
{
Text = "Scale",
Location = new Point(320, 274),
Location = new Point(320, 298),
Size = new Size(76, 24),
Checked = false
};
@@ -237,13 +265,13 @@ public class MainForm : Form
_lblLimitB = new Label
{
Text = "Lim B",
Location = new Point(16, 322),
Location = new Point(16, 346),
Size = new Size(40, 20),
Font = new Font(Font.FontFamily, 9, FontStyle.Bold)
};
_limitBarB = new TrackBar
{
Location = new Point(60, 318),
Location = new Point(60, 342),
Size = new Size(220, 45),
Minimum = 0,
Maximum = 200,
@@ -255,21 +283,21 @@ public class MainForm : Form
_lblLimitValB = new Label
{
Text = "30",
Location = new Point(286, 322),
Location = new Point(286, 346),
Size = new Size(32, 20),
TextAlign = ContentAlignment.MiddleRight
};
_chkScaleB = new CheckBox
{
Text = "Scale",
Location = new Point(320, 322),
Location = new Point(320, 346),
Size = new Size(76, 24),
Checked = false
};
_chkScaleB.CheckedChanged += OnLimitChanged;
OnLimitChanged(null, EventArgs.Empty);
Controls.AddRange(new Control[] { _lblBle, _lblWs, _lblDeviceName, _txtDeviceName, _btnConnect, _chkSwap, _btnTest, _btnMusic, _btnStop, _lblTrack, _lblSendA, _heatA, _lblSendB, _heatB, _lblRecvA, _gaugeRecvA, _lblRecvB, _gaugeRecvB, _lblLimitA, _limitBarA, _lblLimitValA, _chkScaleA, _lblLimitB, _limitBarB, _lblLimitValB, _chkScaleB });
Controls.AddRange(new Control[] { _lblBle, _lblWs, _lblDeviceName, _txtDeviceName, _btnConnect, _chkSwap, _btnTest, _btnMusic, _btnLive, _btnStop, _lblAudioDev, _cboAudioDev, _lblTrack, _lblSendA, _heatA, _lblSendB, _heatB, _lblRecvA, _gaugeRecvA, _lblRecvB, _gaugeRecvB, _lblLimitA, _limitBarA, _lblLimitValA, _chkScaleA, _lblLimitB, _limitBarB, _lblLimitValB, _chkScaleB });
// Tray icon — three cached variants: neutral=gray, active=gold, hot=red-orange
_iconNeutral = CreateVoltageIcon(Color.Gray);
@@ -437,8 +465,9 @@ public class MainForm : Form
void RefreshButtonStates()
{
_btnTest.Enabled = !_server.HasClient && !IsTestRunning;
_btnMusic.Enabled = !_server.HasClient && !_isMusicRunning;
_btnTest.Enabled = !_server.HasClient && !IsTestRunning && !_isMusicRunning && !_isLiveRunning;
_btnMusic.Enabled = !_server.HasClient && !_isMusicRunning && !IsTestRunning && !_isLiveRunning;
_btnLive.Enabled = !_server.HasClient && !_isLiveRunning && !IsTestRunning && !_isMusicRunning;
_btnStop.Enabled = true;
}
@@ -451,6 +480,10 @@ public class MainForm : Form
var elapsed = _musicStopwatch.Elapsed;
_lblTrack.Text = $"Playing {elapsed:mm\\:ss} / {_musicTotalTime:mm\\:ss} {Path.GetFileName(_musicFilePath)}";
}
else if (_isLiveRunning)
{
_lblTrack.Text = "Live capture";
}
UpdateTrayIcon();
}
@@ -516,6 +549,7 @@ public class MainForm : Form
void OnStop(object? sender, EventArgs e)
{
StopMusic();
StopLive();
_state.SetStrength('A', 0);
_state.SetStrength('B', 0);
_state.Stop('A');
@@ -639,6 +673,65 @@ public class MainForm : Form
}
}
void PopulateAudioDevices()
{
using var enumerator = new MMDeviceEnumerator();
var defaultDev = enumerator.GetDefaultAudioEndpoint(DataFlow.Render, Role.Console);
var devices = enumerator.EnumerateAudioEndPoints(DataFlow.Render, DeviceState.Active);
_cboAudioDev.Items.Clear();
int defaultIdx = 0;
for (int i = 0; i < devices.Count; i++)
{
_cboAudioDev.Items.Add(devices[i]);
if (devices[i].ID == defaultDev.ID)
defaultIdx = i;
}
_cboAudioDev.SelectedIndex = defaultIdx;
}
void OnLive(object? sender, EventArgs e)
{
if (_isLiveRunning || _server.HasClient) return;
if (_cboAudioDev.SelectedItem is not MMDevice dev) return;
_isLiveRunning = true;
_btnLive.Enabled = false;
_lblTrack.Text = "Starting live capture...";
_state.SetStrength('A', 60);
_state.SetStrength('B', 60);
_state.Stop('A');
_state.Stop('B');
_liveCapture = new LiveCapture(_state, dev);
_liveCapture.Stopped += () => BeginInvoke(StopLive);
_liveCapture.Start();
_lblTrack.Text = "Live capture";
RefreshButtonStates();
}
void StopLive()
{
if (_liveCapture != null)
{
try { _liveCapture.Stop(); } catch { }
try { _liveCapture.Dispose(); } catch { }
_liveCapture = null;
}
if (_isLiveRunning)
{
_isLiveRunning = false;
if (!IsDisposed)
{
_lblTrack.Text = "";
RefreshButtonStates();
}
}
}
void UpdateTrayIcon()
{
var newState = _state.IsSignaling ? "hot"
+60 -58
View File
@@ -8,27 +8,26 @@ public record MusicPattern(List<WaveFrame> ChannelA, List<WaveFrame> ChannelB, T
public static class MusicAnalyzer
{
const int SampleRate = 44100;
const int WindowSize = 2048;
const int HopSize = 1024;
const double TickDuration = 0.1; // 100ms per e-stim frame
public const int WindowSize = 2048;
public const int HopSize = 1024;
public const double TickDuration = 0.1;
// Frequency band boundaries (Hz)
const double RhythmLow = 20, RhythmHigh = 250;
const double MelodyLow = 300, MelodyHigh = 4000;
public static MusicPattern Analyze(string mp3Path, IProgress<int>? progress = null)
{
var samples = DecodeToMono(mp3Path);
var duration = TimeSpan.FromSeconds((double)samples.Length / SampleRate);
const int sampleRate = 44100;
var samples = DecodeToMono(mp3Path, sampleRate);
var duration = TimeSpan.FromSeconds((double)samples.Length / sampleRate);
var tickCount = (int)Math.Ceiling((double)samples.Length / SampleRate / TickDuration);
var tickCount = (int)Math.Ceiling((double)samples.Length / sampleRate / TickDuration);
var channelA = new List<WaveFrame>(tickCount);
var channelB = new List<WaveFrame>(tickCount);
var window = new FftSharp.Windows.Hanning();
var buffer = new double[WindowSize];
int fftsPerTick = (int)Math.Round(TickDuration * SampleRate / HopSize);
int fftsPerTick = (int)Math.Round(TickDuration * sampleRate / HopSize);
double[]? prevRhythmMag = null;
double maxFlux = 0;
@@ -49,7 +48,7 @@ public static class MusicAnalyzer
var mag = FFT.Magnitude(spectrum);
var (rhythmEnergy, rhythmFlux, melodyEnergy, melodyFreq) =
ExtractFeatures(mag, prevRhythmMag);
ExtractFeatures(mag, prevRhythmMag, sampleRate);
if (rhythmFlux > maxFlux) maxFlux = rhythmFlux;
if (melodyEnergy > maxMelodyEnergy) maxMelodyEnergy = melodyEnergy;
@@ -79,57 +78,20 @@ public static class MusicAnalyzer
progress?.Report(100);
// Second pass: normalize and build WaveFrames
const int rhythmFreqMs = 150; // ~7Hz deep pulse
foreach (var tf in tickFeatures)
{
// Channel A: rhythm onset
double normalizedFlux = maxFlux > 0 ? tf.RhythmFlux / maxFlux : 0;
int onsetIntensity = (int)Math.Round(normalizedFlux * 100);
// Sub-tick attack/decay shape
var intA = new[]
{
(byte)Math.Clamp(onsetIntensity, 0, 100),
(byte)Math.Clamp(onsetIntensity * 6 / 10, 0, 100),
(byte)Math.Clamp(onsetIntensity * 3 / 10, 0, 100),
(byte)0
};
var freqA = Freq.Compress4(new[] { rhythmFreqMs, rhythmFreqMs, rhythmFreqMs, rhythmFreqMs });
channelA.Add(new WaveFrame(freqA, intA));
// Channel B: melody
double avgEnergy = tf.MelodyCount > 0 ? tf.MelodyEnergy / tf.MelodyCount : 0;
double normalizedEnergy = maxMelodyEnergy > 0 ? avgEnergy / maxMelodyEnergy : 0;
int melodyIntensity = (int)Math.Round(normalizedEnergy * 80); // cap at 80 for comfort
melodyIntensity = Math.Clamp(melodyIntensity, 0, 100);
// Weighted average dominant frequency
double weightedFreq = 0;
double totalWeight = 0;
foreach (var f in tf.MelodyFreqSamples)
{
weightedFreq += f * f; // weight by energy (freq already squared mag)
totalWeight += f;
}
double avgMelodyHz = totalWeight > 0 ? weightedFreq / totalWeight : 500;
int estimsMs = MapPitchToPeriod(avgMelodyHz);
var intB = new[]
{
(byte)melodyIntensity, (byte)melodyIntensity,
(byte)melodyIntensity, (byte)melodyIntensity
};
var freqB = Freq.Compress4(new[] { estimsMs, estimsMs, estimsMs, estimsMs });
channelB.Add(new WaveFrame(freqB, intB));
var (frameA, frameB) = BuildWaveFrame(tf, maxFlux, maxMelodyEnergy);
channelA.Add(frameA);
channelB.Add(frameB);
}
return new MusicPattern(channelA, channelB, duration);
}
static (double rhythmEnergy, double rhythmFlux, double melodyEnergy, double melodyFreq) ExtractFeatures(
double[] magnitude, double[]? prevRhythmMag)
public static (double rhythmEnergy, double rhythmFlux, double melodyEnergy, double melodyFreq)
ExtractFeatures(double[] magnitude, double[]? prevRhythmMag, int sampleRate)
{
double binWidth = (double)SampleRate / WindowSize;
double binWidth = (double)sampleRate / WindowSize;
int rhythmLoBin = (int)(RhythmLow / binWidth);
int rhythmHiBin = (int)(RhythmHigh / binWidth);
int melodyLoBin = (int)(MelodyLow / binWidth);
@@ -172,7 +134,7 @@ public static class MusicAnalyzer
return (rhythmEnergy, rhythmFlux, melodyEnergy, melodyFreq);
}
static int MapPitchToPeriod(double hz)
public static int MapPitchToPeriod(double hz)
{
// Map melody frequency (300-4000Hz) to e-stim period (10-1000ms)
// Logarithmic mapping: low notes → deep, high notes → buzzy
@@ -185,15 +147,15 @@ public static class MusicAnalyzer
return Math.Clamp(ms, 10, 1000);
}
static double[] DecodeToMono(string mp3Path)
static double[] DecodeToMono(string mp3Path, int sampleRate)
{
using var reader = new Mp3FileReader(mp3Path);
var format = new WaveFormat(SampleRate, 16, 1);
var format = new WaveFormat(sampleRate, 16, 1);
using var resampler = new MediaFoundationResampler(reader, format);
resampler.ResamplerQuality = 60;
var sampleList = new List<float>();
var buffer = new byte[SampleRate * 2]; // 1s worth of 16-bit mono
var buffer = new byte[sampleRate * 2]; // 1s worth of 16-bit mono
int read;
while ((read = resampler.Read(buffer, 0, buffer.Length)) > 0)
{
@@ -210,11 +172,51 @@ public static class MusicAnalyzer
return result;
}
class TickFeature
public class TickFeature
{
public double RhythmFlux;
public double MelodyEnergy;
public double MelodyCount;
public readonly List<double> MelodyFreqSamples = new();
}
public static (WaveFrame chA, WaveFrame chB) BuildWaveFrame(TickFeature tf, double maxFlux, double maxMelodyEnergy)
{
const int rhythmFreqMs = 150;
double normalizedFlux = maxFlux > 0 ? tf.RhythmFlux / maxFlux : 0;
int onsetIntensity = (int)Math.Round(normalizedFlux * 100);
var intA = new[]
{
(byte)Math.Clamp(onsetIntensity, 0, 100),
(byte)Math.Clamp(onsetIntensity * 6 / 10, 0, 100),
(byte)Math.Clamp(onsetIntensity * 3 / 10, 0, 100),
(byte)0
};
var freqA = Freq.Compress4(new[] { rhythmFreqMs, rhythmFreqMs, rhythmFreqMs, rhythmFreqMs });
double avgEnergy = tf.MelodyCount > 0 ? tf.MelodyEnergy / tf.MelodyCount : 0;
double normalizedEnergy = maxMelodyEnergy > 0 ? avgEnergy / maxMelodyEnergy : 0;
int melodyIntensity = (int)Math.Round(normalizedEnergy * 80);
melodyIntensity = Math.Clamp(melodyIntensity, 0, 100);
double weightedFreq = 0;
double totalWeight = 0;
foreach (var f in tf.MelodyFreqSamples)
{
weightedFreq += f * f;
totalWeight += f;
}
double avgMelodyHz = totalWeight > 0 ? weightedFreq / totalWeight : 500;
int estimsMs = MapPitchToPeriod(avgMelodyHz);
var intB = new[]
{
(byte)melodyIntensity, (byte)melodyIntensity,
(byte)melodyIntensity, (byte)melodyIntensity
};
var freqB = Freq.Compress4(new[] { estimsMs, estimsMs, estimsMs, estimsMs });
return (new WaveFrame(freqA, intA), new WaveFrame(freqB, intB));
}
}