diff --git a/LiveCapture.cs b/LiveCapture.cs new file mode 100644 index 0000000..6fee0c6 --- /dev/null +++ b/LiveCapture.cs @@ -0,0 +1,151 @@ +using System.Numerics; +using FftSharp; +using NAudio.CoreAudioApi; +using NAudio.Wave; + +namespace Substation; + +public class LiveCapture : IDisposable +{ + readonly State _state; + readonly MMDevice _device; + WasapiLoopbackCapture? _capture; + Thread? _processThread; + volatile bool _running; + + readonly Queue _sampleBuffer = new(); + readonly object _bufferLock = new(); + + double[]? _prevRhythmMag; + double _liveMaxFlux; + double _liveMaxMelodyEnergy; + int _sampleRate; + int _fftsPerTick; + int _fftIndexInTick; + + public event Action? Stopped; + + public LiveCapture(State state, MMDevice device) + { + _state = state; + _device = device; + } + + public void Start() + { + _running = true; + _capture = new WasapiLoopbackCapture(_device); + _sampleRate = _capture.WaveFormat.SampleRate; + _fftsPerTick = Math.Max(1, (int)Math.Round(MusicAnalyzer.TickDuration * _sampleRate / MusicAnalyzer.HopSize)); + + _capture.DataAvailable += OnDataAvailable; + _capture.RecordingStopped += OnRecordingStopped; + + _processThread = new Thread(ProcessLoop) { IsBackground = true, Name = "LiveCapture-FFT" }; + _processThread.Start(); + + _capture.StartRecording(); + Console.WriteLine($"[live] capture started: {_device.FriendlyName} ({_sampleRate}Hz, {_capture.WaveFormat.Channels}ch)"); + } + + public void Stop() + { + _running = false; + try { _capture?.StopRecording(); } catch { } + } + + void OnDataAvailable(object? sender, WaveInEventArgs e) + { + int channels = _capture!.WaveFormat.Channels; + int bytesPerSample = _capture.WaveFormat.BitsPerSample / 8; + int frameSize = channels * bytesPerSample; + int sampleCount = e.BytesRecorded / frameSize; + + lock (_bufferLock) + { + for (int i = 0; i < sampleCount; i++) + { + int offset = i * frameSize; + float left = BitConverter.ToSingle(e.Buffer, offset); + float right = channels >= 2 + ? BitConverter.ToSingle(e.Buffer, offset + bytesPerSample) + : left; + _sampleBuffer.Enqueue((left + right) * 0.5); + } + + // Cap buffer size to prevent memory growth if processing falls behind + while (_sampleBuffer.Count > _sampleRate * 2) + _sampleBuffer.Dequeue(); + } + } + + void OnRecordingStopped(object? sender, StoppedEventArgs e) + { + _running = false; + Stopped?.Invoke(); + Console.WriteLine("[live] capture stopped"); + } + + void ProcessLoop() + { + var window = new FftSharp.Windows.Hanning(); + var buffer = new double[MusicAnalyzer.WindowSize]; + var tf = new MusicAnalyzer.TickFeature(); + + while (_running) + { + double[]? windowData = null; + + lock (_bufferLock) + { + if (_sampleBuffer.Count >= MusicAnalyzer.WindowSize) + { + for (int i = 0; i < MusicAnalyzer.WindowSize; i++) + buffer[i] = _sampleBuffer.Dequeue(); + windowData = buffer; + } + } + + if (windowData == null) + { + Thread.Sleep(5); + continue; + } + + window.ApplyInPlace(windowData); + var spectrum = FFT.Forward(windowData); + var mag = FFT.Magnitude(spectrum); + + var (rhythmEnergy, rhythmFlux, melodyEnergy, melodyFreq) = + MusicAnalyzer.ExtractFeatures(mag, _prevRhythmMag, _sampleRate); + + _prevRhythmMag = mag; + + // Adaptive normalization: running max with slow decay + _liveMaxFlux = Math.Max(rhythmFlux, _liveMaxFlux * 0.999); + _liveMaxMelodyEnergy = Math.Max(melodyEnergy, _liveMaxMelodyEnergy * 0.999); + + tf.RhythmFlux = Math.Max(tf.RhythmFlux, rhythmFlux); + tf.MelodyEnergy += melodyEnergy; + tf.MelodyFreqSamples.Add(melodyFreq); + tf.MelodyCount++; + _fftIndexInTick++; + + if (_fftIndexInTick >= _fftsPerTick) + { + var (frameA, frameB) = MusicAnalyzer.BuildWaveFrame(tf, _liveMaxFlux, _liveMaxMelodyEnergy); + _state.EnqueueStream('A', new[] { frameA }); + _state.EnqueueStream('B', new[] { frameB }); + + tf = new MusicAnalyzer.TickFeature(); + _fftIndexInTick = 0; + } + } + } + + public void Dispose() + { + _running = false; + try { _capture?.Dispose(); } catch { } + } +} diff --git a/MainForm.cs b/MainForm.cs index 955ece8..0ec4b4b 100644 --- a/MainForm.cs +++ b/MainForm.cs @@ -1,6 +1,7 @@ using System.ComponentModel; using System.Drawing.Drawing2D; using System.Diagnostics; +using NAudio.CoreAudioApi; using NAudio.Wave; namespace Substation; @@ -48,10 +49,15 @@ public class MainForm : Form string _trayState = ""; readonly Button _btnMusic; + readonly Button _btnLive; readonly Label _lblTrack; + readonly Label _lblAudioDev; + readonly ComboBox _cboAudioDev; bool _isMusicRunning; + bool _isLiveRunning; WaveOutEvent? _audioOut; Stopwatch? _musicStopwatch; + LiveCapture? _liveCapture; public MainForm(CoyoteDevice device, State state, Server server) { @@ -61,7 +67,7 @@ public class MainForm : Form _loopCts = new CancellationTokenSource(); Text = $"Substation {typeof(MainForm).Assembly.GetName().Version}"; - ClientSize = new Size(420, 425); + ClientSize = new Size(420, 450); FormBorderStyle = FormBorderStyle.FixedSingle; MaximizeBox = false; StartPosition = FormStartPosition.CenterScreen; @@ -117,30 +123,52 @@ public class MainForm : Form { Text = "Test", Location = new Point(16, 96), - Size = new Size(100, 32) + Size = new Size(80, 32) }; _btnTest.Click += OnTest; _btnMusic = new Button { Text = "Music", - Location = new Point(130, 96), - Size = new Size(100, 32) + Location = new Point(104, 96), + Size = new Size(80, 32) }; _btnMusic.Click += OnMusic; + _btnLive = new Button + { + Text = "Live", + Location = new Point(192, 96), + Size = new Size(80, 32) + }; + _btnLive.Click += OnLive; + _btnStop = new Button { Text = "Stop All", - Location = new Point(244, 96), - Size = new Size(100, 32) + Location = new Point(280, 96), + Size = new Size(80, 32) }; _btnStop.Click += OnStop; + _lblAudioDev = new Label + { + Text = "Audio:", + Location = new Point(16, 134), + Size = new Size(40, 20) + }; + _cboAudioDev = new ComboBox + { + DropDownStyle = ComboBoxStyle.DropDownList, + Location = new Point(60, 132), + Size = new Size(344, 22) + }; + PopulateAudioDevices(); + _lblTrack = new Label { Text = "", - Location = new Point(16, 134), + Location = new Point(16, 158), Size = new Size(388, 16), ForeColor = Color.DimGray }; @@ -148,37 +176,37 @@ public class MainForm : Form _lblSendA = new Label { Text = "Send A", - Location = new Point(16, 160), + Location = new Point(16, 184), Size = new Size(48, 20), Font = new Font(Font.FontFamily, 9, FontStyle.Bold) }; _heatA = new HeatMap(60) { - Location = new Point(72, 158), + Location = new Point(72, 182), Size = new Size(332, 22) }; _lblSendB = new Label { Text = "Send B", - Location = new Point(16, 188), + Location = new Point(16, 212), Size = new Size(48, 20), Font = new Font(Font.FontFamily, 9, FontStyle.Bold) }; _heatB = new HeatMap(60) { - Location = new Point(72, 186), + Location = new Point(72, 210), Size = new Size(332, 22) }; _lblRecvA = new Label { Text = "Recv A", - Location = new Point(16, 216), + Location = new Point(16, 240), Size = new Size(48, 20), Font = new Font(Font.FontFamily, 9, FontStyle.Bold) }; _gaugeRecvA = new ProgressBar { - Location = new Point(72, 216), + Location = new Point(72, 240), Size = new Size(332, 20), Minimum = 0, Maximum = 200, @@ -187,13 +215,13 @@ public class MainForm : Form _lblRecvB = new Label { Text = "Recv B", - Location = new Point(16, 242), + Location = new Point(16, 266), Size = new Size(48, 20), Font = new Font(Font.FontFamily, 9, FontStyle.Bold) }; _gaugeRecvB = new ProgressBar { - Location = new Point(72, 242), + Location = new Point(72, 266), Size = new Size(332, 20), Minimum = 0, Maximum = 200, @@ -203,13 +231,13 @@ public class MainForm : Form _lblLimitA = new Label { Text = "Lim A", - Location = new Point(16, 274), + Location = new Point(16, 298), Size = new Size(40, 20), Font = new Font(Font.FontFamily, 9, FontStyle.Bold) }; _limitBarA = new TrackBar { - Location = new Point(60, 270), + Location = new Point(60, 294), Size = new Size(220, 45), Minimum = 0, Maximum = 200, @@ -221,14 +249,14 @@ public class MainForm : Form _lblLimitValA = new Label { Text = "30", - Location = new Point(286, 274), + Location = new Point(286, 298), Size = new Size(32, 20), TextAlign = ContentAlignment.MiddleRight }; _chkScaleA = new CheckBox { Text = "Scale", - Location = new Point(320, 274), + Location = new Point(320, 298), Size = new Size(76, 24), Checked = false }; @@ -237,13 +265,13 @@ public class MainForm : Form _lblLimitB = new Label { Text = "Lim B", - Location = new Point(16, 322), + Location = new Point(16, 346), Size = new Size(40, 20), Font = new Font(Font.FontFamily, 9, FontStyle.Bold) }; _limitBarB = new TrackBar { - Location = new Point(60, 318), + Location = new Point(60, 342), Size = new Size(220, 45), Minimum = 0, Maximum = 200, @@ -255,21 +283,21 @@ public class MainForm : Form _lblLimitValB = new Label { Text = "30", - Location = new Point(286, 322), + Location = new Point(286, 346), Size = new Size(32, 20), TextAlign = ContentAlignment.MiddleRight }; _chkScaleB = new CheckBox { Text = "Scale", - Location = new Point(320, 322), + Location = new Point(320, 346), Size = new Size(76, 24), Checked = false }; _chkScaleB.CheckedChanged += OnLimitChanged; OnLimitChanged(null, EventArgs.Empty); - Controls.AddRange(new Control[] { _lblBle, _lblWs, _lblDeviceName, _txtDeviceName, _btnConnect, _chkSwap, _btnTest, _btnMusic, _btnStop, _lblTrack, _lblSendA, _heatA, _lblSendB, _heatB, _lblRecvA, _gaugeRecvA, _lblRecvB, _gaugeRecvB, _lblLimitA, _limitBarA, _lblLimitValA, _chkScaleA, _lblLimitB, _limitBarB, _lblLimitValB, _chkScaleB }); + Controls.AddRange(new Control[] { _lblBle, _lblWs, _lblDeviceName, _txtDeviceName, _btnConnect, _chkSwap, _btnTest, _btnMusic, _btnLive, _btnStop, _lblAudioDev, _cboAudioDev, _lblTrack, _lblSendA, _heatA, _lblSendB, _heatB, _lblRecvA, _gaugeRecvA, _lblRecvB, _gaugeRecvB, _lblLimitA, _limitBarA, _lblLimitValA, _chkScaleA, _lblLimitB, _limitBarB, _lblLimitValB, _chkScaleB }); // Tray icon — three cached variants: neutral=gray, active=gold, hot=red-orange _iconNeutral = CreateVoltageIcon(Color.Gray); @@ -437,8 +465,9 @@ public class MainForm : Form void RefreshButtonStates() { - _btnTest.Enabled = !_server.HasClient && !IsTestRunning; - _btnMusic.Enabled = !_server.HasClient && !_isMusicRunning; + _btnTest.Enabled = !_server.HasClient && !IsTestRunning && !_isMusicRunning && !_isLiveRunning; + _btnMusic.Enabled = !_server.HasClient && !_isMusicRunning && !IsTestRunning && !_isLiveRunning; + _btnLive.Enabled = !_server.HasClient && !_isLiveRunning && !IsTestRunning && !_isMusicRunning; _btnStop.Enabled = true; } @@ -451,6 +480,10 @@ public class MainForm : Form var elapsed = _musicStopwatch.Elapsed; _lblTrack.Text = $"Playing {elapsed:mm\\:ss} / {_musicTotalTime:mm\\:ss} {Path.GetFileName(_musicFilePath)}"; } + else if (_isLiveRunning) + { + _lblTrack.Text = "Live capture"; + } UpdateTrayIcon(); } @@ -516,6 +549,7 @@ public class MainForm : Form void OnStop(object? sender, EventArgs e) { StopMusic(); + StopLive(); _state.SetStrength('A', 0); _state.SetStrength('B', 0); _state.Stop('A'); @@ -639,6 +673,65 @@ public class MainForm : Form } } + void PopulateAudioDevices() + { + using var enumerator = new MMDeviceEnumerator(); + var defaultDev = enumerator.GetDefaultAudioEndpoint(DataFlow.Render, Role.Console); + var devices = enumerator.EnumerateAudioEndPoints(DataFlow.Render, DeviceState.Active); + + _cboAudioDev.Items.Clear(); + int defaultIdx = 0; + for (int i = 0; i < devices.Count; i++) + { + _cboAudioDev.Items.Add(devices[i]); + if (devices[i].ID == defaultDev.ID) + defaultIdx = i; + } + _cboAudioDev.SelectedIndex = defaultIdx; + } + + void OnLive(object? sender, EventArgs e) + { + if (_isLiveRunning || _server.HasClient) return; + if (_cboAudioDev.SelectedItem is not MMDevice dev) return; + + _isLiveRunning = true; + _btnLive.Enabled = false; + _lblTrack.Text = "Starting live capture..."; + + _state.SetStrength('A', 60); + _state.SetStrength('B', 60); + _state.Stop('A'); + _state.Stop('B'); + + _liveCapture = new LiveCapture(_state, dev); + _liveCapture.Stopped += () => BeginInvoke(StopLive); + _liveCapture.Start(); + + _lblTrack.Text = "Live capture"; + RefreshButtonStates(); + } + + void StopLive() + { + if (_liveCapture != null) + { + try { _liveCapture.Stop(); } catch { } + try { _liveCapture.Dispose(); } catch { } + _liveCapture = null; + } + + if (_isLiveRunning) + { + _isLiveRunning = false; + if (!IsDisposed) + { + _lblTrack.Text = ""; + RefreshButtonStates(); + } + } + } + void UpdateTrayIcon() { var newState = _state.IsSignaling ? "hot" diff --git a/MusicAnalyzer.cs b/MusicAnalyzer.cs index be855f9..e90eb7c 100644 --- a/MusicAnalyzer.cs +++ b/MusicAnalyzer.cs @@ -8,27 +8,26 @@ public record MusicPattern(List ChannelA, List ChannelB, T public static class MusicAnalyzer { - const int SampleRate = 44100; - const int WindowSize = 2048; - const int HopSize = 1024; - const double TickDuration = 0.1; // 100ms per e-stim frame + public const int WindowSize = 2048; + public const int HopSize = 1024; + public const double TickDuration = 0.1; - // Frequency band boundaries (Hz) const double RhythmLow = 20, RhythmHigh = 250; const double MelodyLow = 300, MelodyHigh = 4000; public static MusicPattern Analyze(string mp3Path, IProgress? progress = null) { - var samples = DecodeToMono(mp3Path); - var duration = TimeSpan.FromSeconds((double)samples.Length / SampleRate); + const int sampleRate = 44100; + var samples = DecodeToMono(mp3Path, sampleRate); + var duration = TimeSpan.FromSeconds((double)samples.Length / sampleRate); - var tickCount = (int)Math.Ceiling((double)samples.Length / SampleRate / TickDuration); + var tickCount = (int)Math.Ceiling((double)samples.Length / sampleRate / TickDuration); var channelA = new List(tickCount); var channelB = new List(tickCount); var window = new FftSharp.Windows.Hanning(); var buffer = new double[WindowSize]; - int fftsPerTick = (int)Math.Round(TickDuration * SampleRate / HopSize); + int fftsPerTick = (int)Math.Round(TickDuration * sampleRate / HopSize); double[]? prevRhythmMag = null; double maxFlux = 0; @@ -49,7 +48,7 @@ public static class MusicAnalyzer var mag = FFT.Magnitude(spectrum); var (rhythmEnergy, rhythmFlux, melodyEnergy, melodyFreq) = - ExtractFeatures(mag, prevRhythmMag); + ExtractFeatures(mag, prevRhythmMag, sampleRate); if (rhythmFlux > maxFlux) maxFlux = rhythmFlux; if (melodyEnergy > maxMelodyEnergy) maxMelodyEnergy = melodyEnergy; @@ -79,57 +78,20 @@ public static class MusicAnalyzer progress?.Report(100); // Second pass: normalize and build WaveFrames - const int rhythmFreqMs = 150; // ~7Hz deep pulse - foreach (var tf in tickFeatures) { - // Channel A: rhythm onset - double normalizedFlux = maxFlux > 0 ? tf.RhythmFlux / maxFlux : 0; - int onsetIntensity = (int)Math.Round(normalizedFlux * 100); - // Sub-tick attack/decay shape - var intA = new[] - { - (byte)Math.Clamp(onsetIntensity, 0, 100), - (byte)Math.Clamp(onsetIntensity * 6 / 10, 0, 100), - (byte)Math.Clamp(onsetIntensity * 3 / 10, 0, 100), - (byte)0 - }; - var freqA = Freq.Compress4(new[] { rhythmFreqMs, rhythmFreqMs, rhythmFreqMs, rhythmFreqMs }); - channelA.Add(new WaveFrame(freqA, intA)); - - // Channel B: melody - double avgEnergy = tf.MelodyCount > 0 ? tf.MelodyEnergy / tf.MelodyCount : 0; - double normalizedEnergy = maxMelodyEnergy > 0 ? avgEnergy / maxMelodyEnergy : 0; - int melodyIntensity = (int)Math.Round(normalizedEnergy * 80); // cap at 80 for comfort - melodyIntensity = Math.Clamp(melodyIntensity, 0, 100); - - // Weighted average dominant frequency - double weightedFreq = 0; - double totalWeight = 0; - foreach (var f in tf.MelodyFreqSamples) - { - weightedFreq += f * f; // weight by energy (freq already squared mag) - totalWeight += f; - } - double avgMelodyHz = totalWeight > 0 ? weightedFreq / totalWeight : 500; - int estimsMs = MapPitchToPeriod(avgMelodyHz); - - var intB = new[] - { - (byte)melodyIntensity, (byte)melodyIntensity, - (byte)melodyIntensity, (byte)melodyIntensity - }; - var freqB = Freq.Compress4(new[] { estimsMs, estimsMs, estimsMs, estimsMs }); - channelB.Add(new WaveFrame(freqB, intB)); + var (frameA, frameB) = BuildWaveFrame(tf, maxFlux, maxMelodyEnergy); + channelA.Add(frameA); + channelB.Add(frameB); } return new MusicPattern(channelA, channelB, duration); } - static (double rhythmEnergy, double rhythmFlux, double melodyEnergy, double melodyFreq) ExtractFeatures( - double[] magnitude, double[]? prevRhythmMag) + public static (double rhythmEnergy, double rhythmFlux, double melodyEnergy, double melodyFreq) + ExtractFeatures(double[] magnitude, double[]? prevRhythmMag, int sampleRate) { - double binWidth = (double)SampleRate / WindowSize; + double binWidth = (double)sampleRate / WindowSize; int rhythmLoBin = (int)(RhythmLow / binWidth); int rhythmHiBin = (int)(RhythmHigh / binWidth); int melodyLoBin = (int)(MelodyLow / binWidth); @@ -172,7 +134,7 @@ public static class MusicAnalyzer return (rhythmEnergy, rhythmFlux, melodyEnergy, melodyFreq); } - static int MapPitchToPeriod(double hz) + public static int MapPitchToPeriod(double hz) { // Map melody frequency (300-4000Hz) to e-stim period (10-1000ms) // Logarithmic mapping: low notes → deep, high notes → buzzy @@ -185,15 +147,15 @@ public static class MusicAnalyzer return Math.Clamp(ms, 10, 1000); } - static double[] DecodeToMono(string mp3Path) + static double[] DecodeToMono(string mp3Path, int sampleRate) { using var reader = new Mp3FileReader(mp3Path); - var format = new WaveFormat(SampleRate, 16, 1); + var format = new WaveFormat(sampleRate, 16, 1); using var resampler = new MediaFoundationResampler(reader, format); resampler.ResamplerQuality = 60; var sampleList = new List(); - var buffer = new byte[SampleRate * 2]; // 1s worth of 16-bit mono + var buffer = new byte[sampleRate * 2]; // 1s worth of 16-bit mono int read; while ((read = resampler.Read(buffer, 0, buffer.Length)) > 0) { @@ -210,11 +172,51 @@ public static class MusicAnalyzer return result; } - class TickFeature + public class TickFeature { public double RhythmFlux; public double MelodyEnergy; public double MelodyCount; public readonly List MelodyFreqSamples = new(); } + + public static (WaveFrame chA, WaveFrame chB) BuildWaveFrame(TickFeature tf, double maxFlux, double maxMelodyEnergy) + { + const int rhythmFreqMs = 150; + + double normalizedFlux = maxFlux > 0 ? tf.RhythmFlux / maxFlux : 0; + int onsetIntensity = (int)Math.Round(normalizedFlux * 100); + var intA = new[] + { + (byte)Math.Clamp(onsetIntensity, 0, 100), + (byte)Math.Clamp(onsetIntensity * 6 / 10, 0, 100), + (byte)Math.Clamp(onsetIntensity * 3 / 10, 0, 100), + (byte)0 + }; + var freqA = Freq.Compress4(new[] { rhythmFreqMs, rhythmFreqMs, rhythmFreqMs, rhythmFreqMs }); + + double avgEnergy = tf.MelodyCount > 0 ? tf.MelodyEnergy / tf.MelodyCount : 0; + double normalizedEnergy = maxMelodyEnergy > 0 ? avgEnergy / maxMelodyEnergy : 0; + int melodyIntensity = (int)Math.Round(normalizedEnergy * 80); + melodyIntensity = Math.Clamp(melodyIntensity, 0, 100); + + double weightedFreq = 0; + double totalWeight = 0; + foreach (var f in tf.MelodyFreqSamples) + { + weightedFreq += f * f; + totalWeight += f; + } + double avgMelodyHz = totalWeight > 0 ? weightedFreq / totalWeight : 500; + int estimsMs = MapPitchToPeriod(avgMelodyHz); + + var intB = new[] + { + (byte)melodyIntensity, (byte)melodyIntensity, + (byte)melodyIntensity, (byte)melodyIntensity + }; + var freqB = Freq.Compress4(new[] { estimsMs, estimsMs, estimsMs, estimsMs }); + + return (new WaveFrame(freqA, intA), new WaveFrame(freqB, intB)); + } }