using System; using System.Collections.Generic; using T3.Core.DataTypes.DataSet; using T3.Core.IO; using T3.Core.Settings; namespace T3.Core.Audio; /// /// Analyzes the external WASAPI system audio and attempts to "latch onto" the audio timing so that the BPM /// rate and editor timing follow the tempo of the soundtrack — even if the initial BPM rate was imprecise or is changing, as in live performances. /// /// The algorithm requires an initial tempo and downbeat timing from the user. This is provided by tapping a few times and then pressing Resync. /// It then uses expected onset timings (e.g., bass on the 1st and 1/4 beats, snares on the 2nd and 4th, etc.) to calculate a phase error /// and shift the beat timing and BPM to minimize it. /// /// I tested it with all kinds of musical styles (all in 4/4, of course): The usual suspects like dubstep, techno, electro (Plaster, THNTS), but also /// analog (Metallica, Led Zeppelin, Deep Purple), disco (Michael Jackson), and jazz. It appears to work well for electronic music, but heavy tempo shifts /// — like in jazz — are prone to slippage. /// /// The implementation is more complex than expected: /// /// To get maximum precision, we use every audio buffer update from WASAPI and immediately calculate frequency bins, possible onsets, and timing. /// This is normally out of sync with the display update rate of 60Hz. /// /// To avoid jittering, the resulting timing is then smoothed in BeatTiming. /// You can enable CoreSettings.EnableBeatSyncProfiling to get a better understanding of how different musical styles are matched. /// The algorithm uses too many magic numbers to list, but the most relevant ones are: /// - proportionalBpmAdjustment /// - phaseAdjustmentAmount /// /// These can be nicely tweaked via hot code reloading. Also, the definitions of FrequencyBands and RhythmicTemplates are useful targets for tweaking. /// public static class BeatSynchronizer { /// /// Gets the current estimated BPM. /// public static double CurrentBpm => _currentBpm; /// /// Gets the current progress within the bar, from 0.0 to 1.0. /// public static double BarProgress => _barTime; /// /// Initializes the beat synchronizer. Must be called once at application start. /// private static void Initialize() { if (_initialized) return; for (var i = 0; i < FrequencyBandCount; i++) { _recentTypeOnsetStrengths[i] = new Queue(); _totalTypeOnsetStrengths[i] = 0f; } _initialized = true; } /// /// Manually resynchronizes the beat tracker, setting the BPM and forcing the phase to the start of a bar. /// This should be called by the user (e.g., via a "Resync" button). /// /// The BPM to set the beat tracker to initially. public static void Resync(double initialBpm) { Initialize(); _currentBpm = Math.Clamp(initialBpm, MinBpm, MaxBpm); _barTime = (int)(_barTime / 4) * 4; // Reset to beginning of last measure _detectedOnsets.Clear(); for (var index = 0; index < _lastAnyOnsetDetectionTimes.Length; index++) { _lastAnyOnsetDetectionTimes[index] = 0; } var numberOfTrackedTypes = (int)FrequencyRangeType.Hihat + 1; for (var i = 0; i < numberOfTrackedTypes; i++) { _recentTypeOnsetStrengths[i].Clear(); _totalTypeOnsetStrengths[i] = 0f; } } /// /// Updates the beat timer, detects onsets, and adjusts BPM via PID controller. /// This is called by BeatTiming after user tapped and resynced... /// internal static void UpdateBeatTimer() { Initialize(); var currentTimeMs = WasapiAudioInput.LastUpdateTime * 1000; var deltaTimeMs = WasapiAudioInput.TimeSinceLastUpdate * 1000; // Advance time... _barTime += _currentBpm / 60.0 / 1000.0 / 4.0 * deltaTimeMs; var maxAge = (60000.0 / MinBpm * 2); _detectedOnsets.RemoveAll(o => currentTimeMs - o.TimeMs > maxAge); // Find onsets for (var index = 0; index < _bands.Length; index++) { var band = _bands[index]; var currentBassOnsetStrength = SumBandAttacks(band.StartBand, band.EndBand); if (!TryDetectAndQueueOnsetStrength(band, currentBassOnsetStrength, currentTimeMs, out var onset)) continue; if (CoreSettings.Config.EnableBeatSyncProfiling) { DebugDataRecording.KeepTraceData("BPM/OnSet/" + band.Type, onset.Amplitude); } _lastAnyOnsetDetectionTimes[index] = currentTimeMs; _detectedOnsets.Add(onset); } // Adjust bpmRate through phase offset var totalWeightedPhaseError = 0.0; var totalWeight = 0.0; if (_detectedOnsets.Count > 0) { for (var index = 0; index < _detectedOnsets.Count; index++) { var onset = _detectedOnsets[index]; var conceptualBarStartTimeMs = currentTimeMs - _barTime % 1 * BarDurationMs; var offsetFromNearestConceptualBarStartMs = (onset.TimeMs - conceptualBarStartTimeMs) % BarDurationMs; if (offsetFromNearestConceptualBarStartMs < 0) offsetFromNearestConceptualBarStartMs += BarDurationMs; var onsetNormalizedBarPosition = offsetFromNearestConceptualBarStartMs / BarDurationMs; foreach (var template in _onsetRhythmicTemplates[(int)onset.Type]) { var rawErrorToTemplateNormalized = onsetNormalizedBarPosition - template.NormalizedBarPosition; if (rawErrorToTemplateNormalized > 0.5) rawErrorToTemplateNormalized -= 1.0; if (rawErrorToTemplateNormalized < -0.5) rawErrorToTemplateNormalized += 1.0; var errorToTemplateMs = rawErrorToTemplateNormalized * BarDurationMs; if (!(Math.Abs(errorToTemplateMs) <= template.ToleranceMs * 2)) // HACK continue; totalWeightedPhaseError += rawErrorToTemplateNormalized * template.ImpactWeight; totalWeight += template.ImpactWeight * onset.Amplitude; } } } var currentPhaseErrorNormalized = 0.0; var hasRelevantOnsets = totalWeight > 0.0; if (hasRelevantOnsets) currentPhaseErrorNormalized = totalWeightedPhaseError / totalWeight; if (!hasRelevantOnsets) return; // This is basically the P in a PID controller and controls how fast the BPM should be adjusted // -0.1 *** loses track with Led Zeppelin // -0.3 **** faster but jittery // -0.7 ** too jumpy // -1.0 * erratic double proportionalBpmAdjustment = -0.4f; // High values result in a "pumping" effect. double phaseAdjustmentAmount = 0.01f; var bpmCorrection = (proportionalBpmAdjustment * currentPhaseErrorNormalized); if (CoreSettings.Config.EnableBeatSyncProfiling) { DebugDataRecording.KeepTraceData("BPM/barProgress", _barTime % 1); DebugDataRecording.KeepTraceData("BPM/current", _currentBpm); DebugDataRecording.KeepTraceData("BPM/phaseError", currentPhaseErrorNormalized); DebugDataRecording.KeepTraceData("BPM/bpmCorrection", bpmCorrection); } var phaseCorrection = currentPhaseErrorNormalized * phaseAdjustmentAmount; _currentBpm += bpmCorrection; _currentBpm = Math.Clamp(_currentBpm, MinBpm, MaxBpm); _barTime -= phaseCorrection; } /// /// Sums the attack values within a specified band range. /// private static float SumBandAttacks(int startBand, int endBand) { var sum = 0f; // Ensure bounds checking for robustness, although usually constant var actualEndBand = Math.Min(endBand, AudioConfig.FrequencyBandCount - 1); for (var i = startBand; i <= actualEndBand; i++) { sum += AudioAnalysis.FrequencyBandOnSets[i]; } return sum / (actualEndBand - startBand); } /// /// Updates the sliding window average for a given onset type and checks for an onset. /// /// A tuple indicating if an onset was detected for this type, and its current strength. private static bool TryDetectAndQueueOnsetStrength(FrequencyBand band, float currentStrength, double currentTimeMs, out Onset onset) { var typeIndex = (int)band.Type; // Cast enum to int for array indexing onset = new Onset(currentTimeMs, currentStrength, band.Type); // Update sliding window average _recentTypeOnsetStrengths[typeIndex].Enqueue(currentStrength); _totalTypeOnsetStrengths[typeIndex] += currentStrength; if (_recentTypeOnsetStrengths[typeIndex].Count > OnsetHistoryWindowSizeFrames) { _totalTypeOnsetStrengths[typeIndex] -= _recentTypeOnsetStrengths[typeIndex].Dequeue(); } var averageStrength = 0f; if (_recentTypeOnsetStrengths[typeIndex].Count > 0) { averageStrength = _totalTypeOnsetStrengths[typeIndex] / _recentTypeOnsetStrengths[typeIndex].Count; } // Check for onset specific to this type var hasEnoughPause = (currentTimeMs - _lastAnyOnsetDetectionTimes[typeIndex]) > MinOnsetIntervalMs; var hasEnoughStrength = currentStrength > averageStrength * band.OnSetThresholdFactor * 1.4f; return hasEnoughPause && hasEnoughStrength; } private static double BarDurationMs => (60000.0 / _currentBpm) * 4.0; private sealed record FrequencyBand(FrequencyRangeType Type, int StartBand, int EndBand, float OnSetThresholdFactor); private record struct RhythmicTemplate(float NormalizedBarPosition, float ImpactWeight, float ToleranceMs); private sealed record Onset(double TimeMs, float Amplitude, FrequencyRangeType Type); private enum FrequencyRangeType { Bass = 0, Snare = 1, Hihat = 2, Undefined = 3 // Keep Undefined last, as it's not for direct indexing } // Timing private static double _currentBpm = 120.0; private static double _barTime; private static readonly List _detectedOnsets = []; private static readonly Queue[] _recentTypeOnsetStrengths = new Queue[FrequencyBandCount]; private static readonly float[] _totalTypeOnsetStrengths = new float[FrequencyBandCount]; private static readonly double[] _lastAnyOnsetDetectionTimes = new double[FrequencyBandCount]; private const int FrequencyBandCount = (int)FrequencyRangeType.Undefined; private const int OnsetHistoryWindowSizeFrames = 200; private static bool _initialized; // Configuration Constants & PID Gains private const double MinBpm = 50.0; private const double MaxBpm = 190.0; private const double MinOnsetIntervalMs = 50.0; private static readonly FrequencyBand[] _bands = [ new(FrequencyRangeType.Bass, 0, 7, 3.5f), new(FrequencyRangeType.Snare, 4, 26, 3.0f), new(FrequencyRangeType.Hihat, 20, 31, 3.0f), ]; // Rhythmic Template Definition (now an array) private static readonly List[] _onsetRhythmicTemplates = [ // Bass [ new RhythmicTemplate { NormalizedBarPosition = 0.00f, ImpactWeight = 1.3f, ToleranceMs = 50 }, new RhythmicTemplate { NormalizedBarPosition = 0.25f, ImpactWeight = 0.5f, ToleranceMs = 50 }, new RhythmicTemplate { NormalizedBarPosition = 0.50f, ImpactWeight = 0.8f, ToleranceMs = 50 }, new RhythmicTemplate { NormalizedBarPosition = 0.75f, ImpactWeight = 0.5f, ToleranceMs = 50 }, // new RhythmicTemplate { NormalizedBarPosition = 0.125f, ImpactWeight = 0.3f, ToleranceMs = 30 }, // new RhythmicTemplate { NormalizedBarPosition = 0.375f, ImpactWeight = 0.3f, ToleranceMs = 30 }, // new RhythmicTemplate { NormalizedBarPosition = 0.625f, ImpactWeight = 0.3f, ToleranceMs = 30 }, // new RhythmicTemplate { NormalizedBarPosition = 0.875f, ImpactWeight = 0.3f, ToleranceMs = 30 } ], // Snare [ new RhythmicTemplate { NormalizedBarPosition = 0.25f, ImpactWeight = 1.0f, ToleranceMs = 50 }, new RhythmicTemplate { NormalizedBarPosition = 0.75f, ImpactWeight = 1.0f, ToleranceMs = 50 }, new RhythmicTemplate { NormalizedBarPosition = 0.00f, ImpactWeight = 0.3f, ToleranceMs = 30 }, new RhythmicTemplate { NormalizedBarPosition = 0.50f, ImpactWeight = 0.3f, ToleranceMs = 30 } ], // Hihat [ new RhythmicTemplate { NormalizedBarPosition = 0.000f, ImpactWeight = 1.0f, ToleranceMs = 60 }, new RhythmicTemplate { NormalizedBarPosition = 0.125f, ImpactWeight = 0.7f, ToleranceMs = 60 }, new RhythmicTemplate { NormalizedBarPosition = 0.250f, ImpactWeight = 0.5f, ToleranceMs = 60 }, new RhythmicTemplate { NormalizedBarPosition = 0.375f, ImpactWeight = 0.7f, ToleranceMs = 60 }, new RhythmicTemplate { NormalizedBarPosition = 0.500f, ImpactWeight = 0.5f, ToleranceMs = 60 }, new RhythmicTemplate { NormalizedBarPosition = 0.625f, ImpactWeight = 0.7f, ToleranceMs = 60 }, new RhythmicTemplate { NormalizedBarPosition = 0.750f, ImpactWeight = 0.5f, ToleranceMs = 60 }, new RhythmicTemplate { NormalizedBarPosition = 0.875f, ImpactWeight = 0.7f, ToleranceMs = 60 } ] ]; }