private void openToolStripMenuItem_Click(object sender, EventArgs e)
{
var ofd = new OpenFileDialog();
if (ofd.ShowDialog() != DialogResult.OK)
{
return;
}
using (var stream = new FileStream(ofd.FileName, FileMode.Open))
{
var waveFile = new WaveFile(stream);
_signal = waveFile[Channels.Left];
}
_stft = new Stft(_frameSize, _hopSize);
var frameDuration = (double)_frameSize / _signal.SamplingRate;
var hopDuration = (double)_hopSize / _signal.SamplingRate;
var freqs = new[] { 300f, 600, 1000, 2000, 4000, 7000 };
var pitchExtractor = new PitchExtractor(_signal.SamplingRate, frameDuration, hopDuration, high: 900 /*Hz*/);
var pitchTrack = pitchExtractor.ParallelComputeFrom(_signal)
.Select(p => p.Features[0])
.ToArray();
var tdExtractor = new TimeDomainFeaturesExtractor(_signal.SamplingRate, "all", frameDuration, hopDuration);
var spectralExtractor = new SpectralFeaturesExtractor(_signal.SamplingRate, "sc+sn", frameDuration, hopDuration, frequencies: freqs);
var mpeg7Extractor = new Mpeg7SpectralFeaturesExtractor(_signal.SamplingRate, "all", frameDuration, hopDuration);
mpeg7Extractor.IncludeHarmonicFeatures("all");
mpeg7Extractor.SetPitchTrack(pitchTrack);
tdExtractor.AddFeature("pitch_zcr", (signal, start, end) => { return(Pitch.FromZeroCrossingsSchmitt(signal, start, end)); });
//spectralExtractor.AddFeature("pitch_hss", (spectrum, fs) => { return Pitch.FromHss(spectrum, _signal.SamplingRate); } );
var tdVectors = tdExtractor.ParallelComputeFrom(_signal);
var spectralVectors = spectralExtractor.ParallelComputeFrom(_signal);
var mpeg7Vectors = mpeg7Extractor.ParallelComputeFrom(_signal);
_vectors = FeaturePostProcessing.Join(tdVectors, spectralVectors, mpeg7Vectors);
//FeaturePostProcessing.NormalizeMean(_vectors);
//FeaturePostProcessing.AddDeltas(_vectors);
var descriptions = tdExtractor.FeatureDescriptions
.Concat(spectralExtractor.FeatureDescriptions)
.Concat(mpeg7Extractor.FeatureDescriptions);
FillFeaturesList(_vectors, descriptions);
spectrogramPlot.ColorMapName = "afmhot";
spectrogramPlot.MarklineThickness = 2;
spectrogramPlot.Spectrogram = _stft.Spectrogram(_signal);
}
private void PlotLine(DiscreteSignal signal)
{
m_renderer.positionCount = signal.Samples.Length;
for (int i = 0; i < m_renderer.positionCount; i++)
{
m_renderer.SetPosition(i, new Vector3((float)i / m_renderer.positionCount, signal[i], 0));
}
}
/// <summary>
/// Estimates pitch from <paramref name="signal"/> using YIN algorithm.
/// </summary>
/// <param name="signal">Signal</param>
/// <param name="startPos">Index of the first sample in signal for processing</param>
/// <param name="endPos">Index of the last sample in signal for processing</param>>
/// <param name="low">Lower frequency of expected pitch range</param>
/// <param name="high">Upper frequency of expected pitch range</param>
/// <param name="cmdfThreshold">CMDF threshold</param>
public static float FromYin(DiscreteSignal signal,
int startPos = 0,
int endPos = -1,
float low = 80 /*Hz*/,
float high = 400 /*Hz*/,
float cmdfThreshold = 0.2f)
{
return(FromYin(signal.Samples, signal.SamplingRate, startPos, endPos, low, high, cmdfThreshold));
}
/// <summary>
/// Evaluate harmonic and percussive mag-phase spectrograms from given signal.
/// Both spectrogram objects share the same phase array.
/// </summary>
/// <param name="signal"></param>
/// <returns></returns>
public (MagnitudePhaseList, MagnitudePhaseList) EvaluateSpectrograms(DiscreteSignal signal)
{
// spectrogram memory will be reused for harmonic magnitudes
var harmonicSpectrogram = _stft.MagnitudePhaseSpectrogram(signal);
var harmonicMagnitudes = harmonicSpectrogram.Magnitudes;
// median filtering along frequency axis:
var percussiveMagnitudes = new List <float[]>(harmonicMagnitudes.Count);
for (var i = 0; i < harmonicMagnitudes.Count; i++)
{
var mag = new DiscreteSignal(1, harmonicMagnitudes[i]);
percussiveMagnitudes.Add(_medianPercussive.ApplyTo(mag).Samples);
_medianPercussive.Reset();
}
// median filtering along time axis:
for (var j = 0; j <= _stft.Size / 2; j++)
{
int i = 0, k = 0;
for (; k < _medianHarmonic.Size / 2; k++) // feed first Size/2 samples
{
_medianHarmonic.Process(harmonicMagnitudes[k][j]);
}
for (; i < harmonicMagnitudes.Count - _medianHarmonic.Size / 2; i++, k++)
{
var h = _medianHarmonic.Process(harmonicMagnitudes[k][j]);
harmonicMagnitudes[i][j] *= _mask(h, percussiveMagnitudes[k][j]);
percussiveMagnitudes[i][j] *= _mask(percussiveMagnitudes[k][j], h);
}
for (k = 0; k < _medianHarmonic.Size / 2; i++, k++) // don't forget last samples
{
var h = _medianHarmonic.Process(0);
harmonicMagnitudes[i][j] *= _mask(h, percussiveMagnitudes[i][j]);
percussiveMagnitudes[i][j] *= _mask(percussiveMagnitudes[i][j], h);
}
_medianHarmonic.Reset();
}
var percussiveSpectrogram = new MagnitudePhaseList
{
Magnitudes = percussiveMagnitudes,
Phases = harmonicSpectrogram.Phases
};
return(harmonicSpectrogram, percussiveSpectrogram);
}
private static void AssertFilterOutput(DiscreteSignal output)
{
Assert.Multiple(() =>
{
Assert.That(output[0], Is.EqualTo(1.0).Within(1e-7));
Assert.That(output[1], Is.EqualTo(1.0).Within(1e-7));
Assert.That(output[2], Is.EqualTo(0.4).Within(1e-7));
Assert.That(output[3], Is.EqualTo(0.04).Within(1e-7));
});
}
/// <summary>
/// Does ring modulation (RM) and returns RM signal.
/// </summary>
/// <param name="carrier">Carrier signal</param>
/// <param name="modulator">Modulator signal</param>
public static DiscreteSignal Ring(DiscreteSignal carrier, DiscreteSignal modulator)
{
if (carrier.SamplingRate != modulator.SamplingRate)
{
throw new ArgumentException("Sampling rates must be the same!");
}
return(new DiscreteSignal(carrier.SamplingRate,
carrier.Samples.Zip(modulator.Samples, (c, m) => c * m)));
}
public override DiscreteSignal ApplyTo(DiscreteSignal signal,
FilteringMethod method = FilteringMethod.Auto)
{
var input = signal.Samples;
var output = new float[input.Length];
var posSynthesis = 0;
for (var posAnalysis = 0; posAnalysis + _fftSize < input.Length; posAnalysis += _hopSize)
{
input.FastCopyTo(_re, _fftSize, posAnalysis);
_zeroblock.FastCopyTo(_im, _fftSize);
_re.ApplyWindow(_window);
_fft.Direct(_re, _im);
for (var j = 0; j < _fftSize / 2 + 1; j++)
{
var mag = Math.Sqrt(_re[j] * _re[j] + _im[j] * _im[j]);
var phase = Math.Atan2(_im[j], _re[j]);
_re[j] = (float)mag;
_im[j] = 0;
}
for (var j = _fftSize / 2 + 1; j < _fftSize; j++)
{
_re[j] = _im[j] = 0.0f;
}
_fft.Inverse(_re, _im);
for (var j = 0; j < _re.Length; j++)
{
output[posSynthesis + j] += _re[j] * _window[j];
}
for (var j = 0; j < _hopSize; j++)
{
output[posSynthesis + j] *= _gain;
output[j] = Wet * output[j] + Dry * input[j];
}
posSynthesis += _hopSize;
}
for (; posSynthesis < output.Length; posSynthesis++)
{
output[posSynthesis] *= _gain;
output[posSynthesis] = Wet * output[posSynthesis] + Dry * input[posSynthesis];
}
return(new DiscreteSignal(signal.SamplingRate, output));
}
public static List <Complex> WaveletTransformation(DiscreteSignal signal)
{
List <Complex> result = new List <Complex>();
var convolution = new Convolution();
var discreteSignalH = new DiscreteSignal();
var valuesH = new List <Value>();
for (int i = 0; i < H.Count; i++)
{
valuesH.Add(new Value {
Y = H[i]
});
}
discreteSignalH.Values = valuesH;
var discreteSignalG = new DiscreteSignal();
var valuesG = new List <Value>();
for (int i = 0; i < G.Count; i++)
{
valuesG.Add(new Value {
Y = G[i]
});
}
discreteSignalG.Values = valuesG;
List <Value> hSamples = convolution.Process(signal, discreteSignalH).Values;
List <Value> gSamples = convolution.Process(signal, discreteSignalG).Values;
List <Complex> hHalf = new List <Complex>();
List <Complex> gHalf = new List <Complex>();
for (int i = 0; i < hSamples.Count; i++)
{
if (i % 2 == 0)
{
hHalf.Add(hSamples[i].Y);
}
else
{
gHalf.Add(gSamples[i].Y);
}
}
for (int i = 0; i < gHalf.Count; i++)
{
result.Add(new Complex(hHalf[i].Real, gHalf[i].Imaginary));
}
return(result);
}
private void overlapSaveToolStripMenuItem_Click(object sender, EventArgs e)
{
if (_signal == null)
{
return;
}
_filteredSignal = _filter.ApplyTo(_signal, FilteringMethod.OverlapSave);
signalAfterFilteringPanel.Signal = _filteredSignal;
spectrogramAfterFilteringPanel.Spectrogram = _stft.Spectrogram(_filteredSignal);
}
/// <summary>
/// Generates signal by generating all its samples one-by-one.
/// </summary>
protected virtual DiscreteSignal Generate()
{
var signal = new DiscreteSignal(SamplingRate, Length);
for (var i = 0; i < signal.Length; i++)
{
signal[i] = NextSample();
}
return(signal);
}
/// <summary>
/// Applies effect to entire <paramref name="signal"/> and returns tuple of output signals [left signal, right signal].
/// </summary>
/// <param name="signal">Input signal</param>
public virtual (DiscreteSignal, DiscreteSignal) ApplyTo(DiscreteSignal signal)
{
var sr = signal.SamplingRate;
var left = new float[signal.Length];
var right = new float[signal.Length];
Process(signal.Samples, left, right);
return(new DiscreteSignal(sr, left), new DiscreteSignal(sr, right));
}
private void differenceEquationToolStripMenuItem_Click(object sender, EventArgs e)
{
if (_signal == null)
{
return;
}
_filteredSignal = _filter.ApplyTo(_signal, FilteringMethod.DifferenceEquation);
signalAfterFilteringPanel.Signal = _filteredSignal;
spectrogramAfterFilteringPanel.Spectrogram = _stft.Spectrogram(_filteredSignal);
}
/// <summary>
/// Does simple frequency demodulation pf <paramref name="signal"/> based on Hilbert transform.
/// </summary>
public static DiscreteSignal DemodulateFrequency(DiscreteSignal signal)
{
var diff = new float[signal.Length];
MathUtils.Diff(signal.Samples, diff);
var ht = new HilbertTransform(signal.Length);
var mag = ht.AnalyticSignal(diff).Magnitude;
return(new DiscreteSignal(signal.SamplingRate, mag.ToFloats()) - 1.0f);
}
public FirFiltersVersion2Vs5VsZi()
{
_signal = new WhiteNoiseBuilder().OfLength(N).Build();
_filterV5Kernel5 = new FirFilter(DesignFilter.FirWinLp(5, 0.1));
_filterV2Kernel5 = new FirFilterV2(DesignFilter.FirWinLp(5, 0.1));
_filterZiKernel5 = new ZiFilter(DesignFilter.FirWinLp(5, 0.1), new[] { 1.0 });
_filterV5Kernel35 = new FirFilter(DesignFilter.FirWinLp(35, 0.1));
_filterV2Kernel35 = new FirFilterV2(DesignFilter.FirWinLp(35, 0.1));
_filterZiKernel35 = new ZiFilter(DesignFilter.FirWinLp(35, 0.1), new[] { 1.0 });
}
public void LoadAudio(string path)
{
if (path.EndsWith(".wav"))
{
using (var stream = new FileStream(path, FileMode.Open))
{
var waveFile = new WaveFile(stream);
SourceSignal = waveFile[Channels.Left];;
}
}
}
/// <summary>
/// Phase Vocoder algorithm
/// </summary>
/// <param name="signal"></param>
/// <param name="method"></param>
/// <returns></returns>
public DiscreteSignal ApplyTo(DiscreteSignal signal,
FilteringMethod method = FilteringMethod.Auto)
{
var input = signal.Samples;
var output = new float[(int)(input.Length * _stretch) + _fftSize];
var posSynthesis = 0;
for (var posAnalysis = 0; posAnalysis + _fftSize < input.Length; posAnalysis += _hopAnalysis)
{
input.FastCopyTo(_re, _fftSize, posAnalysis);
_zeroblock.FastCopyTo(_im, _fftSize);
_re.ApplyWindow(_window);
_fft.Direct(_re, _im);
for (var j = 0; j < _fftSize / 2 + 1; j++)
{
var mag = Math.Sqrt(_re[j] * _re[j] + _im[j] * _im[j]);
var phase = 2 * Math.PI * _rand.NextDouble();
_re[j] = (float)(mag * Math.Cos(phase));
_im[j] = (float)(mag * Math.Sin(phase));
}
for (var j = _fftSize / 2 + 1; j < _fftSize; j++)
{
_re[j] = _im[j] = 0.0f;
}
_fft.Inverse(_re, _im);
for (var j = 0; j < _re.Length; j++)
{
output[posSynthesis + j] += _re[j] * _window[j];
}
for (var j = 0; j < _hopSynthesis; j++)
{
output[posSynthesis + j] *= _gain;
}
posSynthesis += _hopSynthesis;
}
for (; posSynthesis < output.Length; posSynthesis++)
{
output[posSynthesis] *= _gain;
}
return(new DiscreteSignal(signal.SamplingRate, output));
}
/// <summary>
/// Applies effect to entire signals (in left and right channels)
/// and returns tuple of output signals [left signal, right signal].
/// </summary>
/// <param name="leftSignal">Input signal (left channel)</param>
/// <param name="rightSignal">Input signal (right channel)</param>
public virtual (DiscreteSignal, DiscreteSignal) ApplyTo(DiscreteSignal leftSignal, DiscreteSignal rightSignal)
{
var srl = leftSignal.SamplingRate;
var srr = rightSignal.SamplingRate;
var left = new float[leftSignal.Length];
var right = new float[rightSignal.Length];
Process(leftSignal.Samples, rightSignal.Samples, left, right);
return(new DiscreteSignal(srl, left), new DiscreteSignal(srr, right));
}
/// <summary>
/// Subtracts the "signal" amplitudes from the "mainSignal" amplitudes
/// </summary>
/// <param name="mainSignal"></param>
/// <param name="signal"></param>
public static void CombineSubtract(this DiscreteSignal mainSignal, DiscreteSignal signal)
{
if (mainSignal.Samples.Length != signal.Samples.Length)
{
throw new ArgumentException();
}
for (int i = 0; i < mainSignal.Samples.Length; i++)
{
mainSignal.Samples[i] = mainSignal.Samples[i] - signal.Samples[i];
}
}
/// <summary>
/// Applies an ADSR envelope to mainSignal
/// </summary>
/// <param name="mainSignal"></param>
/// <param name="adsrSignal"></param>
public static void ApplyAdsr(this DiscreteSignal mainSignal, DiscreteSignal adsrSignal)
{
if (mainSignal.Samples.Length != adsrSignal.Samples.Length)
{
throw new ArgumentException();
}
for (int i = 0; i < mainSignal.Samples.Length; i++)
{
mainSignal[i] = mainSignal.Samples[i] * adsrSignal.Samples[i];
}
}
/// <summary>
/// Filters entire <paramref name="signal"/> by processing each signal sample in a loop.
/// </summary>
/// <param name="filter">Online filter</param>
/// <param name="signal">Input signal</param>
public static DiscreteSignal FilterOnline(this IOnlineFilter filter, DiscreteSignal signal)
{
var output = new float[signal.Length];
var samples = signal.Samples;
for (var i = 0; i < samples.Length; i++)
{
output[i] = filter.Process(samples[i]);
}
return(new DiscreteSignal(signal.SamplingRate, output));
}
public void TestSameSamplingRate()
{
var signal = new DiscreteSignal(16000, new [] { 1.0f, -2, 3, 1, 4, -2, 1, -5, 3 });
var resampled = Operation.Resample(signal, 16000);
Assert.Multiple(() =>
{
Assert.That(resampled.Samples, Is.EqualTo(resampled.Samples).Within(1e-10));
Assert.That(resampled, Is.Not.SameAs(signal));
});
}
public static void WordCount(string filePath)
{
Console.WriteLine($"Word detection on file {filePath}");
DiscreteSignal signal = ManualWordCount.LoadAudioFile(filePath);
DiscreteSignal signalPreprocessed = ManualWordCount.PreprocessAudio(signal);
int nWords = ManualWordCount.WordCount(signalPreprocessed);
Console.WriteLine($"{nWords} words detected in speech");
}
/// <summary>
/// Averages two signals together.
/// </summary>
/// <param name="mainSignal"></param>
/// <param name="signal"></param>
public static void CombineAverage(this DiscreteSignal mainSignal, DiscreteSignal signal)
{
if (mainSignal.Samples.Length != signal.Samples.Length)
{
throw new ArgumentException();
}
for (int i = 0; i < mainSignal.Samples.Length; i++)
{
mainSignal.Samples[i] = (mainSignal.Samples[i] + signal.Samples[i]) / 2f;
}
}
/// <summary>
/// Parallel computation (joins chunks of feature vector lists into one list)
/// </summary>
/// <param name="signal"></param>
/// <returns></returns>
public virtual List <FeatureVector> ParallelComputeFrom(DiscreteSignal signal)
{
var chunks = ParallelChunksComputeFrom(signal);
var featureVectors = new List <FeatureVector>();
foreach (var vectors in chunks)
{
featureVectors.AddRange(vectors);
}
return(featureVectors);
}
/// <summary>
/// Compute the sequence of feature vectors from some fragment of a signal
/// </summary>
/// <param name="signal">Signal</param>
/// <param name="startSample">The number (position) of the first sample for processing</param>
/// <param name="endSample">The number (position) of last sample for processing</param>
/// <returns>Sequence of feature vectors</returns>
public override List <FeatureVector> ComputeFrom(DiscreteSignal signal, int startSample, int endSample)
{
var frameSize = (int)(signal.SamplingRate * FrameSize);
var hopSize = (int)(signal.SamplingRate * HopSize);
var fftSize = _fftSize >= frameSize ? _fftSize : MathUtils.NextPowerOfTwo(frameSize);
var resolution = (float)signal.SamplingRate / fftSize;
var frequencies = Enumerable.Range(0, fftSize + 1)
.Select(f => f * resolution)
.ToArray();
var featureVectors = new List <FeatureVector>();
var featureCount = FeatureCount;
var fft = new Fft(fftSize);
// reserve memory for reusable blocks
var spectrum = new float[fftSize / 2 + 1]; // buffer for magnitude spectrum
var block = new float[fftSize]; // buffer for currently processed block
var zeroblock = new float[fftSize]; // just a buffer of zeros for quick memset
var i = startSample;
while (i + frameSize < endSample)
{
// prepare all blocks in memory for the current step:
zeroblock.FastCopyTo(block, fftSize);
signal.Samples.FastCopyTo(block, frameSize, i);
fft.MagnitudeSpectrum(block, spectrum);
var featureVector = new float[featureCount];
for (var j = 0; j < featureCount; j++)
{
featureVector[j] = _extractors[j](spectrum, frequencies);
}
featureVectors.Add(new FeatureVector
{
Features = featureVector,
TimePosition = (double)i / signal.SamplingRate
});
i += hopSize;
}
return(featureVectors);
}
/// <summary>
/// Here is the main function for online processing of chunks:
/// </summary>
private void ProcessNewChunk(object sender, EventArgs e)
{
// =========================== take next chunk of random size ================================
// (random size is chosen carefully)
var randomSize = Math.Min(_randomizer.Next(_blockConvolver.HopSize / 4, _blockConvolver.HopSize * 4),
Math.Min(_signal.Length - _offset,
_filteredLength - _filteredOffset));
_input = _signal[_offset, _offset + randomSize].Samples;
// ===================================== process it ==========================================
_blockConvolver.Process(_input, _output);
// ===================== do what we want with a new portion of data ==========================
_output.FastCopyTo(_filtered.Samples, _input.Length, 0, _filteredOffset);
_offset += _input.Length;
_filteredOffset += _input.Length;
// ================================= visualize signals =======================================
signalPlot.Signal = new DiscreteSignal(_signal.SamplingRate, _input);
filteredSignalPlot.Signal = new DiscreteSignal(_signal.SamplingRate, _output);
if (_filteredOffset >= _filtered.Length - _input.Length)
{
_filteredOffset = 0;
_filtered = new DiscreteSignal(_signal.SamplingRate, Math.Min(_signal.Length + _fftSize, 60 * 16000));
}
filteredFullSignalPlot.Signal = _filtered;
// ====================== reset if we've reached the end of a signal =========================
if (_offset + randomSize >= _signal.Length)
{
_offset = 0;
_filteredOffset = 0;
_chunkNo = 0;
_blockConvolver.Reset();
_filtered = new DiscreteSignal(_signal.SamplingRate, _filteredLength);
}
_chunkNo++;
labelInfo.Text = $"Chunk #{_chunkNo + 1} / Processed {(float)_offset/_signal.SamplingRate} seconds";
}
private void decimateToolStripMenuItem_Click(object sender, EventArgs e)
{
if (_signal == null)
{
return;
}
var factor = int.Parse(resampleTextBox.Text);
_filteredSignal = Operation.Decimate(_signal, factor);
signalAfterFilteringPanel.Signal = _filteredSignal;
spectrogramAfterFilteringPanel.Spectrogram = _stft.Spectrogram(_filteredSignal);
}
public void TestFilterCombinations()
{
var pre = new PreEmphasisFilter();
var de = new DeEmphasisFilter();
var filter = pre * de;
var samples = new[] { 1.0f, 0.1f, -0.4f, 0.2f };
var signal = new DiscreteSignal(1, samples);
var filtered = filter.ApplyTo(signal);
Assert.That(filtered.Samples, Is.EqualTo(signal.Samples).Within(1e-7));
}
/// <summary>
/// Does amplitude modulation (AM) and returns AM signal.
/// </summary>
/// <param name="carrier">Carrier signal</param>
/// <param name="modulatorFrequency">Modulator frequency</param>
/// <param name="modulationIndex">Modulation index (depth)</param>
public static DiscreteSignal Amplitude(DiscreteSignal carrier,
float modulatorFrequency = 20 /*Hz*/,
float modulationIndex = 0.5f)
{
var fs = carrier.SamplingRate;
var mf = modulatorFrequency; // just short aliases //
var mi = modulationIndex;
var output = Enumerable.Range(0, carrier.Length)
.Select(i => carrier[i] * (1 + mi * Math.Cos(2 * Math.PI * mf / fs * i)));
return(new DiscreteSignal(fs, output.ToFloats()));
}
/// <summary>
/// Pitch estimation from zero crossing rate (overloaded for DiscreteSignal)
/// </summary>
/// <param name="signal"></param>
/// <param name="startPos"></param>
/// <param name="endPos"></param>
/// <returns></returns>
public static float FromZeroCrossingsSchmitt(DiscreteSignal signal,
int startPos = 0,
int endPos = -1,
float lowSchmittThreshold = -1e10f,
float highSchmittThreshold = 1e10f)
{
return(FromZeroCrossingsSchmitt(signal.Samples,
signal.SamplingRate,
startPos,
endPos,
lowSchmittThreshold,
highSchmittThreshold));
}
