public void Generate()
{
IAudioStream audioStream = inputTrack.File ?
AudioStreamFactory.FromFileInfoIeee32(inputTrack.FileInfo) :
inputTrack.Stream;
audioStream = new MonoStream(audioStream);
audioStream = new ResamplingStream(audioStream, ResamplingQuality.Medium, profile.SampleRate);
STFT stft = new STFT(audioStream, profile.FrameSize, profile.FrameStep, WindowType.Hann, STFT.OutputFormat.Decibel, this.bufferSize);
index = 0;
indices = stft.WindowCount;
frameBuffer = new float[profile.FrameSize / 2];
List <SubFingerprint> subFingerprints = new List <SubFingerprint>();
while (stft.HasNext())
{
// Get FFT spectrum
stft.ReadFrame(frameBuffer);
// Sum FFT bins into target frequency bands
profile.MapFrequencies(frameBuffer, bands);
CalculateSubFingerprint(bandsPrev, bands, subFingerprints);
CommonUtil.Swap <float[]>(ref bands, ref bandsPrev);
index++;
// Output subfingerprints every once in a while
if (index % this.eventInterval == 0 && SubFingerprintsGenerated != null)
{
SubFingerprintsGenerated(this, new SubFingerprintsGeneratedEventArgs(inputTrack, subFingerprints, index, indices));
subFingerprints.Clear();
}
}
// Output remaining subfingerprints
if (SubFingerprintsGenerated != null)
{
SubFingerprintsGenerated(this, new SubFingerprintsGeneratedEventArgs(inputTrack, subFingerprints, index, indices));
}
if (Completed != null)
{
Completed(this, EventArgs.Empty);
}
audioStream.Close();
}
public void Generate(AudioTrack track)
{
IAudioStream audioStream = new ResamplingStream(
new MonoStream(AudioStreamFactory.FromFileInfoIeee32(track.FileInfo)),
ResamplingQuality.Medium, profile.SamplingRate);
STFT stft = new STFT(audioStream, profile.WindowSize, profile.HopSize, WindowType.Hann, STFT.OutputFormat.Decibel);
int index = 0;
int indices = stft.WindowCount;
int processedFrames = 0;
float[] spectrum = new float[profile.WindowSize / 2];
float[] smoothedSpectrum = new float[spectrum.Length - profile.SpectrumSmoothingLength + 1]; // the smooved frequency spectrum of the current frame
var spectrumSmoother = new SimpleMovingAverage(profile.SpectrumSmoothingLength);
float[] spectrumTemporalAverage = new float[spectrum.Length]; // a running average of each spectrum bin over time
float[] spectrumResidual = new float[spectrum.Length]; // the difference between the current spectrum and the moving average spectrum
var peakHistory = new PeakHistory(1 + profile.TargetZoneDistance + profile.TargetZoneLength, spectrum.Length / 2);
var peakPairs = new List <PeakPair>(profile.PeaksPerFrame * profile.PeakFanout); // keep a single instance of the list to avoid instantiation overhead
var subFingerprints = new List <SubFingerprint>();
while (stft.HasNext())
{
// Get the FFT spectrum
stft.ReadFrame(spectrum);
// Skip frames whose average spectrum volume is below the threshold
// This skips silent frames (zero samples) that only contain very low noise from the FFT
// and that would screw up the temporal spectrum average below for the following frames.
if (spectrum.Average() < spectrumMinThreshold)
{
index++;
continue;
}
// Smooth the frequency spectrum to remove small peaks
if (profile.SpectrumSmoothingLength > 0)
{
spectrumSmoother.Clear();
for (int i = 0; i < spectrum.Length; i++)
{
var avg = spectrumSmoother.Add(spectrum[i]);
if (i >= profile.SpectrumSmoothingLength)
{
smoothedSpectrum[i - profile.SpectrumSmoothingLength] = avg;
}
}
}
// Update the temporal moving bin average
if (processedFrames == 0)
{
// Init averages on first frame
for (int i = 0; i < spectrum.Length; i++)
{
spectrumTemporalAverage[i] = spectrum[i];
}
}
else
{
// Update averages on all subsequent frames
for (int i = 0; i < spectrum.Length; i++)
{
spectrumTemporalAverage[i] = ExponentialMovingAverage.UpdateMovingAverage(
spectrumTemporalAverage[i], profile.SpectrumTemporalSmoothingCoefficient, spectrum[i]);
}
}
// Calculate the residual
// The residual is the difference of the current spectrum to the temporal average spectrum. The higher
// a bin residual is, the steeper the increase in energy in that peak.
for (int i = 0; i < spectrum.Length; i++)
{
spectrumResidual[i] = spectrum[i] - spectrumTemporalAverage[i] - 90f;
}
// Find local peaks in the residual
// The advantage of finding peaks in the residual instead of the spectrum is that spectrum energy is usually
// concentrated in the low frequencies, resulting in a clustering of the highest peaks in the lows. Getting
// peaks from the residual distributes the peaks more evenly across the spectrum.
var peaks = peakHistory.List; // take oldest list,
peaks.Clear(); // clear it, and
FindLocalMaxima(spectrumResidual, peaks); // refill with new peaks
// Pick the largest n peaks
int numMaxima = Math.Min(peaks.Count, profile.PeaksPerFrame);
if (numMaxima > 0)
{
peaks.Sort((p1, p2) => p1.Value == p2.Value ? 0 : p1.Value < p2.Value ? 1 : -1); // order peaks by height
if (peaks.Count > numMaxima)
{
peaks.RemoveRange(numMaxima, peaks.Count - numMaxima); // select the n tallest peaks by deleting the rest
}
peaks.Sort((p1, p2) => p1.Index == p2.Index ? 0 : p1.Index < p2.Index ? -1 : 1); // sort peaks by index (not really necessary)
}
peakHistory.Add(index, peaks);
if (FrameProcessed != null)
{
// Mark peaks as 0dB for spectrogram display purposes
foreach (var peak in peaks)
{
spectrum[peak.Index] = 0;
spectrumResidual[peak.Index] = 0;
}
FrameProcessed(this, new FrameProcessedEventArgs {
AudioTrack = track, Index = index, Indices = indices,
Spectrum = spectrum, SpectrumResidual = spectrumResidual
});
}
processedFrames++;
index++;
if (processedFrames >= peakHistory.Length)
{
peakPairs.Clear();
FindPairsWithMaxEnergy(peakHistory, peakPairs);
ConvertPairsToSubFingerprints(peakPairs, subFingerprints);
}
if (subFingerprints.Count > 512)
{
FireFingerprintHashesGenerated(track, indices, subFingerprints);
subFingerprints.Clear();
}
}
// Flush the remaining peaks of the last frames from the history to get all remaining pairs
for (int i = 0; i < profile.TargetZoneLength; i++)
{
var peaks = peakHistory.List;
peaks.Clear();
peakHistory.Add(-1, peaks);
peakPairs.Clear();
FindPairsWithMaxEnergy(peakHistory, peakPairs);
ConvertPairsToSubFingerprints(peakPairs, subFingerprints);
}
FireFingerprintHashesGenerated(track, indices, subFingerprints);
audioStream.Close();
}
/// <summary>
/// This method generates hash codes from an audio stream in a streaming fashion,
/// which means that it only maintains a small constant-size state and can process
/// streams of arbitrary length.
///
/// Here is a scheme of the data processing flow. After the subband splitting
/// stage, every subband is processed independently.
///
/// +-----------------+ +--------------+ +-----------+
/// audio stream +---> mono conversion +---> downsampling +---> whitening |
/// +-----------------+ +--------------+ +---------+-+
/// |
/// +-------------------+ +------------------+ |
/// | subband splitting <---+ subband analysis <---+
/// +--+---+---+---+----+ +------------------+
/// | | | |
/// | v v v
/// | ... ... ...
/// |
/// | +------------------+ +-----------------+
/// +---> RMS downsampling +---> onset detection |
/// +------------------+ +----------+------+
/// |
/// +-----------------+ |
/// hash codes <-------------------+ hash generation <----------+
/// +-----------------+
///
/// The hash codes from the hash generators of each band are then sent though a
/// sorter which brings them into sequential temporal order before they are stored
/// in the final list.
/// </summary>
/// <param name="track"></param>
public void Generate(AudioTrack track)
{
IAudioStream audioStream = new ResamplingStream(
new MonoStream(AudioStreamFactory.FromFileInfoIeee32(track.FileInfo)),
ResamplingQuality.Medium, profile.SamplingRate);
var whiteningStream = new WhiteningStream(audioStream,
profile.WhiteningNumPoles, profile.WhiteningDecaySecs, profile.WhiteningBlockLength);
var subbandAnalyzer = new SubbandAnalyzer(whiteningStream);
float[] analyzedFrame = new float[profile.SubBands];
var bandAnalyzers = new BandAnalyzer[profile.SubBands];
for (int i = 0; i < profile.SubBands; i++)
{
bandAnalyzers[i] = new BandAnalyzer(profile, i);
}
List <SubFingerprint> hashes = new List <SubFingerprint>();
HashTimeSorter hashSorter = new HashTimeSorter(profile.SubBands);
var sw = new Stopwatch();
sw.Start();
int totalFrames = subbandAnalyzer.WindowCount;
int currentFrame = 0;
while (subbandAnalyzer.HasNext())
{
subbandAnalyzer.ReadFrame(analyzedFrame);
for (int i = 0; i < profile.SubBands; i++)
{
bandAnalyzers[i].ProcessSample(analyzedFrame[i], hashSorter.Queues[i]);
}
if (currentFrame % 4096 == 0)
{
hashSorter.Fill(hashes, false);
if (SubFingerprintsGenerated != null)
{
SubFingerprintsGenerated(this, new SubFingerprintsGeneratedEventArgs(track, hashes, currentFrame, totalFrames));
hashes.Clear();
}
}
currentFrame++;
}
for (int i = 0; i < bandAnalyzers.Length; i++)
{
bandAnalyzers[i].Flush(hashSorter.Queues[i]);
}
hashSorter.Fill(hashes, true);
if (SubFingerprintsGenerated != null)
{
SubFingerprintsGenerated(this, new SubFingerprintsGeneratedEventArgs(track, hashes, currentFrame, totalFrames));
hashes.Clear();
}
sw.Stop();
audioStream.Close();
Console.WriteLine("time: " + sw.Elapsed);
}
public float Run()
{
IAudioStream audioStream = new ResamplingStream(
new MonoStream(AudioStreamFactory.FromFileInfoIeee32(audioTrack.FileInfo)),
ResamplingQuality.Medium, 11000);
ContinuousFrequencyActivationQuantifier cfaq = new ContinuousFrequencyActivationQuantifier(audioStream);
float[] cfaValue = new float[1];
float[] cfaValues = new float[cfaq.WindowCount];
Label[] cfaLabels = new Label[cfaq.WindowCount];
int count = 0;
int musicCount = 0;
while (cfaq.HasNext())
{
cfaq.ReadFrame(cfaValue);
cfaValues[count] = cfaValue[0];
if (cfaValue[0] > threshold)
{
musicCount++;
cfaLabels[count] = Label.MUSIC;
}
Console.WriteLine("cfa {0,3}% {3} {1,5:0.00} {2}", (int)(Math.Round((float)count++ / cfaq.WindowCount * 100)), cfaValue[0], cfaValue[0] > threshold ? "MUSIC" : "", TimeUtil.BytesToTimeSpan(audioStream.Position, audioStream.Properties));
}
audioStream.Close();
if (smoothing)
{
// 3.3 Smoothing
/* majority filtering with sliding window ~5 secs
* 1 frame = ~2,4 secs, at least 3 frames are needed for majority filtering -> 3 * ~2,4 secs = ~7,2 secs */
// filter out single NO_MUSIC frames
for (int i = 2; i < cfaLabels.Length; i++)
{
if (cfaLabels[i - 2] == Label.MUSIC && cfaLabels[i - 1] == Label.NO_MUSIC && cfaLabels[i] == Label.MUSIC)
{
cfaLabels[i - 1] = Label.MUSIC;
}
}
// filter out single MUSIC frames
for (int i = 2; i < cfaLabels.Length; i++)
{
if (cfaLabels[i - 2] == Label.NO_MUSIC && cfaLabels[i - 1] == Label.MUSIC && cfaLabels[i] == Label.NO_MUSIC)
{
cfaLabels[i - 1] = Label.NO_MUSIC;
}
}
// swap ~5 secs NO_MUSIC segments to MUSIC
for (int i = 3; i < cfaLabels.Length; i++)
{
if (cfaLabels[i - 3] == Label.MUSIC && cfaLabels[i - 2] == Label.NO_MUSIC && cfaLabels[i - 1] == Label.NO_MUSIC && cfaLabels[i] == Label.MUSIC)
{
cfaLabels[i - 1] = Label.MUSIC;
cfaLabels[i - 2] = Label.MUSIC;
}
}
// swap ~5 secs NMUSIC segments to NO_MUSIC
for (int i = 3; i < cfaLabels.Length; i++)
{
if (cfaLabels[i - 3] == Label.NO_MUSIC && cfaLabels[i - 2] == Label.MUSIC && cfaLabels[i - 1] == Label.MUSIC && cfaLabels[i] == Label.NO_MUSIC)
{
cfaLabels[i - 1] = Label.NO_MUSIC;
cfaLabels[i - 2] = Label.NO_MUSIC;
}
}
}
float musicRatio = (float)musicCount / count;
float musicRatioSmoothed = -1f;
Console.WriteLine("'" + audioTrack.FileInfo.FullName + "' contains " + ((int)(Math.Round(musicRatio * 100))) + "% music");
if (smoothing)
{
musicCount = cfaLabels.Count <Label>(l => l == Label.MUSIC);
musicRatioSmoothed = (float)musicCount / count;
Console.WriteLine("smoothed: " + ((int)(Math.Round(musicRatioSmoothed * 100))) + "% music");
}
if (writeLog)
{
FileInfo logFile = new FileInfo(audioTrack.FileInfo.FullName + ".music");
StreamWriter writer = logFile.CreateText();
writer.WriteLine(musicRatio + "; " + musicRatioSmoothed);
writer.WriteLine(threshold);
for (int i = 0; i < cfaValues.Length; i++)
{
writer.WriteLine("{0:0.00000}; {1}; \t{2}", cfaValues[i], cfaValues[i] > threshold ? Label.MUSIC : Label.NO_MUSIC, cfaLabels[i]);
}
writer.Flush();
writer.Close();
}
return(0);
}
public void Generate(AudioTrack track)
{
IAudioStream audioStream = new ResamplingStream(
new MonoStream(AudioStreamFactory.FromFileInfoIeee32(track.FileInfo)),
ResamplingQuality.Medium, profile.SamplingRate);
var chroma = new Chroma(audioStream, profile.WindowSize, profile.HopSize, profile.WindowType,
profile.ChromaMinFrequency, profile.ChromaMaxFrequency, false, profile.ChromaMappingMode);
float[] chromaFrame;
var chromaBuffer = new RingBuffer <float[]>(profile.ChromaFilterCoefficients.Length);
var chromaFilterCoefficients = profile.ChromaFilterCoefficients;
var filteredChromaFrame = new double[Chroma.Bins];
var classifiers = profile.Classifiers;
var maxFilterWidth = classifiers.Max(c => c.Filter.Width);
var integralImage = new IntegralImage(maxFilterWidth, Chroma.Bins);
int index = 0;
int indices = chroma.WindowCount;
var subFingerprints = new List <SubFingerprint>();
while (chroma.HasNext())
{
// Get chroma frame buffer
// When the chroma buffer is full, we can take and reuse the oldest array
chromaFrame = chromaBuffer.Count == chromaBuffer.Length ? chromaBuffer[0] : new float[Chroma.Bins];
// Read chroma frame into buffer
chroma.ReadFrame(chromaFrame);
// ChromaFilter
chromaBuffer.Add(chromaFrame);
if (chromaBuffer.Count < chromaBuffer.Length)
{
// Wait for the buffer to fill completely for the filtering to start
continue;
}
Array.Clear(filteredChromaFrame, 0, filteredChromaFrame.Length);
for (int i = 0; i < chromaFilterCoefficients.Length; i++)
{
var frame = chromaBuffer[i];
for (int j = 0; j < frame.Length; j++)
{
filteredChromaFrame[j] += frame[j] * chromaFilterCoefficients[i];
}
}
// ChromaNormalizer
double euclideanNorm = 0;
for (int i = 0; i < filteredChromaFrame.Length; i++)
{
var value = filteredChromaFrame[i];
euclideanNorm += value * value;
}
euclideanNorm = Math.Sqrt(euclideanNorm);
if (euclideanNorm < profile.ChromaNormalizationThreshold)
{
Array.Clear(filteredChromaFrame, 0, filteredChromaFrame.Length);
}
else
{
for (int i = 0; i < filteredChromaFrame.Length; i++)
{
filteredChromaFrame[i] /= euclideanNorm;
}
}
// ImageBuilder
// ... just add one feature vector after another as rows to the image
integralImage.AddColumn(filteredChromaFrame);
// FingerprintCalculator
if (integralImage.Columns < maxFilterWidth)
{
// Wait for the image to fill completely before hashes can be generated
continue;
}
// Calculate subfingerprint hash
uint hash = 0;
for (int i = 0; i < classifiers.Length; i++)
{
hash = (hash << 2) | grayCodeMapping[classifiers[i].Classify(integralImage, 0)];
}
// We have a SubFingerprint@frameTime
subFingerprints.Add(new SubFingerprint(index, new SubFingerprintHash(hash), false));
index++;
if (index % 512 == 0 && SubFingerprintsGenerated != null)
{
SubFingerprintsGenerated(this, new SubFingerprintsGeneratedEventArgs(track, subFingerprints, index, indices));
subFingerprints.Clear();
}
}
if (SubFingerprintsGenerated != null)
{
SubFingerprintsGenerated(this, new SubFingerprintsGeneratedEventArgs(track, subFingerprints, index, indices));
}
if (Completed != null)
{
Completed(this, EventArgs.Empty);
}
audioStream.Close();
}