public void SubFingerprintsAreBuiltCorrectlyFromAudioSamplesForTrack()
{
const int TenSeconds = 10;
var samples = new AudioSamples {
Samples = TestUtilities.GenerateRandomFloatArray(SampleRate * TenSeconds)
};
const int ThreeFingerprints = 3;
var rawFingerprints = GetGenericFingerprints(ThreeFingerprints);
fingerprintService.Setup(service => service.CreateFingerprints(samples, It.IsAny <DefaultFingerprintConfiguration>())).Returns(rawFingerprints);
lshAlgorithm.Setup(service => service.Hash(It.IsAny <Fingerprint>(), NumberOfHashTables, NumberOfHashKeysPerTable, It.IsAny <IEnumerable <string> >())).Returns(new HashedFingerprint(GenericSignature(), GenericHashBuckets(), 0, 0, Enumerable.Empty <string>()));
var hashDatas = fingerprintCommandBuilder.BuildFingerprintCommand()
.From(samples)
.UsingServices(audioService.Object)
.Hash()
.Result;
Assert.AreEqual(ThreeFingerprints, hashDatas.Count);
byte[] genericSignature = GenericSignature();
long[] genericHashBuckets = GenericHashBuckets();
foreach (var hashData in hashDatas)
{
CollectionAssert.AreEqual(genericSignature, hashData.SubFingerprint);
CollectionAssert.AreEqual(genericHashBuckets, hashData.HashBins);
}
}
public void ShouldGenerateLogSpectrumWithBiggerOverlap()
{
int Fs = 5512;
int seconds = 10;
float[] samples = new float[Fs * seconds];
for (int i = 0; i < samples.Length; ++i)
{
float value = (float)(1.3 * System.Math.Sin(2 * System.Math.PI * 15 * i / Fs));
samples[i] = value;
}
var audio = new AudioSamples(samples, "test", Fs);
var config = new DefaultSpectrogramConfig
{
Stride = new IncrementalStaticStride(Fs),
Overlap = 32,
ImageLength = 128
};
var spectralImages = spectrumService.CreateLogSpectrogram(audio, config);
Assert.AreEqual((10 * Fs - config.WdftSize) / Fs, spectralImages.Count);
}
public FingerprintSignature CreateAudioFingerprint(string key, string filename, int startPositionInMS, int toReadInMS)
{
SpectrogramConfig spectrogramConfig = new DefaultSpectrogramConfig();
AudioSamples samples = null;
try
{
// First read audio file and downsample it to mono 5512hz
samples = audioEngine.ReadMonoFromFile(filename, spectrogramConfig.SampleRate, startPositionInMS, toReadInMS);
}
catch
{
return(null);
}
// No slice the audio is chunks seperated by 11,6 ms (5512hz 11,6ms = 64 samples!)
// An with length of 371ms (5512kHz 371ms = 2048 samples [rounded])
FingerprintSignature fingerprint = audioEngine.CreateFingerprint(samples, spectrogramConfig);
if (fingerprint != null)
{
fingerprint.Reference = key;
}
return(fingerprint);
}
public void AddAudioPacket(AudioSamples samples)
{
lock (_dataMutex) {
_packets.Add(samples);
PacketsCount = _packets.Count;
}
}
public void SubFingerprintsAreBuiltCorrectlyFromFileForTrack()
{
const int TenSeconds = 10;
var samples = new AudioSamples {
Samples = TestUtilities.GenerateRandomFloatArray(SampleRate * TenSeconds)
};
const int ThreeFingerprints = 3;
var rawFingerprints = GetGenericFingerprints(ThreeFingerprints);
const string PathToAudioFile = "path-to-audio-file";
audioService.Setup(service => service.ReadMonoSamplesFromFile(PathToAudioFile, SampleRate)).Returns(samples);
fingerprintService.Setup(
service => service.CreateFingerprints(samples, It.IsAny <DefaultFingerprintConfiguration>())).Returns(
rawFingerprints);
lshAlgorithm.Setup(service => service.Hash(It.IsAny <Fingerprint>(), NumberOfHashTables, NumberOfHashKeysPerTable)).Returns(new HashedFingerprint(GenericSignature, GenericHashBuckets, 0, 0.928));
var hashDatas = fingerprintCommandBuilder.BuildFingerprintCommand()
.From(PathToAudioFile)
.UsingServices(audioService.Object)
.Hash()
.Result;
Assert.AreEqual(ThreeFingerprints, hashDatas.Count);
foreach (var hashData in hashDatas)
{
Assert.AreSame(GenericSignature, hashData.SubFingerprint);
}
}
bool _ProcessPackets()
{
//check if we have packets
if (!_Process())
{
return(false); //no packets available
}
else
{
AudioPacket p;
uint channelsCount = 2;
lock (_dataMutex) {
p = GetExistingPacket();
channelsCount = Grabber.GetChannels();
}
//Broadcast the grabbed audio packets to the attached players
foreach (var player in AttachedPlayers)
{
AudioSamples samples = new AudioSamples();
samples.channels = channelsCount;
samples.startIndex = p.startIndex;
samples.samples = p.data;
player.AddAudioPacket(samples);
}
}
return(true);
}
private void NormalizeAudioIfNecessary(AudioSamples samples, FingerprintConfiguration configuration)
{
if (configuration.NormalizeSignal)
{
audioSamplesNormalizer.NormalizeInPlace(samples.Samples);
}
}
public List <HashedFingerprint> CreateFingerprints(AudioSamples samples, FingerprintConfiguration configuration)
{
var spectrum = spectrumService.CreateLogSpectrogram(samples, configuration.SpectrogramConfig);
var fingerprints = CreateFingerprintsFromLogSpectrum(spectrum, configuration);
return(HashFingerprints(fingerprints, configuration));
}
public List <Fingerprint> CreateFingerprints(AudioSamples samples, FingerprintConfiguration configuration)
{
NormalizeAudioIfNecessary(samples, configuration);
var spectrum = spectrumService.CreateLogSpectrogram(samples, configuration.SpectrogramConfig);
return(CreateFingerprintsFromLogSpectrum(spectrum, configuration));
}
public List <SpectralImage> CreateLogSpectrogram(AudioSamples audioSamples, SpectrogramConfig configuration)
{
int wdftSize = configuration.WdftSize;
int width = (audioSamples.Samples.Length - wdftSize) / configuration.Overlap;
if (width < 1)
{
return(new List <SpectralImage>());
}
float[] frames = new float[width * configuration.LogBins];
ushort[] logFrequenciesIndexes = logUtility.GenerateLogFrequenciesRanges(audioSamples.SampleRate, configuration);
float[] window = configuration.Window.GetWindow(wdftSize);
float[] samples = audioSamples.Samples;
unsafe
{
Parallel.For(0, width, index =>
{
float *fftArray = stackalloc float[wdftSize];
CopyAndWindow(fftArray, samples, index * configuration.Overlap, window);
fftServiceUnsafe.FFTForwardInPlace(fftArray, wdftSize);
ExtractLogBins(fftArray, logFrequenciesIndexes, configuration.LogBins, wdftSize, frames, index);
});
}
var images = CutLogarithmizedSpectrum(frames, audioSamples.SampleRate, configuration);
ScaleFullSpectrum(images, configuration);
return(images);
}
public float[][] CreateSpectrogram(AudioSamples audioSamples, int overlap, int wdftSize)
{
float[] samples = audioSamples.Samples;
int width = (samples.Length - wdftSize) / overlap;
float[][] frames = new float[width][];
for (int i = 0; i < width; i++)
{
float[] complexSignal = fftService.FFTForward(samples, i * overlap, wdftSize);
float[] band = new float[(wdftSize / 2) + 1];
for (int j = 0; j < (wdftSize / 2) + 1; j++)
{
double re = complexSignal[2 * j];
double img = complexSignal[(2 * j) + 1];
re /= (float)wdftSize / 2;
img /= (float)wdftSize / 2;
band[j] = (float)((re * re) + (img * img));
}
frames[i] = band;
}
return(frames);
}
public void SubFingerprintsAreBuiltCorrectlyFromFileForTrackStartingAtSpecificSecond()
{
const string PathToAudioFile = "path-to-audio-file";
const int StartSecond = 10;
const int SecondsToProcess = 20;
AudioSamples samples = new AudioSamples
{
Samples = TestUtilities.GenerateRandomFloatArray(SampleRate * 10)
};
const int ThreeFingerprints = 3;
var rawFingerprints = GetGenericFingerprints(ThreeFingerprints);
audioService.Setup(service => service.ReadMonoSamplesFromFile(PathToAudioFile, SampleRate, SecondsToProcess, StartSecond)).Returns(samples);
fingerprintService.Setup(service => service.CreateFingerprints(samples, It.IsAny <DefaultFingerprintConfiguration>())).Returns(rawFingerprints);
byte[] genericSignature = GenericSignature();
lshAlgorithm.Setup(service => service.Hash(It.IsAny <Fingerprint>(), NumberOfHashTables, NumberOfHashKeysPerTable, It.IsAny <IEnumerable <string> >())).Returns(
new HashedFingerprint(genericSignature, GenericHashBuckets(), 0, 0, Enumerable.Empty <string>()));
var hashDatas = fingerprintCommandBuilder.BuildFingerprintCommand()
.From(PathToAudioFile, SecondsToProcess, StartSecond)
.UsingServices(audioService.Object)
.Hash()
.Result;
Assert.AreEqual(ThreeFingerprints, hashDatas.Count);
foreach (var hashData in hashDatas)
{
CollectionAssert.AreEqual(genericSignature, hashData.SubFingerprint);
}
audioService.Verify(service => service.ReadMonoSamplesFromFile(PathToAudioFile, SampleRate, SecondsToProcess, StartSecond));
}
public unsafe void do_output_frames(int strideCount)
{
task1.strideCount = strideCount;
if (!task1.UseMappedMemory)
{
task1.openCLCQ.EnqueueWriteBuffer(task1.clSamplesBytes, false, 0, sizeof(int) * stride * strideCount, task1.clSamplesBytesPtr);
}
//task.openCLCQ.EnqueueUnmapMemObject(task.clSamplesBytes, task.clSamplesBytes.HostPtr);
//task.openCLCQ.EnqueueMapBuffer(task.clSamplesBytes, true, MapFlags.WRITE, 0, task.samplesBufferLen / 2);
task1.EnqueueKernels();
if (task2.strideCount > 0)
{
task2.openCLCQ.Finish();
}
int bs = stride * strideCount;
samplesInBuffer -= bs;
if (samplesInBuffer > 0)
{
AudioSamples.MemCpy(
((byte *)task2.clSamplesBytesPtr),
((byte *)task1.clSamplesBytesPtr) + bs * _pcm.BlockAlign,
samplesInBuffer * _pcm.BlockAlign);
}
CLParityTask tmp = task1;
task1 = task2;
task2 = tmp;
task1.strideCount = 0;
}
private static float[] GetQuerySamples(AudioSamples audioSamples, int startAtSecond, int secondsToProcess)
{
int sampleRate = audioSamples.SampleRate;
float[] querySamples = new float[sampleRate * secondsToProcess];
int startAt = startAtSecond * sampleRate;
Array.Copy(audioSamples.Samples, startAt, querySamples, 0, querySamples.Length);
return querySamples;
}
internal void read_bytes(byte *dst, int len)
{
if (ptr_m + len > end_m)
{
throw new IndexOutOfRangeException();
}
AudioSamples.MemCpy(dst, ptr_m, len);
ptr_m += len;
}
public IWithFingerprintConfiguration From(string pathToAudioFile)
{
createFingerprintsMethod = () =>
{
AudioSamples audioSamples = audioService.ReadMonoSamplesFromFile(pathToAudioFile, fingerprintConfiguration.SampleRate);
return(fingerprintService.CreateFingerprints(audioSamples, fingerprintConfiguration));
};
return(this);
}
AudioSamples GetExistingPacket()
{
if (_packets.Count > 0)
{
AudioSamples p = _packets [0];
_packets.RemoveAt(0);
return(p);
}
return(null);
}
protected float[] GetQuerySamples(AudioSamples audioSamples, int startAtSecond, int secondsToProcess)
{
int sampleRate = audioSamples.SampleRate;
float[] querySamples = new float[sampleRate * secondsToProcess];
int startAt = startAtSecond * sampleRate;
Array.Copy(audioSamples.Samples, startAt, querySamples, 0, querySamples.Length);
return(querySamples);
}
public IWithFingerprintConfiguration From(string pathToAudioFile, int secondsToProcess, int startAtSecond)
{
createFingerprintsMethod = () =>
{
AudioSamples audioSamples = audioService.ReadMonoSamplesFromFile(pathToAudioFile, FingerprintConfiguration.SampleRate, secondsToProcess, startAtSecond);
return(fingerprintService.CreateFingerprints(audioSamples, FingerprintConfiguration));
};
return(this);
}
void OnDataArrived(float[] data, uint len)
{
AudioSamples samples = new AudioSamples();
samples.samples = (float[])data.Clone();
samples.channels = _grabber.GetChannels();
lock (_samples) {
_samples.Add(samples);
}
}
unsafe void restore_samples_fixed(FlacFrame frame, int ch)
{
FlacSubframeInfo sub = frame.subframes[ch];
AudioSamples.MemCpy(sub.samples, sub.best.residual, sub.best.order);
int *data = sub.samples + sub.best.order;
int *residual = sub.best.residual + sub.best.order;
int data_len = frame.blocksize - sub.best.order;
int s0, s1, s2;
switch (sub.best.order)
{
case 0:
AudioSamples.MemCpy(data, residual, data_len);
break;
case 1:
s1 = data[-1];
for (int i = data_len; i > 0; i--)
{
s1 += *(residual++);
*(data++) = s1;
}
//data[i] = residual[i] + data[i - 1];
break;
case 2:
s2 = data[-2];
s1 = data[-1];
for (int i = data_len; i > 0; i--)
{
s0 = *(residual++) + (s1 << 1) - s2;
*(data++) = s0;
s2 = s1;
s1 = s0;
}
//data[i] = residual[i] + data[i - 1] * 2 - data[i - 2];
break;
case 3:
for (int i = 0; i < data_len; i++)
{
data[i] = residual[i] + (((data[i - 1] - data[i - 2]) << 1) + (data[i - 1] - data[i - 2])) + data[i - 3];
}
break;
case 4:
for (int i = 0; i < data_len; i++)
{
data[i] = residual[i] + ((data[i - 1] + data[i - 3]) << 2) - ((data[i - 2] << 2) + (data[i - 2] << 1)) - data[i - 4];
}
break;
}
}
public Hashes CreateFingerprintsFromAudioSamples(AudioSamples samples, FingerprintConfiguration configuration)
{
var spectrumFrames = spectrumService.CreateLogSpectrogram(samples, configuration.SpectrogramConfig);
var hashes = CreateOriginalFingerprintsFromFrames(spectrumFrames, configuration)
.AsParallel()
.ToList()
.Select(fingerprint => lshAlgorithm.Hash(fingerprint, configuration.HashingConfig))
.ToList();
return(new Hashes(hashes, samples.Duration, MediaType.Audio, samples.RelativeTo, new[] { samples.Origin }));
}
private void SetupFftService(DefaultSpectrogramConfig configuration, AudioSamples samples)
{
logUtility.Setup(utility => utility.GenerateLogFrequenciesRanges(SampleRate, configuration))
.Returns(new[]
{
118, 125, 133, 141, 149, 158, 167, 177, 187, 198, 210, 223, 236, 250, 264, 280, 297, 314,
333, 352, 373, 395, 419, 443, 470, 497, 527, 558, 591, 626, 663, 702, 744,
});
fftService.Setup(service => service.FFTForward(samples.Samples, It.IsAny <int>(), configuration.WdftSize, It.IsAny <float[]>()))
.Returns(TestUtilities.GenerateRandomFloatArray(2048));
}
AudioSamples CreatePacket()
{
if (_graveYard.Count > 0)
{
AudioSamples p = _graveYard [0];
_graveYard.RemoveAt(0);
p.startIndex = 0;
return(p);
}
return(new AudioSamples());
}
private void DetectAudio(float[] inputAudioSample, int inputSampleRate, int inputChannels)
{
TimeSpan wait = (DateTime.Now - waitUntil);
if (wait.TotalDays < 0 && wait.TotalSeconds < 0)
{
return;
}
// Run separeted task, ignore when one is allready running
lock (lockObject)
{
// if detection task is runnign don't run a new one (otherwhise als er al een detectie draait dan niet verder gaan
if (detectTask != null)
{
return;
}
detectTask = new WSRecognize();
} //lock
Task task = Task.Factory.StartNew(() =>
{
using (AudioEngine ae = new AudioEngine())
{
AudioSamples audio = new AudioSamples();
audio.Channels = inputChannels;
audio.SampleRate = inputSampleRate;
audio.Samples = inputAudioSample;
audio.Origin = "MEMORY";
if (doTimeStretching)
{
Console.WriteLine("Timestretching with " + timeStretchRateFactor.ToString("#0.00") + "f");
audio = ae.TimeStretch(audio.Samples, audio.SampleRate, audio.Channels, timeStretchRateFactor);
}
audio = ae.Resample(audio.Samples, audio.SampleRate, audio.Channels, 5512);
FingerprintSignature fsQuery = ae.CreateFingerprint(audio, SpectrogramConfig.Default);
switch (webServiceCall)
{
case "SLOW":
detectTask.DetectAudioFragmentSlow(fsQuery, null, REST_ResultFingerDetect);
break;
default:
detectTask.DetectAudioFragmentFast(fsQuery, null, REST_ResultFingerDetect);
break;
}
//detectTask.DetectAudioFragment(fsQuery, null, REST_ResultFingerDetect);
} //using
});
}
public void Flush()
{
if (mixbuff != null)
{
if (mixbuff.source[iSource].Length < mixbuff.source[iSource].Size)
{
AudioSamples.MemSet(mixbuff.source[iSource].Bytes, 0, mixbuff.source[iSource].Length * Settings.PCM.BlockAlign, (mixbuff.source[iSource].Size - mixbuff.source[iSource].Length) * Settings.PCM.BlockAlign);
}
mixer.UnlockEmptyBuffer(mixbuff, iSource, volume);
mixbuff = null;
}
}
public void CreateFingerprintFromSamplesWhichAreExactlyEqualToMinimumLength()
{
DefaultFingerprintConfiguration config = new DefaultFingerprintConfiguration();
AudioSamples samples = GenerateRandomAudioSamples(config.SamplesPerFingerprint + config.SpectrogramConfig.WdftSize);
var hash = fingerprintCommandBuilder.BuildFingerprintCommand()
.From(samples)
.UsingServices(bassAudioService)
.Hash()
.Result;
Assert.AreEqual(1, hash.Count);
}
public void ShouldGenerateLogSpectrumFromAudioSamples()
{
int Fs = 5512;
int seconds = 10;
float[] samples = new float[Fs * seconds];
float f1 = 410;
float f2 = 1400;
for (int t = 0; t < samples.Length; ++t)
{
samples[t] = (float)System.Math.Sin(2 * System.Math.PI * f1 / Fs * t)
+ (float)System.Math.Sin(2 * System.Math.PI * f2 / Fs * t);
}
var audio = new AudioSamples(samples, "410Hz", 5512);
var config = new DefaultSpectrogramConfig
{
Stride = new IncrementalStaticStride(5512)
};
var spectralImages = spectrumService.CreateLogSpectrogram(audio, config);
Assert.AreEqual((seconds * Fs - config.WdftSize) / Fs, spectralImages.Count);
// check with logspace(log10(318), log10(2000), 33), 410Hz are located in 4th bin, 1400Hz at 25th (0 indexed)
int tf1 = 4;
int tf2 = 25;
foreach (var image in spectralImages)
{
float[] spectrum = image.Image;
for (int row = 0; row < image.Rows; ++row)
{
for (int col = 0; col < image.Cols; ++col)
{
int index = row * image.Cols + col;
if (col == tf1 || col == tf2)
{
Assert.AreEqual(col == tf1 ? 1 : 0.78, spectrum[index], 0.01);
}
else
{
Assert.AreEqual(0, spectrum[index], 0.001);
}
}
}
}
}
private static void VerifyEndingsAreAttached(float[] prev, float[] next, AudioSamples current, int minSize, int strideSize)
{
int prefixLength = minSize - strideSize;
Assert.AreEqual(minSize + prefixLength, current.Samples.Length);
for (int i = 0; i < prefixLength; i++)
{
Assert.AreEqual(prev[prev.Length - prefixLength + i], current.Samples[i]);
}
for (int i = 0; i < next.Length; ++i)
{
Assert.AreEqual(next[i], current.Samples[prefixLength + i]);
}
}
public static Sample DecodeData(DataMessage dmsg)
{
var returnSample = new Sample(dmsg.DataType, null);
if (dmsg.DataType == DataMessage.Types.DataType.Image)
{
var msg = new ImageSamples();
msg.MergeFrom(dmsg.Data);
returnSample.Data = msg;
return(returnSample);
}
else if (dmsg.DataType == DataMessage.Types.DataType.Audio)
{
var msg = new AudioSamples();
msg.MergeFrom(dmsg.Data);
returnSample.Data = msg;
return(returnSample);
}
else if (dmsg.DataType == DataMessage.Types.DataType.String)
{
var msg = new StringSample();
msg.MergeFrom(dmsg.Data);
returnSample.Data = msg;
return(returnSample);
}
else if (dmsg.DataType == DataMessage.Types.DataType.Float32Arr)
{
var msg = new Float32Array();
msg.MergeFrom(dmsg.Data);
returnSample.Data = msg;
return(returnSample);
}
else if (dmsg.DataType == DataMessage.Types.DataType.Int64Arr)
{
var msg = new Int64Array();
msg.MergeFrom(dmsg.Data);
returnSample.Data = msg;
return(returnSample);
}
else
{
Debug.Log("Unknown message");
return(null);
}
}
public List <SpectralImage> CreateLogSpectrogram(AudioSamples audioSamples, SpectrogramConfig configuration)
{
int width = (audioSamples.Samples.Length - configuration.WdftSize) / configuration.Overlap;
if (width < 1)
{
return(new List <SpectralImage>());
}
float[][] frames = new float[width][];
//int[] logFrequenciesIndexes = logUtility.GenerateLogFrequenciesRanges(audioSamples.SampleRate, configuration);
for (int i = 0; i < width; i++)
{
float[] complexSignal = fftService.FFTForward(audioSamples.Samples, i * configuration.Overlap, configuration.WdftSize);
frames[i] = ExtractLogBins(complexSignal, logFrequenciesIndexes, configuration.LogBins);
}
return(CutLogarithmizedSpectrum(frames, audioSamples.SampleRate, configuration));
}
